diff --git a/browser/cli/src/commands/schema.test.ts b/browser/cli/src/commands/schema.test.ts new file mode 100644 index 000000000..fa53384c4 --- /dev/null +++ b/browser/cli/src/commands/schema.test.ts @@ -0,0 +1,85 @@ +import { mkdtemp, readFile, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { Agent } from '@tomic/lib'; + +const originalCwd = process.cwd(); + +describe('schema command', () => { + afterEach(() => { + process.chdir(originalCwd); + vi.restoreAllMocks(); + }); + + it('registers a code schema and generates ontology bindings from the same Store', async () => { + vi.spyOn(console, 'log').mockImplementation(() => undefined); + + const dir = await mkdtemp(path.join(tmpdir(), 'atomic-schema-cli-')); + const keys = await Agent.generateKeyPair(); + const agentSecret = Agent.buildSecret( + keys.privateKey, + `did:ad:agent:${keys.publicKey}`, + ); + + await writeFile( + path.join(dir, 'atomic.config.json'), + JSON.stringify( + { + outputFolder: './src/ontologies', + ontologies: [], + agentSecret, + _ISLIB_: false, + }, + null, + 2, + ), + ); + await writeFile( + path.join(dir, 'todo-schema.mjs'), + `export default { + name: 'TodoApp', + version: '1.0.0', + classes: { + todo: { + type: 'object', + required: ['title'], + properties: { + title: { type: 'string', description: 'Task title' }, + done: { type: 'boolean' } + } + } + } + };`, + ); + + process.chdir(dir); + + const { schemaCommand } = await import('./schema.js'); + await schemaCommand(['./todo-schema.mjs', '--local', '--generate', '--lock']); + + const generated = await readFile( + path.join(dir, 'src/ontologies/todoApp.ts'), + 'utf8', + ); + const index = await readFile( + path.join(dir, 'src/ontologies/index.ts'), + 'utf8', + ); + + expect(generated).toContain('export const todoapp'); + expect(generated).toContain('classes:'); + expect(generated).toContain('properties:'); + expect(generated).toContain('interface PropTypeMapping'); + expect(generated).toContain('did:ad:'); + expect(index).toContain("from './todoapp.js'"); + + // The committed, self-verifying lockfile is written and verifies. + const { verifySchemaLock } = await import('@tomic/lib'); + const lock = JSON.parse( + await readFile(path.join(dir, 'src/ontologies/TodoApp.schema.lock.json'), 'utf8'), + ); + expect(verifySchemaLock(lock)).toEqual({ ok: true, errors: [] }); + expect(lock.ontology).toMatch(/^did:ad:frozen:[0-9a-f]{64}$/); + }); +}); diff --git a/browser/cli/src/commands/schema.ts b/browser/cli/src/commands/schema.ts new file mode 100644 index 000000000..9b3b23312 --- /dev/null +++ b/browser/cli/src/commands/schema.ts @@ -0,0 +1,249 @@ +/* eslint-disable no-console */ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; +import chalk from 'chalk'; +import { + resolveConfig as prettierResolveConfig, + format as prettierFormat, +} from 'prettier'; +import { + buildSchemaLock, + verifySchemaLock, + type DefinedSchema, + type AtomicSchemaPackage, + type RegisteredSchema, +} from '@tomic/lib'; +import { createConfiguredStore } from '../store.js'; +import { atomicConfig } from '../config.js'; +import { generateOntology } from '../generateOntology.js'; +import { PropertyRecord } from '../PropertyRecord.js'; +import { generateExternals } from '../generateExternals.js'; +import { generateIndex } from '../generateIndex.js'; + +type SchemaModule = Record; +type SchemaInput = DefinedSchema | AtomicSchemaPackage; + +interface SchemaCommandOptions { + exportName: string; + save: boolean; + addToConfig: boolean; + generateBindings: boolean; + lock: boolean; +} + +export const schemaCommand = async (args: string[]) => { + const [schemaPath, ...rest] = args; + + if (!schemaPath) { + console.error( + chalk.red('ERROR: Missing schema module path.'), + '\nUsage: ad-generate schema ./schema.js [--export todoSchema] [--local] [--generate]', + ); + process.exit(1); + } + + const opts = parseOptions(rest); + const schema = await loadSchema(schemaPath, opts.exportName); + const store = await createConfiguredStore(); + + if (opts.save) { + store.setServerConnected(true); + } + + const registered = await store.registerSchema(schema, { save: opts.save }); + + if (opts.addToConfig) { + await addOntologyToConfig(registered.ontology.subject); + } + + if (opts.generateBindings) { + await writeOntologyBindings(registered, store); + } + + if (opts.lock) { + await writeSchemaLock(schema); + } + + printRegisteredSchema(registered, opts.save); +}; + +async function writeSchemaLock(schema: SchemaInput): Promise { + const lock = buildSchemaLock(schema); + const verification = verifySchemaLock(lock); + + if (!verification.ok) { + throw new Error( + `Refusing to write an invalid schema lock:\n${verification.errors.join('\n')}`, + ); + } + + const outputFolder = path.resolve(process.cwd(), atomicConfig.outputFolder); + const filePath = path.join(outputFolder, `${lock.name}.schema.lock.json`); + await fs.mkdir(outputFolder, { recursive: true }); + await fs.writeFile(filePath, `${JSON.stringify(lock, null, 2)}\n`); + + console.log(chalk.blue('Wrote lockfile'), chalk.cyan(filePath)); +} + +function parseOptions(args: string[]): SchemaCommandOptions { + let exportName = 'default'; + let save = true; + let addToConfig = false; + let generateBindings = false; + let lock = false; + + for (let i = 0; i < args.length; i++) { + const arg = args[i]; + + if (arg === '--export' || arg === '-e') { + exportName = args[i + 1]; + i++; + continue; + } + + if (arg === '--local') { + save = false; + continue; + } + + if (arg === '--add-to-config') { + addToConfig = true; + continue; + } + + if (arg === '--generate') { + generateBindings = true; + continue; + } + + if (arg === '--lock') { + lock = true; + continue; + } + + throw new Error(`Unknown schema command option: ${arg}`); + } + + if (!exportName) { + throw new Error('Missing value for --export'); + } + + return { exportName, save, addToConfig, generateBindings, lock }; +} + +async function loadSchema( + schemaPath: string, + exportName: string, +): Promise { + const absolutePath = path.resolve(process.cwd(), schemaPath); + const moduleUrl = pathToFileURL(absolutePath).href; + const module = (await import(moduleUrl)) as SchemaModule; + const value = module[exportName]; + + if (!isSchemaInput(value)) { + throw new Error( + `Export ${exportName} from ${schemaPath} is not a defineSchema() result or schema package.`, + ); + } + + return value; +} + +function isSchemaInput(value: unknown): value is SchemaInput { + if (typeof value !== 'object' || value === null) { + return false; + } + + if ('schemaHash' in value && 'normalized' in value) { + return true; + } + + return 'name' in value && 'classes' in value; +} + +function printRegisteredSchema( + registered: RegisteredSchema, + saved: boolean, +): void { + console.log(chalk.green(saved ? 'Published schema' : 'Registered schema')); + console.log(`${chalk.blue('Ontology:')} ${registered.ontology.subject}`); + console.log(`${chalk.blue('Hash:')} ${registered.model.ontology.schemaHash}`); + console.log( + `${chalk.blue('Classes:')} ${Object.keys(registered.classes).length}`, + ); + console.log( + `${chalk.blue('Properties:')} ${Object.keys(registered.properties).length}`, + ); +} + +async function addOntologyToConfig(subject: string): Promise { + const configPath = path.resolve(process.cwd(), './atomic.config.json'); + const raw = await fs.readFile(configPath, 'utf8'); + const config = JSON.parse(raw) as { ontologies?: string[] }; + const ontologies = config.ontologies ?? []; + + if (!ontologies.includes(subject)) { + config.ontologies = [...ontologies, subject]; + await fs.writeFile(configPath, `${JSON.stringify(config, null, 2)}\n`); + } + + console.log(`${chalk.blue('Config:')} added ${subject}`); +} + +async function writeOntologyBindings( + registered: RegisteredSchema, + store: Awaited>, +): Promise { + const propertyRecord = new PropertyRecord(); + const ontology = await generateOntology( + registered.ontology.subject, + propertyRecord, + store, + ); + + await writeGeneratedFile(ontology); + + const missingProps = propertyRecord.getMissingProperties(); + + if (missingProps.length > 0) { + const externalsContent = await generateExternals(missingProps, store); + await writeGeneratedFile({ + filename: 'externals.ts', + content: externalsContent, + }); + } + + await writeGeneratedFile( + generateIndex( + [registered.ontology.subject], + missingProps.length > 0, + store, + ), + ); +} + +async function writeGeneratedFile({ + filename, + content, +}: { + filename: string; + content: string; +}): Promise { + const outputFolder = path.resolve(process.cwd(), atomicConfig.outputFolder); + const filePath = path.join(outputFolder, filename); + await fs.mkdir(outputFolder, { recursive: true }); + + let formatted = content; + const prettierConfig = await prettierResolveConfig(filePath); + + if (prettierConfig) { + formatted = await prettierFormat(content, { + ...prettierConfig, + parser: 'typescript', + }); + } + + await fs.writeFile(filePath, formatted); + console.log(chalk.blue('Wrote to'), chalk.cyan(filePath)); +} diff --git a/browser/cli/src/config.ts b/browser/cli/src/config.ts index 22115b44a..919354c3f 100644 --- a/browser/cli/src/config.ts +++ b/browser/cli/src/config.ts @@ -20,6 +20,8 @@ export interface AtomicConfig { * If left empty the public agent is used. */ agentSecret?: string; + /** Server used for relative subjects and publishing code-first schemas. */ + serverUrl?: string; /** The list of subjects of your ontologies */ ontologies: string[]; diff --git a/browser/cli/src/generateBaseObject.ts b/browser/cli/src/generateBaseObject.ts index eabca06fe..7692aa17a 100644 --- a/browser/cli/src/generateBaseObject.ts +++ b/browser/cli/src/generateBaseObject.ts @@ -1,4 +1,4 @@ -import { Resource, type Core, core } from '@tomic/lib'; +import { Resource, type Core, core, type Store } from '@tomic/lib'; import { store } from './store.js'; import { camelCaseify, dedupe } from './utils.js'; import chalk from 'chalk'; @@ -13,6 +13,7 @@ type BaseObject = { export const generateBaseObject = async ( ontology: Resource, + activeStore: Store = store, ): Promise<[string, ReverseMapping]> => { if (ontology.error) { throw ontology.error; @@ -23,9 +24,9 @@ export const generateBaseObject = async ( const name = camelCaseify(ontology.props.shortname); const baseObj = { - classes: await listToObj(classes, 'classes'), - properties: await listToObj(properties, 'properties'), - __classDefs: await createClassDefs(classes), + classes: await listToObj(classes, 'classes', activeStore), + properties: await listToObj(properties, 'properties', activeStore), + __classDefs: await createClassDefs(classes, activeStore), }; const objStr = `export const ${name} = { @@ -40,10 +41,11 @@ export const generateBaseObject = async ( const listToObj = async ( list: string[], type: string, + activeStore: Store, ): Promise> => { const entries = await Promise.all( list.map(async subject => { - const resource = await store.getResource(subject); + const resource = await activeStore.getResource(subject); return [camelCaseify(resource.get(core.properties.shortname)), subject]; }), @@ -76,9 +78,10 @@ const listToObj = async ( const createClassDefs = async ( classes: string[], + activeStore: Store, ): Promise> => { const classResources = await Promise.all( - classes.map(async c => await store.getResource(c)), + classes.map(async c => await activeStore.getResource(c)), ); const entries = classResources.map(resource => { diff --git a/browser/cli/src/generateClassExports.ts b/browser/cli/src/generateClassExports.ts index 2d375f748..ea9967b81 100644 --- a/browser/cli/src/generateClassExports.ts +++ b/browser/cli/src/generateClassExports.ts @@ -1,4 +1,4 @@ -import { Resource, urls, type Core } from '@tomic/lib'; +import { Resource, urls, type Core, type Store } from '@tomic/lib'; import { atomicConfig } from './config.js'; import { ReverseMapping } from './generateBaseObject.js'; import { store } from './store.js'; @@ -19,12 +19,13 @@ const NAMESPACE_TEMPLATE = ` export const generateClassExports = ( ontology: Resource, reverseMapping: ReverseMapping, + activeStore: Store = store, ): string => { const classes = ontology.getArray(urls.properties.classes) as string[]; const body = classes .map(subject => { - const res = store.getResourceLoading(subject); + const res = activeStore.getResourceLoading(subject); const objectPath = reverseMapping[subject]; return createExportLine(res.props.shortname, objectPath); diff --git a/browser/cli/src/generateClasses.ts b/browser/cli/src/generateClasses.ts index 00af3d767..5a43fff0e 100644 --- a/browser/cli/src/generateClasses.ts +++ b/browser/cli/src/generateClasses.ts @@ -1,4 +1,4 @@ -import { type Core, type Resource } from '@tomic/lib'; +import { type Core, type Resource, type Store } from '@tomic/lib'; import { store } from './store.js'; import { ReverseMapping } from './generateBaseObject.js'; import { PropertyRecord } from './PropertyRecord.js'; @@ -8,11 +8,12 @@ export const generateClasses = ( ontology: Resource, reverseMapping: ReverseMapping, propertyRecord: PropertyRecord, + activeStore: Store = store, ): string => { const classes = dedupe(ontology.props.classes ?? []); const classStringList = classes.map(subject => { - return generateClass(subject, reverseMapping, propertyRecord); + return generateClass(subject, reverseMapping, propertyRecord, activeStore); }); const innerStr = classStringList.join('\n'); @@ -26,8 +27,9 @@ const generateClass = ( subject: string, reverseMapping: ReverseMapping, propertyRecord: PropertyRecord, + activeStore: Store, ): string => { - const resource = store.getResourceLoading(subject); + const resource = activeStore.getResourceLoading(subject); const transformSubject = (str: string) => { const name = reverseMapping[str]; diff --git a/browser/cli/src/generateExternals.ts b/browser/cli/src/generateExternals.ts index 4a80f56f7..57f2da83f 100644 --- a/browser/cli/src/generateExternals.ts +++ b/browser/cli/src/generateExternals.ts @@ -1,4 +1,4 @@ -import { Core, Datatype, Resource } from '@tomic/lib'; +import { Core, Datatype, Resource, type Store } from '@tomic/lib'; import { atomicConfig } from './config.js'; import { DatatypeToTSTypeMap } from './DatatypeToTSTypeMap.js'; import { store } from './store.js'; @@ -65,9 +65,12 @@ const generateBaseObjectProperties = ( return lines.join('\n'); }; -export const generateExternals = async (props: string[]) => { +export const generateExternals = async ( + props: string[], + activeStore: Store = store, +) => { const properties: Resource[] = await Promise.all( - props.map(p => store.getResource(p)), + props.map(p => activeStore.getResource(p)), ); const baseOjbectProperties = generateBaseObjectProperties(properties); diff --git a/browser/cli/src/generateIndex.ts b/browser/cli/src/generateIndex.ts index 1cb9119fd..e05f4e2ab 100644 --- a/browser/cli/src/generateIndex.ts +++ b/browser/cli/src/generateIndex.ts @@ -1,7 +1,7 @@ import { store } from './store.js'; import { camelCaseify, getExtension } from './utils.js'; import { atomicConfig } from './config.js'; -import type { Core } from '@tomic/lib'; +import type { Core, Store } from '@tomic/lib'; enum Inserts { MODULE_ALIAS = '{{1}}', @@ -30,9 +30,10 @@ export function initOntologies(): void { export const generateIndex = ( ontologies: string[], inludeExternals: boolean, + activeStore: Store = store, ) => { const names = ontologies.map(x => { - const res = store.getResourceLoading(x); + const res = activeStore.getResourceLoading(x); return camelCaseify(res.props.shortname); }); diff --git a/browser/cli/src/generateOntology.ts b/browser/cli/src/generateOntology.ts index 9fc7c5549..789a2794d 100644 --- a/browser/cli/src/generateOntology.ts +++ b/browser/cli/src/generateOntology.ts @@ -8,7 +8,7 @@ import { generateClassExports } from './generateClassExports.js'; import { atomicConfig } from './config.js'; import { PropertyRecord } from './PropertyRecord.js'; -import { Core } from '@tomic/lib'; +import { Core, type Store } from '@tomic/lib'; enum Inserts { MODULE_ALIAS = '{{1}}', @@ -44,11 +44,12 @@ declare module '${Inserts.MODULE_ALIAS}' { export const generateOntology = async ( subject: string, propertyRecord: PropertyRecord, + activeStore: Store = store, ): Promise<{ filename: string; content: string; }> => { - const ontology = await store.getResource(subject); + const ontology = await activeStore.getResource(subject); const properties = dedupe(ontology.props.properties ?? []); @@ -56,14 +57,27 @@ export const generateOntology = async ( propertyRecord.reportPropertyDefined(prop); } - const [baseObjStr, reverseMapping] = await generateBaseObject(ontology); - const classesStr = generateClasses(ontology, reverseMapping, propertyRecord); + const [baseObjStr, reverseMapping] = await generateBaseObject( + ontology, + activeStore, + ); + const classesStr = generateClasses( + ontology, + reverseMapping, + propertyRecord, + activeStore, + ); const [propertiesStr, propertiesImports] = generatePropTypeMapping( ontology, reverseMapping, + activeStore, ); const subToNameStr = generateSubjectToNameMapping(ontology, reverseMapping); - const classExportsStr = generateClassExports(ontology, reverseMapping); + const classExportsStr = generateClassExports( + ontology, + reverseMapping, + activeStore, + ); const content = TEMPLATE.replaceAll( Inserts.MODULE_ALIAS, diff --git a/browser/cli/src/generatePropTypeMapping.ts b/browser/cli/src/generatePropTypeMapping.ts index 87cbf8278..5154e8bdd 100644 --- a/browser/cli/src/generatePropTypeMapping.ts +++ b/browser/cli/src/generatePropTypeMapping.ts @@ -1,4 +1,4 @@ -import { Datatype, Resource, type Core } from '@tomic/lib'; +import { Datatype, Resource, type Core, type Store } from '@tomic/lib'; import { store } from './store.js'; import { ReverseMapping } from './generateBaseObject.js'; import { DatatypeToTSTypeMap } from './DatatypeToTSTypeMap.js'; @@ -7,11 +7,12 @@ import { dedupe } from './utils.js'; export const generatePropTypeMapping = ( ontology: Resource, reverseMapping: ReverseMapping, + activeStore: Store = store, ): [mappingString: string, usedImports: string[]] => { const properties = dedupe(ontology.props.properties ?? []); const lines = properties - .map(subject => generateLine(subject, reverseMapping)) + .map(subject => generateLine(subject, reverseMapping, activeStore)) .join('\n'); const mappingString = `interface PropTypeMapping { @@ -23,8 +24,12 @@ export const generatePropTypeMapping = ( return [mappingString, imports]; }; -const generateLine = (subject: string, reverseMapping: ReverseMapping) => { - const resource = store.getResourceLoading(subject); +const generateLine = ( + subject: string, + reverseMapping: ReverseMapping, + activeStore: Store, +) => { + const resource = activeStore.getResourceLoading(subject); const datatype = resource.props.datatype as Datatype; const type = DatatypeToTSTypeMap[datatype]; diff --git a/browser/cli/src/index.ts b/browser/cli/src/index.ts index 2ca1e9547..0c97c16b5 100644 --- a/browser/cli/src/index.ts +++ b/browser/cli/src/index.ts @@ -18,6 +18,12 @@ commands.set('init', () => import('./commands/init.js').then(m => m.initCommand(process.argv.slice(3))), ); +commands.set('schema', () => + import('./commands/schema.js').then(m => + m.schemaCommand(process.argv.slice(3)), + ), +); + if (commands.has(command)) { commands.get(command)?.(); } else { diff --git a/browser/cli/src/store.ts b/browser/cli/src/store.ts index ca122cbdc..fd8da68f8 100644 --- a/browser/cli/src/store.ts +++ b/browser/cli/src/store.ts @@ -1,6 +1,8 @@ import { Agent, Store } from '@tomic/lib'; import { atomicConfig } from './config.js'; +const DEFAULT_SERVER_URL = 'http://localhost:9883'; + const getCommandIndex = (): number | undefined => { const agentIndex = process.argv.indexOf('--agent'); if (agentIndex !== -1) return agentIndex; @@ -11,7 +13,7 @@ const getCommandIndex = (): number | undefined => { return undefined; }; -const getAgent = async (): Promise => { +export const getAgent = async (): Promise => { let secret; const agentCommandIndex = getCommandIndex(); @@ -26,10 +28,25 @@ const getAgent = async (): Promise => { return Agent.fromSecret(secret, 'js'); }; -export const store = new Store(); +export const store = new Store({ + serverUrl: atomicConfig.serverUrl ?? DEFAULT_SERVER_URL, +}); getAgent().then(agent => { if (agent) { store.setAgent(agent); } }); + +export const createConfiguredStore = async (): Promise => { + const configuredStore = new Store({ + serverUrl: atomicConfig.serverUrl ?? DEFAULT_SERVER_URL, + }); + const agent = await getAgent(); + + if (agent) { + configuredStore.setAgent(agent); + } + + return configuredStore; +}; diff --git a/browser/cli/src/usage.ts b/browser/cli/src/usage.ts index 0169269ee..a4721f660 100644 --- a/browser/cli/src/usage.ts +++ b/browser/cli/src/usage.ts @@ -3,5 +3,11 @@ ad-generate Commands: ontologies Generates typescript files for ontologies specified in the config file. + schema Registers or publishes a code-first schema module. init Creates a template config file. + +Schema command: + ad-generate schema ./schema.js [--export schemaName] [--local] [--add-to-config] [--generate] [--lock] + + --lock Write a committed *.schema.lock.json (the self-verifying, shareable frozen artifact). `; diff --git a/browser/data-browser/src/components/NavBar.tsx b/browser/data-browser/src/components/NavBar.tsx index ba80cdcd8..9acee2d91 100644 --- a/browser/data-browser/src/components/NavBar.tsx +++ b/browser/data-browser/src/components/NavBar.tsx @@ -27,6 +27,7 @@ import { FaBars, FaMagnifyingGlass, FaShare, + FaSnowflake, FaTags, } from 'react-icons/fa6'; import * as RadixPopover from '@radix-ui/react-popover'; @@ -252,6 +253,15 @@ export function NavBar({ resource: resourceProp }: NavBarProps): JSX.Element { {parent && } + {resource.subject.startsWith('did:ad:frozen:') && ( + + + Frozen + + )} p.theme.radius}; + background-color: ${p => p.theme.colors.bg1}; + border: 1px solid ${p => p.theme.colors.bg2}; + color: ${p => p.theme.colors.textLight}; + font-size: 0.8rem; + white-space: nowrap; +`; + const NavBarWrapper = styled.nav` height: 100%; width: 100%; diff --git a/browser/data-browser/src/components/ResourceContextMenu/FreezeDialog.tsx b/browser/data-browser/src/components/ResourceContextMenu/FreezeDialog.tsx new file mode 100644 index 000000000..6bc890609 --- /dev/null +++ b/browser/data-browser/src/components/ResourceContextMenu/FreezeDialog.tsx @@ -0,0 +1,205 @@ +import { useEffect, useState } from 'react'; +import { useResource, useStore } from '@tomic/react'; +import toast from 'react-hot-toast'; +import { styled } from 'styled-components'; +import { + Dialog, + DialogActions, + DialogContent, + DialogTitle, + useDialog, +} from '../Dialog'; +import { Button } from '../Button'; + +interface FreezeDialogProps { + subject: string; + show: boolean; + bindShow: (open: boolean) => void; +} + +type FreezeMode = 'json-ad' | 'loro'; + +/** + * Freezes a resource (and, by default, the structure it references) into + * immutable, content-addressed `did:ad:frozen` JSON-AD. Shows the result with + * copy / download, and can publish it to the server's `/frozen` store. + */ +export function FreezeDialog({ + subject, + show: open, + bindShow, +}: FreezeDialogProps): React.JSX.Element { + const store = useStore(); + const resource = useResource(subject); + const [dialogProps, show, hide, isOpen] = useDialog({ bindShow }); + + const [mode, setMode] = useState('json-ad'); + const [closure, setClosure] = useState(true); + const [json, setJson] = useState(''); + const [error, setError] = useState(); + const [publishing, setPublishing] = useState(false); + + useEffect(() => { + if (open) { + show(); + } else { + hide(); + } + }, [open]); + + useEffect(() => { + if (!isOpen || mode !== 'json-ad') { + return; + } + + let active = true; + setError(undefined); + + store + .freezeStructure(subject, { closure }) + .then(result => { + if (active) { + setJson(JSON.stringify(result.frozen, null, 2)); + } + }) + .catch((e: Error) => { + if (active) { + setError(e.message); + } + }); + + return () => { + active = false; + }; + }, [isOpen, subject, closure, mode, store]); + + const handleCopy = async () => { + await navigator.clipboard.writeText(json); + toast.success('Copied frozen JSON-AD'); + }; + + const handleDownload = () => { + const blob = new Blob([json], { type: 'application/ad+json' }); + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = `${resource.title || 'resource'}.frozen.json`; + a.click(); + URL.revokeObjectURL(url); + }; + + const handlePublish = async () => { + setPublishing(true); + + try { + const result = await store.freezeStructure(subject, { + closure, + save: true, + }); + const count = Object.keys(result.frozen).length; + toast.success(`Published ${count} frozen resource${count === 1 ? '' : 's'}`); + } catch (e) { + toast.error(`Publish failed: ${(e as Error).message}`); + } finally { + setPublishing(false); + } + }; + + return ( + + {isOpen && ( + <> + +

Freeze {resource.title}

+ + + + + + setMode('json-ad')} + /> + JSON-AD reproducible, no history + + + + Loro coming soon + + + + setClosure(e.target.checked)} + /> + Include referenced structure + + + {error ? {error} : 
{json}
} + + + + + + + + )} + + ); +} + +const StyledContent = styled(DialogContent)` + max-height: 90vh; + overflow-x: hidden; +`; + +const Controls = styled.div` + display: flex; + flex-wrap: wrap; + gap: 1rem 2rem; + align-items: center; + margin-bottom: 1rem; +`; + +const ModeGroup = styled.div` + display: flex; + gap: 1.5rem; +`; + +const ModeOption = styled.label<{ $disabled?: boolean }>` + display: inline-flex; + align-items: center; + gap: 0.4rem; + cursor: ${p => (p.$disabled ? 'not-allowed' : 'pointer')}; + opacity: ${p => (p.$disabled ? 0.5 : 1)}; +`; + +const Hint = styled.small` + color: ${p => p.theme.colors.textLight}; +`; + +const Pre = styled.pre` + background: ${p => p.theme.colors.bg1}; + border: 1px solid ${p => p.theme.colors.bg2}; + border-radius: ${p => p.theme.radius}; + padding: 1rem; + overflow: auto; + max-height: 60vh; + font-size: 0.8rem; +`; + +const ErrorText = styled.p` + color: ${p => p.theme.colors.alert}; +`; diff --git a/browser/data-browser/src/components/ResourceContextMenu/index.tsx b/browser/data-browser/src/components/ResourceContextMenu/index.tsx index 3f5a6b7a1..44ef91663 100644 --- a/browser/data-browser/src/components/ResourceContextMenu/index.tsx +++ b/browser/data-browser/src/components/ResourceContextMenu/index.tsx @@ -26,6 +26,7 @@ import { FaPlus, FaArrowUpRightFromSquare, FaMessage, + FaSnowflake, } from 'react-icons/fa6'; import { useQueryScopeHandler } from '../../hooks/useQueryScope'; import { @@ -36,6 +37,7 @@ import { ResourceInline } from '../../views/ResourceInline'; import { ResourceUsage } from '../ResourceUsage'; import { useCurrentSubject } from '../../helpers/useCurrentSubject'; import { ResourceCodeUsageDialog } from '../../views/CodeUsage/ResourceCodeUsageDialog'; +import { FreezeDialog } from './FreezeDialog'; import { useNewRoute } from '../../helpers/useNewRoute'; import { addIf } from '../../helpers/addIf'; import { useNavigateWithTransition } from '../../hooks/useNavigateWithTransition'; @@ -64,6 +66,7 @@ export const ContextMenuOptions = { Export: 'export', Open: 'open', AddToChat: 'addToChat', + Freeze: 'freeze', } as const; export type ContextMenuOptionsUnion = @@ -101,9 +104,14 @@ export function ResourceContextMenu({ const resource = useResource(subject); const [showDeleteDialog, setShowDeleteDialog] = useState(false); const [showCodeUsageDialog, setShowCodeUsageDialog] = useState(false); + const [showFreezeDialog, setShowFreezeDialog] = useState(false); const handleAddClick = useNewRoute(subject); const [currentSubject] = useCurrentSubject(); - const canWrite = useCanWrite(resource); + const writeRight = useCanWrite(resource); + // Frozen resources are content-addressed and immutable — never editable, + // regardless of rights. Hide all write actions and show a frozen badge. + const isFrozen = subject.startsWith('did:ad:frozen:'); + const canWrite = writeRight && !isFrozen; const { enableScope } = useQueryScopeHandler(subject); const { setContextItems, isOpen, setIsOpen } = useAISidebar(); const { items: customItems } = useCustomContextItemsContext(); @@ -202,6 +210,14 @@ export function ResourceContextMenu({ icon: , onClick: () => setShowCodeUsageDialog(true), }, + { + id: ContextMenuOptions.Freeze, + label: 'Freeze', + helper: + 'Create an immutable, content-addressed (did:ad:frozen) copy of this resource and the structure it references.', + icon: , + onClick: () => setShowFreezeDialog(true), + }, { id: ContextMenuOptions.AddToChat, label: 'Add to chat', @@ -302,6 +318,11 @@ export function ResourceContextMenu({ bindShow={setShowCodeUsageDialog} /> )} + ); } diff --git a/browser/data-browser/src/locales/de.po b/browser/data-browser/src/locales/de.po index 9e92ad8ca..b60dc2afa 100644 --- a/browser/data-browser/src/locales/de.po +++ b/browser/data-browser/src/locales/de.po @@ -212,6 +212,7 @@ msgstr "Fehler in die Zwischenablage kopiert" msgid "Clear" msgstr "Leeren" +#: src/components/ResourceContextMenu/FreezeDialog.tsx #: src/components/Toaster.tsx #: src/routes/SyncRoute.tsx msgid "Copy" @@ -4376,3 +4377,67 @@ msgstr "Build-Commit" #: src/routes/AboutRoute.tsx msgid "built {0}" msgstr "erstellt {0}" + +#: src/components/ResourceContextMenu/index.tsx +msgid "Freeze" +msgstr "Einfrieren" + +#: src/components/ResourceContextMenu/index.tsx +msgid "Create an immutable, content-addressed (did:ad:frozen) copy of this resource and the structure it references." +msgstr "Erstellen Sie eine unveränderliche, inhaltsadressierte (did:ad:frozen) Kopie dieser Ressource und der von ihr referenzierten Struktur." + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Copied frozen JSON-AD" +msgstr "Gefrorenes JSON-AD kopiert" + +#. placeholder {0}: count +#. placeholder {1}: count === 1 ? '' : 's' +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Published {0} frozen resource{1}" +msgstr "{0} eingefrorene Ressource{1} veröffentlicht" + +#. placeholder {0}: (e as Error).message +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Publish failed: {0}" +msgstr "Veröffentlichung fehlgeschlagen: {0}" + +#. placeholder {0}: resource.title +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Freeze {0}" +msgstr "{0} einfrieren" + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "<0/> JSON-AD <1>reproducible, no history" +msgstr "<0/> JSON-AD <1>reproduzierbar, keine Historie" + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "<0/> Loro <1>coming soon" +msgstr "<0/> Loro <1>demnächst verfügbar" + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Coming soon — keeps CRDT history, binary, id not reproducible" +msgstr "Bald verfügbar – behält CRDT-Verlauf, binär, ID nicht reproduzierbar" + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Freeze format" +msgstr "Format einfrieren" + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "<0/> Include referenced structure" +msgstr "<0/> Referenzierte Struktur einschließen" + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Download" +msgstr "Herunterladen" + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Publish to server" +msgstr "Auf Server veröffentlichen" + +#: src/components/NavBar.tsx +msgid "Frozen" +msgstr "Eingefroren" + +#: src/components/NavBar.tsx +msgid "Content-addressed and immutable — verified by hash" +msgstr "Inhaltsadressiert und unveränderlich – verifiziert durch Hash" diff --git a/browser/data-browser/src/locales/en.po b/browser/data-browser/src/locales/en.po index 9c85bb5c1..34ebc65a6 100644 --- a/browser/data-browser/src/locales/en.po +++ b/browser/data-browser/src/locales/en.po @@ -2236,6 +2236,7 @@ msgstr "Nothing to copy." msgid "Copied error to clipboard" msgstr "Copied error to clipboard" +#: src/components/ResourceContextMenu/FreezeDialog.tsx #: src/components/Toaster.tsx #: src/routes/SyncRoute.tsx msgid "Copy" @@ -4392,3 +4393,67 @@ msgstr "Build commit" #: src/routes/AboutRoute.tsx msgid "built {0}" msgstr "built {0}" + +#: src/components/ResourceContextMenu/index.tsx +msgid "Freeze" +msgstr "Freeze" + +#: src/components/ResourceContextMenu/index.tsx +msgid "Create an immutable, content-addressed (did:ad:frozen) copy of this resource and the structure it references." +msgstr "Create an immutable, content-addressed (did:ad:frozen) copy of this resource and the structure it references." + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Copied frozen JSON-AD" +msgstr "Copied frozen JSON-AD" + +#. placeholder {0}: count +#. placeholder {1}: count === 1 ? '' : 's' +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Published {0} frozen resource{1}" +msgstr "Published {0} frozen resource{1}" + +#. placeholder {0}: (e as Error).message +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Publish failed: {0}" +msgstr "Publish failed: {0}" + +#. placeholder {0}: resource.title +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Freeze {0}" +msgstr "Freeze {0}" + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "<0/> JSON-AD <1>reproducible, no history" +msgstr "<0/> JSON-AD <1>reproducible, no history" + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "<0/> Loro <1>coming soon" +msgstr "<0/> Loro <1>coming soon" + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Coming soon — keeps CRDT history, binary, id not reproducible" +msgstr "Coming soon — keeps CRDT history, binary, id not reproducible" + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Freeze format" +msgstr "Freeze format" + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "<0/> Include referenced structure" +msgstr "<0/> Include referenced structure" + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Download" +msgstr "Download" + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Publish to server" +msgstr "Publish to server" + +#: src/components/NavBar.tsx +msgid "Frozen" +msgstr "Frozen" + +#: src/components/NavBar.tsx +msgid "Content-addressed and immutable — verified by hash" +msgstr "Content-addressed and immutable — verified by hash" diff --git a/browser/data-browser/src/locales/es.po b/browser/data-browser/src/locales/es.po index b6dbb4578..151d5e28f 100644 --- a/browser/data-browser/src/locales/es.po +++ b/browser/data-browser/src/locales/es.po @@ -197,6 +197,7 @@ msgstr "Error copiado al portapapeles" msgid "Clear" msgstr "Borrar" +#: src/components/ResourceContextMenu/FreezeDialog.tsx #: src/components/Toaster.tsx #: src/routes/SyncRoute.tsx msgid "Copy" @@ -4371,3 +4372,67 @@ msgstr "Compilación commit" #: src/routes/AboutRoute.tsx msgid "built {0}" msgstr "compilado el {0}" + +#: src/components/ResourceContextMenu/index.tsx +msgid "Freeze" +msgstr "Congelar" + +#: src/components/ResourceContextMenu/index.tsx +msgid "Create an immutable, content-addressed (did:ad:frozen) copy of this resource and the structure it references." +msgstr "Crea una copia inmutable, direccionada por contenido (did:ad:frozen) de este recurso y la estructura a la que hace referencia." + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Copied frozen JSON-AD" +msgstr "JSON-AD congelado copiado" + +#. placeholder {0}: count +#. placeholder {1}: count === 1 ? '' : 's' +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Published {0} frozen resource{1}" +msgstr "Publicado {0} recurso{1} congelado" + +#. placeholder {0}: (e as Error).message +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Publish failed: {0}" +msgstr "Error al publicar: {0}" + +#. placeholder {0}: resource.title +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Freeze {0}" +msgstr "Congelar {0}" + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "<0/> JSON-AD <1>reproducible, no history" +msgstr "<0/> JSON-AD <1>reproducible, sin historial" + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "<0/> Loro <1>coming soon" +msgstr "<0/> Loro <1>próximamente" + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Coming soon — keeps CRDT history, binary, id not reproducible" +msgstr "Próximamente — conserva el historial CRDT, binario, id no reproducible" + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Freeze format" +msgstr "Formato de congelación" + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "<0/> Include referenced structure" +msgstr "<0/> Incluir estructura referenciada" + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Download" +msgstr "Descargar" + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Publish to server" +msgstr "Publicar en el servidor" + +#: src/components/NavBar.tsx +msgid "Frozen" +msgstr "Congelado" + +#: src/components/NavBar.tsx +msgid "Content-addressed and immutable — verified by hash" +msgstr "Dirigido por contenido e inmutable — verificado por hash" diff --git a/browser/data-browser/src/locales/fr.po b/browser/data-browser/src/locales/fr.po index 860aaf375..ba75d32d4 100644 --- a/browser/data-browser/src/locales/fr.po +++ b/browser/data-browser/src/locales/fr.po @@ -197,6 +197,7 @@ msgstr "Erreur copiée dans le presse-papier" msgid "Clear" msgstr "Effacer" +#: src/components/ResourceContextMenu/FreezeDialog.tsx #: src/components/Toaster.tsx #: src/routes/SyncRoute.tsx msgid "Copy" @@ -4384,3 +4385,67 @@ msgstr "Construire le commit" #: src/routes/AboutRoute.tsx msgid "built {0}" msgstr "construit {0}" + +#: src/components/ResourceContextMenu/index.tsx +msgid "Freeze" +msgstr "Geler" + +#: src/components/ResourceContextMenu/index.tsx +msgid "Create an immutable, content-addressed (did:ad:frozen) copy of this resource and the structure it references." +msgstr "Créez une copie immuable et adressée par le contenu (did:ad:frozen) de cette ressource et de la structure qu'elle référence." + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Copied frozen JSON-AD" +msgstr "JSON-AD gelé copié" + +#. placeholder {0}: count +#. placeholder {1}: count === 1 ? '' : 's' +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Published {0} frozen resource{1}" +msgstr "{0} ressource{1} gelée{1} publiée{1}" + +#. placeholder {0}: (e as Error).message +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Publish failed: {0}" +msgstr "Échec de la publication : {0}" + +#. placeholder {0}: resource.title +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Freeze {0}" +msgstr "Geler {0}" + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "<0/> JSON-AD <1>reproducible, no history" +msgstr "<0/> JSON-AD <1>reproductible, pas d'historique" + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "<0/> Loro <1>coming soon" +msgstr "<0/> Loro <1>bientôt disponible" + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Coming soon — keeps CRDT history, binary, id not reproducible" +msgstr "Bientôt disponible — conserve l'historique CRDT, binaire, id non reproductible" + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Freeze format" +msgstr "Format de gel" + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "<0/> Include referenced structure" +msgstr "<0/> Inclure la structure référencée" + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Download" +msgstr "Télécharger" + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Publish to server" +msgstr "Publier sur le serveur" + +#: src/components/NavBar.tsx +msgid "Frozen" +msgstr "Frozen" + +#: src/components/NavBar.tsx +msgid "Content-addressed and immutable — verified by hash" +msgstr "Adressable par contenu et immuable — vérifié par hachage" diff --git a/browser/e2e/tests/frozen.spec.ts b/browser/e2e/tests/frozen.spec.ts new file mode 100644 index 000000000..df4bd12cb --- /dev/null +++ b/browser/e2e/tests/frozen.spec.ts @@ -0,0 +1,51 @@ +import { expect, test } from '@playwright/test'; +import { before, contextMenuClick, openSubject } from './test-utils'; + +/** + * The frozen (`did:ad:frozen`) feature, end to end through the data-browser: + * 1. Freeze a resource into immutable, content-addressed JSON-AD. + * 2. Publish it to the server and resolve it back by its frozen id. + * 3. A frozen resource is immutable — no edit affordance. + */ +test.describe('freeze', () => { + test.beforeEach(before); + + test('freezes a resource, publishes it, and resolves the frozen copy', async ({ + page, + }) => { + // The current resource is the dev drive. Freeze it. + await contextMenuClick('freeze', page); + + const dialog = page.locator('dialog[open]'); + await expect( + dialog.getByRole('heading', { name: /^Freeze/ }), + ).toBeVisible(); + + // The frozen JSON-AD is content-addressed: a did:ad:frozen id appears. + const body = dialog.locator('pre'); + await expect(body).toContainText(/did:ad:frozen:[0-9a-f]{64}/, { + timeout: 15000, + }); + + const frozenId = (await body.innerText()).match( + /did:ad:frozen:[0-9a-f]{64}/, + )?.[0]; + expect(frozenId).toBeTruthy(); + + // Publish to the server, then resolve the frozen resource by its id. + await dialog.getByRole('button', { name: 'Publish to server' }).click(); + await expect( + page.getByText(/Published \d+ frozen resource/), + ).toBeVisible({ timeout: 15000 }); + + await openSubject(page, frozenId!); + await expect( + page.locator(`main[about="${frozenId}"]`).first(), + ).toBeVisible({ timeout: 20000 }); + + // A frozen resource is immutable: a Frozen badge, and no Edit affordance. + await expect(page.getByTestId('frozen-badge')).toBeVisible(); + await page.click('[data-test="context-menu"]'); + await expect(page.getByTestId('menu-item-edit')).toHaveCount(0); + }); +}); diff --git a/browser/lib/package.json b/browser/lib/package.json index 866cdfbe0..812c1cbdc 100644 --- a/browser/lib/package.json +++ b/browser/lib/package.json @@ -62,6 +62,13 @@ }, "require": "./dist/ontologies/*.cjs" }, + "./schema.js": { + "import": { + "types": "./dist/src/schema.d.ts", + "default": "./dist/schema.js" + }, + "require": "./dist/schema.cjs" + }, "./client-db.worker.js": "./dist/client-db.worker.js" }, "main-dev": "src/index.ts", diff --git a/browser/lib/src/freeze.test.ts b/browser/lib/src/freeze.test.ts new file mode 100644 index 000000000..a04f9a0ac --- /dev/null +++ b/browser/lib/src/freeze.test.ts @@ -0,0 +1,276 @@ +import { describe, it } from 'vitest'; + +import { + freezeResources, + SELF_PREFIX, + UNIT_MEMBERS_KEY, + type FreezableResource, + type JsonValue, +} from './freeze.js'; + +const CORE_DATATYPE = 'https://atomicdata.dev/properties/datatype'; +const CORE_SHORTNAME = 'https://atomicdata.dev/properties/shortname'; +const CORE_DESCRIPTION = 'https://atomicdata.dev/properties/description'; +const CORE_REQUIRES = 'https://atomicdata.dev/properties/requires'; +const CORE_CLASSTYPE = 'https://atomicdata.dev/properties/classtype'; +const CORE_CLASSES = 'https://atomicdata.dev/properties/classes'; +const CORE_PROPERTIES = 'https://atomicdata.dev/properties/properties'; + +/** prop -> class -> ontology, the common acyclic schema shape. */ +function acyclicSchema(): FreezableResource[] { + return [ + { + localId: 'p:title', + content: { + [CORE_SHORTNAME]: 'title', + [CORE_DATATYPE]: 'string', + [CORE_DESCRIPTION]: 'Task title', + }, + }, + { + localId: 'c:todo', + content: { + [CORE_SHORTNAME]: 'todo', + [CORE_REQUIRES]: ['p:title'], + }, + }, + { + localId: 'o:todoApp', + content: { + [CORE_SHORTNAME]: 'todoApp', + [CORE_CLASSES]: ['c:todo'], + [CORE_PROPERTIES]: ['p:title'], + }, + }, + ]; +} + +describe('freezeResources — acyclic', () => { + it('assigns each resource a distinct did:ad:frozen id', ({ expect }) => { + const { byLocalId } = freezeResources(acyclicSchema()); + + for (const id of byLocalId.values()) { + expect(id).toMatch(/^did:ad:frozen:[0-9a-f]{64}$/); + } + + expect(new Set(byLocalId.values()).size).toBe(3); + }); + + it('rewrites references to the referent frozen id', ({ expect }) => { + const { resources, byLocalId } = freezeResources(acyclicSchema()); + const byId = (localId: string) => + resources.find(r => r.frozenId === byLocalId.get(localId))!; + const todoClass = byId('c:todo'); + const ontology = byId('o:todoApp'); + + expect((todoClass.content as Record)[CORE_REQUIRES]).toEqual( + [byLocalId.get('p:title')], + ); + expect((ontology.content as Record)[CORE_CLASSES]).toEqual([ + byLocalId.get('c:todo'), + ]); + expect((ontology.content as Record)[CORE_PROPERTIES]).toEqual( + [byLocalId.get('p:title')], + ); + }); + + it('is independent of input order', ({ expect }) => { + const forward = freezeResources(acyclicSchema()); + const reversed = freezeResources([...acyclicSchema()].reverse()); + + for (const localId of ['p:title', 'c:todo', 'o:todoApp']) { + expect(reversed.byLocalId.get(localId)).toBe( + forward.byLocalId.get(localId), + ); + } + }); + + it('is independent of property key order', ({ expect }) => { + const reordered: FreezableResource[] = [ + { + localId: 'p:title', + content: { + [CORE_DESCRIPTION]: 'Task title', + [CORE_DATATYPE]: 'string', + [CORE_SHORTNAME]: 'title', + }, + }, + ]; + + expect(freezeResources(reordered).byLocalId.get('p:title')).toBe( + freezeResources([ + { + localId: 'p:title', + content: { + [CORE_SHORTNAME]: 'title', + [CORE_DATATYPE]: 'string', + [CORE_DESCRIPTION]: 'Task title', + }, + }, + ]).byLocalId.get('p:title'), + ); + }); + + it('dedupes identical content across separate runs', ({ expect }) => { + const make = (localId: string): FreezableResource => ({ + localId, + content: { [CORE_SHORTNAME]: 'title', [CORE_DATATYPE]: 'string' }, + }); + + expect(freezeResources([make('a')]).byLocalId.get('a')).toBe( + freezeResources([make('b')]).byLocalId.get('b'), + ); + }); + + it('changes the id when any content (incl. description) changes', ({ + expect, + }) => { + const base = freezeResources(acyclicSchema()).byLocalId.get('p:title'); + const edited = acyclicSchema(); + (edited[0].content as Record)[CORE_DESCRIPTION] = 'Changed'; + + expect(freezeResources(edited).byLocalId.get('p:title')).not.toBe(base); + }); + + it('leaves external (non-localId) references untouched', ({ expect }) => { + const externalProp = 'https://atomicdata.dev/properties/parent'; + const { resources } = freezeResources([ + { + localId: 'c:todo', + content: { [CORE_SHORTNAME]: 'todo', [externalProp]: 'did:ad:someDrive' }, + }, + ]); + + expect( + (resources[0].content as Record)[externalProp], + ).toBe('did:ad:someDrive'); + }); +}); + +/** Person.friend (classtype Person) <-> Person.requires friend: a 2-cycle. */ +function cyclicSchema(): FreezableResource[] { + return [ + { + localId: 'p:friend', + content: { + [CORE_SHORTNAME]: 'friend', + [CORE_DATATYPE]: 'atomicURL', + [CORE_CLASSTYPE]: 'c:person', + }, + }, + { + localId: 'c:person', + content: { + [CORE_SHORTNAME]: 'person', + [CORE_REQUIRES]: ['p:friend'], + }, + }, + ]; +} + +describe('freezeResources — cycles', () => { + it('freezes a cycle as one unit whose members share its id', ({ expect }) => { + const { resources, byLocalId } = freezeResources(cyclicSchema()); + + expect(byLocalId.get('p:friend')).toMatch(/^did:ad:frozen:[0-9a-f]{64}$/); + // Both members resolve to the same unit id; the unit is one frozen object. + expect(byLocalId.get('p:friend')).toBe(byLocalId.get('c:person')); + expect(resources).toHaveLength(1); + expect([...resources[0].unit].sort()).toEqual(['c:person', 'p:friend']); + }); + + it('wraps members under the unit key with intra-cycle self tokens', ({ + expect, + }) => { + const { resources } = freezeResources(cyclicSchema()); + const members = (resources[0].content as Record)[ + UNIT_MEMBERS_KEY + ] as Array>; + + expect(members).toHaveLength(2); + + const friend = members.find(m => m[CORE_SHORTNAME] === 'friend')!; + const person = members.find(m => m[CORE_SHORTNAME] === 'person')!; + + expect(friend[CORE_CLASSTYPE]).toMatch( + new RegExp(`^${SELF_PREFIX}\\d+$`), + ); + expect((person[CORE_REQUIRES] as string[])[0]).toMatch( + new RegExp(`^${SELF_PREFIX}\\d+$`), + ); + }); + + it('is independent of input order', ({ expect }) => { + const forward = freezeResources(cyclicSchema()); + const reversed = freezeResources([...cyclicSchema()].reverse()); + + expect(reversed.byLocalId.get('p:friend')).toBe( + forward.byLocalId.get('p:friend'), + ); + expect(reversed.byLocalId.get('c:person')).toBe( + forward.byLocalId.get('c:person'), + ); + }); + + it('re-hashes the whole unit when one member changes', ({ expect }) => { + const before = freezeResources(cyclicSchema()); + const edited = cyclicSchema(); + (edited[0].content as Record)[CORE_DESCRIPTION] = 'A friend'; + const after = freezeResources(edited); + + expect(after.byLocalId.get('p:friend')).not.toBe( + before.byLocalId.get('p:friend'), + ); + }); + + it('verifies by re-hash: the unit id is blake3(JCS(content))', async ({ + expect, + }) => { + const { blake3 } = await import('@noble/hashes/blake3.js'); + const { bytesToHex, utf8ToBytes } = await import( + '@noble/hashes/utils.js' + ); + const { jcsCanonicalize } = await import('./jcs.js'); + const { resources } = freezeResources(cyclicSchema()); + const unit = resources[0]; + + const recomputed = `did:ad:frozen:${bytesToHex( + blake3(utf8ToBytes(jcsCanonicalize(unit.content))), + )}`; + + expect(recomputed).toBe(unit.frozenId); + }); + + it('handles a self-referential resource as a unit', ({ expect }) => { + const { resources, byLocalId } = freezeResources([ + { + localId: 'c:node', + content: { + [CORE_SHORTNAME]: 'node', + [CORE_CLASSTYPE]: 'c:node', + }, + }, + ]); + + expect(byLocalId.get('c:node')).toMatch(/^did:ad:frozen:[0-9a-f]{64}$/); + expect(resources[0].unit).toEqual(['c:node']); + + const members = (resources[0].content as Record)[ + UNIT_MEMBERS_KEY + ] as Array>; + expect(members[0][CORE_CLASSTYPE]).toMatch( + new RegExp(`^${SELF_PREFIX}\\d+$`), + ); + }); +}); + +describe('freezeResources — validation', () => { + it('rejects duplicate localIds', ({ expect }) => { + expect(() => + freezeResources([ + { localId: 'dup', content: {} }, + { localId: 'dup', content: {} }, + ]), + ).toThrow('unique'); + }); +}); diff --git a/browser/lib/src/freeze.ts b/browser/lib/src/freeze.ts new file mode 100644 index 000000000..003011c60 --- /dev/null +++ b/browser/lib/src/freeze.ts @@ -0,0 +1,451 @@ +import { blake3 } from '@noble/hashes/blake3.js'; +import { bytesToHex, utf8ToBytes } from '@noble/hashes/utils.js'; + +import { jcsCanonicalize } from './jcs.js'; + +export type JsonValue = + | string + | number + | boolean + | null + | JsonValue[] + | { [key: string]: JsonValue }; + +export type FrozenId = `did:ad:frozen:${string}`; + +const FROZEN_PREFIX = 'did:ad:frozen:'; + +/** + * Placeholder for an intra-cycle reference, by the referent's canonical index. + * Part of the frozen-unit format: a materializer rewires `did:ad:frozen:self:{i}` + * to the i-th member of the unit. + */ +export const SELF_PREFIX = 'did:ad:frozen:self:'; + +/** Reserved key carrying the ordered members of a cycle's frozen unit object. */ +export const UNIT_MEMBERS_KEY = 'urn:atomic-freeze:unit'; + +export interface FreezableResource { + /** + * Temporary, unique id. It is used both as this resource's bookkeeping key and + * as the token other resources put inside their `content` to reference it. Any + * string value anywhere in another resource's `content` that equals this + * `localId` is treated as a reference and rewritten to the computed + * {@link FrozenId}. `content` must NOT contain a self-identifier — the subject + * IS the hash, so it is excluded from the hashed bytes. + */ + localId: string; + /** The resource body to freeze. References to other resources are localIds. */ + content: JsonValue; +} + +export interface FrozenResource { + frozenId: FrozenId; + /** + * JSON-AD body with internal references resolved to FrozenIds. For a cycle + * unit this is a `{ [UNIT_MEMBERS_KEY]: [...members] }` wrapper whose members + * reference each other by canonical index. + */ + content: JsonValue; + /** + * The localIds this frozen object covers: one for an ordinary resource, the + * whole cycle for a unit. Members of a cycle are not individually addressable + * — they share the unit's FrozenId and resolve together. + */ + unit: string[]; +} + +export interface FreezeResult { + /** One entry per distinct frozen object (ordinary resource or cycle unit). */ + resources: FrozenResource[]; + /** localId -> FrozenId for every input resource (cycle members map to the unit). */ + byLocalId: Map; +} + +/** + * Content-addresses a set of resources that may reference each other, producing + * a Merkle DAG of `did:ad:frozen:{blake3}` identifiers. A reference is rewritten + * to the referent's hash before hashing, so a parent's id depends on its + * children's ids (the whole content is hashed, including descriptions). + * + * Cyclic references (e.g. a `Person` class with a `friend` property whose + * classtype is `Person`) have no leaf to start from. Each strongly-connected + * group is therefore frozen as a single **unit** object so the id stays + * `blake3(canonical bytes)` and remains verifiable by re-hashing. The cycle's + * members share that unit id and resolve together. + */ +export function freezeResources(input: FreezableResource[]): FreezeResult { + const ids = new Set(input.map(r => r.localId)); + + if (ids.size !== input.length) { + throw new Error('freezeResources: localId values must be unique'); + } + + const byId = new Map(input.map(r => [r.localId, r] as const)); + const edges = new Map>( + input.map(r => [r.localId, collectRefs(r.content, ids)] as const), + ); + + // Tarjan emits SCCs in reverse topological order (sinks first), which is + // exactly the bottom-up order we need: every out-edge of a component points + // at an already-frozen component. + const sccs = stronglyConnectedComponents([...ids], edges); + const frozenIdByLocal = new Map(); + const resources = new Map(); + + for (const scc of sccs) { + const isCycle = + scc.length > 1 || (edges.get(scc[0])?.has(scc[0]) ?? false); + + if (isCycle) { + freezeCycle(scc, byId, edges, frozenIdByLocal, resources); + } else { + freezeSingleton(scc[0], byId, edges, frozenIdByLocal, resources); + } + } + + return { resources: [...resources.values()], byLocalId: frozenIdByLocal }; +} + +function freezeSingleton( + localId: string, + byId: Map, + edges: Map>, + frozenIdByLocal: Map, + out: Map, +): void { + // All references point at earlier (already-frozen) components. + const content = substitute( + byId.get(localId)!.content, + resolvedRefMap(edges.get(localId), frozenIdByLocal), + ); + const frozenId = frozenIdFor(content); + + frozenIdByLocal.set(localId, frozenId); + + const existing = out.get(frozenId); + + if (existing) { + existing.unit.push(localId); + } else { + out.set(frozenId, { frozenId, content, unit: [localId] }); + } +} + +function freezeCycle( + scc: string[], + byId: Map, + edges: Map>, + frozenIdByLocal: Map, + out: Map, +): void { + const sccSet = new Set(scc); + const order = canonicalOrder(scc, sccSet, byId, edges, frozenIdByLocal); + const indexOf = new Map(order.map((id, i) => [id, i] as const)); + + // The unit wraps its members in canonical order; intra-cycle refs become a + // self token (by index), refs that leave the cycle become their FrozenId. + const members = order.map(localId => + substitute( + byId.get(localId)!.content, + cycleRefMap(edges.get(localId), sccSet, indexOf, frozenIdByLocal), + ), + ); + const content: JsonValue = { [UNIT_MEMBERS_KEY]: members }; + const frozenId = frozenIdFor(content); + + for (const localId of order) { + frozenIdByLocal.set(localId, frozenId); + } + + out.set(frozenId, { frozenId, content, unit: [...order] }); +} + +/** + * Deterministic ordering of a cycle's members, independent of input order, via + * color refinement: start each member colored by its content (intra-cycle refs + * blanked), then repeatedly recolor using neighbors' colors until the partition + * stabilizes. Ties (true structural automorphisms — vanishingly rare for + * schemas) are broken by localId, which makes those — and only those — cases + * input-dependent. + */ +function canonicalOrder( + scc: string[], + sccSet: Set, + byId: Map, + edges: Map>, + frozenIdByLocal: Map, +): string[] { + let color = new Map( + scc.map(localId => [ + localId, + hashCanonical( + substitute( + byId.get(localId)!.content, + cycleRefMap(edges.get(localId), sccSet, undefined, frozenIdByLocal), + ), + ), + ]), + ); + + for (let round = 0; round < scc.length; round++) { + const next = new Map( + scc.map(localId => [ + localId, + hashCanonical( + substitute( + byId.get(localId)!.content, + neighborColorRefMap( + edges.get(localId), + sccSet, + color, + frozenIdByLocal, + ), + ), + ), + ]), + ); + + if (partitionSignature(scc, next) === partitionSignature(scc, color)) { + color = next; + break; + } + + color = next; + } + + return [...scc].sort((a, b) => { + const ca = color.get(a)!; + const cb = color.get(b)!; + + if (ca !== cb) { + return ca < cb ? -1 : 1; + } + + return a < b ? -1 : a > b ? 1 : 0; + }); +} + +/** Maps every reference to its already-computed FrozenId (drops unknowns). */ +function resolvedRefMap( + refs: Set | undefined, + frozenIdByLocal: Map, +): Map { + const map = new Map(); + + for (const ref of refs ?? []) { + const frozenId = frozenIdByLocal.get(ref); + + if (frozenId) { + map.set(ref, frozenId); + } + } + + return map; +} + +/** + * For hashing a cycle: intra-cycle refs become a self token (by canonical index, + * or a constant when `indexOf` is omitted during initial coloring); refs that + * leave the cycle become their FrozenId. + */ +function cycleRefMap( + refs: Set | undefined, + sccSet: Set, + indexOf: Map | undefined, + frozenIdByLocal: Map, +): Map { + const map = new Map(); + + for (const ref of refs ?? []) { + if (sccSet.has(ref)) { + map.set(ref, indexOf ? `${SELF_PREFIX}${indexOf.get(ref)}` : SELF_PREFIX); + } else { + const frozenId = frozenIdByLocal.get(ref); + + if (frozenId) { + map.set(ref, frozenId); + } + } + } + + return map; +} + +/** Like {@link cycleRefMap} but intra-cycle refs carry the neighbor's color. */ +function neighborColorRefMap( + refs: Set | undefined, + sccSet: Set, + color: Map, + frozenIdByLocal: Map, +): Map { + const map = new Map(); + + for (const ref of refs ?? []) { + if (sccSet.has(ref)) { + map.set(ref, `${SELF_PREFIX}${color.get(ref)}`); + } else { + const frozenId = frozenIdByLocal.get(ref); + + if (frozenId) { + map.set(ref, frozenId); + } + } + } + + return map; +} + +/** Canonical signature of the equivalence classes induced by `color`. */ +function partitionSignature( + scc: string[], + color: Map, +): string { + const groups = new Map(); + + for (const id of scc) { + const key = color.get(id)!; + const group = groups.get(key); + + if (group) { + group.push(id); + } else { + groups.set(key, [id]); + } + } + + return [...groups.values()] + .map(group => [...group].sort().join(',')) + .sort() + .join('|'); +} + +function collectRefs(value: JsonValue, ids: Set): Set { + const out = new Set(); + + const walk = (node: JsonValue): void => { + if (typeof node === 'string') { + if (ids.has(node)) { + out.add(node); + } + } else if (Array.isArray(node)) { + node.forEach(walk); + } else if (node !== null && typeof node === 'object') { + Object.values(node).forEach(walk); + } + }; + + walk(value); + + return out; +} + +function substitute(value: JsonValue, map: Map): JsonValue { + if (typeof value === 'string') { + return map.get(value) ?? value; + } + + if (Array.isArray(value)) { + return value.map(item => substitute(item, map)); + } + + if (value !== null && typeof value === 'object') { + return Object.fromEntries( + Object.entries(value).map(([key, child]) => [ + key, + substitute(child, map), + ]), + ); + } + + return value; +} + +function hashCanonical(value: JsonValue): string { + return bytesToHex(blake3(utf8ToBytes(jcsCanonicalize(value)))); +} + +/** + * The canonical frozen id for a JSON-AD body: `did:ad:frozen:{blake3(JCS(content))}`. + * The single source of truth shared by production and verification — re-hashing a + * stored frozen object with this must reproduce its id. + */ +export function frozenIdFor(content: JsonValue): FrozenId { + return (FROZEN_PREFIX + hashCanonical(content)) as FrozenId; +} + +/** + * Process-wide registry of frozen bodies by id. Populated whenever frozen + * resources are computed locally (`defineSchema`, `registerFrozenSchema`, + * `loadSchemaLock`), so the Store can lazily PUT a referenced definition to the + * server on save without the caller ever publishing it explicitly. Bodies are + * immutable and content-addressed, so first-write-wins is always safe. + */ +const frozenBodyRegistry = new Map(); + +export function registerFrozenBodies( + resources: ReadonlyArray<{ frozenId: FrozenId; content: JsonValue }>, +): void { + for (const { frozenId, content } of resources) { + if (!frozenBodyRegistry.has(frozenId)) { + frozenBodyRegistry.set(frozenId, content); + } + } +} + +/** The locally-known body for a frozen id, if one has been registered. */ +export function getRegisteredFrozenBody( + frozenId: string, +): JsonValue | undefined { + return frozenBodyRegistry.get(frozenId as FrozenId); +} + +function stronglyConnectedComponents( + nodes: string[], + edges: Map>, +): string[][] { + let counter = 0; + const index = new Map(); + const low = new Map(); + const onStack = new Set(); + const stack: string[] = []; + const result: string[][] = []; + + const connect = (v: string): void => { + index.set(v, counter); + low.set(v, counter); + counter++; + stack.push(v); + onStack.add(v); + + for (const w of edges.get(v) ?? []) { + if (!index.has(w)) { + connect(w); + low.set(v, Math.min(low.get(v)!, low.get(w)!)); + } else if (onStack.has(w)) { + low.set(v, Math.min(low.get(v)!, index.get(w)!)); + } + } + + if (low.get(v) === index.get(v)) { + const component: string[] = []; + let w: string; + + do { + w = stack.pop()!; + onStack.delete(w); + component.push(w); + } while (w !== v); + + result.push(component); + } + }; + + for (const v of nodes) { + if (!index.has(v)) { + connect(v); + } + } + + return result; +} diff --git a/browser/lib/src/frozen-resolve.test.ts b/browser/lib/src/frozen-resolve.test.ts new file mode 100644 index 000000000..5f5e2f668 --- /dev/null +++ b/browser/lib/src/frozen-resolve.test.ts @@ -0,0 +1,298 @@ +import { afterEach, describe, it, vi } from 'vitest'; + +import { Agent } from './agent.js'; +import { JSCryptoProvider } from './CryptoProvider.js'; +import { Datatype } from './datatypes.js'; +import { frozenIdFor } from './freeze.js'; +import { core } from './ontologies/core.js'; +import { JSONADParser } from './parse.js'; +import { buildSchemaLock } from './schema-lock.js'; +import { Store } from './store.js'; + +const body = { + [core.properties.isA]: [core.classes.property], + [core.properties.shortname]: 'title', + [core.properties.datatype]: Datatype.STRING, +}; +const id = frozenIdFor(body); +const hash = id.replace('did:ad:frozen:', ''); + +const ok = (obj: unknown) => ({ + ok: true, + status: 200, + text: async () => JSON.stringify(obj), +}); + +function connectedStore(): Store { + const store = new Store({ serverUrl: 'https://example.com' }); + store.setServerConnected(true); + + return store; +} + +describe('Store frozen resolution', () => { + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it('fetches, verifies, and materializes a frozen resource', async ({ + expect, + }) => { + const fetchMock = vi.fn(async () => ok(body)); + vi.stubGlobal('fetch', fetchMock); + + const resource = await connectedStore().getResource(id); + + expect(resource.subject).toBe(id); + expect(resource.get(core.properties.shortname)).toBe('title'); + expect(resource.get(core.properties.datatype)).toBe(Datatype.STRING); + expect(fetchMock).toHaveBeenCalledWith( + `https://example.com/frozen/${hash}`, + expect.objectContaining({ headers: { Accept: 'application/ad+json' } }), + ); + }); + + it('rejects a body that fails hash verification', async ({ expect }) => { + vi.stubGlobal('fetch', vi.fn(async () => ok({ tampered: true }))); + + await expect(connectedStore().getResource(id)).rejects.toThrow( + /hash verification/i, + ); + }); + + it('errors when the frozen resource is absent', async ({ expect }) => { + vi.stubGlobal( + 'fetch', + vi.fn(async () => ({ ok: false, status: 404, text: async () => '' })), + ); + + await expect(connectedStore().getResource(id)).rejects.toThrow(/404/); + }); +}); + +const todoPackage = { + name: 'TodoApp', + classes: { + todo: { + type: 'object' as const, + required: ['title'], + properties: { + title: { type: 'string' as const, description: 'Task title' }, + done: { type: 'boolean' as const }, + }, + }, + }, +}; + +describe('Store.registerFrozenSchema', () => { + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it('materializes frozen resources locally so a property resolves offline', async ({ + expect, + }) => { + // Any network call would throw — local materialization must need none. + vi.stubGlobal( + 'fetch', + vi.fn(async () => { + throw new Error('should not hit the network'); + }), + ); + + const store = new Store({ serverUrl: 'https://example.com' }); + const frozen = await store.registerFrozenSchema(todoPackage); + const titleId = frozen.properties['todo.title']; + + expect(titleId).toMatch(/^did:ad:frozen:[0-9a-f]{64}$/); + + const prop = await store.getProperty(titleId); + expect(prop.shortname).toBe('title'); + expect(prop.datatype).toBe(Datatype.STRING); + // Description is presentation — excluded from the frozen body. + expect(prop.description).toBe(''); + expect(frozen.presentation.properties['todo.title'].description).toBe( + 'Task title', + ); + }); + + it('publishes every frozen resource to /frozen with { save: true }', async ({ + expect, + }) => { + const calls: string[] = []; + vi.stubGlobal( + 'fetch', + vi.fn(async (url: string, init: { method: string }) => { + calls.push(`${init.method} ${url}`); + + return { ok: true, status: 204, text: async () => '' }; + }), + ); + + const store = new Store({ serverUrl: 'https://example.com' }); + const frozen = await store.registerFrozenSchema(todoPackage, { + save: true, + }); + + expect(calls.length).toBe(frozen.resources.length); + expect( + calls.every(call => + call.startsWith('PUT https://example.com/frozen/'), + ), + ).toBe(true); + }); +}); + +describe('Store.loadSchemaLock', () => { + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it('makes a bundled lockfile resolve offline with no server', async ({ + expect, + }) => { + // The lockfile is the only input — any network call would throw. + vi.stubGlobal( + 'fetch', + vi.fn(async () => { + throw new Error('should not hit the network'); + }), + ); + + const lock = buildSchemaLock(todoPackage); + const store = new Store({ serverUrl: 'https://example.com' }); + store.loadSchemaLock(lock); + + const titleId = lock.presentation.properties['todo.title'].id; + const prop = await store.getProperty(titleId); + + expect(prop.shortname).toBe('title'); + expect(prop.datatype).toBe(Datatype.STRING); + + const todoClass = await store.getResource(lock.presentation.classes.todo.id); + expect(todoClass.get(core.properties.shortname)).toBe('todo'); + }); + + it('rejects a tampered lockfile', ({ expect }) => { + const lock = buildSchemaLock(todoPackage); + const [firstId] = Object.keys(lock.frozen); + const tampered = { + ...lock, + frozen: { + ...lock.frozen, + [firstId]: { ...(lock.frozen[firstId] as object), tampered: true }, + }, + }; + + const store = new Store({ serverUrl: 'https://example.com' }); + expect(() => store.loadSchemaLock(tampered as typeof lock)).toThrow( + /invalid schema lock/i, + ); + }); +}); + +describe('Store.createSchemaPointer', () => { + it('creates a signed Ontology pointing at the frozen ids', async ({ + expect, + }) => { + const keys = await Agent.generateKeyPair(); + const store = new Store({ serverUrl: 'https://example.com' }); + store.setAgent( + new Agent( + new JSCryptoProvider(keys.privateKey), + `did:ad:agent:${keys.publicKey}`, + ), + ); + + const frozen = await store.registerFrozenSchema({ + ...todoPackage, + version: '1.0.0', + }); + const pointer = await store.createSchemaPointer(frozen); + + // Stable, signed DID — the durable "name" for the latest version. + expect(pointer.subject).toMatch(/^did:ad:/); + expect(pointer.hasClasses(core.classes.ontology)).toBe(true); + // Its members point at the immutable frozen ids. + expect(pointer.props.classes).toEqual([frozen.classes.todo]); + expect(pointer.props.properties).toEqual( + expect.arrayContaining([ + frozen.properties['todo.title'], + frozen.properties['todo.done'], + ]), + ); + }); +}); + +describe('Store.freezeStructure', () => { + const P = 'https://atomicdata.dev/properties/'; + const propSubject = 'https://my.drive/p/title'; + const classSubject = 'https://my.drive/c/todo'; + const ontSubject = 'https://my.drive/o/todoapp'; + + const seeded = (): Store => { + const store = new Store({ serverUrl: 'https://my.drive' }); + + const add = (obj: Record) => { + const [res] = new JSONADParser().parse(obj, obj['@id'] as string); + res.loading = false; + store.addResource(res, { skipCommitCompare: true }); + }; + + add({ + '@id': propSubject, + [`${P}isA`]: ['https://atomicdata.dev/classes/Property'], + [`${P}shortname`]: 'title', + [`${P}datatype`]: Datatype.STRING, + }); + add({ + '@id': classSubject, + [`${P}isA`]: ['https://atomicdata.dev/classes/Class'], + [`${P}shortname`]: 'todo', + [`${P}requires`]: [propSubject], + }); + add({ + '@id': ontSubject, + [`${P}isA`]: ['https://atomicdata.dev/class/ontology'], + [`${P}shortname`]: 'todoapp', + [`${P}classes`]: [classSubject], + [`${P}properties`]: [propSubject], + [`${P}parent`]: 'https://my.drive', + }); + + return store; + }; + + it('freezes a resource and the structure it references', async ({ + expect, + }) => { + const frozen = await seeded().freezeStructure(ontSubject); + + expect(Object.keys(frozen.frozen)).toHaveLength(3); + expect(frozen.root).toBe(frozen.bySubject[ontSubject]); + expect(frozen.root).toMatch(/^did:ad:frozen:[0-9a-f]{64}$/); + + // Cross-references rewritten to frozen ids; hierarchy stripped. + const ontBody = frozen.frozen[frozen.root] as Record; + expect(ontBody[`${P}classes`]).toEqual([frozen.bySubject[classSubject]]); + expect(ontBody[`${P}properties`]).toEqual([frozen.bySubject[propSubject]]); + expect(ontBody[`${P}parent`]).toBeUndefined(); + + const classBody = frozen.frozen[frozen.bySubject[classSubject]] as Record< + string, + unknown + >; + expect(classBody[`${P}requires`]).toEqual([frozen.bySubject[propSubject]]); + }); + + it('freezes only the root with { closure: false }', async ({ expect }) => { + const frozen = await seeded().freezeStructure(ontSubject, { + closure: false, + }); + + expect(Object.keys(frozen.frozen)).toHaveLength(1); + // The reference stays as the original subject, not rewritten. + const ontBody = frozen.frozen[frozen.root] as Record; + expect(ontBody[`${P}classes`]).toEqual([classSubject]); + }); +}); diff --git a/browser/lib/src/frozen-vectors.test.ts b/browser/lib/src/frozen-vectors.test.ts new file mode 100644 index 000000000..26ed255fb --- /dev/null +++ b/browser/lib/src/frozen-vectors.test.ts @@ -0,0 +1,31 @@ +import { readFileSync } from 'node:fs'; +import path from 'node:path'; +import { describe, it } from 'vitest'; + +import { frozenIdFor, type JsonValue } from './freeze.js'; + +interface Vector { + name: string; + body: JsonValue; + id: string; +} + +const fixturePath = path.resolve( + process.cwd(), + '../../test-vectors/frozen.json', +); +const { vectors } = JSON.parse(readFileSync(fixturePath, 'utf8')) as { + vectors: Vector[]; +}; + +describe('cross-language frozen vectors', () => { + it('has vectors to check', ({ expect }) => { + expect(vectors.length).toBeGreaterThan(0); + }); + + for (const vector of vectors) { + it(`reproduces the id for "${vector.name}"`, ({ expect }) => { + expect(frozenIdFor(vector.body)).toBe(vector.id); + }); + } +}); diff --git a/browser/lib/src/index.ts b/browser/lib/src/index.ts index cf9e96294..1e06b4e2a 100644 --- a/browser/lib/src/index.ts +++ b/browser/lib/src/index.ts @@ -45,6 +45,8 @@ export * from './error.js'; export * from './datatypes.js'; export * from './parse.js'; export * from './search.js'; +export * from './schema.js'; +export * from './schema-lock.js'; export * from './resource.js'; export * from './store.js'; export * from './subject.js'; diff --git a/browser/lib/src/jcs.test.ts b/browser/lib/src/jcs.test.ts new file mode 100644 index 000000000..0080eb5d3 --- /dev/null +++ b/browser/lib/src/jcs.test.ts @@ -0,0 +1,42 @@ +import { describe, it } from 'vitest'; + +import { jcsCanonicalize } from './jcs.js'; + +describe('jcsCanonicalize (RFC 8785)', () => { + it('sorts object keys by UTF-16 code unit', ({ expect }) => { + expect(jcsCanonicalize({ b: 1, a: 2 })).toBe('{"a":2,"b":1}'); + }); + + it('sorts nested objects and preserves array order', ({ expect }) => { + expect(jcsCanonicalize({ z: [3, 1, 2], a: { d: 1, c: 2 } })).toBe( + '{"a":{"c":2,"d":1},"z":[3,1,2]}', + ); + }); + + it('orders ascii before higher code points', ({ expect }) => { + expect(jcsCanonicalize({ 'é': 1, a: 2 })).toBe('{"a":2,"é":1}'); + }); + + it('serializes numbers with ECMAScript semantics', ({ expect }) => { + expect(jcsCanonicalize(1.0)).toBe('1'); + expect(jcsCanonicalize(1.5)).toBe('1.5'); + expect(jcsCanonicalize(-0)).toBe('0'); + }); + + it('escapes strings minimally like JSON.stringify', ({ expect }) => { + expect(jcsCanonicalize('a"b\\c')).toBe('"a\\"b\\\\c"'); + }); + + it('serializes primitives', ({ expect }) => { + expect(jcsCanonicalize(null)).toBe('null'); + expect(jcsCanonicalize(true)).toBe('true'); + expect(jcsCanonicalize(false)).toBe('false'); + }); + + it('rejects non-finite numbers', ({ expect }) => { + expect(() => jcsCanonicalize(Number.NaN)).toThrow('non-finite'); + expect(() => jcsCanonicalize(Number.POSITIVE_INFINITY)).toThrow( + 'non-finite', + ); + }); +}); diff --git a/browser/lib/src/jcs.ts b/browser/lib/src/jcs.ts new file mode 100644 index 000000000..f048b6359 --- /dev/null +++ b/browser/lib/src/jcs.ts @@ -0,0 +1,58 @@ +/** + * RFC 8785 JSON Canonicalization Scheme (JCS). + * + * Produces the canonical serialization used for all content-addressed + * `did:ad:frozen` hashing, so the same value hashes identically here and in any + * other conformant implementation — notably the `serde_jcs` crate on the Rust + * side. Using a named standard (rather than an ad-hoc stable stringify) is what + * makes cross-language frozen ids byte-for-byte reproducible. + * + * Covers the JSON value space we hash: + * - objects: keys sorted by UTF-16 code unit (`Array.prototype.sort` default, + * which RFC 8785 §3.2.3 mandates) + * - arrays: order preserved + * - strings and finite numbers: ECMAScript `JSON.stringify` semantics, which + * RFC 8785 §3.2.2.2/§3.2.2.3 reference directly + * - booleans and null + * + * Non-finite numbers are rejected (JSON has no representation for them). + */ +export type JcsValue = + | string + | number + | boolean + | null + | JcsValue[] + | { [key: string]: JcsValue }; + +export function jcsCanonicalize(value: JcsValue): string { + if (value === null) { + return 'null'; + } + + if (typeof value === 'boolean') { + return value ? 'true' : 'false'; + } + + if (typeof value === 'number') { + if (!Number.isFinite(value)) { + throw new Error(`JCS cannot serialize a non-finite number: ${value}`); + } + + return JSON.stringify(value); + } + + if (typeof value === 'string') { + return JSON.stringify(value); + } + + if (Array.isArray(value)) { + return `[${value.map(jcsCanonicalize).join(',')}]`; + } + + const keys = Object.keys(value).sort(); + + return `{${keys + .map(key => `${JSON.stringify(key)}:${jcsCanonicalize(value[key])}`) + .join(',')}}`; +} diff --git a/browser/lib/src/schema-lock.test.ts b/browser/lib/src/schema-lock.test.ts new file mode 100644 index 000000000..f8dfb6a74 --- /dev/null +++ b/browser/lib/src/schema-lock.test.ts @@ -0,0 +1,100 @@ +import { describe, it } from 'vitest'; + +import { buildSchemaLock, verifySchemaLock } from './schema-lock.js'; +import { type AtomicSchemaPackage } from './schema.js'; + +const todoSchema: AtomicSchemaPackage = { + name: 'TodoApp', + version: '1.0.0', + classes: { + todo: { + title: 'Todo', + description: 'A task', + type: 'object', + required: ['title'], + properties: { + title: { type: 'string', description: 'Task title' }, + done: { type: 'boolean' }, + }, + }, + }, +}; + +describe('buildSchemaLock', () => { + it('produces a self-verifying lockfile', ({ expect }) => { + const lock = buildSchemaLock(todoSchema); + + expect(verifySchemaLock(lock)).toEqual({ ok: true, errors: [] }); + }); + + it('decodes every frozen id in @index', ({ expect }) => { + const lock = buildSchemaLock(todoSchema); + + for (const id of Object.keys(lock.frozen)) { + expect(lock['@index'][id]).toBeDefined(); + } + + expect(lock['@index'][lock.ontology]).toBe('TodoApp'); + expect( + lock['@index'][lock.presentation.properties['todo.title'].id], + ).toBe('TodoApp.title'); + }); + + it('keeps descriptions in presentation, not in the hashed objects', ({ + expect, + }) => { + const lock = buildSchemaLock(todoSchema); + const titleId = lock.presentation.properties['todo.title'].id; + + expect(lock.presentation.properties['todo.title'].description).toBe( + 'Task title', + ); + expect( + JSON.stringify(lock.frozen[titleId]).includes('Task title'), + ).toBe(false); + }); + + it('is deterministic across runs', ({ expect }) => { + expect(JSON.stringify(buildSchemaLock(todoSchema))).toBe( + JSON.stringify(buildSchemaLock(todoSchema)), + ); + }); + + it('keeps frozen ids stable when only a description changes', ({ expect }) => { + const base = buildSchemaLock(todoSchema); + const reworded = buildSchemaLock({ + ...todoSchema, + classes: { + todo: { + ...todoSchema.classes.todo, + properties: { + ...todoSchema.classes.todo.properties, + title: { type: 'string', description: 'Different' }, + }, + }, + }, + }); + + expect(Object.keys(reworded.frozen).sort()).toEqual( + Object.keys(base.frozen).sort(), + ); + }); +}); + +describe('verifySchemaLock', () => { + it('fails when a frozen object is tampered with', ({ expect }) => { + const lock = buildSchemaLock(todoSchema); + const [firstId] = Object.keys(lock.frozen); + const tampered = { + ...lock, + frozen: { + ...lock.frozen, + [firstId]: { ...(lock.frozen[firstId] as object), tampered: true }, + }, + }; + + const result = verifySchemaLock(tampered as typeof lock); + expect(result.ok).toBe(false); + expect(result.errors[0]).toContain(firstId); + }); +}); diff --git a/browser/lib/src/schema-lock.ts b/browser/lib/src/schema-lock.ts new file mode 100644 index 000000000..b01ea7100 --- /dev/null +++ b/browser/lib/src/schema-lock.ts @@ -0,0 +1,126 @@ +import { frozenIdFor, type FrozenId, type JsonValue } from './freeze.js'; +import { + freezeSchema, + type AtomicSchemaPackage, + type DefinedSchema, + type SchemaHash, +} from './schema.js'; + +/** + * The committed, shareable schema artifact. A frozen id is a deterministic + * function of the schema source, so this file makes a schema *available* without + * any server: any implementation re-hashes the `frozen` objects to verify, and + * registers them locally. See `verifySchemaLock`. + * + * Only `frozen` is hashed (identity-only JSON-AD). `@index` and `presentation` + * are non-hashed human/metadata aids — editing them never moves an id. + */ +export interface SchemaLock { + readonly name: string; + readonly version?: string; + readonly ontology: FrozenId; + /** Frozen id -> `Ontology.shortname`, a human decoder. Not hashed. */ + readonly '@index': Record; + /** Frozen id -> identity-only canonical JSON-AD body. The hashed objects. */ + readonly frozen: Record; + readonly presentation: SchemaLockPresentation; +} + +export interface SchemaLockPresentationEntry { + readonly id: FrozenId; + readonly description: string; +} + +export interface SchemaLockPresentation { + readonly ontology: { + readonly description: string; + readonly version?: string; + readonly schemaHash: SchemaHash; + readonly jsonSchema: AtomicSchemaPackage; + }; + readonly classes: Record; + readonly properties: Record; +} + +/** Builds the committed lockfile for a schema (deterministic, server-free). */ +export function buildSchemaLock( + input: AtomicSchemaPackage | DefinedSchema, +): SchemaLock { + const frozen = freezeSchema(input); + const name = frozen.model.ontology.shortname; + + const index: Record = {}; + + const addIndex = (id: FrozenId, label: string): void => { + // Cycle members share a unit id; join their labels rather than clobber. + index[id] = index[id] ? `${index[id]} + ${label}` : label; + }; + + addIndex(frozen.ontology, name); + + for (const klass of frozen.model.classes) { + addIndex(frozen.classes[klass.key], `${name}.${klass.shortname}`); + } + + for (const property of frozen.model.properties) { + addIndex(frozen.properties[property.key], `${name}.${property.shortname}`); + } + + const frozenObjects: Record = {}; + + for (const resource of frozen.resources) { + frozenObjects[resource.frozenId] = resource.content; + } + + return { + name, + version: frozen.model.ontology.version, + ontology: frozen.ontology, + '@index': index, + frozen: frozenObjects, + presentation: { + ontology: frozen.presentation.ontology, + classes: mapPresentation(frozen.presentation.classes), + properties: mapPresentation(frozen.presentation.properties), + }, + }; +} + +export interface SchemaLockVerification { + readonly ok: boolean; + readonly errors: string[]; +} + +/** + * Re-hashes every frozen object and checks it matches its id. This is the + * language-neutral verification a consumer (or a CI stale-lockfile guard) runs; + * it depends only on JCS + blake3, never on the freeze algorithm. + */ +export function verifySchemaLock(lock: SchemaLock): SchemaLockVerification { + const errors: string[] = []; + + for (const [id, object] of Object.entries(lock.frozen)) { + const actual = frozenIdFor(object); + + if (actual !== id) { + errors.push(`Frozen object "${id}" hashes to "${actual}"`); + } + } + + if (!(lock.ontology in lock.frozen)) { + errors.push(`Ontology id "${lock.ontology}" is missing from "frozen"`); + } + + return { ok: errors.length === 0, errors }; +} + +function mapPresentation( + entries: Record, +): Record { + return Object.fromEntries( + Object.entries(entries).map(([key, value]) => [ + key, + { id: value.frozenId, description: value.description }, + ]), + ); +} diff --git a/browser/lib/src/schema.test.ts b/browser/lib/src/schema.test.ts new file mode 100644 index 000000000..e4487f1f9 --- /dev/null +++ b/browser/lib/src/schema.test.ts @@ -0,0 +1,913 @@ +import { describe, it, vi } from 'vitest'; + +import { Agent } from './agent.js'; +import type { Commit } from './commit.js'; +import { JSCryptoProvider } from './CryptoProvider.js'; +import { + canonicalizeSchemaPackage, + defineSchema, + freezeSchema, + hashSchemaPackage, + SCHEMA_HASH_PROPERTY, + schemaToOntologyModel, + type AtomicSchemaPackage, +} from './schema.js'; +import { Datatype } from './datatypes.js'; +import { core } from './ontologies/core.js'; +import { getKnownNameBySubject } from './ontology.js'; +import { Store } from './store.js'; + +interface MockClientDbEntry { + json: string; + snapshot?: Uint8Array; +} + +function attachMockClientDb( + store: Store, + dbState = new Map(), +): Map { + store.setClientDb({ + isReady: true, + isInitialized: true, + initError: undefined, + putResourceWithSnapshot: vi.fn( + async (subject: string, json: string, snapshot?: Uint8Array) => { + dbState.set(subject, { json, snapshot }); + }, + ), + getResource: async (subject: string) => dbState.get(subject)?.json ?? null, + getResourceWithSnapshot: async (subject: string) => { + const entry = dbState.get(subject); + + return { jsonAd: entry?.json ?? null, snapshot: entry?.snapshot }; + }, + getLoroSnapshot: async (subject: string) => dbState.get(subject)?.snapshot, + waitForInit: async () => true, + waitForReady: async () => true, + } as unknown as Parameters[0]); + + return dbState; +} + +const todoSchema: AtomicSchemaPackage = { + name: 'TodoApp', + version: '1.0.0', + classes: { + todo: { + title: 'Todo', + description: 'A task', + type: 'object', + required: ['title'], + properties: { + title: { + type: 'string', + description: 'Task title', + }, + done: { + type: 'boolean', + default: false, + }, + }, + }, + }, +}; + +describe('schema package hashing', () => { + it('hashes equivalent schemas the same regardless of object key order', ({ + expect, + }) => { + const differentlyOrdered: AtomicSchemaPackage = { + classes: { + todo: { + properties: { + done: { + default: false, + type: 'boolean', + }, + title: { + description: 'Task title', + type: 'string', + }, + }, + required: ['title'], + type: 'object', + description: 'A task', + title: 'Todo', + }, + }, + version: '1.0.0', + name: 'TodoApp', + }; + + expect(hashSchemaPackage(differentlyOrdered)).toBe( + hashSchemaPackage(todoSchema), + ); + }); + + it('changes the hash when a property datatype changes', ({ expect }) => { + const changed: AtomicSchemaPackage = { + ...todoSchema, + classes: { + todo: { + ...todoSchema.classes.todo, + properties: { + ...todoSchema.classes.todo.properties, + title: { + type: 'integer', + description: 'Task title', + }, + }, + }, + }, + }; + + expect(hashSchemaPackage(changed)).not.toBe(hashSchemaPackage(todoSchema)); + }); + + it('normalizes away undefined fields before canonicalization', ({ + expect, + }) => { + const withUndefined = { + ...todoSchema, + description: undefined, + } as AtomicSchemaPackage; + + expect(canonicalizeSchemaPackage(withUndefined)).toBe( + canonicalizeSchemaPackage(todoSchema), + ); + }); + + it('rejects non-finite numbers', ({ expect }) => { + const invalid = { + ...todoSchema, + classes: { + todo: { + ...todoSchema.classes.todo, + properties: { + ratio: { + type: 'number', + default: Number.NaN, + }, + }, + }, + }, + } as AtomicSchemaPackage; + + expect(() => hashSchemaPackage(invalid)).toThrow('non-finite number'); + }); +}); + +describe('defineSchema', () => { + it('returns the original schema, normalized schema, and schema hash', ({ + expect, + }) => { + const defined = defineSchema(todoSchema); + + expect(defined.schema).toBe(todoSchema); + expect(defined.normalized).toEqual(todoSchema); + expect(defined.schemaHash).toMatch(/^blake3:[0-9a-f]{64}$/); + }); + + it('supports pinned external ontology imports', ({ expect }) => { + const external = defineSchema({ + name: 'ProjectApp', + imports: { + todo: { + subject: 'did:ad:todoOntology', + expectedHash: defineSchema(todoSchema).schemaHash, + }, + }, + classes: { + project: { + type: 'object', + properties: { + title: { + $ref: 'todo.properties.title', + }, + }, + }, + }, + }); + + expect(external.normalized.imports?.todo.subject).toBe( + 'did:ad:todoOntology', + ); + expect(external.normalized.imports?.todo.expectedHash).toMatch( + /^blake3:[0-9a-f]{64}$/, + ); + }); + + it('exposes content-addressed frozen ids as typed class/property handles', ({ + expect, + }) => { + // Inline literal (not annotated) so the generic captures the literal keys — + // `.classes.todo` / `.properties.title` are typed, autocompleted handles. + const defined = defineSchema({ + name: 'TodoApp', + version: '1.0.0', + classes: { + todo: { + type: 'object', + required: ['title'], + properties: { + title: { type: 'string' }, + done: { type: 'boolean' }, + }, + }, + }, + }); + + expect(defined.classes.todo).toMatch(/^did:ad:frozen:[0-9a-f]{64}$/); + expect(defined.properties.title).toMatch(/^did:ad:frozen:[0-9a-f]{64}$/); + expect(defined.properties.done).toMatch(/^did:ad:frozen:[0-9a-f]{64}$/); + + // The handles are exactly the frozen ids, addressable with no server. + const frozen = freezeSchema(defined); + expect(defined.classes.todo).toBe(frozen.classes.todo); + expect(defined.properties.title).toBe(frozen.properties['todo.title']); + }); + + it('computes handles deterministically and locally (no server)', ({ + expect, + }) => { + const a = defineSchema(todoSchema).classes.todo; + const b = defineSchema(todoSchema).classes.todo; + + expect(a).toBe(b); + }); + + it('registers the schema at runtime so props resolve by shortname', ({ + expect, + }) => { + const defined = defineSchema(todoSchema); + // Accessing a handle triggers lazy freeze + runtime registration. + const titleId = defined.properties.title; + + expect(getKnownNameBySubject(titleId)).toBe('title'); + }); + + it('does not freeze on definition, and throws on handle access for imports', ({ + expect, + }) => { + // Defining an import-containing schema must not throw (no eager freeze)... + const external = defineSchema({ + name: 'ProjectApp', + imports: { + todo: { + subject: 'did:ad:todoOntology', + expectedHash: defineSchema(todoSchema).schemaHash, + }, + }, + classes: { + project: { + type: 'object', + properties: { title: { $ref: 'todo.properties.title' } }, + }, + }, + }); + + // ...but reaching for a local handle (which needs a store) throws clearly. + expect(() => external.classes.project).toThrow(/store-backed registration/); + }); +}); + +describe('schemaToOntologyModel', () => { + it('converts schema classes and properties to Atomic ontology model entries', ({ + expect, + }) => { + const model = schemaToOntologyModel(todoSchema); + + expect(model.ontology.shortname).toBe('TodoApp'); + expect(model.ontology.schemaHash).toMatch(/^blake3:[0-9a-f]{64}$/); + expect(model.classes).toEqual([ + { + key: 'todo', + subject: undefined, + shortname: 'todo', + description: 'A task', + requires: ['todo.title'], + recommends: ['todo.done'], + }, + ]); + expect( + model.properties.toSorted((a, b) => + a.propertyKey.localeCompare(b.propertyKey), + ), + ).toMatchObject([ + { + key: 'todo.done', + classKey: 'todo', + propertyKey: 'done', + shortname: 'done', + description: 'done', + datatype: Datatype.BOOLEAN, + }, + { + key: 'todo.title', + classKey: 'todo', + propertyKey: 'title', + shortname: 'title', + description: 'Task title', + datatype: Datatype.STRING, + }, + ]); + }); + + it('maps common JSON Schema datatypes to Atomic datatypes', ({ expect }) => { + const model = schemaToOntologyModel({ + name: 'Types', + classes: { + item: { + type: 'object', + properties: { + dueAt: { type: 'string', format: 'date' }, + url: { type: 'string', format: 'uri' }, + count: { type: 'integer' }, + price: { type: 'number' }, + payload: { type: 'object' }, + tags: { type: 'array', items: { type: 'string' } }, + owner: { $ref: '#/$defs/Agent' }, + files: { type: 'array', items: { $ref: '#/$defs/File' } }, + }, + }, + }, + }); + + expect( + Object.fromEntries( + model.properties.map(prop => [prop.propertyKey, prop.datatype]), + ), + ).toEqual({ + dueAt: Datatype.DATE, + url: Datatype.URI, + count: Datatype.INTEGER, + price: Datatype.FLOAT, + payload: Datatype.JSON, + tags: Datatype.JSON, + owner: Datatype.ATOMIC_URL, + files: Datatype.RESOURCEARRAY, + }); + }); + + it('keeps atomic extension metadata for generated properties and classes', ({ + expect, + }) => { + const model = schemaToOntologyModel({ + name: 'People', + classes: { + person: { + type: 'object', + 'atomic:subject': 'did:ad:personClass', + 'atomic:shortname': 'person', + required: ['friend'], + properties: { + friend: { + type: 'string', + 'atomic:subject': 'did:ad:friendProperty', + 'atomic:datatype': Datatype.ATOMIC_URL, + 'atomic:classType': 'did:ad:personClass', + 'atomic:isLocked': true, + }, + }, + }, + }, + }); + + expect(model.classes[0].subject).toBe('did:ad:personClass'); + expect(model.properties[0]).toMatchObject({ + subject: 'did:ad:friendProperty', + datatype: Datatype.ATOMIC_URL, + classType: 'did:ad:personClass', + isLocked: true, + }); + }); +}); + +describe('freezeSchema', () => { + it('produces did:ad:frozen ids for the ontology, classes, and properties', ({ + expect, + }) => { + const frozen = freezeSchema(todoSchema); + + expect(frozen.ontology).toMatch(/^did:ad:frozen:[0-9a-f]{64}$/); + expect(frozen.classes.todo).toMatch(/^did:ad:frozen:[0-9a-f]{64}$/); + expect(frozen.properties['todo.title']).toMatch( + /^did:ad:frozen:[0-9a-f]{64}$/, + ); + expect(frozen.properties['todo.done']).toMatch( + /^did:ad:frozen:[0-9a-f]{64}$/, + ); + }); + + it('rewrites class requires to the frozen property id', ({ expect }) => { + const frozen = freezeSchema(todoSchema); + const todoClass = frozen.resources.find( + resource => resource.frozenId === frozen.classes.todo, + )!; + + expect( + (todoClass.content as Record)[core.properties.requires], + ).toEqual([frozen.properties['todo.title']]); + }); + + it('rewrites ontology members to frozen ids and keeps the schema hash', ({ + expect, + }) => { + const frozen = freezeSchema(todoSchema); + const ontology = frozen.resources.find( + resource => resource.frozenId === frozen.ontology, + )!; + const content = ontology.content as Record; + + expect(content[core.properties.classes]).toEqual([frozen.classes.todo]); + expect(content[core.properties.properties]).toEqual( + expect.arrayContaining([ + frozen.properties['todo.title'], + frozen.properties['todo.done'], + ]), + ); + // Presentation (schemaHash, descriptions) is NOT part of frozen identity. + expect(content[SCHEMA_HASH_PROPERTY]).toBeUndefined(); + expect(content[core.properties.description]).toBeUndefined(); + expect(frozen.presentation.ontology.schemaHash).toBe( + frozen.model.ontology.schemaHash, + ); + }); + + it('is deterministic and content-addressed across runs', ({ expect }) => { + expect(freezeSchema(todoSchema).properties['todo.title']).toBe( + freezeSchema(defineSchema(todoSchema)).properties['todo.title'], + ); + }); + + it('keeps the property id stable when only its description changes', ({ + expect, + }) => { + const base = freezeSchema(todoSchema).properties['todo.title']; + const reworded = freezeSchema({ + ...todoSchema, + classes: { + todo: { + ...todoSchema.classes.todo, + properties: { + ...todoSchema.classes.todo.properties, + title: { type: 'string', description: 'A different title' }, + }, + }, + }, + }); + + // Description is presentation, not identity — the id must not move... + expect(reworded.properties['todo.title']).toBe(base); + // ...but the new wording is captured in the presentation layer. + expect(reworded.presentation.properties['todo.title'].description).toBe( + 'A different title', + ); + }); + + it('changes the property id when its datatype changes', ({ expect }) => { + const base = freezeSchema(todoSchema).properties['todo.title']; + const changed = freezeSchema({ + ...todoSchema, + classes: { + todo: { + ...todoSchema.classes.todo, + properties: { + ...todoSchema.classes.todo.properties, + title: { type: 'integer', description: 'Task title' }, + }, + }, + }, + }); + + expect(changed.properties['todo.title']).not.toBe(base); + }); + + it('keeps a class id stable when a property description changes', ({ + expect, + }) => { + const base = freezeSchema(todoSchema).classes.todo; + const reworded = freezeSchema({ + ...todoSchema, + classes: { + todo: { + ...todoSchema.classes.todo, + properties: { + ...todoSchema.classes.todo.properties, + title: { type: 'string', description: 'Reworded' }, + }, + }, + }, + }); + + // No cascade: a cosmetic edit deep in the tree moves nothing. + expect(reworded.classes.todo).toBe(base); + expect(reworded.ontology).toBe(freezeSchema(todoSchema).ontology); + }); + + it('allows the same shortname across classes when definitions are identical', ({ + expect, + }) => { + const frozen = freezeSchema({ + name: 'Notes', + classes: { + note: { + type: 'object', + properties: { title: { type: 'string', description: 'Title' } }, + }, + memo: { + type: 'object', + properties: { title: { type: 'string', description: 'Title' } }, + }, + }, + }); + + // Identical "title" definitions dedupe to a single frozen id. + expect(frozen.properties['note.title']).toBe(frozen.properties['memo.title']); + }); + + it('rejects different definitions that share a shortname', ({ expect }) => { + expect(() => + freezeSchema({ + name: 'Notes', + classes: { + note: { + type: 'object', + properties: { title: { type: 'string', description: 'A' } }, + }, + memo: { + type: 'object', + properties: { title: { type: 'integer', description: 'B' } }, + }, + }, + }), + ).toThrow('shortname "title"'); + }); + + it('throws on imported property references it cannot resolve', ({ + expect, + }) => { + expect(() => + freezeSchema({ + name: 'ProjectApp', + imports: { + todo: { subject: 'did:ad:todoOntology' }, + }, + classes: { + project: { + type: 'object', + required: ['title'], + properties: { + title: { $ref: 'todo.properties.title' }, + }, + }, + }, + }), + ).toThrow('Imported properties require store-backed registration'); + }); +}); + +describe('Store.registerSchema', () => { + it('creates local DID ontology, class, and property resources that resolve', async ({ + expect, + }) => { + const keys = await Agent.generateKeyPair(); + const store = new Store({ serverUrl: 'https://example.com' }); + store.setAgent( + new Agent( + new JSCryptoProvider(keys.privateKey), + `did:ad:agent:${keys.publicKey}`, + ), + ); + + const registered = await store.registerSchema(defineSchema(todoSchema)); + const titleProperty = registered.properties['todo.title']; + const todoClass = registered.classes.todo; + + expect(registered.ontology.subject).toMatch(/^did:ad:/); + expect(titleProperty.subject).toMatch(/^did:ad:/); + expect(todoClass.subject).toMatch(/^did:ad:/); + expect(registered.ontology.props.classes).toEqual([todoClass.subject]); + expect(registered.ontology.props.properties).toEqual( + expect.arrayContaining([ + titleProperty.subject, + registered.properties['todo.done'].subject, + ]), + ); + expect(todoClass.props.requires).toEqual([titleProperty.subject]); + + const resolvedProperty = await store.getProperty(titleProperty.subject); + + expect(resolvedProperty.shortname).toBe('title'); + expect(resolvedProperty.datatype).toBe(Datatype.STRING); + expect( + store.getRegisteredSchemaSubject(registered.model.ontology.schemaHash), + ).toBe(registered.ontology.subject); + expect(registered.ontology.hasClasses(core.classes.ontology)).toBe(true); + }); + + it('can save registered schemas through the normal commit path', async ({ + expect, + }) => { + const keys = await Agent.generateKeyPair(); + const store = new Store({ serverUrl: 'https://example.com' }); + store.setServerConnected(true); + store.setAgent( + new Agent( + new JSCryptoProvider(keys.privateKey), + `did:ad:agent:${keys.publicKey}`, + ), + ); + const posted: Commit[] = []; + const postCommit = vi.fn(async (commit: Commit) => { + const created = { + ...commit, + id: `https://example.com/commits/${commit.signature}`, + } as Commit; + posted.push(created); + + return created; + }); + ( + store as unknown as { client: { postCommit: typeof postCommit } } + ).client.postCommit = postCommit; + + const registered = await store.registerSchema(defineSchema(todoSchema), { + save: true, + }); + + expect(postCommit).toHaveBeenCalled(); + expect(posted.map(commit => commit.subject)).toEqual( + expect.arrayContaining([ + registered.ontology.subject, + registered.classes.todo.subject, + registered.properties['todo.title'].subject, + registered.properties['todo.done'].subject, + ]), + ); + }); + + it('reloads saved schema resources from the local DB in a fresh Store', async ({ + expect, + }) => { + const keys = await Agent.generateKeyPair(); + const agent = new Agent( + new JSCryptoProvider(keys.privateKey), + `did:ad:agent:${keys.publicKey}`, + ); + const producer = new Store({ serverUrl: 'https://example.com', agent }); + producer.setServerConnected(false); + const dbState = attachMockClientDb(producer); + + const registered = await producer.registerSchema(defineSchema(todoSchema), { + save: true, + }); + const ontologySubject = registered.ontology.subject; + const classSubject = registered.classes.todo.subject; + const titlePropertySubject = registered.properties['todo.title'].subject; + + expect(dbState.has(ontologySubject)).toBe(true); + expect(dbState.has(classSubject)).toBe(true); + expect(dbState.has(titlePropertySubject)).toBe(true); + + const consumer = new Store({ serverUrl: 'https://example.com', agent }); + consumer.setServerConnected(false); + attachMockClientDb(consumer, dbState); + + const ontology = await consumer.getResource(ontologySubject); + expect(ontology.get(SCHEMA_HASH_PROPERTY)).toBe( + registered.model.ontology.schemaHash, + ); + expect(ontology.props.classes).toEqual([classSubject]); + expect(ontology.props.properties).toEqual( + expect.arrayContaining([titlePropertySubject]), + ); + + const todoClass = await consumer.getResource(classSubject); + expect(todoClass.props.requires).toEqual([titlePropertySubject]); + + const titleProperty = await consumer.getProperty(titlePropertySubject); + expect(titleProperty.shortname).toBe('title'); + expect(titleProperty.datatype).toBe(Datatype.STRING); + }); + + it('checks expected hashes for imported ontologies', async ({ expect }) => { + const keys = await Agent.generateKeyPair(); + const store = new Store({ serverUrl: 'https://example.com' }); + store.setAgent( + new Agent( + new JSCryptoProvider(keys.privateKey), + `did:ad:agent:${keys.publicKey}`, + ), + ); + + const todo = await store.registerSchema(defineSchema(todoSchema)); + const projectSchema = defineSchema({ + name: 'ProjectApp', + imports: { + todo: { + subject: todo.ontology.subject, + expectedHash: todo.model.ontology.schemaHash, + }, + }, + classes: { + project: { + type: 'object', + properties: { + title: { type: 'string' }, + }, + }, + }, + }); + + await expect(store.registerSchema(projectSchema)).resolves.toMatchObject({ + ontology: expect.objectContaining({ subject: expect.any(String) }), + }); + + const mismatched = defineSchema({ + ...projectSchema.schema, + imports: { + todo: { + subject: todo.ontology.subject, + expectedHash: + 'blake3:0000000000000000000000000000000000000000000000000000000000000000', + }, + }, + }); + + await expect(store.registerSchema(mismatched)).rejects.toThrow( + 'Schema import todo expected', + ); + }); + + it('can create an instance using returned schema subjects', async ({ + expect, + }) => { + const keys = await Agent.generateKeyPair(); + const store = new Store({ serverUrl: 'https://example.com' }); + store.setAgent( + new Agent( + new JSCryptoProvider(keys.privateKey), + `did:ad:agent:${keys.publicKey}`, + ), + ); + const registered = await store.registerSchema(defineSchema(todoSchema)); + const resource = await store.newResource({ + isA: registered.classes.todo.subject, + propVals: { + [registered.properties['todo.title'].subject]: 'Buy milk', + [registered.properties['todo.done'].subject]: false, + }, + }); + + expect(resource.getClasses()).toContain(registered.classes.todo.subject); + expect(resource.get(registered.properties['todo.title'].subject)).toBe( + 'Buy milk', + ); + expect(resource.get(registered.properties['todo.done'].subject)).toBe( + false, + ); + }); + + it('rejects datatype changes for an explicitly reused Property subject', async ({ + expect, + }) => { + const keys = await Agent.generateKeyPair(); + const store = new Store({ serverUrl: 'https://example.com' }); + store.setAgent( + new Agent( + new JSCryptoProvider(keys.privateKey), + `did:ad:agent:${keys.publicKey}`, + ), + ); + const propertySubject = 'did:ad:sharedTitleProperty'; + + await store.registerSchema( + defineSchema({ + name: 'TodoApp', + classes: { + todo: { + type: 'object', + properties: { + title: { + type: 'string', + 'atomic:subject': propertySubject, + }, + }, + }, + }, + }), + ); + + await expect( + store.registerSchema( + defineSchema({ + name: 'ChangedTodoApp', + classes: { + todo: { + type: 'object', + properties: { + title: { + type: 'integer', + 'atomic:subject': propertySubject, + }, + }, + }, + }, + }), + ), + ).rejects.toThrow('already registered with a different'); + }); + + it('can reuse one imported Property in another Class', async ({ expect }) => { + const keys = await Agent.generateKeyPair(); + const store = new Store({ serverUrl: 'https://example.com' }); + store.setAgent( + new Agent( + new JSCryptoProvider(keys.privateKey), + `did:ad:agent:${keys.publicKey}`, + ), + ); + const todo = await store.registerSchema(defineSchema(todoSchema)); + const project = await store.registerSchema( + defineSchema({ + name: 'ProjectApp', + imports: { + todo: { + subject: todo.ontology.subject, + expectedHash: todo.model.ontology.schemaHash, + }, + }, + classes: { + project: { + type: 'object', + required: ['title'], + properties: { + title: { $ref: 'todo.properties.title' }, + }, + }, + }, + }), + ); + + expect(project.properties['project.title']).toBeUndefined(); + expect(project.classes.project.props.requires).toEqual([ + todo.properties['todo.title'].subject, + ]); + }); +}); + +describe('Store lazy frozen publish on save', () => { + it('PUTs referenced frozen definitions to /frozen when an instance is saved', async ({ + expect, + }) => { + const keys = await Agent.generateKeyPair(); + const store = new Store({ serverUrl: 'https://example.com' }); + store.setServerConnected(true); + store.setAgent( + new Agent( + new JSCryptoProvider(keys.privateKey), + `did:ad:agent:${keys.publicKey}`, + ), + ); + + // Commits go to the mocked client; nothing hits the network there. + ( + store as unknown as { client: { postCommit: typeof vi.fn } } + ).client.postCommit = vi.fn(async (commit: Commit) => ({ + ...commit, + id: `https://example.com/commits/${commit.signature}`, + })) as never; + + // Capture the `/frozen` PUTs the store makes on save. + const puts: string[] = []; + const fetchMock = vi.fn(async (url: string, init?: { method?: string }) => { + if (init?.method === 'PUT') { + puts.push(url); + } + + return { ok: true, status: 200 } as Response; + }); + vi.stubGlobal('fetch', fetchMock); + + try { + // No registerSchema, no CLI — just define and use. + const schema = defineSchema(todoSchema); + + const todo = await store.newResource({ + isA: schema.classes.todo, + propVals: { [schema.properties.title]: 'Buy milk' }, + }); + await todo.save(); + + const classHash = schema.classes.todo.replace('did:ad:frozen:', ''); + const titleHash = schema.properties.title.replace('did:ad:frozen:', ''); + + // The class and its property definitions were published automatically. + expect(puts.some(url => url.endsWith(`/frozen/${classHash}`))).toBe(true); + expect(puts.some(url => url.endsWith(`/frozen/${titleHash}`))).toBe(true); + } finally { + vi.unstubAllGlobals(); + } + }); +}); diff --git a/browser/lib/src/schema.ts b/browser/lib/src/schema.ts new file mode 100644 index 000000000..412724cab --- /dev/null +++ b/browser/lib/src/schema.ts @@ -0,0 +1,707 @@ +import { blake3 } from '@noble/hashes/blake3.js'; +import { bytesToHex, utf8ToBytes } from '@noble/hashes/utils.js'; + +import { Datatype } from './datatypes.js'; +import { + freezeResources, + registerFrozenBodies, + type FreezableResource, + type FrozenId, + type JsonValue as FreezeJsonValue, +} from './freeze.js'; +import { jcsCanonicalize } from './jcs.js'; +import { core } from './ontologies/core.js'; +import { server } from './ontologies/server.js'; +import { registerOntologies } from './ontology.js'; + +type JsonPrimitive = string | number | boolean | null; + +export type JsonValue = + | JsonPrimitive + | JsonValue[] + | { [key: string]: JsonValue }; + +// Re-export the content-addressing primitive (but not its `JsonValue`, which +// would collide with the one above). +export { + freezeResources, + frozenIdFor, + type FrozenId, + type FreezableResource, + type FrozenResource, + type FreezeResult, +} from './freeze.js'; + +export type SchemaHash = `blake3:${string}`; + +export const SCHEMA_HASH_PROPERTY = + 'https://atomicdata.dev/properties/schemaHash'; + +export interface SchemaImportReference { + /** Ontology subject, usually `did:ad:{genesis}` or an HTTP URL. */ + subject: string; + /** Expected Ontology package hash. Generation/registration must fail on mismatch. */ + expectedHash?: SchemaHash; +} + +export interface AtomicSchemaPropertyDefinition { + type?: 'string' | 'integer' | 'number' | 'boolean' | 'object' | 'array'; + format?: string; + description?: string; + default?: JsonValue; + enum?: JsonValue[]; + items?: AtomicSchemaPropertyDefinition | SchemaRef; + properties?: Record; + required?: string[]; + additionalProperties?: boolean | AtomicSchemaPropertyDefinition | SchemaRef; + $ref?: string; + 'atomic:subject'?: string; + 'atomic:shortname'?: string; + 'atomic:datatype'?: Datatype | string; + 'atomic:classType'?: string; + 'atomic:recommends'?: string[]; + 'atomic:allowsOnly'?: JsonValue[]; + 'atomic:isDynamic'?: boolean; + 'atomic:isLocked'?: boolean; +} + +export interface SchemaRef { + $ref: string; +} + +export interface AtomicSchemaClassDefinition { + title?: string; + description?: string; + type: 'object'; + required?: string[]; + properties: Record; + $defs?: Record; + 'atomic:subject'?: string; + 'atomic:shortname'?: string; + 'atomic:recommends'?: string[]; +} + +export interface AtomicSchemaPackage { + name: string; + version?: string; + description?: string; + imports?: Record; + classes: Record; + $defs?: Record; +} + +/** The class keys of a schema package, mapped to their `did:ad:frozen:` ids. */ +export type SchemaClassHandles = { + readonly [C in keyof S['classes']]: string; +}; + +/** Union of every property key across all classes of a schema package. */ +export type SchemaPropertyKey = { + [C in keyof S['classes']]: keyof S['classes'][C]['properties'] & string; +}[keyof S['classes']]; + +/** The property keys of a schema package, mapped to their `did:ad:frozen:` ids. */ +export type SchemaPropertyHandles = { + readonly [P in SchemaPropertyKey]: string; +}; + +export interface DefinedSchema< + Schema extends AtomicSchemaPackage = AtomicSchemaPackage, +> { + readonly schema: Schema; + readonly normalized: AtomicSchemaPackage; + readonly schemaHash: SchemaHash; + /** + * Content-addressed `did:ad:frozen:` id for each class, keyed by class key. + * Computed lazily and locally (no server) the first time it is read — use + * directly as `isA` when creating resources. Self-contained schemas only; + * accessing this on a schema with `$ref` imports throws. + */ + readonly classes: SchemaClassHandles; + /** + * Content-addressed `did:ad:frozen:` id for each property, keyed by property + * key. Use directly as `propVals` keys. + */ + readonly properties: SchemaPropertyHandles; +} + +export interface ConvertedSchemaProperty { + readonly key: string; + readonly classKey: string; + readonly propertyKey: string; + readonly subject?: string; + readonly shortname: string; + readonly description: string; + readonly datatype: Datatype | string; + readonly classType?: string; + readonly allowsOnly?: JsonValue[]; + readonly isDynamic?: boolean; + readonly isLocked?: boolean; +} + +export interface ConvertedSchemaClass { + readonly key: string; + readonly subject?: string; + readonly shortname: string; + readonly description: string; + readonly requires: string[]; + readonly recommends: string[]; +} + +export interface ConvertedSchemaOntology { + readonly shortname: string; + readonly description: string; + readonly version?: string; + readonly schemaHash: SchemaHash; + readonly jsonSchema: AtomicSchemaPackage; +} + +export interface ConvertedSchemaPackage { + readonly ontology: ConvertedSchemaOntology; + readonly classes: ConvertedSchemaClass[]; + readonly properties: ConvertedSchemaProperty[]; +} + +export function defineSchema( + schema: Schema, +): DefinedSchema { + const normalized = normalizeSchemaPackage(schema); + + // `.classes` / `.properties` are content-addressed frozen ids. They are + // computed lazily — on first access — so that (a) merely defining a schema + // (or converting one to a model internally) does no hashing work, and (b) a + // schema with `$ref` imports, which cannot be frozen without a store, only + // errors if a caller actually reaches for the local handles. + let handles: SchemaHandles | undefined; + const getHandles = (): SchemaHandles => (handles ??= computeSchemaHandles(defined)); + + const defined = Object.freeze({ + schema, + normalized, + schemaHash: hashSchemaPackage(normalized), + get classes() { + return getHandles().classes; + }, + get properties() { + return getHandles().properties; + }, + }) as unknown as DefinedSchema; + + return defined; +} + +interface SchemaHandles { + readonly classes: Record; + readonly properties: Record; +} + +/** + * Freezes a self-contained schema into `did:ad:frozen:` ids and exposes them as + * flat `{ classKey -> id }` / `{ propertyKey -> id }` handles. Also registers the + * schema in the global runtime mapping so `resource.props.` resolves + * without any generated bindings. + */ +function computeSchemaHandles(defined: DefinedSchema): SchemaHandles { + const frozen = freezeSchema(defined); + + const properties: Record = {}; + + for (const property of frozen.model.properties) { + properties[property.propertyKey] = frozen.properties[property.key]; + } + + registerFrozenSchemaRuntime(frozen); + // Make the frozen bodies publishable so the Store can lazily PUT them on save. + registerFrozenBodies(frozen.resources); + + return { classes: { ...frozen.classes }, properties }; +} + +/** + * Teaches `@tomic/lib`'s runtime about a frozen schema (subject -> shortname and + * the per-class property set), so quick-access `resource.props` works for + * code-first schemas the same way it does for generated ontologies. + */ +function registerFrozenSchemaRuntime(frozen: FrozenSchema): void { + const properties: Record = {}; + + for (const property of frozen.model.properties) { + properties[property.shortname] = frozen.properties[property.key]; + } + + const classes: Record = {}; + const classDefs: Record = {}; + + for (const klass of frozen.model.classes) { + const classId = frozen.classes[klass.key]; + classes[klass.shortname] = classId; + classDefs[classId] = [...klass.requires, ...klass.recommends] + .map(modelKey => frozen.properties[modelKey]) + .filter((id): id is FrozenId => Boolean(id)); + } + + registerOntologies({ classes, properties, __classDefs: classDefs }); +} + +export function schemaToOntologyModel( + input: AtomicSchemaPackage | DefinedSchema, +): ConvertedSchemaPackage { + const defined = isDefinedSchema(input) ? input : defineSchema(input); + const schema = defined.normalized; + const properties: ConvertedSchemaProperty[] = []; + const classes = Object.entries(schema.classes).map(([classKey, klass]) => { + const required = new Set(klass.required ?? []); + const classPropertyKeys: Array<{ modelKey: string; propertyKey: string }> = + []; + + for (const [propertyKey, propertyDefinition] of Object.entries( + klass.properties, + )) { + if ( + isSchemaRef(propertyDefinition) && + isImportedPropertyRef(propertyDefinition.$ref) + ) { + classPropertyKeys.push({ + modelKey: `ref:${propertyDefinition.$ref}`, + propertyKey, + }); + continue; + } + + const convertedProperty = convertPropertyDefinition( + classKey, + propertyKey, + propertyDefinition, + ); + + properties.push(convertedProperty); + classPropertyKeys.push({ + modelKey: convertedProperty.key, + propertyKey, + }); + } + + const requiredKeys = classPropertyKeys.filter(key => + required.has(key.propertyKey), + ); + const recommendedKeys = classPropertyKeys.filter( + key => !required.has(key.propertyKey), + ); + + return { + key: classKey, + subject: klass['atomic:subject'], + shortname: klass['atomic:shortname'] ?? classKey, + description: klass.description ?? klass.title ?? classKey, + requires: requiredKeys.map(key => key.modelKey), + recommends: recommendedKeys.map(key => key.modelKey), + } satisfies ConvertedSchemaClass; + }); + + return { + ontology: { + shortname: schema.name, + description: schema.description ?? schema.name, + version: schema.version, + schemaHash: defined.schemaHash, + jsonSchema: schema, + }, + classes, + properties, + }; +} + +export interface FrozenSchemaResource { + readonly frozenId: FrozenId; + /** Identity-only JSON-AD body with internal references resolved to FrozenIds. */ + readonly content: JsonValue; +} + +export interface FrozenSchemaMetadata { + readonly frozenId: FrozenId; + /** Human description — presentation, not part of the frozen identity. */ + readonly description: string; +} + +/** + * Mutable, human-facing metadata that is deliberately NOT hashed into frozen + * ids, so editing it (typo fixes, rewording, translations) never churns an id. + * Keyed by the developer-facing model key so each usage keeps its own text even + * when identical definitions dedupe to one frozen id. + */ +export interface FrozenSchemaPresentation { + readonly ontology: { + readonly description: string; + readonly version?: string; + readonly schemaHash: SchemaHash; + readonly jsonSchema: AtomicSchemaPackage; + }; + readonly classes: Record; + readonly properties: Record; +} + +export interface FrozenSchema { + readonly ontology: FrozenId; + readonly classes: Record; + readonly properties: Record; + readonly resources: FrozenSchemaResource[]; + readonly presentation: FrozenSchemaPresentation; + readonly model: ConvertedSchemaPackage; +} + +const FREEZE_NS = 'urn:atomic-freeze:'; +const ONTOLOGY_LOCAL_ID = `${FREEZE_NS}ontology`; +const classLocalId = (key: string): string => `${FREEZE_NS}class:${key}`; +const propLocalId = (key: string): string => `${FREEZE_NS}prop:${key}`; + +/** + * Converts a schema package into content-addressed `did:ad:frozen` resources: + * one materialized Ontology, Class, and Property JSON-AD body each, with every + * cross-reference resolved to the referent's frozen hash. The whole body is + * hashed (descriptions included), so the same definition always yields the same + * id and any edit yields a new one. + * + * Self-contained schemas only: an imported property (`$ref: + * "alias.properties.x"`) cannot be resolved to a frozen id without the + * referenced Ontology, so callers needing imports must go through the + * store-backed registration path. + */ +export function freezeSchema( + input: AtomicSchemaPackage | DefinedSchema, +): FrozenSchema { + const model = schemaToOntologyModel(input); + const propertyKeys = new Set(model.properties.map(property => property.key)); + + const resolveModelKey = (key: string): string => { + if (propertyKeys.has(key)) { + return propLocalId(key); + } + + throw new Error( + `freezeSchema cannot resolve property reference "${key}". Imported properties require store-backed registration.`, + ); + }; + + const freezable: FreezableResource[] = []; + + // Frozen bodies hold IDENTITY only — the machine contract that decides how + // data is validated/interpreted. Presentation (descriptions, labels, + // translations) is excluded so cosmetic edits never churn a frozen id; it + // rides in the mutable package layer (`presentation`, below). + for (const property of model.properties) { + freezable.push({ + localId: propLocalId(property.key), + content: compactContent({ + [core.properties.isA]: [core.classes.property], + [core.properties.shortname]: property.shortname, + [core.properties.datatype]: property.datatype, + [core.properties.classtype]: property.classType, + [core.properties.allowsOnly]: property.allowsOnly as + | FreezeJsonValue + | undefined, + [core.properties.isDynamic]: property.isDynamic, + [core.properties.isLocked]: property.isLocked, + }), + }); + } + + for (const klass of model.classes) { + freezable.push({ + localId: classLocalId(klass.key), + content: compactContent({ + [core.properties.isA]: [core.classes.class], + [core.properties.shortname]: klass.shortname, + [core.properties.requires]: klass.requires.map(resolveModelKey), + [core.properties.recommends]: klass.recommends.map(resolveModelKey), + }), + }); + } + + freezable.push({ + localId: ONTOLOGY_LOCAL_ID, + content: compactContent({ + [core.properties.isA]: [core.classes.ontology], + [core.properties.shortname]: model.ontology.shortname, + [core.properties.classes]: model.classes.map(klass => + classLocalId(klass.key), + ), + [core.properties.properties]: model.properties.map(property => + propLocalId(property.key), + ), + [server.properties.version]: model.ontology.version, + }), + }); + + const { resources, byLocalId } = freezeResources(freezable); + + const requireId = (localId: string): FrozenId => { + const frozenId = byLocalId.get(localId); + + if (!frozenId) { + throw new Error(`freezeSchema did not produce an id for ${localId}`); + } + + return frozenId; + }; + + const classes: Record = {}; + const properties: Record = {}; + + for (const klass of model.classes) { + classes[klass.key] = requireId(classLocalId(klass.key)); + } + + for (const property of model.properties) { + properties[property.key] = requireId(propLocalId(property.key)); + } + + // A shortname must map to a single definition per ontology, so the lockfile + // `@index` and generated bindings can key on it. Content-addressing makes this + // forgiving: identical definitions across classes dedupe to one frozen id and + // pass; only genuinely different definitions sharing a shortname are rejected. + assertUniqueShortnames( + model.ontology.shortname, + 'property', + model.properties.map(p => [p.shortname, properties[p.key]] as const), + ); + assertUniqueShortnames( + model.ontology.shortname, + 'class', + model.classes.map(k => [k.shortname, classes[k.key]] as const), + ); + + const presentation: FrozenSchemaPresentation = { + ontology: { + description: model.ontology.description, + version: model.ontology.version, + schemaHash: model.ontology.schemaHash, + jsonSchema: model.ontology.jsonSchema, + }, + classes: Object.fromEntries( + model.classes.map(klass => [ + klass.key, + { frozenId: classes[klass.key], description: klass.description }, + ]), + ), + properties: Object.fromEntries( + model.properties.map(property => [ + property.key, + { frozenId: properties[property.key], description: property.description }, + ]), + ), + }; + + return { + ontology: requireId(ONTOLOGY_LOCAL_ID), + classes, + properties, + resources: resources.map(resource => ({ + frozenId: resource.frozenId, + content: resource.content as JsonValue, + })), + presentation, + model, + }; +} + +function assertUniqueShortnames( + ontologyShortname: string, + kind: 'class' | 'property', + entries: ReadonlyArray, +): void { + const idsByShortname = new Map>(); + + for (const [shortname, frozenId] of entries) { + const ids = idsByShortname.get(shortname) ?? new Set(); + ids.add(frozenId); + idsByShortname.set(shortname, ids); + } + + for (const [shortname, ids] of idsByShortname) { + if (ids.size > 1) { + throw new Error( + `Schema "${ontologyShortname}" defines ${ids.size} different ${kind} resources with shortname "${shortname}". A shortname must map to one ${kind} per ontology — make the definitions identical to share, or rename.`, + ); + } + } +} + +function compactContent( + entries: Record, +): FreezeJsonValue { + return Object.fromEntries( + Object.entries(entries).filter(([, value]) => value !== undefined), + ) as FreezeJsonValue; +} + +export function hashSchemaPackage(schema: AtomicSchemaPackage): SchemaHash { + const canonical = canonicalizeSchemaPackage(schema); + const hash = blake3(utf8ToBytes(canonical)); + + return `blake3:${bytesToHex(hash)}`; +} + +export function canonicalizeSchemaPackage(schema: AtomicSchemaPackage): string { + return jcsCanonicalize( + normalizeSchemaPackage(schema) as unknown as FreezeJsonValue, + ); +} + +export function normalizeSchemaPackage( + schema: Schema, +): Schema { + return normalizeValue(schema) as unknown as Schema; +} + +function normalizeValue(value: unknown): JsonValue | undefined { + if (value === undefined) { + return undefined; + } + + if (value === null) { + return null; + } + + if (typeof value === 'string' || typeof value === 'boolean') { + return value; + } + + if (typeof value === 'number') { + if (!Number.isFinite(value)) { + throw new Error(`Schema contains a non-finite number: ${value}`); + } + + return value; + } + + if (Array.isArray(value)) { + return value.map(item => { + const normalized = normalizeValue(item); + + return normalized === undefined ? null : normalized; + }); + } + + if (typeof value === 'object') { + const normalizedEntries = Object.entries( + value as Record, + ).flatMap(([key, child]) => { + const normalized = normalizeValue(child); + + return normalized === undefined ? [] : [[key, normalized] as const]; + }); + + return Object.fromEntries( + normalizedEntries.sort(([a], [b]) => a.localeCompare(b)), + ); + } + + throw new Error(`Schema contains an unsupported value: ${String(value)}`); +} + +function convertPropertyDefinition( + classKey: string, + propertyKey: string, + propertyDefinition: AtomicSchemaPropertyDefinition | SchemaRef, +): ConvertedSchemaProperty { + return { + key: `${classKey}.${propertyKey}`, + classKey, + propertyKey, + subject: + 'atomic:subject' in propertyDefinition + ? propertyDefinition['atomic:subject'] + : undefined, + shortname: + ('atomic:shortname' in propertyDefinition + ? propertyDefinition['atomic:shortname'] + : undefined) ?? propertyKey, + description: + 'description' in propertyDefinition + ? (propertyDefinition.description ?? propertyKey) + : propertyKey, + datatype: + ('atomic:datatype' in propertyDefinition + ? propertyDefinition['atomic:datatype'] + : undefined) ?? datatypeFromJsonSchema(propertyDefinition), + classType: + 'atomic:classType' in propertyDefinition + ? propertyDefinition['atomic:classType'] + : undefined, + allowsOnly: + ('atomic:allowsOnly' in propertyDefinition + ? propertyDefinition['atomic:allowsOnly'] + : undefined) ?? + ('enum' in propertyDefinition ? propertyDefinition.enum : undefined), + isDynamic: + 'atomic:isDynamic' in propertyDefinition + ? propertyDefinition['atomic:isDynamic'] + : undefined, + isLocked: + 'atomic:isLocked' in propertyDefinition + ? propertyDefinition['atomic:isLocked'] + : undefined, + }; +} + +function datatypeFromJsonSchema( + propertyDefinition: AtomicSchemaPropertyDefinition | SchemaRef, +): Datatype { + if (isSchemaRef(propertyDefinition)) { + return Datatype.ATOMIC_URL; + } + + switch (propertyDefinition.type) { + case 'string': + if (propertyDefinition.format === 'date') { + return Datatype.DATE; + } + + if (propertyDefinition.format === 'uri') { + return Datatype.URI; + } + + return Datatype.STRING; + case 'integer': + return Datatype.INTEGER; + case 'number': + return Datatype.FLOAT; + case 'boolean': + return Datatype.BOOLEAN; + case 'array': + return isSchemaRef(propertyDefinition.items) + ? Datatype.RESOURCEARRAY + : Datatype.JSON; + case 'object': + return Datatype.JSON; + default: + if (propertyDefinition.$ref) { + return Datatype.ATOMIC_URL; + } + + return Datatype.JSON; + } +} + +function isDefinedSchema(value: unknown): value is DefinedSchema { + return ( + typeof value === 'object' && + value !== null && + 'schema' in value && + 'normalized' in value && + 'schemaHash' in value + ); +} + +function isSchemaRef(value: unknown): value is SchemaRef { + return ( + typeof value === 'object' && + value !== null && + '$ref' in value && + typeof (value as SchemaRef).$ref === 'string' + ); +} + +function isImportedPropertyRef(ref: string): boolean { + return /^[^.]+\.properties\.[^.]+$/.test(ref); +} diff --git a/browser/lib/src/store.ts b/browser/lib/src/store.ts index 6404b4df1..7bb0cefd8 100644 --- a/browser/lib/src/store.ts +++ b/browser/lib/src/store.ts @@ -16,12 +16,33 @@ import { EventManager } from './EventManager.js'; import { hasBrowserAPI } from './hasBrowserAPI.js'; import { collections } from './ontologies/collections.js'; import { commits } from './ontologies/commits.js'; -import { core } from './ontologies/core.js'; +import { core, type Core } from './ontologies/core.js'; import { server, type Server } from './ontologies/server.js'; import type { OptionalClass, UnknownClass } from './ontology.js'; import { JSONADParser } from './parse.js'; import { Resource, unknownSubject } from './resource.js'; import { type SearchOpts, buildSearchSubject } from './search.js'; +import { + freezeSchema, + SCHEMA_HASH_PROPERTY, + schemaToOntologyModel, + type AtomicSchemaPackage, + type ConvertedSchemaPackage, + type DefinedSchema, + type FrozenSchema, +} from './schema.js'; +import { + freezeResources, + frozenIdFor, + getRegisteredFrozenBody, + registerFrozenBodies, + UNIT_MEMBERS_KEY, + type FrozenId, + type FreezableResource, + type JsonValue as FrozenJsonValue, +} from './freeze.js'; +import { jcsCanonicalize } from './jcs.js'; +import { verifySchemaLock, type SchemaLock } from './schema-lock.js'; import { stringToSlug } from './stringToSlug.js'; import { bytesToHex, hexToBytes, type JSONValue } from './value.js'; import { WSClient } from './websockets.js'; @@ -72,10 +93,45 @@ type CreateResourceOptions = { isA?: string | string[]; /** Any additional properties the resource should have */ propVals?: Record; + /** Validate `propVals` while creating the resource. Defaults to true. */ + validatePropVals?: boolean; /** Set to true if the resource should have a DID as subject. Defaults to `true` for `did:ad` agents, otherwise `false`. */ did?: boolean; }; +export interface RegisteredSchema { + ontology: Resource; + classes: Record>; + properties: Record>; + model: ConvertedSchemaPackage; +} + +export interface RegisterSchemaOptions { + /** Save generated resources through the normal Commit/outbox path. */ + save?: boolean; +} + +export interface FreezeStructureOptions { + /** + * Follow value references to other already-loaded resources, freezing the + * whole structure (e.g. an Ontology + its Classes + Properties). Defaults to + * true; set false to freeze only the root resource. + */ + closure?: boolean; + /** Also publish each frozen body to `/frozen` on the server. Defaults to false. */ + save?: boolean; +} + +/** The result of freezing a resource (and the structure it references). */ +export interface FrozenStructure { + /** The root resource's frozen id. */ + root: FrozenId; + /** Original subject -> frozen id, for every resource included. */ + bySubject: Record; + /** Frozen id -> identity JSON-AD body (what a consumer re-hashes and materializes). */ + frozen: Record; +} + export interface StoreOpts { /** The default store URL, where to send commits and where to create new instances */ serverUrl?: string; @@ -268,7 +324,6 @@ function yieldToEventLoop(): Promise { }); } - /** * Cheap equality for commit-log property values. Strict `===` would always * report arrays/objects as different even when their contents match, so the @@ -318,6 +373,9 @@ export class Store { private _resources: Map; /** Mapping from HTTP aliases to primary subjects (e.g. DIDs) */ private aliases: Map = new Map(); + private schemaHashIndex: Map = new Map(); + /** Frozen ids already PUT to the server this session — skip redundant publishes. */ + private publishedFrozenIds: Set = new Set(); /** List of resources that have parents that are not saved to the server, when a parent is saved it should also save its children */ private batchedResources: Map> = new Map(); @@ -871,6 +929,19 @@ export class Store { const endpoint = new URL('/commit', this.serverUrl).toString(); + // Lazy publish-on-save: ensure any content-addressed `did:ad:frozen` + // definitions this resource references (its class, a code-first property) + // exist on the server before the commit lands — so code-first schemas need + // no separate build/publish step. Best-effort: a failure here must not block + // the user's save. + const draining = this.resources.get(subject); + + if (draining) { + await this.publishReferencedFrozen(draining).catch(err => + console.warn('[Store] auto-publish of frozen references failed:', err), + ); + } + // Step 1: POST the pre-signed genesis if present. The genesis // commit's `loroUpdate` was captured in `signChanges` at sign // time; `_loroVersionAtLastSave` was advanced THERE to the same @@ -1570,6 +1641,7 @@ export class Store { parent, isA, propVals, + validatePropVals = true, noParent, did, }: CreateResourceOptions = {}): Promise> { @@ -1603,7 +1675,7 @@ export class Store { if (propVals) { for (const [key, value] of Object.entries(propVals)) { - await resource.set(key, value); + await resource.set(key, value, validatePropVals); } } @@ -1636,6 +1708,608 @@ export class Store { return resource; } + public async registerSchema( + schema: AtomicSchemaPackage | DefinedSchema, + opts: RegisterSchemaOptions = {}, + ): Promise { + const model = schemaToOntologyModel(schema); + + await this.validateSchemaImports(model.ontology.jsonSchema); + + const ontology = await this.newResource({ + isA: core.classes.ontology, + validatePropVals: false, + propVals: { + [core.properties.shortname]: stringToSlug(model.ontology.shortname), + [core.properties.description]: model.ontology.description, + [core.properties.classes]: [], + [core.properties.properties]: [], + [core.properties.instances]: [], + }, + }); + + const properties: Record> = {}; + + for (const property of model.properties) { + this.assertCompatibleExistingSchemaProperty(property); + + const resource = await this.newResource({ + subject: property.subject, + parent: ontology.subject, + isA: core.classes.property, + validatePropVals: false, + propVals: { + [core.properties.shortname]: property.shortname, + [core.properties.description]: property.description, + [core.properties.datatype]: property.datatype, + ...(property.classType + ? { [core.properties.classtype]: property.classType } + : {}), + ...(property.allowsOnly + ? { [core.properties.allowsOnly]: property.allowsOnly as string[] } + : {}), + ...(property.isDynamic !== undefined + ? { [core.properties.isDynamic]: property.isDynamic } + : {}), + ...(property.isLocked !== undefined + ? { [core.properties.isLocked]: property.isLocked } + : {}), + }, + }); + + properties[property.key] = resource; + } + + const classes: Record> = {}; + + for (const klass of model.classes) { + const requires = await Promise.all( + klass.requires.map(key => + this.resolveRegisteredSchemaPropertyKey( + key, + model.ontology.jsonSchema, + properties, + ), + ), + ); + const recommends = await Promise.all( + klass.recommends.map(key => + this.resolveRegisteredSchemaPropertyKey( + key, + model.ontology.jsonSchema, + properties, + ), + ), + ); + const resource = await this.newResource({ + subject: klass.subject, + parent: ontology.subject, + isA: core.classes.class, + validatePropVals: false, + propVals: { + [core.properties.shortname]: klass.shortname, + [core.properties.description]: klass.description, + [core.properties.requires]: requires, + [core.properties.recommends]: recommends, + }, + }); + + classes[klass.key] = resource; + } + + await ontology.set( + core.properties.classes, + Object.values(classes).map(resource => resource.subject), + false, + ); + await ontology.set( + core.properties.properties, + Object.values(properties).map(resource => resource.subject), + false, + ); + await ontology.set( + server.properties.jsonSchema, + model.ontology.jsonSchema as unknown as JSONValue, + false, + ); + await ontology.set(SCHEMA_HASH_PROPERTY, model.ontology.schemaHash, false); + + if (model.ontology.version) { + await ontology.set( + server.properties.version, + model.ontology.version, + false, + ); + } + + this.schemaHashIndex.set(model.ontology.schemaHash, ontology.subject); + + if (opts.save) { + await ontology.save(); + + for (const property of Object.values(properties)) { + await property.save(); + } + + for (const klass of Object.values(classes)) { + await klass.save(); + } + } + + return { + ontology, + classes, + properties, + model, + }; + } + + public getRegisteredSchemaSubject(schemaHash: string): string | undefined { + return this.schemaHashIndex.get(schemaHash); + } + + /** + * Content-addressed schema registration: freezes the schema into immutable + * `did:ad:frozen` resources, materializes them into the local store so + * `getResource`/`getProperty` resolve them immediately (offline), and — with + * `{ save: true }` — publishes the canonical bytes to `/frozen/{hash}` so other + * stores can resolve them. Returns the {@link FrozenSchema} (frozen ids per + * developer key, plus the mutable `presentation` layer). Unlike + * {@link registerSchema}, identity is the content hash, not a signed DID. + */ + public async registerFrozenSchema( + schema: AtomicSchemaPackage | DefinedSchema, + opts: RegisterSchemaOptions = {}, + ): Promise { + const frozen = freezeSchema(schema); + + registerFrozenBodies(frozen.resources); + + for (const { frozenId, content } of frozen.resources) { + const [resource] = new JSONADParser().parse(content, frozenId); + resource.loading = false; + this.addResource(resource, { skipCommitCompare: true }); + } + + if (opts.save) { + await Promise.all( + frozen.resources.map(({ frozenId, content }) => + this.publishFrozenResource(frozenId, content), + ), + ); + } + + return frozen; + } + + /** + * Creates the mutable, signed "latest version" pointer for a frozen schema: a + * normal Ontology resource (genesis DID) on the author's drive whose + * `classes`/`properties` point at the immutable frozen ids. Its stable subject + * is the durable name ("the current TodoApp"), and its signed commit history is + * the version log — re-running this when the schema changes records a new + * version while old frozen ids stay permanently resolvable. With + * `{ save: true }` it is signed and committed. + */ + public async createSchemaPointer( + frozen: FrozenSchema, + opts: { parent?: string; save?: boolean } = {}, + ): Promise> { + const version = frozen.presentation.ontology.version; + const ontology = await this.newResource({ + parent: opts.parent, + isA: core.classes.ontology, + validatePropVals: false, + propVals: { + [core.properties.shortname]: stringToSlug( + frozen.model.ontology.shortname, + ), + [core.properties.description]: frozen.presentation.ontology.description, + [core.properties.classes]: Object.values(frozen.classes) as string[], + [core.properties.properties]: Object.values( + frozen.properties, + ) as string[], + [core.properties.instances]: [], + ...(version ? { [server.properties.version]: version } : {}), + }, + }); + + if (opts.save) { + await ontology.save(); + } + + return ontology; + } + + /** + * Freezes a resource — and, by default, the structure it references — into + * immutable, content-addressed `did:ad:frozen` JSON-AD. Generic: works on any + * resource (Ontology, Document, Folder, …), not just schemas. References + * between included resources are rewritten to frozen ids; references outside + * the structure (core schema, drives, agents) stay as their normal subjects. + * Hierarchy/server metadata (`parent`, `lastCommit`, `localId`) is stripped. + * With `{ save: true }` each frozen body is published to `/frozen`. + */ + public async freezeStructure( + subject: string, + opts: FreezeStructureOptions = {}, + ): Promise { + const { closure = true, save = false } = opts; + const root = await this.getResource(subject); + + if (root.error) { + throw new Error(`Cannot freeze ${subject}: ${root.error}`); + } + + const bodies = new Map(); + const queue: string[] = [subject]; + + while (queue.length > 0) { + const current = queue.shift()!; + + if (bodies.has(current)) { + continue; + } + + const resource = + current === subject ? root : this.resources.get(current); + + if (!resource || !resource.isReady()) { + continue; + } + + const body = this.frozenBodyOf(resource); + bodies.set(current, body); + + if (closure) { + for (const ref of this.structureReferences(body)) { + if (!bodies.has(ref) && this.isFreezableResource(ref)) { + queue.push(ref); + } + } + } + } + + const freezable: FreezableResource[] = [...bodies].map( + ([localId, content]) => ({ localId, content }), + ); + const { resources, byLocalId } = freezeResources(freezable); + + const rootId = byLocalId.get(subject); + + if (!rootId) { + throw new Error(`Freezing produced no id for ${subject}`); + } + + if (save) { + await Promise.all( + resources.map(resource => + this.publishFrozenResource(resource.frozenId, resource.content), + ), + ); + } + + return { + root: rootId, + bySubject: Object.fromEntries(byLocalId), + frozen: Object.fromEntries( + resources.map(resource => [resource.frozenId, resource.content]), + ), + }; + } + + /** A resource's propvals as a frozen body — strips subject and mutable metadata. */ + private frozenBodyOf(resource: Resource): FrozenJsonValue { + const strip = new Set([ + core.properties.parent, + 'https://atomicdata.dev/properties/lastCommit', + core.properties.localId, + ]); + const body: Record = {}; + + for (const [key, value] of resource.getEntries()) { + if (strip.has(key) || value instanceof Uint8Array) { + continue; + } + + body[key] = value as FrozenJsonValue; + } + + return body; + } + + /** Subjects referenced by `body` that are already loaded resources. */ + private structureReferences(body: FrozenJsonValue): string[] { + const out: string[] = []; + + const walk = (value: FrozenJsonValue): void => { + if (typeof value === 'string') { + if (this.resources.has(value)) { + out.push(value); + } + } else if (Array.isArray(value)) { + value.forEach(walk); + } else if (value !== null && typeof value === 'object') { + Object.values(value).forEach(walk); + } + }; + + walk(body); + + return out; + } + + /** Whether a referenced subject should be pulled into a frozen structure. */ + private isFreezableResource(subject: string): boolean { + if ( + subject.startsWith('https://atomicdata.dev/') || + subject.startsWith('did:ad:agent:') || + subject.startsWith('did:ad:commit:') || + subject.startsWith('did:ad:frozen:') || + subject.startsWith('did:ad:blob:') + ) { + return false; + } + + const resource = this.resources.get(subject); + + return ( + !!resource && + resource.isReady() && + !resource.hasClasses(server.classes.drive) + ); + } + + /** + * Registers an app-bundled `*.schema.lock.json` into the store: verifies every + * frozen object by re-hash, then materializes each as a read-only Resource so + * the schema resolves offline with no server. This is "available without a + * host" — the lockfile travels with the code, and a frozen id is reproducible + * from it. Returns the lock so callers can read its id maps. Cycle "unit" + * objects are skipped (not yet individually materializable). + */ + public loadSchemaLock(lock: SchemaLock): SchemaLock { + const verification = verifySchemaLock(lock); + + if (!verification.ok) { + throw new Error( + `Refusing to load an invalid schema lock: ${verification.errors.join('; ')}`, + ); + } + + for (const [frozenId, content] of Object.entries(lock.frozen)) { + if ( + content && + typeof content === 'object' && + UNIT_MEMBERS_KEY in content + ) { + continue; + } + + registerFrozenBodies([ + { frozenId: frozenId as FrozenId, content: content as FrozenJsonValue }, + ]); + + const [resource] = new JSONADParser().parse(content, frozenId); + resource.loading = false; + this.addResource(resource, { skipCommitCompare: true }); + } + + return lock; + } + + private async publishFrozenResource( + frozenId: FrozenId, + content: unknown, + ): Promise { + const hash = frozenId.replace('did:ad:frozen:', ''); + const base = this.getServerUrl().replace(/\/$/, ''); + const response = await fetch(`${base}/frozen/${hash}`, { + method: 'PUT', + headers: { 'Content-Type': 'application/ad+json' }, + body: jcsCanonicalize(content as Parameters[0]), + }); + + if (!response.ok) { + throw new Error( + `Failed to publish frozen resource ${frozenId} (HTTP ${response.status})`, + ); + } + } + + /** + * Lazy publish-on-save: if a resource about to be committed references any + * `did:ad:frozen:` definitions (its class, a code-first property, an enum + * value…) whose bodies are locally known but not yet on the server, PUT them + * first — together with the transitive closure they reference (a class points + * at its property ids). This is what makes code-first schemas need no build or + * publish step: defining a schema registers its bodies, and the first save of + * an instance pushes the definitions. Publishing is idempotent (hash-keyed) and + * de-duplicated per session, so unchanged saves do no extra work. + */ + private async publishReferencedFrozen(resource: Resource): Promise { + const discovered = new Set(); + const queue: string[] = []; + + for (const [key, value] of resource.getEntries()) { + this.collectFrozenRefs(key, discovered, queue); + this.collectFrozenRefs(value as FrozenJsonValue, discovered, queue); + } + + if (queue.length === 0) { + return; + } + + const toPublish = new Map(); + + while (queue.length > 0) { + const id = queue.shift()!; + + if (toPublish.has(id) || this.publishedFrozenIds.has(id)) { + continue; + } + + const body = getRegisteredFrozenBody(id); + + if (body === undefined) { + // Not locally known — assume it is already hosted (or external). + continue; + } + + toPublish.set(id, body as FrozenJsonValue); + // Expand the closure: a class body references its property ids, etc. + this.collectFrozenRefs(body as FrozenJsonValue, discovered, queue); + } + + if (toPublish.size === 0) { + return; + } + + await Promise.all( + [...toPublish].map(([id, body]) => + this.publishFrozenResource(id as FrozenId, body).then(() => { + this.publishedFrozenIds.add(id); + }), + ), + ); + } + + /** + * Walks a JSON-AD value, collecting every `did:ad:frozen:` string into `into` + * and (optionally) appending newly-seen ids to `queue` for closure expansion. + */ + private collectFrozenRefs( + value: FrozenJsonValue, + into: Set, + queue?: string[], + ): void { + if (typeof value === 'string') { + if (value.startsWith('did:ad:frozen:') && !into.has(value)) { + into.add(value); + queue?.push(value); + } + } else if (Array.isArray(value)) { + for (const item of value) { + this.collectFrozenRefs(item, into, queue); + } + } else if (value !== null && typeof value === 'object') { + for (const item of Object.values(value)) { + this.collectFrozenRefs(item as FrozenJsonValue, into, queue); + } + } + } + + private assertCompatibleExistingSchemaProperty( + property: ConvertedSchemaPackage['properties'][number], + ): void { + if (!property.subject) { + return; + } + + const existing = this.resources.get(property.subject); + + if (!existing?.isReady() || !existing.hasClasses(core.classes.property)) { + return; + } + + const checks: Array< + [string, JSONValue | undefined, JSONValue | undefined] + > = [ + [ + core.properties.datatype, + existing.get(core.properties.datatype), + property.datatype, + ], + [ + core.properties.classtype, + existing.get(core.properties.classtype), + property.classType, + ], + [ + core.properties.allowsOnly, + existing.get(core.properties.allowsOnly), + property.allowsOnly as JSONValue | undefined, + ], + ]; + + for (const [field, actual, expected] of checks) { + if (JSON.stringify(actual ?? null) !== JSON.stringify(expected ?? null)) { + throw new Error( + `Schema property ${property.subject} is already registered with a different ${field}. Publish a new Property subject for datatype or semantic changes.`, + ); + } + } + } + + private async validateSchemaImports( + schema: AtomicSchemaPackage, + ): Promise { + for (const [name, schemaImport] of Object.entries(schema.imports ?? {})) { + if (!schemaImport.expectedHash) { + continue; + } + + const resource = await this.getResource(schemaImport.subject); + const actualHash = resource.get(SCHEMA_HASH_PROPERTY)?.toString(); + + if (actualHash !== schemaImport.expectedHash) { + throw new Error( + `Schema import ${name} expected ${schemaImport.expectedHash} but resolved ${actualHash ?? 'no schema hash'} from ${schemaImport.subject}`, + ); + } + } + } + + private async resolveRegisteredSchemaPropertyKey( + key: string, + schema: AtomicSchemaPackage, + generatedProperties: Record>, + ): Promise { + const generated = generatedProperties[key]; + + if (generated) { + return generated.subject; + } + + if (!key.startsWith('ref:')) { + throw new Error(`Could not resolve generated schema property key ${key}`); + } + + const ref = key.slice('ref:'.length); + const [importName, collectionName, propertyShortname] = ref.split('.'); + + if (collectionName !== 'properties' || !propertyShortname) { + throw new Error(`Unsupported schema property reference ${ref}`); + } + + const schemaImport = schema.imports?.[importName]; + + if (!schemaImport) { + throw new Error( + `Schema property reference ${ref} uses unknown import ${importName}`, + ); + } + + const importedOntology = await this.getResource( + schemaImport.subject, + ); + const propertySubjects = importedOntology.getArray( + core.properties.properties, + ) as string[]; + + for (const subject of propertySubjects) { + const property = await this.getResource(subject); + + if (property.get(core.properties.shortname) === propertyShortname) { + return property.subject; + } + } + + throw new Error( + `Schema property reference ${ref} did not match any Property in ${schemaImport.subject}`, + ); + } + /** * Creates a new personal Drive for the current Agent, saves it, and links * it to the Agent resource. Returns the Drive's Resource (already saved). @@ -2117,6 +2791,48 @@ export class Store { return work; } + /** + * Resolves a `did:ad:frozen:` subject: fetches the JSON-AD bytes from + * `/frozen/{hash}`, verifies they hash to the id (trustless — the server is + * just a cache), and materializes a read-only Resource. Cycle "unit" objects + * are not yet materializable individually. + */ + private async fetchFrozenResource( + subject: string, + ): Promise> { + const pureId = subject.split('?')[0].split('#')[0]; + const hash = pureId.replace('did:ad:frozen:', ''); + const base = this.getServerUrl().replace(/\/$/, ''); + const response = await fetch(`${base}/frozen/${hash}`, { + headers: { Accept: 'application/ad+json' }, + }); + + if (!response.ok) { + throw new Error( + `Could not resolve frozen resource ${pureId} (HTTP ${response.status})`, + ); + } + + const body = JSON.parse(await response.text()); + + // Trustless: the bytes must hash to the requested id. + if (frozenIdFor(body) !== pureId) { + throw new Error(`Frozen resource ${pureId} failed hash verification`); + } + + if (body && typeof body === 'object' && UNIT_MEMBERS_KEY in body) { + throw new Error( + `Frozen resource ${pureId} is a reference-cycle unit; individual materialization is not yet supported.`, + ); + } + + const [resource] = new JSONADParser().parse(body, pureId); + resource.loading = false; + this.addResource(resource as Resource, { skipCommitCompare: true }); + + return resource as Resource; + } + private async _fetchResourceFromServerImpl< C extends OptionalClass = UnknownClass, >( @@ -2152,6 +2868,13 @@ export class Store { return local; } + // Frozen resources are content-addressed and immutable: fetch their bytes + // from `/frozen/{hash}`, verify by re-hash, and materialize locally — no + // commits, no auth, no trust in the source. + if (normalizedSubject.startsWith('did:ad:frozen:')) { + return this.fetchFrozenResource(normalizedSubject); + } + if (opts.setLoading) { const newR = new Resource(subject); newR.loading = true; @@ -2449,21 +3172,18 @@ export class Store { ); } + // Description is presentation, not identity — content-addressed `did:ad:frozen` + // properties omit it deliberately. Default to empty rather than rejecting, + // so a frozen property still resolves and validates. const description = resource.get(core.properties.description); - if (description === undefined) { - throw Error( - `Property ${subject} has no description: ${resource.debugValueSummary()}`, - ); - } - const classTypeURL = resource.get(core.properties.classtype)?.toString(); const propery: Property = { subject, classType: classTypeURL, shortname: shortname.toString(), - description: description.toString(), + description: description?.toString() ?? '', datatype: datatypeFromUrl(datatypeUrl.toString()), allowsOnly: resource.get(core.properties.allowsOnly), }; diff --git a/browser/lib/tests/frozen-e2e.integration.test.ts b/browser/lib/tests/frozen-e2e.integration.test.ts new file mode 100644 index 000000000..315d17ae9 --- /dev/null +++ b/browser/lib/tests/frozen-e2e.integration.test.ts @@ -0,0 +1,76 @@ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { setTimeout as delay } from 'node:timers/promises'; + +import { Agent } from '../src/agent.js'; +import { Store } from '../src/store.js'; +import { Datatype } from '../src/datatypes.js'; +import { core } from '../src/ontologies/core.js'; +import { startServer, type ServerHandle } from './server-fixture.js'; + +const todoSchema = { + name: 'FrozenTodoApp', + classes: { + todo: { + type: 'object' as const, + required: ['title'], + properties: { + title: { type: 'string' as const, description: 'Task title' }, + done: { type: 'boolean' as const }, + }, + }, + }, +}; + +describe('frozen schema: publish + resolve through a real server', () => { + let server: ServerHandle; + + beforeAll(async () => { + server = await startServer(); + }, 60_000); + + afterAll(async () => { + await server?.stop(); + }); + + it('producer publishes frozen resources; a fresh consumer resolves and uses them', async () => { + const agent = await Agent.fromSecret(server.agentSecret); + const producer = new Store({ serverUrl: server.serverUrl, agent }); + producer.setServerConnected(true); + + // Freeze + PUT every frozen body to /frozen on the real server. + const frozen = await producer.registerFrozenSchema(todoSchema, { + save: true, + }); + const titleId = frozen.properties['todo.title']; + const classId = frozen.classes.todo; + + expect(titleId).toMatch(/^did:ad:frozen:[0-9a-f]{64}$/); + + // A brand-new consumer with no shared memory and no agent. + const consumer = new Store({ serverUrl: server.serverUrl }); + consumer.setServerConnected(true); + + // Give the server a beat to have the bytes durably available. + await delay(250); + + // Resolves over HTTP: GET /frozen -> re-hash verify -> materialize. + const prop = await consumer.getProperty(titleId); + expect(prop.shortname).toBe('title'); + expect(prop.datatype).toBe(Datatype.STRING); + + const todoClass = await consumer.getResource(classId); + expect(todoClass.get(core.properties.shortname)).toBe('todo'); + expect(todoClass.get(core.properties.requires)).toEqual([titleId]); + + // The consumer can build an instance against the frozen class + property. + const todo = await consumer.newResource({ + isA: classId, + propVals: { [titleId]: 'Buy milk' }, + }); + expect(todo.getClasses()).toContain(classId); + expect(todo.get(titleId)).toBe('Buy milk'); + + producer.disconnect(); + consumer.disconnect(); + }, 60_000); +}); diff --git a/browser/lib/tests/schema-code-first.integration.test.ts b/browser/lib/tests/schema-code-first.integration.test.ts new file mode 100644 index 000000000..4a7d33ffb --- /dev/null +++ b/browser/lib/tests/schema-code-first.integration.test.ts @@ -0,0 +1,89 @@ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { setTimeout as delay } from 'node:timers/promises'; + +import { Agent } from '../src/agent.js'; +import { Store } from '../src/store.js'; +import { SCHEMA_HASH_PROPERTY } from '../src/schema.js'; +import { startServer, type ServerHandle } from './server-fixture.js'; +import { todoSchema } from './schema-code-first/producer-schema.js'; +import { defineProjectSchema } from './schema-code-first/consumer-schema.js'; + +async function waitForResource(store: Store, subject: string) { + const deadline = Date.now() + 15_000; + let lastError: unknown; + + while (Date.now() < deadline) { + try { + return await store.getResource(subject); + } catch (e) { + lastError = e; + await delay(250); + } + } + + throw lastError instanceof Error + ? lastError + : new Error(`Timed out fetching ${subject}`); +} + +describe('code-first schema publish and reuse', () => { + let server: ServerHandle; + + beforeAll(async () => { + server = await startServer(); + }, 60_000); + + afterAll(async () => { + await server?.stop(); + }); + + it('publishes a JS-defined schema, then another Store fetches and reuses one Property', async () => { + const agent = await Agent.fromSecret(server.agentSecret); + const producer = new Store({ serverUrl: server.serverUrl, agent }); + producer.setServerConnected(true); + + const published = await producer.registerSchema(todoSchema, { save: true }); + + const consumer = new Store({ serverUrl: server.serverUrl, agent }); + consumer.setServerConnected(true); + + const fetchedOntology = await waitForResource( + consumer, + published.ontology.subject, + ); + + expect(fetchedOntology.get(SCHEMA_HASH_PROPERTY)).toBe( + published.model.ontology.schemaHash, + ); + + const consumerSchema = defineProjectSchema( + published.ontology.subject, + published.model.ontology.schemaHash, + ); + const registeredConsumerSchema = + await consumer.registerSchema(consumerSchema); + + expect(registeredConsumerSchema.classes.project.props.requires).toEqual([ + published.properties['todo.title'].subject, + ]); + + const reusedProperty = await consumer.getProperty( + published.properties['todo.title'].subject, + ); + expect(reusedProperty.shortname).toBe('title'); + + const project = await consumer.newResource({ + isA: registeredConsumerSchema.classes.project.subject, + propVals: { + [published.properties['todo.title'].subject]: 'Shared title', + }, + }); + + expect(project.get(published.properties['todo.title'].subject)).toBe( + 'Shared title', + ); + + producer.disconnect(); + consumer.disconnect(); + }, 60_000); +}); diff --git a/browser/lib/tests/schema-code-first/consumer-schema.ts b/browser/lib/tests/schema-code-first/consumer-schema.ts new file mode 100644 index 000000000..76fd496d3 --- /dev/null +++ b/browser/lib/tests/schema-code-first/consumer-schema.ts @@ -0,0 +1,31 @@ +import { defineSchema, type SchemaHash } from '../../src/schema.js'; + +export function defineProjectSchema( + todoOntologySubject: string, + expectedHash: SchemaHash, +) { + return defineSchema({ + name: 'ProjectConsumer', + version: '1.0.0', + description: 'A consumer schema that reuses a producer property', + imports: { + todo: { + subject: todoOntologySubject, + expectedHash, + }, + }, + classes: { + project: { + title: 'Project', + description: 'A project that reuses todo.title', + type: 'object', + required: ['title'], + properties: { + title: { + $ref: 'todo.properties.title', + }, + }, + }, + }, + }); +} diff --git a/browser/lib/tests/schema-code-first/producer-schema.ts b/browser/lib/tests/schema-code-first/producer-schema.ts new file mode 100644 index 000000000..8fa8287a9 --- /dev/null +++ b/browser/lib/tests/schema-code-first/producer-schema.ts @@ -0,0 +1,26 @@ +import { defineSchema } from '../../src/schema.js'; + +export const todoSchema = defineSchema({ + name: 'TodoProject', + version: '1.0.0', + description: 'A tiny producer-owned todo schema', + classes: { + todo: { + title: 'Todo', + description: 'A task in a todo list', + type: 'object', + required: ['title'], + properties: { + title: { + type: 'string', + description: 'Task title', + }, + done: { + type: 'boolean', + description: 'Whether the task is complete', + default: false, + }, + }, + }, + }, +}); diff --git a/browser/lib/tsup.config.ts b/browser/lib/tsup.config.ts index 7ef0a56d8..86ee4934b 100644 --- a/browser/lib/tsup.config.ts +++ b/browser/lib/tsup.config.ts @@ -7,6 +7,7 @@ export default defineConfig(options => ({ minify: !options.watch, entry: { index: 'src/index.ts', + schema: 'src/schema.ts', // The DedicatedWorker that hosts the WASM ClientDb. Bundled as its own // entry so consumers can load it via `new Worker(new URL(..., // import.meta.url))`. Without this, the previous setup relied on a diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index 2ac073acb..c53c23c3d 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -71,6 +71,7 @@ - [Schema](schema/intro.md) - [Classes](schema/classes.md) - [Datatypes](schema/datatypes.md) + - [Code-first schemas](schema/code-first.md) - [FAQ](schema/faq.md) - [Atomic Data Extended](extended.md) diff --git a/docs/src/schema/code-first.md b/docs/src/schema/code-first.md new file mode 100644 index 000000000..3dd7dbbcd --- /dev/null +++ b/docs/src/schema/code-first.md @@ -0,0 +1,149 @@ +{{#title Code-first schemas: define your data model in JavaScript}} + +# Code-first schemas + +Instead of clicking together Classes and Properties in the browser first, you can +declare your data model **in code** and just use it. There is **no build step, no +codegen, and no publish command** — your schema is a value in your repo, and the +Classes and Properties are published automatically the first time you save data +that uses them. + +This works because schemas are **content-addressed**: a Class or Property's +identifier is a hash of its definition (see [`did:ad:frozen`](../did.md)). The +identifier is computed locally, in-process, so your code — not a server — is the +source of truth. + +> You'll need a running [Atomic Server](../atomic-server.md) and an +> [Agent](../agents.md) (the data-browser creates one for you on first run — copy +> its secret from **User Settings**). + +## 1. Define a schema + +`defineSchema` takes a small, JSON-Schema-like object that maps onto Atomic +[Classes](classes.md), [Properties](classes.md), and [Datatypes](datatypes.md): + +```ts +// schema.ts +import { defineSchema } from '@tomic/lib'; + +export const todoSchema = defineSchema({ + name: 'TodoApp', + version: '1.0.0', + classes: { + todo: { + type: 'object', + description: 'A task in a todo list', + required: ['title'], + properties: { + title: { type: 'string', description: 'Task title' }, + done: { type: 'boolean', description: 'Whether the task is complete' }, + dueAt: { type: 'string', format: 'date', description: 'Due date' }, + }, + }, + }, +}); +``` + +Each `properties` entry becomes a **Property**, each class a **Class**, and the +whole thing an **Ontology**. `type`/`format` map to Atomic datatypes +(`string`→string, `boolean`→boolean, `string`+`format: date`→date, +`integer`→integer, `number`→float, …). Anything not in `required` is *recommended*. + +## 2. Use it — no build step + +`todoSchema.classes` and `todoSchema.properties` are typed handles to the +content-addressed ids. Use them directly. The first time you `save()` a resource +that references them, the Store publishes the Class and Property definitions to +your server automatically: + +```ts +import { Agent, Store } from '@tomic/lib'; +import { todoSchema } from './schema'; + +const agent = await Agent.fromSecret(process.env.AGENT_SECRET); +const store = new Store({ serverUrl: 'http://localhost:9883', agent }); +store.setServerConnected(true); + +const todo = await store.newResource({ + isA: todoSchema.classes.todo, // typed; autocompletes 'todo' + propVals: { + [todoSchema.properties.title]: 'Buy milk', + [todoSchema.properties.done]: false, + }, +}); + +await todo.save(); // ← publishes the todo Class + its Properties if the server lacks them +``` + +That's the whole loop: **define, use, save.** No `ad-generate`, no generated +files to commit, no separate publish step. Publishing is idempotent (hash-keyed), +so an unchanged schema re-saves with no extra work. + +The handles give you autocomplete and typo-safety on class and property keys. +(Per-field inference of `resource.props.title` still requires the generated +bindings below; without them, `props` is loosely typed but works at runtime.) + +## 3. Updating your schema + +Your schema file is the source of truth. To change the model, **just edit the +file.** The next time your app saves a resource using the changed Class or +Property, the new definition publishes itself — there's nothing else to run. + +What happens to identity when you change something is the important part, and +it's designed to keep existing data safe: + +- **Add a Class or Property** → new resources are created; everything else is + untouched. +- **Change only a description, label, or translation** → identifiers stay the + same. Presentation is not part of a Property's identity, so cosmetic edits never + churn ids or invalidate data. +- **Change a Property's datatype** (or other machine meaning) → this is a + breaking change, so it produces a **new** Property. The old one stays valid + forever, so resources created with it keep working. You're never silently + reinterpreting existing data. + +You get this for free, because `defineSchema` already gives you content-addressed +(`did:ad:frozen:`) ids: every identifier is the hash of its machine meaning. The +*same* definition always yields the *same* id (idempotent; identical definitions +across apps even dedupe), and any change yields a new id deterministically. A +frozen definition is immutable and read-only, and shows a **❄ Frozen** badge in +the data-browser. + +### Versioning across releases + +- Bump `version` in your schema for a new release; old ids remain resolvable, so + old data is never orphaned. +- To make a release resolvable **offline, with no server**, commit a + `*.schema.lock.json` and load it with + [`Store.loadSchemaLock(lock)`](../js-lib/store.md) — the lockfile travels with + your code. +- For a stable, *editable* "latest" handle that also renders in the GUI, keep a + normal Ontology pointing at the current frozen ids — `createSchemaPointer` + builds one, and its commit history is your version log. + +## Optional: `@tomic/cli` + +You never *need* the CLI, but [`@tomic/cli`](../js-cli.md) is handy for two things: + +- **Pre-publish** a schema so it's browsable in the data-browser before any app + has saved data: `npx ad-generate schema ./schema.ts`. +- **Generate committed `.ts` bindings** (and a `--lock` file) if you prefer those + over the inline `defineSchema` handles — this is also what enables fully-typed + `resource.props.title` access. + +## What you keep from the GUI + +Code-first does **not** replace the [Ontology editor](../atomicserver/gui.md) or +[table view](../atomicserver/gui/tables.md) — +they still create the same Classes and Properties, and a code-first Ontology +remains fully viewable and editable in the browser. Use whichever fits: define in +code for repeatable, reviewable, version-controlled schemas; use the GUI for quick +exploration. + +## Cross-language + +A schema's frozen identifiers are a pure function of its content (RFC 8785 JCS + +BLAKE3), so they are reproducible in any language. The Rust SDK +(`atomic_lib::frozen::freeze_schema`) produces byte-for-byte identical ids — so a +schema authored in JavaScript and one authored in Rust converge on the same +resources. diff --git a/examples/code-first-schema/publish-schema.mjs b/examples/code-first-schema/publish-schema.mjs new file mode 100644 index 000000000..aa041871c --- /dev/null +++ b/examples/code-first-schema/publish-schema.mjs @@ -0,0 +1,63 @@ +// A tiny @tomic/lib "code-first schema" app — no build step, no codegen. +// +// AGENT_SECRET= node publish-schema.mjs +// +// Defines a schema in code, creates a resource that uses it, and saves it. +// The Class and Property definitions are content-addressed and publish +// themselves to the server on the first save — nothing else to run. +import { Agent, Store, defineSchema } from '@tomic/lib'; + +const SERVER = process.env.SERVER_URL || 'http://localhost:9883'; +const SECRET = process.env.AGENT_SECRET; + +if (!SECRET) { + throw new Error('Set AGENT_SECRET (your agent secret from the data-browser).'); +} + +// Define the data model in code — JSON-Schema-like, mapped to Atomic. +// `todoSchema.classes` / `.properties` are content-addressed `did:ad:frozen` ids, +// computed locally (no server needed to have an identity). +const todoSchema = defineSchema({ + name: 'TodoApp', + version: '1.0.0', + classes: { + todo: { + title: 'Todo', + description: 'A task in a todo list', + type: 'object', + required: ['title'], + properties: { + title: { type: 'string', description: 'Task title' }, + done: { type: 'boolean', description: 'Whether the task is complete' }, + dueAt: { type: 'string', format: 'date', description: 'Due date' }, + }, + }, + }, +}); + +console.log('Schema ids (computed locally, no server):'); +console.log(' CLASS todo ' + todoSchema.classes.todo); +console.log(' PROPERTY title ' + todoSchema.properties.title); +console.log(' PROPERTY done ' + todoSchema.properties.done); + +const agent = await Agent.fromSecret(SECRET); +const store = new Store({ serverUrl: SERVER, agent }); +store.setServerConnected(true); + +// Create and save a resource that uses the schema. Saving auto-publishes the +// Class + Property definitions to the server (idempotent, hash-keyed). +const todo = await store.newResource({ + isA: todoSchema.classes.todo, + propVals: { + [todoSchema.properties.title]: 'Buy milk', + [todoSchema.properties.done]: false, + }, +}); + +await todo.save(); + +console.log('\nSaved a todo — its Class & Properties are now on the server:'); +console.log(' TODO ' + todo.subject); +console.log('Open the class id above in the data-browser to see the schema.'); + +process.exit(0); diff --git a/lib/src/commit.rs b/lib/src/commit.rs index b6090f2c6..5f283b0e1 100644 --- a/lib/src/commit.rs +++ b/lib/src/commit.rs @@ -353,6 +353,16 @@ impl Commit { let commit = self; let subject = commit.subject.clone(); + // Frozen resources are immutable, content-addressed, and signatureless — + // they live in Tree::Frozen and are verified by re-hash, never edited. + if subject.is_frozen_did() { + return Err(format!( + "Cannot commit to {}: `did:ad:frozen` resources are immutable and content-addressed.", + subject + ) + .into()); + } + if subject.is_did() && subject.as_str().starts_with("did:ad:") { let pure_id = subject.pure_id(); let b64_part = if subject.is_agent_did() { @@ -1870,6 +1880,42 @@ mod test { ); } + #[tokio::test] + async fn commit_to_frozen_subject_is_rejected() { + let (store, _agent) = store_with_known_agent().await; + let commit = Commit { + subject: + "did:ad:frozen:0000000000000000000000000000000000000000000000000000000000000000" + .into(), + created_at: 0, + signer: "did:ad:agent:placeholder".into(), + loro_update: None, + destroy: None, + signature: Some("placeholder".to_string()), + previous_commit: None, + is_genesis: None, + url: None, + }; + let opts = CommitOpts { + validate_signature: false, + validate_timestamp: false, + validate_previous_commit: false, + validate_rights: false, + ..CommitOpts::no_validations_no_index() + }; + // The frozen check fires before signature validation, so an invalid + // signature here is irrelevant — immutability is enforced first. + let err = commit + .validate_and_build_response(&opts, &store) + .await + .unwrap_err(); + assert!( + err.to_string().to_lowercase().contains("immutable"), + "expected an immutability error, got: {}", + err + ); + } + /// Signing a commit with agent B but writing to a resource that agent A /// created (and owns) must fail signature validation. #[tokio::test] diff --git a/lib/src/db.rs b/lib/src/db.rs index d4cfae23b..91dffbac0 100644 --- a/lib/src/db.rs +++ b/lib/src/db.rs @@ -850,6 +850,51 @@ impl Db { store.get_resource_extended(subject, false, for_agent).await } + /// Resolves a `did:ad:frozen:` subject from [`Tree::Frozen`]: loads the + /// stored JSON-AD bytes, **verifies they hash to the requested id** + /// (trustless — no signature, no trust in the source), and parses them into + /// a read-only Resource. Immutability is enforced elsewhere by rejecting + /// commits to frozen subjects. Cycle "unit" objects are not yet + /// materializable as individual resources. + async fn materialize_frozen(&self, subject: &Subject) -> AtomicResult { + let id = subject.pure_id(); + let hash_hex = subject + .frozen_hash_hex() + .ok_or_else(|| AtomicError::not_found(format!("Invalid frozen subject: {}", id)))?; + let bytes = self + .kv + .get(Tree::Frozen, hash_hex.as_bytes())? + .ok_or_else(|| AtomicError::not_found(format!("Frozen resource not found: {}", id)))?; + let body: serde_json::Value = serde_json::from_slice(&bytes) + .map_err(|e| format!("Stored frozen body for {} is not valid JSON: {}", id, e))?; + + // Trustless verification: the stored bytes must hash to the requested id. + crate::frozen::verify_frozen(&id, &body)?; + + if crate::frozen::is_unit(&body) { + return Err(format!( + "Frozen subject {} is a reference-cycle unit; materializing individual members is not yet supported.", + id + ) + .into()); + } + + let serde_json::Value::Object(map) = body else { + return Err(format!("Frozen body for {} must be a JSON object", id).into()); + }; + + // DontSave: a frozen resource is read-only — never write it to + // Tree::Resources and never run class-`requires` validation (a frozen + // definition deliberately omits presentation like `description`; its + // validity is the hash, not class completeness). + let parse_opts = crate::parse::ParseOpts { + save: crate::parse::SaveOpts::DontSave, + skip_unknown_props: true, + ..Default::default() + }; + crate::parse::parse_json_ad_map_to_resource(map, self, Some(id), &parse_opts).await + } + pub fn add_class_extender(&self, class_extender: ClassExtender) -> AtomicResult<()> { let mut extenders = self .class_extenders @@ -2027,6 +2072,11 @@ impl Storelike for Db { #[instrument(skip_all)] async fn get_resource(&self, subject: &Subject) -> AtomicResult { let normalized = self.normalize_subject(subject); + // Frozen resources are content-addressed and immutable; they live in + // Tree::Frozen, not Tree::Resources, and materialize by re-hash + parse. + if normalized.is_frozen_did() { + return self.materialize_frozen(&normalized).await; + } let subject_str = normalized.pure_id(); if let Ok(propvals) = self.get_propvals(&subject_str) { let mut res_subject = normalized.clone(); diff --git a/lib/src/db/redb_store.rs b/lib/src/db/redb_store.rs index eeaef3508..127fbd4ca 100644 --- a/lib/src/db/redb_store.rs +++ b/lib/src/db/redb_store.rs @@ -31,6 +31,7 @@ const TABLE_DRIVE_MAPPING: TableDefinition<&[u8], &[u8]> = TableDefinition::new( const TABLE_DID_MAPPING: TableDefinition<&[u8], &[u8]> = TableDefinition::new("did_mapping"); const TABLE_LORO_SNAPSHOTS: TableDefinition<&[u8], &[u8]> = TableDefinition::new("loro_snapshots"); const TABLE_BLOBS: TableDefinition<&[u8], &[u8]> = TableDefinition::new("blobs"); +const TABLE_FROZEN: TableDefinition<&[u8], &[u8]> = TableDefinition::new("frozen_v1"); fn table_def(tree: Tree) -> TableDefinition<'static, &'static [u8], &'static [u8]> { match tree { @@ -44,6 +45,7 @@ fn table_def(tree: Tree) -> TableDefinition<'static, &'static [u8], &'static [u8 Tree::DidMapping => TABLE_DID_MAPPING, Tree::LoroSnapshots => TABLE_LORO_SNAPSHOTS, Tree::Blobs => TABLE_BLOBS, + Tree::Frozen => TABLE_FROZEN, } } @@ -151,6 +153,7 @@ impl RedbStore { let _ = tx.open_table(TABLE_DID_MAPPING); let _ = tx.open_table(TABLE_LORO_SNAPSHOTS); let _ = tx.open_table(TABLE_BLOBS); + let _ = tx.open_table(TABLE_FROZEN); tx.commit() .map_err(|e| format!("Failed to commit initial tables: {e}"))?; } @@ -186,6 +189,7 @@ impl RedbStore { let _ = tx.open_table(TABLE_DID_MAPPING); let _ = tx.open_table(TABLE_LORO_SNAPSHOTS); let _ = tx.open_table(TABLE_BLOBS); + let _ = tx.open_table(TABLE_FROZEN); tx.commit() .map_err(|e| format!("Failed to commit initial tables: {e}"))?; } @@ -224,6 +228,7 @@ impl RedbStore { let _ = tx.open_table(TABLE_DID_MAPPING); let _ = tx.open_table(TABLE_LORO_SNAPSHOTS); let _ = tx.open_table(TABLE_BLOBS); + let _ = tx.open_table(TABLE_FROZEN); tx.commit() .map_err(|e| format!("Failed to commit initial tables: {e}"))?; } diff --git a/lib/src/db/sled_store.rs b/lib/src/db/sled_store.rs index 283402217..b28e6e7c9 100644 --- a/lib/src/db/sled_store.rs +++ b/lib/src/db/sled_store.rs @@ -24,6 +24,7 @@ pub struct SledStore { did_mapping: sled::Tree, loro_snapshots: sled::Tree, blobs: sled::Tree, + frozen: sled::Tree, } impl SledStore { @@ -47,6 +48,7 @@ impl SledStore { let did_mapping = db.open_tree(Tree::DidMapping)?; let loro_snapshots = db.open_tree(Tree::LoroSnapshots)?; let blobs = db.open_tree(Tree::Blobs)?; + let frozen = db.open_tree(Tree::Frozen)?; Ok(SledStore { db, @@ -60,6 +62,7 @@ impl SledStore { did_mapping, loro_snapshots, blobs, + frozen, }) } @@ -80,6 +83,7 @@ impl SledStore { Tree::DidMapping => &self.did_mapping, Tree::LoroSnapshots => &self.loro_snapshots, Tree::Blobs => &self.blobs, + Tree::Frozen => &self.frozen, } } } @@ -159,6 +163,7 @@ impl KvStore for SledStore { let mut batch_did_mapping = sled::Batch::default(); let mut batch_loro_snapshots = sled::Batch::default(); let mut batch_blobs = sled::Batch::default(); + let mut batch_frozen = sled::Batch::default(); for op in operations { let batch = match op.tree { @@ -172,6 +177,7 @@ impl KvStore for SledStore { Tree::DidMapping => &mut batch_did_mapping, Tree::LoroSnapshots => &mut batch_loro_snapshots, Tree::Blobs => &mut batch_blobs, + Tree::Frozen => &mut batch_frozen, }; match op.method { Method::Insert => { @@ -226,6 +232,10 @@ impl KvStore for SledStore { .apply_batch(batch_blobs) .map_err(|e| format!("Failed to apply blobs batch: {}", e))?; + self.frozen + .apply_batch(batch_frozen) + .map_err(|e| format!("Failed to apply frozen batch: {}", e))?; + Ok(()) } diff --git a/lib/src/db/test.rs b/lib/src/db/test.rs index a73d08582..0c20f8f14 100644 --- a/lib/src/db/test.rs +++ b/lib/src/db/test.rs @@ -656,6 +656,70 @@ async fn blobs_storage() { assert_eq!(data.to_vec(), retrieved); } +#[tokio::test] +async fn frozen_storage() { + let store = Db::init_temp("frozen_storage").await.unwrap(); + // A frozen object is content-addressed by the BLAKE3 of its JCS bytes. + let body = serde_json::json!({ + "https://atomicdata.dev/properties/shortname": "title", + "https://atomicdata.dev/properties/datatype": "https://atomicdata.dev/datatypes/string" + }); + let id = crate::frozen::frozen_id(&body).unwrap(); + let hash_hex = id.strip_prefix("did:ad:frozen:").unwrap(); + let bytes = serde_jcs::to_string(&body).unwrap().into_bytes(); + + store + .kv + .insert(Tree::Frozen, hash_hex.as_bytes(), &bytes) + .unwrap(); + let retrieved = store + .kv + .get(Tree::Frozen, hash_hex.as_bytes()) + .unwrap() + .unwrap(); + + // Round-trips, and the stored bytes still verify against the id. + let parsed: serde_json::Value = serde_json::from_slice(&retrieved).unwrap(); + assert!(crate::frozen::verify_frozen(&id, &parsed).is_ok()); +} + +#[tokio::test] +async fn frozen_materialization() { + let store = Db::init_temp("frozen_materialization").await.unwrap(); + let body = serde_json::json!({ + "https://atomicdata.dev/properties/isA": ["https://atomicdata.dev/classes/Property"], + "https://atomicdata.dev/properties/shortname": "title", + "https://atomicdata.dev/properties/datatype": "https://atomicdata.dev/datatypes/string" + }); + let id = crate::frozen::frozen_id(&body).unwrap(); + let hash_hex = id.strip_prefix("did:ad:frozen:").unwrap(); + let bytes = serde_jcs::to_string(&body).unwrap().into_bytes(); + store + .kv + .insert(Tree::Frozen, hash_hex.as_bytes(), &bytes) + .unwrap(); + + // Resolving the frozen subject materializes a read-only Resource. + let subject = Subject::from_raw(&id, None); + let resource = store.get_resource(&subject).await.unwrap(); + + assert_eq!(resource.get_subject().as_str(), id); + assert_eq!(resource.get(urls::SHORTNAME).unwrap().to_string(), "title"); + assert_eq!( + resource.get(urls::DATATYPE_PROP).unwrap().to_string(), + "https://atomicdata.dev/datatypes/string" + ); + + // Bytes that don't hash to the addressed id are rejected on resolve. + let bogus_hex = "00".repeat(32); + store + .kv + .insert(Tree::Frozen, bogus_hex.as_bytes(), b"{\"x\":1}") + .unwrap(); + let bogus_subject = Subject::from_raw(&format!("did:ad:frozen:{}", bogus_hex), None); + assert!(store.get_resource(&bogus_subject).await.is_err()); +} + #[tokio::test] /// Changing these values actually correctly updates the index. async fn invalidate_cache() { diff --git a/lib/src/db/trees.rs b/lib/src/db/trees.rs index e505eb2d6..f717ec79b 100644 --- a/lib/src/db/trees.rs +++ b/lib/src/db/trees.rs @@ -27,6 +27,10 @@ pub enum Tree { LoroSnapshots, /// Content-addressed storage for binary files, keyed by BLAKE3 hash. Blobs, + /// Content-addressed storage for immutable `did:ad:frozen` JSON-AD bodies, + /// keyed by the 32-byte BLAKE3 hash of their JCS canonicalization. Unlike + /// `Blobs` (opaque bytes), these materialize into read-only Resources. + Frozen, } const RESOURCES: &str = "resources_v3"; @@ -44,6 +48,7 @@ const DRIVE_MAPPING: &str = "drive_mapping"; const DID_MAPPING: &str = "did_mapping"; const LORO_SNAPSHOTS: &str = "loro_snapshots"; const BLOBS: &str = "blobs"; +const FROZEN: &str = "frozen_v1"; impl std::fmt::Display for Tree { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { @@ -58,6 +63,7 @@ impl std::fmt::Display for Tree { Tree::DidMapping => f.write_str(DID_MAPPING), Tree::LoroSnapshots => f.write_str(LORO_SNAPSHOTS), Tree::Blobs => f.write_str(BLOBS), + Tree::Frozen => f.write_str(FROZEN), } } } @@ -76,6 +82,7 @@ impl AsRef<[u8]> for Tree { Tree::DidMapping => DID_MAPPING.as_bytes(), Tree::LoroSnapshots => LORO_SNAPSHOTS.as_bytes(), Tree::Blobs => BLOBS.as_bytes(), + Tree::Frozen => FROZEN.as_bytes(), } } } diff --git a/lib/src/frozen.rs b/lib/src/frozen.rs new file mode 100644 index 000000000..d6545880d --- /dev/null +++ b/lib/src/frozen.rs @@ -0,0 +1,800 @@ +//! Content-addressed `did:ad:frozen` resources. +//! +//! A frozen resource is identified by `did:ad:frozen:{blake3-hex}` over the +//! RFC 8785 (JCS) canonicalization of its JSON-AD body. This is the exact hash +//! the TypeScript producer computes +//! (`browser/lib/src/freeze.ts#frozenIdFor`), so ids are byte-for-byte +//! reproducible across languages. The shared contract is pinned by +//! `test-vectors/frozen.json`. +//! +//! Frozen objects are immutable and signatureless: they are verified by +//! re-hashing, never by a commit signature. See `planning/did-ad-frozen-server.md`. + +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; + +use serde_json::Value; + +use crate::errors::AtomicResult; +use crate::subject::DID_AD_FROZEN_PREFIX; + +/// Placeholder for an intra-cycle reference, by the referent's canonical index. +/// Matches `browser/lib/src/freeze.ts#SELF_PREFIX`. +const SELF_PREFIX: &str = "did:ad:frozen:self:"; + +/// Reserved top-level key marking a frozen **unit** object: the materialized form +/// of a reference cycle, whose value is an ordered array of member bodies that +/// reference each other by `did:ad:frozen:self:{index}`. Matches +/// `browser/lib/src/freeze.ts#UNIT_MEMBERS_KEY`. +pub const FROZEN_UNIT_KEY: &str = "urn:atomic-freeze:unit"; + +/// True if a frozen body is a cycle unit (vs. a single resource body). +pub fn is_unit(body: &serde_json::Value) -> bool { + body.get(FROZEN_UNIT_KEY).is_some() +} + +/// Computes the `did:ad:frozen:{blake3-hex}` id for a JSON-AD body. +pub fn frozen_id(body: &serde_json::Value) -> AtomicResult { + let canonical = serde_jcs::to_string(body) + .map_err(|e| format!("Failed to JCS-canonicalize frozen body: {}", e))?; + let hash = blake3::hash(canonical.as_bytes()); + + Ok(format!("{}{}", DID_AD_FROZEN_PREFIX, hash.to_hex())) +} + +/// Returns `Ok(())` when `body` hashes to `id`, otherwise an error. This is the +/// verify-by-rehash check the server runs on store and serve; no signature or +/// trust in the source is required. +pub fn verify_frozen(id: &str, body: &serde_json::Value) -> AtomicResult<()> { + let actual = frozen_id(body)?; + + if actual == id { + Ok(()) + } else { + Err(format!( + "Frozen body hashes to {} but was addressed as {}", + actual, id + ) + .into()) + } +} + +/// A resource to be frozen, identified by a temporary `local_id`. Any string +/// value inside another resource's `content` that equals this `local_id` is a +/// reference and is rewritten to the computed frozen id. +pub struct FreezableResource { + pub local_id: String, + pub content: Value, +} + +/// A content-addressed result: one per distinct frozen object (ordinary resource +/// or cycle unit). `unit` lists the local_ids it covers. +pub struct FrozenResource { + pub frozen_id: String, + pub content: Value, + pub unit: Vec, +} + +pub struct FreezeResult { + pub resources: Vec, + pub by_local_id: HashMap, +} + +/// Content-addresses a set of mutually-referencing resources into a +/// `did:ad:frozen` Merkle DAG, byte-for-byte identical to +/// `browser/lib/src/freeze.ts#freezeResources`. References are rewritten to the +/// referent's hash before hashing; each reference cycle is frozen as one unit +/// object so every stored object stays verifiable by re-hash. +pub fn freeze_resources(input: Vec) -> AtomicResult { + let ids: HashSet = input.iter().map(|r| r.local_id.clone()).collect(); + + if ids.len() != input.len() { + return Err("freeze_resources: local_id values must be unique".into()); + } + + let by_id: HashMap = + input.into_iter().map(|r| (r.local_id, r.content)).collect(); + let nodes: Vec = by_id.keys().cloned().collect(); + let edges: HashMap> = by_id + .iter() + .map(|(id, content)| (id.clone(), collect_refs(content, &ids))) + .collect(); + + // Tarjan emits SCCs sinks-first (reverse topological), so every out-edge + // points at an already-frozen component. + let sccs = strongly_connected_components(&nodes, &edges); + let mut frozen_by_local: HashMap = HashMap::new(); + let mut out: Vec = Vec::new(); + + for scc in sccs { + let is_cycle = scc.len() > 1 + || edges + .get(&scc[0]) + .map(|e| e.contains(&scc[0])) + .unwrap_or(false); + + if is_cycle { + freeze_cycle(&scc, &by_id, &edges, &mut frozen_by_local, &mut out)?; + } else { + freeze_singleton(&scc[0], &by_id, &edges, &mut frozen_by_local, &mut out)?; + } + } + + Ok(FreezeResult { + resources: out, + by_local_id: frozen_by_local, + }) +} + +fn freeze_singleton( + local_id: &str, + by_id: &HashMap, + edges: &HashMap>, + frozen_by_local: &mut HashMap, + out: &mut Vec, +) -> AtomicResult<()> { + let map = resolved_ref_map(edges.get(local_id), frozen_by_local); + let content = substitute(&by_id[local_id], &map); + let frozen_id = frozen_id(&content)?; + + frozen_by_local.insert(local_id.to_string(), frozen_id.clone()); + + if let Some(existing) = out.iter_mut().find(|r| r.frozen_id == frozen_id) { + existing.unit.push(local_id.to_string()); + } else { + out.push(FrozenResource { + frozen_id, + content, + unit: vec![local_id.to_string()], + }); + } + + Ok(()) +} + +fn freeze_cycle( + scc: &[String], + by_id: &HashMap, + edges: &HashMap>, + frozen_by_local: &mut HashMap, + out: &mut Vec, +) -> AtomicResult<()> { + let scc_set: HashSet = scc.iter().cloned().collect(); + let order = canonical_order(scc, &scc_set, by_id, edges, frozen_by_local)?; + let index_of: HashMap = order + .iter() + .enumerate() + .map(|(i, id)| (id.clone(), i)) + .collect(); + + let mut members: Vec = Vec::with_capacity(order.len()); + for local_id in &order { + let map = cycle_ref_map( + edges.get(local_id), + &scc_set, + Some(&index_of), + frozen_by_local, + ); + members.push(substitute(&by_id[local_id], &map)); + } + let content = serde_json::json!({ FROZEN_UNIT_KEY: members }); + let frozen_id = frozen_id(&content)?; + + for local_id in &order { + frozen_by_local.insert(local_id.clone(), frozen_id.clone()); + } + + out.push(FrozenResource { + frozen_id, + content, + unit: order, + }); + + Ok(()) +} + +/// Deterministic, input-order-independent ordering of a cycle's members via +/// color refinement. Mirrors `freeze.ts#canonicalOrder`. +fn canonical_order( + scc: &[String], + scc_set: &HashSet, + by_id: &HashMap, + edges: &HashMap>, + frozen_by_local: &HashMap, +) -> AtomicResult> { + let mut color: HashMap = HashMap::new(); + for local_id in scc { + let map = cycle_ref_map(edges.get(local_id), scc_set, None, frozen_by_local); + color.insert( + local_id.clone(), + hash_canonical(&substitute(&by_id[local_id], &map))?, + ); + } + + for _ in 0..scc.len() { + let mut next: HashMap = HashMap::new(); + for local_id in scc { + let map = neighbor_color_ref_map(edges.get(local_id), scc_set, &color, frozen_by_local); + next.insert( + local_id.clone(), + hash_canonical(&substitute(&by_id[local_id], &map))?, + ); + } + + if partition_signature(scc, &next) == partition_signature(scc, &color) { + color = next; + break; + } + + color = next; + } + + let mut order: Vec = scc.to_vec(); + order.sort_by(|a, b| color[a].cmp(&color[b]).then_with(|| a.cmp(b))); + + Ok(order) +} + +fn resolved_ref_map( + refs: Option<&BTreeSet>, + frozen_by_local: &HashMap, +) -> HashMap { + let mut map = HashMap::new(); + for r in refs.into_iter().flatten() { + if let Some(fid) = frozen_by_local.get(r) { + map.insert(r.clone(), fid.clone()); + } + } + map +} + +fn cycle_ref_map( + refs: Option<&BTreeSet>, + scc_set: &HashSet, + index_of: Option<&HashMap>, + frozen_by_local: &HashMap, +) -> HashMap { + let mut map = HashMap::new(); + for r in refs.into_iter().flatten() { + if scc_set.contains(r) { + let token = match index_of { + Some(idx) => format!("{}{}", SELF_PREFIX, idx[r]), + None => SELF_PREFIX.to_string(), + }; + map.insert(r.clone(), token); + } else if let Some(fid) = frozen_by_local.get(r) { + map.insert(r.clone(), fid.clone()); + } + } + map +} + +fn neighbor_color_ref_map( + refs: Option<&BTreeSet>, + scc_set: &HashSet, + color: &HashMap, + frozen_by_local: &HashMap, +) -> HashMap { + let mut map = HashMap::new(); + for r in refs.into_iter().flatten() { + if scc_set.contains(r) { + map.insert(r.clone(), format!("{}{}", SELF_PREFIX, color[r])); + } else if let Some(fid) = frozen_by_local.get(r) { + map.insert(r.clone(), fid.clone()); + } + } + map +} + +fn partition_signature(scc: &[String], color: &HashMap) -> String { + let mut groups: BTreeMap> = BTreeMap::new(); + for id in scc { + groups + .entry(color[id].clone()) + .or_default() + .push(id.clone()); + } + let mut parts: Vec = groups + .into_values() + .map(|mut g| { + g.sort(); + g.join(",") + }) + .collect(); + parts.sort(); + parts.join("|") +} + +fn collect_refs(value: &Value, ids: &HashSet) -> BTreeSet { + let mut out = BTreeSet::new(); + walk_refs(value, ids, &mut out); + out +} + +fn walk_refs(value: &Value, ids: &HashSet, out: &mut BTreeSet) { + match value { + Value::String(s) => { + if ids.contains(s) { + out.insert(s.clone()); + } + } + Value::Array(a) => a.iter().for_each(|v| walk_refs(v, ids, out)), + Value::Object(o) => o.values().for_each(|v| walk_refs(v, ids, out)), + _ => {} + } +} + +fn substitute(value: &Value, map: &HashMap) -> Value { + match value { + Value::String(s) => map + .get(s) + .map(|r| Value::String(r.clone())) + .unwrap_or_else(|| value.clone()), + Value::Array(a) => Value::Array(a.iter().map(|v| substitute(v, map)).collect()), + Value::Object(o) => Value::Object( + o.iter() + .map(|(k, v)| (k.clone(), substitute(v, map))) + .collect(), + ), + _ => value.clone(), + } +} + +fn hash_canonical(value: &Value) -> AtomicResult { + let canonical = + serde_jcs::to_string(value).map_err(|e| format!("Failed to JCS-canonicalize: {}", e))?; + Ok(blake3::hash(canonical.as_bytes()).to_hex().to_string()) +} + +fn strongly_connected_components( + nodes: &[String], + edges: &HashMap>, +) -> Vec> { + struct State<'a> { + edges: &'a HashMap>, + index: HashMap, + low: HashMap, + on_stack: HashSet, + stack: Vec, + counter: usize, + result: Vec>, + } + + fn connect(s: &mut State, v: &str) { + s.index.insert(v.to_string(), s.counter); + s.low.insert(v.to_string(), s.counter); + s.counter += 1; + s.stack.push(v.to_string()); + s.on_stack.insert(v.to_string()); + + let neighbors: Vec = s + .edges + .get(v) + .map(|e| e.iter().cloned().collect()) + .unwrap_or_default(); + + for w in neighbors { + if !s.index.contains_key(&w) { + connect(s, &w); + let lw = s.low[&w]; + let lv = s.low[v]; + s.low.insert(v.to_string(), lv.min(lw)); + } else if s.on_stack.contains(&w) { + let iw = s.index[&w]; + let lv = s.low[v]; + s.low.insert(v.to_string(), lv.min(iw)); + } + } + + if s.low[v] == s.index[v] { + let mut component = Vec::new(); + loop { + let w = s.stack.pop().unwrap(); + s.on_stack.remove(&w); + let is_root = w == v; + component.push(w); + if is_root { + break; + } + } + s.result.push(component); + } + } + + let mut state = State { + edges, + index: HashMap::new(), + low: HashMap::new(), + on_stack: HashSet::new(), + stack: Vec::new(), + counter: 0, + result: Vec::new(), + }; + + for v in nodes { + if !state.index.contains_key(v) { + connect(&mut state, v); + } + } + + state.result +} + +// --- Schema authoring DSL --------------------------------------------------- +// +// `freeze_schema` is the Rust counterpart of `browser/lib/src/schema.ts# +// freezeSchema`: it builds identity-only JSON-AD bodies for an Ontology and its +// Classes/Properties and content-addresses them, producing frozen ids +// byte-for-byte identical to the TS producer. Descriptions and other +// presentation are deliberately excluded from identity. Input is order- +// preserving (Vec-based), because an Ontology's `classes`/`properties` array +// order is significant (JCS does not sort arrays). + +const P_ISA: &str = "https://atomicdata.dev/properties/isA"; +const P_SHORTNAME: &str = "https://atomicdata.dev/properties/shortname"; +const P_DATATYPE: &str = "https://atomicdata.dev/properties/datatype"; +const P_CLASSTYPE: &str = "https://atomicdata.dev/properties/classtype"; +const P_REQUIRES: &str = "https://atomicdata.dev/properties/requires"; +const P_RECOMMENDS: &str = "https://atomicdata.dev/properties/recommends"; +const P_CLASSES: &str = "https://atomicdata.dev/properties/classes"; +const P_PROPERTIES: &str = "https://atomicdata.dev/properties/properties"; +const P_VERSION: &str = "https://atomicdata.dev/properties/version"; +const C_PROPERTY: &str = "https://atomicdata.dev/classes/Property"; +const C_CLASS: &str = "https://atomicdata.dev/classes/Class"; +const C_ONTOLOGY: &str = "https://atomicdata.dev/class/ontology"; + +/// An Atomic datatype, the machine contract of a Property. +#[derive(Clone, Copy)] +pub enum SchemaDatatype { + String, + Integer, + Float, + Boolean, + Date, + Uri, + AtomicUrl, + ResourceArray, + Json, +} + +impl SchemaDatatype { + pub fn url(&self) -> &'static str { + match self { + SchemaDatatype::String => "https://atomicdata.dev/datatypes/string", + SchemaDatatype::Integer => "https://atomicdata.dev/datatypes/integer", + SchemaDatatype::Float => "https://atomicdata.dev/datatypes/float", + SchemaDatatype::Boolean => "https://atomicdata.dev/datatypes/boolean", + SchemaDatatype::Date => "https://atomicdata.dev/datatypes/date", + SchemaDatatype::Uri => "https://atomicdata.dev/datatypes/uri", + SchemaDatatype::AtomicUrl => "https://atomicdata.dev/datatypes/atomicURL", + SchemaDatatype::ResourceArray => "https://atomicdata.dev/datatypes/resourceArray", + SchemaDatatype::Json => "https://atomicdata.dev/datatypes/json", + } + } +} + +pub struct SchemaProperty { + /// Developer key, also the default shortname. + pub key: String, + pub datatype: SchemaDatatype, + pub shortname: Option, + pub class_type: Option, +} + +pub struct SchemaClass { + pub key: String, + pub shortname: Option, + /// Property keys that are required (the rest become `recommends`). + pub required: Vec, + pub properties: Vec, +} + +pub struct SchemaDef { + pub name: String, + pub version: Option, + pub classes: Vec, +} + +/// Frozen ids for a schema, keyed by developer key (`"classKey.propKey"` for +/// properties). +pub struct FrozenSchema { + pub ontology: String, + pub classes: HashMap, + pub properties: HashMap, +} + +/// Freezes a code-first schema into `did:ad:frozen` ids identical to the TS +/// producer. Acyclic by construction (Ontology -> Classes -> Properties). +pub fn freeze_schema(def: &SchemaDef) -> AtomicResult { + let prop_local = |class_key: &str, prop_key: &str| format!("prop:{}.{}", class_key, prop_key); + let class_local = |class_key: &str| format!("class:{}", class_key); + let ontology_local = "ontology".to_string(); + + // The TS producer normalizes the schema (sorts object keys) before freezing, + // so ids are independent of declaration order. Mirror that by processing + // classes and properties sorted by key. (Byte order; TS uses `localeCompare`, + // which coincides for the lowercase-ASCII shortnames that are the convention.) + let mut classes: Vec<&SchemaClass> = def.classes.iter().collect(); + classes.sort_by(|a, b| a.key.cmp(&b.key)); + let prepared: Vec<(&SchemaClass, Vec<&SchemaProperty>)> = classes + .iter() + .map(|c| { + let mut props: Vec<&SchemaProperty> = c.properties.iter().collect(); + props.sort_by(|a, b| a.key.cmp(&b.key)); + (*c, props) + }) + .collect(); + + let mut freezable: Vec = Vec::new(); + + for (class, props) in &prepared { + for property in props { + let shortname = property.shortname.clone().unwrap_or(property.key.clone()); + let mut body = serde_json::Map::new(); + body.insert( + P_ISA.to_string(), + Value::Array(vec![Value::String(C_PROPERTY.into())]), + ); + body.insert(P_SHORTNAME.to_string(), Value::String(shortname)); + body.insert( + P_DATATYPE.to_string(), + Value::String(property.datatype.url().into()), + ); + if let Some(ct) = &property.class_type { + body.insert(P_CLASSTYPE.to_string(), Value::String(ct.clone())); + } + + freezable.push(FreezableResource { + local_id: prop_local(&class.key, &property.key), + content: Value::Object(body), + }); + } + } + + for (class, props) in &prepared { + let required: HashSet<&str> = class.required.iter().map(|s| s.as_str()).collect(); + let mut requires = Vec::new(); + let mut recommends = Vec::new(); + for property in props { + let local = prop_local(&class.key, &property.key); + if required.contains(property.key.as_str()) { + requires.push(Value::String(local)); + } else { + recommends.push(Value::String(local)); + } + } + + let shortname = class.shortname.clone().unwrap_or(class.key.clone()); + let mut body = serde_json::Map::new(); + body.insert( + P_ISA.to_string(), + Value::Array(vec![Value::String(C_CLASS.into())]), + ); + body.insert(P_SHORTNAME.to_string(), Value::String(shortname)); + body.insert(P_REQUIRES.to_string(), Value::Array(requires)); + body.insert(P_RECOMMENDS.to_string(), Value::Array(recommends)); + + freezable.push(FreezableResource { + local_id: class_local(&class.key), + content: Value::Object(body), + }); + } + + let class_refs: Vec = prepared + .iter() + .map(|(c, _)| Value::String(class_local(&c.key))) + .collect(); + let property_refs: Vec = prepared + .iter() + .flat_map(|(c, props)| { + props + .iter() + .map(move |p| Value::String(prop_local(&c.key, &p.key))) + }) + .collect(); + + let mut ontology_body = serde_json::Map::new(); + ontology_body.insert( + P_ISA.to_string(), + Value::Array(vec![Value::String(C_ONTOLOGY.into())]), + ); + ontology_body.insert(P_SHORTNAME.to_string(), Value::String(def.name.clone())); + ontology_body.insert(P_CLASSES.to_string(), Value::Array(class_refs)); + ontology_body.insert(P_PROPERTIES.to_string(), Value::Array(property_refs)); + if let Some(version) = &def.version { + ontology_body.insert(P_VERSION.to_string(), Value::String(version.clone())); + } + freezable.push(FreezableResource { + local_id: ontology_local.clone(), + content: Value::Object(ontology_body), + }); + + let result = freeze_resources(freezable)?; + let require_id = |local: &str| -> AtomicResult { + result + .by_local_id + .get(local) + .cloned() + .ok_or_else(|| format!("freeze_schema produced no id for {}", local).into()) + }; + + let mut classes = HashMap::new(); + let mut properties = HashMap::new(); + for (class, props) in &prepared { + classes.insert(class.key.clone(), require_id(&class_local(&class.key))?); + for property in props { + properties.insert( + format!("{}.{}", class.key, property.key), + require_id(&prop_local(&class.key, &property.key))?, + ); + } + } + + Ok(FrozenSchema { + ontology: require_id(&ontology_local)?, + classes, + properties, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[derive(serde::Deserialize)] + struct Vector { + name: String, + body: serde_json::Value, + id: String, + } + + #[derive(serde::Deserialize)] + struct Vectors { + vectors: Vec, + } + + /// Proves the Rust frozen id matches the TypeScript producer for every + /// shared vector. A failure here is a cross-language identity break. + #[test] + fn matches_cross_language_vectors() { + let raw = include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../test-vectors/frozen.json" + )); + let parsed: Vectors = serde_json::from_str(raw).expect("valid fixture"); + + assert!(!parsed.vectors.is_empty(), "fixture has no vectors"); + + for vector in parsed.vectors { + assert_eq!( + frozen_id(&vector.body).unwrap(), + vector.id, + "frozen id mismatch for vector {}", + vector.name + ); + } + } + + #[test] + fn verify_frozen_rejects_a_mismatch() { + let body = serde_json::json!({ "a": 1 }); + let wrong = + "did:ad:frozen:0000000000000000000000000000000000000000000000000000000000000000"; + + assert!(verify_frozen(&frozen_id(&body).unwrap(), &body).is_ok()); + assert!(verify_frozen(wrong, &body).is_err()); + } + + #[derive(serde::Deserialize)] + struct FreezeInput { + #[serde(rename = "localId")] + local_id: String, + content: serde_json::Value, + } + + #[derive(serde::Deserialize)] + struct FreezeCase { + name: String, + input: Vec, + expected: HashMap, + } + + #[derive(serde::Deserialize)] + struct FreezeCases { + cases: Vec, + } + + /// Proves the Rust `freeze_resources` graph algorithm (topological hashing + + /// cycle units + color refinement) is byte-for-byte identical to the + /// TypeScript producer — the foundation for authoring schemas in Rust. + #[test] + fn freeze_resources_matches_cross_language_vectors() { + let raw = include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../test-vectors/freeze-resources.json" + )); + let parsed: FreezeCases = serde_json::from_str(raw).expect("valid fixture"); + + assert!(!parsed.cases.is_empty(), "fixture has no cases"); + + for case in parsed.cases { + let input: Vec = case + .input + .into_iter() + .map(|i| FreezableResource { + local_id: i.local_id, + content: i.content, + }) + .collect(); + let result = freeze_resources(input).unwrap(); + + assert_eq!( + result.by_local_id, case.expected, + "freeze_resources mismatch for case {}", + case.name + ); + } + } + + #[derive(serde::Deserialize)] + struct SchemaExpected { + ontology: String, + classes: HashMap, + properties: HashMap, + } + + #[derive(serde::Deserialize)] + struct SchemaVector { + expected: SchemaExpected, + } + + /// Proves the Rust `freeze_schema` authoring DSL produces ids identical to + /// the TS `freezeSchema` for the same schema — the multi-language authoring + /// guarantee. + #[test] + fn freeze_schema_matches_cross_language_vector() { + let raw = include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../test-vectors/freeze-schema.json" + )); + let vector: SchemaVector = serde_json::from_str(raw).expect("valid fixture"); + + let def = SchemaDef { + name: "FrozenTodoApp".into(), + version: Some("1.0.0".into()), + classes: vec![SchemaClass { + key: "todo".into(), + shortname: None, + required: vec!["title".into()], + properties: vec![ + SchemaProperty { + key: "title".into(), + datatype: SchemaDatatype::String, + shortname: None, + class_type: None, + }, + SchemaProperty { + key: "done".into(), + datatype: SchemaDatatype::Boolean, + shortname: None, + class_type: None, + }, + SchemaProperty { + key: "dueAt".into(), + datatype: SchemaDatatype::Date, + shortname: None, + class_type: None, + }, + ], + }], + }; + + let frozen = freeze_schema(&def).unwrap(); + + assert_eq!(frozen.ontology, vector.expected.ontology, "ontology id"); + assert_eq!(frozen.classes, vector.expected.classes, "class ids"); + assert_eq!( + frozen.properties, vector.expected.properties, + "property ids" + ); + } +} diff --git a/lib/src/lib.rs b/lib/src/lib.rs index 5ccc5e70a..1c75e23c9 100644 --- a/lib/src/lib.rs +++ b/lib/src/lib.rs @@ -80,6 +80,7 @@ pub mod discovery; #[cfg(feature = "db")] pub mod endpoints; pub mod errors; +pub mod frozen; pub mod hierarchy; /// Resource version history (time-travel reads). Prefer this over `loro` in app code. pub mod history { diff --git a/lib/src/parse.rs b/lib/src/parse.rs index 23c11954d..9b32c4319 100644 --- a/lib/src/parse.rs +++ b/lib/src/parse.rs @@ -568,7 +568,11 @@ pub fn parse_propval<'a>( /// Parse a single Json AD string, convert to Atoms /// Adds to the store if `add` is true. #[tracing::instrument(skip_all)] -async fn parse_json_ad_map_to_resource( +/// Parse a JSON-AD object (property-URL -> value map) into a [Resource]. +/// `overwrite_subject` forces the resource's subject (and forbids an inline +/// `@id`), which is what frozen materialization uses — the subject is the +/// content hash, never stored in the body. +pub async fn parse_json_ad_map_to_resource( json: Map, store: &impl crate::Storelike, overwrite_subject: Option, diff --git a/lib/src/subject.rs b/lib/src/subject.rs index 957b2e6cc..6137f683d 100644 --- a/lib/src/subject.rs +++ b/lib/src/subject.rs @@ -11,6 +11,13 @@ pub const DID_AD_COMMIT_PREFIX: &str = "did:ad:commit:"; /// 32-byte BLAKE3 hash of the bytes, hex-encoded (64 chars). pub const DID_AD_BLOB_PREFIX: &str = "did:ad:blob:"; +/// The prefix for Frozen DIDs: `did:ad:frozen:`. The remainder is the +/// 32-byte BLAKE3 hash, hex-encoded (64 chars), of the RFC 8785 (JCS) +/// canonicalization of an immutable JSON-AD body. Unlike a blob, a frozen +/// subject resolves to structured JSON-AD that materializes into a read-only +/// Resource. See `crate::frozen`. +pub const DID_AD_FROZEN_PREFIX: &str = "did:ad:frozen:"; + /// The Subject of a Resource. /// /// In Atomic Data, every subject is a URI. @@ -236,6 +243,45 @@ impl Subject { } } + /// Returns true if this is a DID Frozen subject (did:ad:frozen:). + pub fn is_frozen_did(&self) -> bool { + match self { + Subject::Did { url, .. } => url.as_str().starts_with(DID_AD_FROZEN_PREFIX), + _ => false, + } + } + + /// If this is a `did:ad:frozen:` subject, returns the hex-encoded BLAKE3 + /// hash (the part after the prefix, with any `?drive=` hint stripped). + /// Returns `None` for any other variant. + pub fn frozen_hash_hex(&self) -> Option<&str> { + match self { + Subject::Did { url, .. } => { + let rest = url.as_str().strip_prefix(DID_AD_FROZEN_PREFIX)?; + // Drop query (`?drive=...`) / fragment if present. + let end = rest.find(['?', '#']).unwrap_or(rest.len()); + Some(&rest[..end]) + } + _ => None, + } + } + + /// Construct a `did:ad:frozen:` subject from a 32-byte BLAKE3 hash. + pub fn from_frozen_hash(hash: &[u8; 32]) -> Self { + let mut hex = String::with_capacity(DID_AD_FROZEN_PREFIX.len() + 64); + hex.push_str(DID_AD_FROZEN_PREFIX); + for byte in hash { + // Inline lowercase-hex; avoids pulling in the `hex` crate just for this. + hex.push(char::from_digit((byte >> 4) as u32, 16).unwrap()); + hex.push(char::from_digit((byte & 0xf) as u32, 16).unwrap()); + } + // Url::parse on a `did:ad:frozen:` always succeeds (hex is RFC-3986 safe). + Subject::Did { + url: Url::parse(&hex).expect("valid did:ad:frozen: URL"), + drive_hint: None, + } + } + /// Returns true if this is an internal subject (mapped to the server's base domain). pub fn is_internal(&self) -> bool { matches!(self, Subject::Internal { .. }) @@ -585,6 +631,44 @@ mod tests { ); } + #[test] + fn test_frozen_did_parsing() { + let frozen_did = + "did:ad:frozen:af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262"; + let subject = Subject::from_raw(frozen_did, None); + + assert!(matches!(subject, Subject::Did { .. })); + assert!(subject.is_frozen_did()); + assert!(!subject.is_blob_did()); + assert!(!subject.is_commit_did()); + assert_eq!( + subject.frozen_hash_hex(), + Some("af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262") + ); + + // Roundtrip via raw bytes. + let mut bytes = [0u8; 32]; + for (i, byte) in bytes.iter_mut().enumerate() { + *byte = i as u8; + } + let from_bytes = Subject::from_frozen_hash(&bytes); + assert!(from_bytes.is_frozen_did()); + assert_eq!( + from_bytes.frozen_hash_hex(), + Some("000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f") + ); + + // Drive hint is preserved, hash extraction strips it. + let with_drive = format!("{}?drive=did:ad:abc", frozen_did); + let routed = Subject::from_raw(&with_drive, None); + assert!(routed.is_frozen_did()); + assert_eq!(routed.drive_hint(), Some("did:ad:abc")); + assert_eq!( + routed.frozen_hash_hex(), + Some("af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262") + ); + } + #[test] fn test_did_drive_hint_parsing() { let did_with_drive = "did:ad:123?drive=abc"; diff --git a/planning/did-ad-frozen-server.md b/planning/did-ad-frozen-server.md new file mode 100644 index 000000000..d54ad4251 --- /dev/null +++ b/planning/did-ad-frozen-server.md @@ -0,0 +1,267 @@ +# Server-side `did:ad:frozen` plan + +> **Status: Phases A, B, and C are done and verified.** Storage, parsing, +> materialization, commit-rejection, `GET/PUT /frozen`, browser resolution, +> `registerFrozenSchema`/`loadSchemaLock`/`createSchemaPointer`/`freezeStructure`, +> the data-browser "Freeze" UI (+ immutability badge), and a browser e2e all +> land. Cross-language hashing (TS `jcs.ts` ↔ Rust `serde_jcs`) is proven by +> shared `test-vectors/`. Remaining: **Phase D (sync over iroh)** and the +> optional polish in the companion doc (Loro mode, ClientDb persistence, CI +> lock-guard). + +Companion to [json-schema-code-first.md](./json-schema-code-first.md). That doc +decided the **identity model**: schema definitions are immutable, so they are +content-addressed as `did:ad:frozen:{blake3-hex}` instead of signed genesis DIDs. +The TypeScript producer side is built (`browser/lib/src/freeze.ts`, +`schema.ts#freezeSchema`). This doc plans the Rust/server and Store work needed +to **store, serve, resolve, and sync** frozen objects. + +## Core decision: frozen objects are blob-like, not resource-like + +A frozen object must NOT go through the Commit/Resource pipeline. The commit +validator hard-codes that a genesis resource's subject equals its signature +(`lib/src/commit.rs:318-341`); a signatureless content-addressed object cannot +satisfy that. Rather than bend the genesis path, model frozen objects on the +existing **blob** mechanism, which is already a commit-free, content-addressed +byte store: + +| Concern | Blob (today) | Frozen (new, parallel) | +| --- | --- | --- | +| Storage | `Tree::Blobs`, key = 32-byte BLAKE3, value = raw bytes (`lib/src/db/trees.rs:29`) | `Tree::Frozen`, key = 32-byte BLAKE3, value = canonical JSON-AD bytes | +| Write | `PUT /blob/{hash}`, verify-by-rehash (`server/src/handlers/blob.rs:14-41`) | `PUT /frozen/{hash}`, verify-by-rehash | +| Read | raw bytes (`/download/...`) | **materialized read-only Resource** (parse JSON-AD) | +| Identity parse | `subject.rs#is_blob_did/blob_hash_hex/from_blob_hash` | mirror as `is_frozen_did/frozen_hash_hex/from_frozen_hash` | +| Sync | `BLOB_REQUEST/RESPONSE` frames | `FROZEN_REQUEST/RESPONSE` (or reuse the blob "fetch-if-missing" path) | + +**The invariant that makes this simple:** every stored frozen object satisfies +`blake3(its canonical bytes) == its hash`. The server verifies on write and can +re-verify on read; the client always re-verifies. No host is ever trusted — a +frozen object is self-authenticating by content. Rust needs only BLAKE3 + a +JSON-AD parser; it never needs the freeze/SCC algorithm. + +The one difference from blobs: a frozen object **resolves to a parsed Resource**, +not opaque bytes, so `get_resource`, datatype/required validation, forms, and +`getProperty` work against frozen schema resources with no special-casing at +those call sites. + +## How resolution stays decentralized (and atomicdata.dev stays optional) + +Because every fetch is verify-by-rehash, the source is irrelevant to +correctness. A well-known default server (e.g. `atomicdata.dev`) is therefore a +**cache/CDN, never a dependency**. Resolution order for a `did:ad:frozen` subject: + +1. local `Tree::Frozen` / client ClientDb (OPFS) cache +2. app-bundled frozen objects (JSON-AD shipped with the app) +3. `?drive=` routing hint → pkarr (`discovery.rs`) → iroh peer → `FROZEN_REQUEST` + by hash (the working p2p path today) +4. a **configurable** default-server list (defaults to `atomicdata.dev`) → + `GET /frozen/{hash}` → verify-by-rehash. Removable; the spec never names it. +5. _(future)_ content-hash discovery: announce frozen hashes directly on a DHT so + holders can be found without a drive hint. The dormant `mainline` crate + (`server/Cargo.toml:65`, currently unused) or a pkarr record are the + candidates. Not needed for v1. + +"Not formally relying on atomicdata.dev" = the default-server entry is config, +documented as optional, and the protocol/spec never references it. If it is down +and the bytes exist in a bundle, on the author's drive, or on any peer, resolution +still succeeds and verifies. + +## Availability & incentives: who stores and serves schemas, and why + +A frozen id is **self-certifying but not self-locating** (like a git object hash +or an IPFS CID): it proves the content but doesn't say where to get the bytes. So +the open question isn't correctness — verify-by-rehash already gives us that — it's +**availability**: who keeps the bytes and serves them? The design answer is to make +storing/serving a *byproduct of self-interest* and *trustlessly cheap*, rather than +to bolt on a reward. In priority order: + +1. **Storing is a side effect of using your own data.** A server cannot validate + or serve a resource of class `did:ad:frozen:X` without holding `X` — so every + data-hosting server already holds the schemas its data depends on (the + browser-side publish-on-save in [json-schema-code-first.md](./json-schema-code-first.md), + `Store.publishReferencedFrozen`, pushes them there before the commit lands). + Data and its schema travel together; no altruism required. + +2. **Serving to others is trustless and ~free.** `GET /frozen/{hash}` is public and + re-hashable, so a holder *cannot* serve a wrong answer — closing the endpoint + buys nothing. Frozen objects are tiny, immutable (infinite cache TTL, CDN-able), + and deduplicated (a reused property is stored once). Cost ≈ 0 beats any reward. + _Action:_ set `Cache-Control: public, immutable, max-age=31536000` on + `GET /frozen` responses. + +3. **The reuse flywheel.** Identical definitions hash to the same id, so reuse *is* + convergence: the more an "email"/"name"/… property is reused, the more servers + independently hold it, the more available and canonical it becomes — a Schelling + point. Popular vocabularies get *more* available with use, with zero coordination; + private variants stay isolated on one server (self-punishing divergence). + +4. **A registry/index for discovery + the long tail** — the high-leverage build, and + the same "index all ontologies on atomicdata.dev" goal noted elsewhere. Think + crates.io / npm / schema.org for ontologies: pin-on-push + search over + shortnames/descriptions + a "used by N ontologies" usage signal that surfaces the + flywheel. Incentives align: publishers get discoverability + a durable home, + consumers get one reliable resolve/search endpoint, the operator becomes the + ecosystem's discovery layer. Because apps ship lockfiles (below), the registry is + a convenience and flywheel — **not** a dependency or single point of failure. + +5. **Lockfiles make clients self-sufficient seeders.** A committed + `*.schema.lock.json` travels with the code, so an app *runs* with no server at + all, and any client holding the bytes can re-seed any server (`publishFrozenResource`). + Availability stops being a hard dependency for *running* — it's only needed for + *discovery/interop*, which de-risks the whole scheme. + +6. **Mesh + (optional) paid permanence, later.** Phase D sync makes this + BitTorrent-like: any peer with the bytes serves them, verified by re-hash + (incentive = reciprocity from a shared pool). Pay-once-store-forever + (Arweave-style) is a natural fit for *guaranteed* permanence of critical + vocabularies but is over-engineering for objects this cheap until there's demand — + flag as future, don't build. + +**Net:** don't pay anyone to store schemas. Make storing a side effect of using +your own data, make serving trustless and free, ship lockfiles so nothing *depends* +on a host, and let reuse-convergence plus a discovery registry turn "available +because someone happens to host it" into "available because everyone who uses it +hosts it." + +### Follow-up tasks + +- [ ] `GET /frozen/{hash}`: add long-lived immutable `Cache-Control` headers. +- [ ] Registry/index on a default server: pin-on-push, search by + shortname/description, and a `used-by` count per frozen id. +- [ ] Promote lockfiles to the documented default for shipping a schema with an app. + +## Decided + implemented: cyclic members freeze as one unit + +This was the one place the producer and server models disagreed; it is now +resolved in `freeze.ts` so every stored frozen object satisfies +`blake3(canonical bytes) == hash`. + +- **Acyclic** resources (the common case: ontology → classes → properties) each + freeze to one self-verifying blob, `id = blake3(JCS(content))`. +- A **cycle** (e.g. `Person` class with a `friend` property whose classtype is + `Person`) freezes to a single **unit** object, + `{ "urn:atomic-freeze:unit": [ ...members ] }` (`UNIT_MEMBERS_KEY`), members in + a deterministic canonical order with intra-cycle references rewritten to + `did:ad:frozen:self:{index}` self tokens (`SELF_PREFIX`). The unit id is + `blake3(JCS(unit))` and all members share it; they resolve together. This keeps + the server invariant exact (one blob, verify-by-rehash) and respects "DIDs have + no subpaths" (`docs/src/did.md`). + +Consequence for the **server materializer**: when it parses a frozen object whose +top-level key is `urn:atomic-freeze:unit`, it must expand the array into multiple +read-only Resources and rewire each `did:ad:frozen:self:{index}` token to the +i-th member. The constants are exported from `browser/lib/src/freeze.ts` as the +frozen-format contract. (The Rust side still only ever verifies a hash; it does +not run the freeze algorithm.) + +Deferred: independent addressing/reuse of a single member *inside* a cycle (would +need fragment addressing like `did:ad:frozen:{unit}#{shortname}`, which the DID +spec currently disallows). Co-dependent members travel together, so this is rarely +needed. + +## Work breakdown + +### Phase A — lib + server storage, parsing, materialization + +- [x] `lib/src/frozen.rs`: `frozen_id` / `verify_frozen`. Done + cross-language + tested. +- [x] `lib/src/subject.rs`: `is_frozen_did`, `frozen_hash_hex`, + `from_frozen_hash` (+ `DID_AD_FROZEN_PREFIX`), mirroring the blob helpers. + Done + tested (`test_frozen_did_parsing`, incl. `?drive=` hint stripping). +- [x] `lib/src/db/trees.rs`: `Tree::Frozen` (key = BLAKE3 hex, value = canonical + JSON-AD bytes), wired through both backends (`sled_store`, `redb_store`). + Done + storage round-trip test (`db::test::frozen_storage`). +- [x] `lib/src/db.rs#get_resource`: if `is_frozen_did(subject)`, + `materialize_frozen` looks up `Tree::Frozen`, **verifies by re-hash** + (trustless), and parses the JSON-AD into a read-only Resource via + `parse_json_ad_map_to_resource` with `SaveOpts::DontSave` — bypassing + `Tree::Resources`/Loro and, crucially, class-`requires` validation (a frozen + definition is valid by its hash, not by completeness, so omitting + `description` is fine). Cycle "unit" objects error for now. Done + tested + (`db::test::frozen_materialization`, incl. re-hash rejection of a mismatched + body). +- [x] Reject any Commit whose subject is `did:ad:frozen:` (immutable). Done: + `commit.rs#validate_and_build_response` rejects frozen subjects up front + (before signature checks). Tested (`commit_to_frozen_subject_is_rejected`). +- [x] Use **RFC 8785 JCS** for the canonical JSON-AD bytes on the Rust side. + Done: `lib/src/frozen.rs#frozen_id` uses the already-present `serde_jcs` + + `blake3`, and `test-vectors/frozen.json` pins the contract. Both + `browser/lib/src/frozen-vectors.test.ts` (TS) and the `frozen.rs` test + assert identical ids; verified byte-for-byte across all vectors (incl. + unicode key ordering and a cycle unit). **The sharpest correctness risk is + retired.** + +### Phase B — server endpoints + +- [x] `PUT /frozen/{hash}`: parses the JSON-AD body, verifies + `frozen_id(body) == did:ad:frozen:{hash}` (blake3 of JCS, not raw bytes), + stores the canonical bytes in `Tree::Frozen` idempotently. Public, like + `/blob` (the hash is the capability). Done: `server/src/handlers/frozen.rs`. +- [x] `GET /frozen/{hash}`: returns the stored JSON-AD bytes + (`content-type: application/ad+json`), 404 if absent. +- [x] End-to-end HTTP test (`server::tests::frozen_endpoint_roundtrip`): + PUT -> GET round-trips, a wrong-hash PUT is rejected, and the stored body + resolves through the normal `get_resource` materialization path. +- [ ] _(optional)_ Wire `did:ad:frozen:` into the WebSocket `GET` frame / sync so + frozen objects travel with drives (Phase D), not only the dedicated route. + `get_resource` already resolves them for the HTTP `/` path. + +### Phase C — browser Store + registerSchema switch-over + +- [x] Store: `fetchFrozenResource` detects `did:ad:frozen:` in the fetch path, + fetches `GET /frozen/{hash}`, **re-hashes to verify** (`frozenIdFor`), and + materializes a read-only Resource via `JSONADParser`. `getProperty` was + relaxed to treat `description` as optional (presentation, not identity), so + frozen properties resolve. Tested (`frozen-resolve.test.ts`). +- [x] `Store.registerFrozenSchema` (additive — leaves the signed-DID + `registerSchema` intact): freezes the schema, materializes the frozen bodies + into the local store (offline-resolvable immediately), and with + `{ save: true }` PUTs each to `/frozen/{hash}`. Returns the `FrozenSchema` + (frozen ids per key + `presentation`). Tested. +- [x] **Capstone e2e** (`tests/frozen-e2e.integration.test.ts`, real server): + producer `registerFrozenSchema(..., { save: true })` PUTs frozen bodies; a + fresh agent-less consumer `getProperty`/`getResource` resolves them over + `GET /frozen` (verify-by-rehash), refs intact, and builds an instance + against the frozen class + property. Proves the whole producer→server→ + consumer loop live. +- [ ] Persist frozen objects in ClientDb (OPFS) keyed by hash for offline reload + (currently in-memory only). +- [x] The signed **"latest version" pointer**: `Store.createSchemaPointer(frozen)` + builds a mutable, signed Ontology (genesis DID) on the author's drive whose + `classes`/`properties` reference the immutable frozen ids. Its stable subject + is the durable name and its commit history is the version log; old frozen + ids stay permanently resolvable. Tested (construction). Explicit `replaces` + links would need a new bootstrapped property — deferred. + +### Phase D — sync (frozen travels with drives) + +- [ ] Add `FROZEN_REQUEST/FROZEN_RESPONSE` frames (or generalize the blob + fetch-if-missing path) in `lib/src/sync/protocol.rs` + + `sync/engine.rs#import_sync_push`: when an imported resource references a + `did:ad:frozen:` subject the receiver lacks, request it by hash — exactly + how missing blobs are pulled today. Frozen objects are immutable and + self-verifying, so they do **not** join version vectors or Loro merge. +- [ ] Route frozen fetches over iroh via the `?drive=` hint → pkarr → peer. + +### Phase E — spec, docs, discovery polish + +- [ ] `docs/src/did.md`: add `did:ad:frozen` as the next `did:ad` form — + content-addressed, resolves to canonical JSON-AD (vs. `blob`'s opaque + bytes), immutable, verify-by-rehash, optional `?drive=` hint. +- [ ] Update `docs/src/schema/*` per the companion doc's Phase 5. +- [ ] _(optional/future)_ content-hash DHT announce using the dormant `mainline` + crate or pkarr, for hint-free discovery. + +## Risks / open questions + +- **Cross-language canonicalization** must be byte-identical (Phase A): RFC 8785 + JCS on both sides (`jcs.ts` ↔ `serde_jcs`). Lock it with shared test vectors + before anything else depends on the hash. +- ~~**Cyclic addressing**~~ — decided: one unit per cycle (see above), implemented + in `freeze.ts`. +- **Garbage collection / retention.** Frozen objects are immutable and + accumulate. Refcount from referencing resources, or keep schema frozens + forever? Open. +- **PUT /frozen auth + abuse.** Open vs. authenticated vs. quota'd. +- **`mainline` is dead weight today** — either wire it for content-hash discovery + or drop the dependency; don't leave it implying capability that isn't there. diff --git a/planning/json-schema-code-first.md b/planning/json-schema-code-first.md index 6f421e6f1..09ae9f57e 100644 --- a/planning/json-schema-code-first.md +++ b/planning/json-schema-code-first.md @@ -8,7 +8,8 @@ code, without first publishing Classes and Properties at HTTP URLs. The desired workflow: 1. An app declares a JSON Schema-like model in TypeScript, Rust, or another SDK. -2. Atomic turns that declaration into local Atomic Class and Property resources. +2. Atomic turns that declaration into one local Ontology resource plus local + Atomic Class and Property resources. 3. Those schema resources get `did:ad` subjects and are signed like normal data. 4. The app can immediately create, validate, render, query, and sync resources using those Classes and Properties. @@ -50,9 +51,236 @@ This means import/export must be first-class: plugin config schemas. - `planning/SDK-API-design.md` already names "Schema creation in-code" as a future SDK capability. +- `@tomic/cli` already generates TypeScript ontology bindings from existing + Ontology resources. +- The data browser already creates Ontologies, Classes, and Properties through + the Ontology editor and table editor. + +The missing piece is a coherent code-first Ontology model and SDK API that +produces locally available DID-backed schema resources from code. + +## Current Status + +**Full-stack working and verified end to end.** The content-addressed +`did:ad:frozen` schema system runs producer → server → client across both +TypeScript and Rust, with a data-browser UI driving it. Verified live: a +code-first `@tomic/lib` app publishes a schema and it renders in the browser; the +"Freeze" action produces immutable content-addressed resources that resolve back +by hash. (`registerSchema` keeps the signed-DID path for editable ontologies; +`registerFrozenSchema`/`freezeStructure` are the content-addressed path.) + +Implemented so far: + +- `browser/lib/src/schema.ts` exports `defineSchema()`, schema package types, + canonical normalization, BLAKE3 `schemaHash`, and a pinned import shape. +- `schemaToOntologyModel()` converts the supported schema subset to an + in-memory Ontology/Class/Property model. +- `Store.registerSchema()` creates local DID Ontology, Class, and Property + resources in memory and registers a `schemaHash -> ontology subject` index. +- `Store.registerSchema(schema, { save: true })` pushes generated schema + resources through the normal Commit/outbox path. +- `Store.registerSchema()` validates pinned imports with `expectedHash` and + fails when the resolved Ontology hash does not match. +- Imported individual Properties can be reused with `$ref: + "importAlias.properties.shortname"`; generated Classes point at the imported + Property subject instead of creating a duplicate Property. +- Generated local Properties resolve through `store.getProperty()` after + registration. +- `browser/lib/tests/schema-code-first.integration.test.ts` verifies the + end-to-end target with a real server: one JS producer module defines and + publishes a schema, another JS consumer module fetches the Ontology by DID, + validates the expected hash, reuses one imported Property, and creates an + instance with that Property. +- `@tomic/cli` has an initial `schema` command that loads a JS schema module + and calls `Store.registerSchema()`, publishing by default or registering + locally with `--local`. It can append the published Ontology subject to + `atomic.config.json` with `--add-to-config`. +- `ad-generate schema --generate` now writes TypeScript ontology bindings + directly from the just-registered Ontology resources, using the same Store so + local code-first schemas do not need a server round-trip before codegen. +- The CLI accepts an optional `serverUrl` in `atomic.config.json` and falls back + to `http://localhost:9883` for relative resource creation/publishing. +- `browser/cli/src/commands/schema.test.ts` verifies a minimal JS project can + define a raw schema package, run `ad-generate schema --local --generate`, and + produce TypeScript bindings with generated DID subjects. +- `browser/lib/src/schema.test.ts` verifies saved generated Ontology, Class, + and Property DID resources can be resolved by a fresh offline `Store` from + local DB state. +- `Store.registerSchema()` now rejects an explicit `atomic:subject` Property + reuse when immutable fields such as datatype, classtype, or `allowsOnly` + differ from an already-loaded Property resource. +- Focused SDK tests cover stable schema hashing, datatype-change hash changes, + undefined normalization, invalid numeric values, pinned import metadata, + schema conversion, local DID schema registration, and local property + resolution, instance creation using returned schema subjects, import hash + mismatch failures, and single imported Property reuse. +- `browser/lib/src/jcs.ts` implements RFC 8785 JCS canonicalization, used for all + frozen hashing (and the schema-package hash), so ids are byte-reproducible + across languages. Covered by `browser/lib/src/jcs.test.ts`. +- `browser/lib/src/freeze.ts` implements `freezeResources()`: a generic, + schema-agnostic primitive that content-addresses a set of mutually-referencing + resources into a `did:ad:frozen` Merkle DAG over JCS bytes, freezing each + strongly-connected cycle as one self-verifying unit object (Tarjan SCC + + color-refinement canonical ordering). Covered by + `browser/lib/src/freeze.test.ts` (acyclic determinism/dedup/order-independence, + reference rewriting, external refs, cycle-as-unit, verify-by-rehash, + self-reference, validation). +- `browser/lib/src/schema.ts#freezeSchema()` builds **identity-only** frozen + Ontology/Class/Property JSON-AD bodies (machine contract: shortname, datatype, + classtype, allowsOnly, requires/recommends) with cross-references resolved to + frozen ids, and returns descriptions + ontology version/schemaHash/jsonSchema + separately as `presentation`. So editing a description does not churn any id. + It enforces per-ontology shortname uniqueness (content-aware: identical + definitions dedupe and pass, genuinely different ones sharing a shortname are + rejected). Tested for id format, reference rewriting, identity/presentation + split (description edit keeps ids stable, datatype edit changes them, no + cascade), determinism, shortname dedupe/conflict, and the imported-property + guard. +- `browser/lib/src/schema-lock.ts`: `buildSchemaLock()`/`verifySchemaLock()` — the + committed, self-verifying lockfile (`frozen` objects + `@index` + non-hashed + `presentation`). Verify = re-hash each frozen object via the shared + `frozenIdFor()`. +- **Rust producer (cross-language authoring)** — `lib/src/frozen.rs`: + `frozen_id`/`verify_frozen`, `freeze_resources` (the full content-addressing + core: Tarjan SCC + color refinement + one-unit-per-cycle), and `freeze_schema` + (the order-preserving schema DSL). All **byte-for-byte identical to TS**, pinned + by `test-vectors/{frozen,freeze-resources,freeze-schema}.json` and asserted in + both languages. +- **Server (`did:ad:frozen`)** — `Tree::Frozen` storage (both backends), + `subject.rs` parsing, `db.rs#materialize_frozen` (resolve via re-hash → read-only + Resource), commit-rejection for frozen subjects, and `GET/PUT /frozen/{hash}` + endpoints. Tested incl. a server round-trip + `frozen_endpoint_roundtrip`. +- **Browser client** — `Store.fetchFrozenResource` (resolve `did:ad:frozen` over + HTTP, verify-by-rehash, materialize), `registerFrozenSchema` (freeze + publish), + `loadSchemaLock` (offline, server-free availability from a bundled lockfile), + `createSchemaPointer` (signed mutable Ontology referencing frozen ids), and the + generic `freezeStructure` (freeze any resource + its reference closure). + `getProperty` treats `description` as optional (presentation). Covered by + `frozen-resolve.test.ts`. +- **Capstone integration** — `tests/frozen-e2e.integration.test.ts`: producer + publishes frozen resources to a real server, a fresh consumer resolves and uses + them. +- **Data-browser UI** — a "Freeze" resource context-menu action + + `FreezeDialog` (copy/download/publish, JSON-AD mode, Loro toggle as coming-soon, + closure checkbox); frozen resources are immutable (write actions hidden) with a + "❄ Frozen" badge. Browser e2e `browser/e2e/tests/frozen.spec.ts` drives + freeze → publish → resolve → immutable against a real server. +- `examples/code-first-schema/publish-schema.mjs` — a minimal `@tomic/lib` app + that defines and publishes a schema (verified rendering in the data-browser). + +Not implemented yet (all optional/follow-up — none block the core flows): + +- **Loro freeze mode** (the disabled dialog toggle): a separate feature — + `did:ad:blob` over the (non-reproducible) Loro snapshot, materialized by + Loro-load, keeping history. +- ClientDb/OPFS persistence of frozen objects (offline resolution currently + in-memory; survives via the bundled lockfile but not a page reload). +- `ad-generate` emitting the lockfile + a CI stale-lock guard; CLI-side + import/hash-pin checks and a CLI consumer e2e. +- Phase D sync (frozen objects travelling with drives over iroh). +- Native-Rust-struct codegen from frozen schemas; JSON Schema validation + (richer keywords beyond the Atomic subset) in browser/Rust. +- Drive-context / app-registry discovery of schemas by hash before explicit + registration; round-trip of editor/table-created schemas into frozen. +- **Availability & incentives** (who stores/serves schemas, and a discovery + registry / "index all ontologies" hub) — see the + [Availability & incentives](./did-ad-frozen-server.md#availability--incentives-who-stores-and-serves-schemas-and-why) + section of the server plan. + +## Existing Touch Points + +### TypeScript SDK + +- `browser/lib/src/ontology.ts` defines `OntologyBaseObject`, global + `registerOntologies()`, quick prop name lookup, and class/property type + inference. +- `browser/lib/src/store.ts#getProperty` fetches a Property resource by subject + and converts it to the lightweight `Property` interface used by forms, table + cells, and validation. +- `browser/lib/src/resource.ts#set` calls `store.getProperty()` before datatype + validation. If a generated DID property is not locally registered or synced, + edits can only skip client-side validation and rely on server rejection. +- `browser/lib/src/store.ts#newResource` already supports locally signed + genesis DID resources, which is the primitive needed for code-first + Ontology/Class/Property creation. + +### CLI + +- `browser/cli/src/commands/ontologies.ts` validates configured Ontology + subjects and writes generated TypeScript files. +- `browser/cli/src/generateOntology.ts` is the main Ontology resource -> + generated code pipeline. +- `browser/cli/src/generateBaseObject.ts` reads Ontology `classes` and + `properties`, creates the exported `OntologyBaseObject`, and builds + `__classDefs`. +- `browser/cli/src/generateClasses.ts` and + `browser/cli/src/generatePropTypeMapping.ts` generate native TypeScript + class/property typings from materialized Class and Property resources. +- `browser/cli/src/validateOntologies.ts` currently requires every configured + subject to resolve to `core.classes.ontology`. + +The CLI is therefore not obsolete. It should become part of the code-first loop: + +```text +code schema -> Ontology/Class/Property resources -> @tomic/cli bindings +existing Ontology resources -> @tomic/cli bindings +``` -The missing piece is a coherent schema-bundle model and SDK API that produces -locally available DID-backed schema resources from code. +### Data Browser + +- `browser/data-browser/src/views/OntologyPage` is the existing editor for + Ontology resources. `OntologyContext` mutates the Ontology `classes` and + `properties` arrays. +- `browser/data-browser/src/components/forms/.../NewOntologyDialog.tsx` + creates Ontology resources with empty `classes`, `properties`, and + `instances` arrays. +- `browser/data-browser/src/chunks/TablePage/PropertyForm/NewPropertyDialog.tsx` + creates Property resources from table columns and adds them to the parent + Ontology when the table class is inside an Ontology. +- `browser/data-browser/src/chunks/TablePage/useTableColumns.tsx` resolves table + columns through `store.getProperty()`, so table rendering needs generated DID + properties to be materialized or locally indexed. +- `browser/data-browser/src/components/forms/ResourceForm.tsx` renders required + and recommended fields from a Class resource's `requires` and `recommends` + arrays. + +Code-first schemas must not bypass these paths. Ontology editor and table +editor output should remain valid input to codegen/export, and code-first +output should remain editable and inspectable in these views. + +### Rust / Server + +- `lib/src/schema.rs` has the canonical Rust `Property` and `Class` structs for + materializing schema resources. +- `lib/src/validate.rs` and `Resource` validation fetch Properties and Classes + by subject to check datatypes and required properties. +- `lib/src/populate.rs` defines the bootstrap schema resources and currently + describes the `Property` and `Class` contracts. + +Rust validation should continue to validate against materialized Class and +Property resources. JSON Schema validation can be layered on later for extra +constraints; it should not be required for basic Atomic datatype and required +property validation. + +### Docs / Spec + +- `docs/src/schema/intro.md` currently says Classes and Properties are resolved + using HTTP and that Property URLs should resolve. +- `docs/src/schema/classes.md` documents Property, Datatype, and Class as + first-class resources. +- `docs/src/schema/compare.md` explicitly contrasts Atomic Schema with JSON + Schema by noting that JSON Schema scopes properties to a schema, while Atomic + Properties are reusable. +- `docs/src/schema/migrations.md` already recommends adding new properties + instead of changing existing relationships in place. +- `docs/src/did.md` defines DID resource resolution through Drive context and + says `did:ad:` identifiers have no subpaths. Code-first schema references + should therefore use normal DID resource subjects, not path-like property + members inside one DID. + +The public docs need a coordinated update after the API shape stabilizes: +resolvability should become "resolvable through URL, local store, Drive sync, or +app-bundled schema registry" rather than "must be HTTP-resolvable". ## Proposed Developer API @@ -116,49 +344,154 @@ const todo = await store.newResource({ ## Identity Model -Schema resources need stable local identity without HTTP. - -Recommended first step: signed genesis DID resources. +Schema resources need stable identity without HTTP. -- Each generated Class is a normal Resource with `isA = core.classes.class`. -- Each generated Property is a normal Resource with - `isA = core.classes.property`. -- The first registration signs each resource as a DID genesis resource. -- The returned ontology object stores those DID subjects. -- The schema bundle stores a mapping from developer keys to DID subjects: - `todo -> did:ad:...`, `title -> did:ad:...`. +**Decided: content-addressed `did:ad:frozen` identifiers.** A schema definition is +immutable by intent, so it does not need a signature (provenance) or an owner +who can edit it (authoritative mutability). Both of those are what signing buys; +neither applies to a frozen definition. So schema resources are identified by a +content hash, not a genesis signature: -This is compatible with the current DID model and avoids inventing a new -content-addressed DID form before the rest of the stack is ready. +```text +did:ad:frozen:{blake3-hex} +did:ad:frozen:{blake3-hex}?drive=did:ad:{your_drive} // optional routing hint +``` -Open question for later: content-derived schema IDs. A future version may add a -canonical `did:ad:schema:` or a signed statement that binds a content hash -to the schema. Do not block the first implementation on that. +- A `did:ad:frozen` subject resolves to **canonical JSON-AD** (not opaque bytes, + which is what `did:ad:blob` is for). The resolver fetches, **re-hashes to + verify**, parses, and materializes a read-only Resource. +- `id = blake3(JCS(content))`, where `content` is the **identity** of the + resource — the machine contract only. For a Property that is `shortname`, + `datatype`, `classtype`, `allowsOnly`, `isDynamic`, `isLocked`; for a Class, + `shortname` + `requires`/`recommends`. **Presentation is excluded** — + description, label, translations, icon, ordering live in the mutable package + layer (see Identity vs Presentation below), so cosmetic edits never churn an + id. Same identity -> same id (global dedup); a change to the machine contract + (datatype, rename, add/remove a required property) -> a new id, which is a real + new version to link. +- No commit, no signature, no history. Immutable by construction. No keypair is + needed to mint one; ids can be computed offline and deterministically. +- Each Property, Class, and Ontology is its own frozen resource. The Ontology + references its members by their frozen ids, so single-property reuse still + works by pointing at a property's frozen id. Efficient ontology-level + resolution comes from **shipping the members bundled together**, not from + collapsing them into one blob. + +Canonical bytes use **RFC 8785 JCS** (`browser/lib/src/jcs.ts`) so a frozen id is +reproducible byte-for-byte across languages (the Rust side uses a conformant JCS +crate). `id = blake3(JCS(content))`. + +Because frozen resources reference each other by hash, the ids are +interdependent (an Ontology's id depends on its Classes' ids, which depend on +their Properties' ids) — a Merkle DAG built by topological hashing. Mutually +referencing definitions (e.g. a `Person` class with a `friend` property whose +classtype is `Person`) form a cycle with no leaf to start from; each +strongly-connected group is frozen **as a single unit object** +(`{ "urn:atomic-freeze:unit": [...members] }`, members in canonical order with +intra-cycle refs as `did:ad:frozen:self:{index}` tokens). All members share the +unit's id, so `blake3(JCS(bytes)) == id` holds for every stored object and stays +verifiable by re-hashing. + +This is implemented as a generic, schema-agnostic primitive: +`browser/lib/src/freeze.ts#freezeResources(resources)` content-addresses any set +of mutually-referencing resources (Tarjan SCC + color-refinement canonical +ordering for cycles, one unit per cycle), and +`browser/lib/src/schema.ts#freezeSchema(schema)` builds the frozen +Ontology/Class/Property JSON-AD bodies from a defined schema. + +### Identity vs presentation + +Hashing the *whole* body — descriptions included — was the original plan, but it +makes identity churn on every cosmetic edit: a reworded description changes a +property's id, which cascades up through the class and ontology, producing piles +of near-duplicate versions. Descriptions change constantly; identity must not. + +So the boundary is set by one test: **does it change how data is validated or +interpreted?** + +- **Identity (hashed):** `shortname`, `datatype`, `classtype`, `allowsOnly`, + `isDynamic`, `isLocked`; for a Class, `shortname` + `requires`/`recommends`. +- **Presentation (not hashed):** description, label, translations, icon, + ordering, examples — plus the Ontology's `jsonSchema` source and `schemaHash` + (both encode descriptions). These ride in the **mutable package layer**: the + Ontology resource and the lockfile, keyed by frozen id / model key. + +Consequences: + +- Cosmetic edits cause **zero** id churn — no cascade. New ids appear only on a + real machine-contract change (datatype, rename, add/remove a required + property), which is exactly when a new version is warranted; the mutable + name-pointer absorbs that churn for consumers. +- A frozen property is **not self-describing**: its human text comes from the + package you got it through. This is arguably correct — meaning is universal, + wording is contextual and localizable. +- The **index dedupes on meaning**, not wording, so "title: string" is one entry + no matter how differently apps describe it. + +`freezeSchema` returns this split: `resources` (identity-only frozen bodies) and +`presentation` (descriptions + ontology `version`/`schemaHash`/`jsonSchema`, +keyed by model key so each usage keeps its own text even when identical +definitions dedupe to one id). `browser/lib/src/schema-lock.ts#buildSchemaLock()` +assembles both into the committed lockfile, and `verifySchemaLock()` re-hashes +every frozen object to confirm it matches its id — the language-neutral +verification a consumer or CI guard runs, depending only on JCS + blake3. + +### Two layers: frozen definitions + signed pointers + +Not everything can be frozen. Three things are inherently mutable and stay +**normal signed genesis-DID resources on the author's own drive**: + +1. The "latest version" pointer — `name -> latest frozen ontology id` — so an app + can mean "the current TodoApp" even as it evolves. +2. Editable display metadata that should not be frozen into identity, when an app + chooses to keep it mutable (labels, translations, ordering, icons). +3. Endorsement — "this is the official Ontola schema" is provenance, i.e. a + signature, attached at the pointer layer. + +There is no central schema host. Authors publish frozen definitions and signed +pointers on **their own drive**; the `?drive=` hint routes resolution to it over +HTTP, Mainline DHT, or Reticulum, and any node can replicate that drive. + +### Migration from the current implementation + +The shipped code currently mints **signed genesis DID** resources for Ontologies, +Classes, and Properties (`Store.registerSchema`). Moving to `did:ad:frozen` keeps +the conversion and validation code but replaces identity minting with +`freezeSchema`, and requires server-side support for storing, serving, and +resolving `did:ad:frozen` resources (verify-by-rehash, read-only). That server +work is the main remaining gap before `registerSchema` can switch over, and is +planned in detail in [did-ad-frozen-server.md](./did-ad-frozen-server.md) +(model frozen as blob-like, not resource-like; resolve over local cache / +bundle / iroh+pkarr / optional default server; resolve the cyclic-addressing +question first). ## Local Availability The main behavior change is that Property and Class URLs no longer imply HTTP availability. -Resolution order should be: +Resolution order for `did:ad` schema resources should be: 1. in-memory store 2. local persistent store / OPFS / native DB -3. synced drive schema registry +3. the currently opened Drive, when the subject carries `?drive=` or the caller + has an active Drive context 4. bundled app schema registry -5. network fetch, only if the subject is fetchable +5. synced Ontology resources referenced by the Drive, for example the Drive's + `defaultOntology` or app/plugin configuration +6. network / peer discovery, only when there is enough routing context to find + a Drive replica For `did:ad` schema resources, network fetch is not the primary mechanism. The resource must travel with the app, the drive, or sync. -## Schema Bundle Resource +## Ontology as Bundle -Add an Atomic resource that groups generated schema resources. This could reuse -the existing Ontology class or introduce a narrower `SchemaBundle` class. +Use the existing Ontology class as the schema bundle resource. Minimum useful fields: -- `isA`: Ontology or SchemaBundle +- `isA`: Ontology - `shortname` / name - `version` - `classes`: ResourceArray of generated Class resources @@ -167,10 +500,118 @@ Minimum useful fields: - `schemaHash`: canonical hash of the JSON Schema document - `replaces` / `previousVersion`: optional pointer to older bundle -Using a bundle solves two problems: +Using an Ontology as the bundle solves three problems: - app startup can register one thing and get every generated subject - sync can discover the schema resources needed to interpret app data +- the existing OntologyPage, `defaultOntology`, code generation, and generated + `OntologyBaseObject` shape keep working + +The Ontology should not embed Properties and Classes as anonymous JSON-only +children in the first implementation. Keeping them materialized as standalone +resources preserves Atomic's semantic reuse model and avoids creating a second +schema runtime for forms, table columns, validation, graph views, usage views, +and query building. + +## Schema Hash + +`schemaHash` is a version fingerprint, not a resolver. + +Define it as the hash of a canonical normalized schema document, for example: + +```text +schemaHash = blake3(canonical_json(normalized_json_schema)) +``` + +It answers "is this exactly the schema version I expected?", not "where do I +fetch this from?" Resolution still starts from the Ontology subject, app-bundled +registry, Drive context, or local/synced store. + +The hash should be stored as a normal Atomic property on the Ontology resource. +Because the Ontology is backed by Loro like any other Resource, the hash is +already part of the resource state and signed history. It should not be stored +as a special resolver entry in the Loro oplog. + +If all a caller has is a `schemaHash`, lookup requires an index: + +```text +schemaHash -> ontology subject +``` + +That index can be derived from app-bundled schemas, local storage, and synced +Drive Ontologies. It is a cache/discovery aid, not authoritative state. + +Canonical schema bytes may optionally be stored as a `did:ad:blob:{blake3}` for +package-lock style pinning. That blob is immutable byte content and cannot +replace the Ontology resource, because blobs have no class, parent, ACL, +history, or links to materialized Class and Property resources. + +## Frozen lockfile (the shareable artifact) + +A frozen id is a pure, deterministic function of the schema source (JCS → +blake3). So the schema does not need to be *hosted* to be *available*: ship a +self-verifying copy of the canonical bytes alongside the code. `ad-generate` +emits a `*.schema.lock.json` next to the generated bindings; it is committed and +guarded by a CI "regenerate and diff" check (stale-lockfile guard). This is +resolution step 2 ("app-bundled frozen objects") made concrete, and it makes any +default server (atomicdata.dev or a drive) a pure cache, never a dependency. + +### Format + +```jsonc +{ + "name": "TodoApp", + "version": "1.0.0", + "ontology": "did:ad:frozen:7c1…", + "@index": { // human aid, NOT hashed + "did:ad:frozen:7c1…": "TodoApp.todo", // class + "did:ad:frozen:9f2…": "TodoApp.title", // property + "did:ad:frozen:a3b…": "TodoApp.done" + }, + "frozen": { // the verbatim, hashed objects (identity only) + "did:ad:frozen:9f2…": { + "https://atomicdata.dev/properties/isA": ["https://atomicdata.dev/classes/Property"], + "https://atomicdata.dev/properties/shortname": "title", + "https://atomicdata.dev/properties/datatype": "https://atomicdata.dev/datatypes/string" + // no description here — that is presentation, below + } + // …one entry per frozen object; a cycle is one `urn:atomic-freeze:unit` entry + }, + "presentation": { // mutable, NOT hashed + "TodoApp.title": { "id": "did:ad:frozen:9f2…", "description": "Task title" } + // …plus ontology-level description / version / jsonSchema + } +} +``` + +### The hashing invariant + +Only the values of `frozen` are hashed, and they hold **identity only** (the +machine contract), serialized as **verbatim canonical JSON-AD** (full property +URLs, `did:ad:frozen:` refs, JCS order). Descriptions/labels live in +`presentation`, which is never hashed — so re-wording text changes the lockfile +diff but not a single id. Verification is trivial and language-neutral: for each +`id -> object` in `frozen`, assert `"did:ad:frozen:" + blake3(JCS(object)) == id`; +everything else (`name`, `version`, `@index`, `presentation`) is a non-hashed +aid. We deliberately do **not** make references shortnames in the hashed form — +that would reopen the cross-language canonicalization surface JCS just closed. +Shortnames live only in `@index`. + +### Why this shape + +- **Verbose, not gibberish.** The "noise" is the frozen ids (hashes), which are + intrinsic to content-addressing; `@index` decodes every one to + `Ontology.shortname`, so a reviewer reads the index, and a changed hash in a + diff is the visible "this identity moved" signal. +- **Consume without reimplementing freeze.** Any language embeds the file (Rust + `include_str!`, etc.), iterates `frozen`, re-hashes to verify, and materializes + read-only Resources. Reimplementing `freezeResources` is only needed to + *author* schemas in another language — and the committed `frozen` bytes are then + the cross-implementation conformance check. +- **Unambiguous index.** `freezeSchema` enforces per-ontology shortname + uniqueness (content-aware: identical definitions across classes dedupe to one + frozen id and pass; only genuinely different definitions sharing a shortname are + rejected), so each `@index` value is a clean `Ontology.shortname`. ## JSON Schema Mapping @@ -198,6 +639,10 @@ JSON object properties map to Atomic Property resources. - scalar enum -> store in JSON Schema first; map to `allowsOnly` only when the values are valid Atomic values for the Property +Property subjects should normally be generated once and then reused across +schema versions when the property's machine meaning is unchanged. Changes that +alter datatype or semantics should produce a new Property subject. + ### Datatypes Initial mapping: @@ -263,10 +708,10 @@ JSON Schema validation can be added incrementally: ## Versioning -Default to immutable schema versions. +Default to immutable schema versions and immutable property meaning. Editing code should not silently mutate the meaning of existing data on another -device. A changed schema should normally produce a new schema bundle version and +device. A changed schema should normally produce a new Ontology version and possibly new Class/Property DID subjects. Developer API: @@ -283,9 +728,21 @@ Supported policies: - `update-in-place`: edit existing DID resources, useful during development - `fail-if-changed`: production-safe mode for apps that expect exact schemas -Open question: Property identity. If a developer changes only a description, the -Property subject can probably remain stable. If datatype or meaning changes, use -a new Property subject. +If a developer changes only display metadata, such as description, label, +translation, icon, or ordering, the Property subject can remain stable. If +datatype or meaning changes, use a new Property subject. + +Working rule: + +- Ontologies are versioned packages. They may link to previous versions and can + change by publishing a new version. +- Classes are versioned shapes over Properties. Changing required/recommended + structure should generally create a new Class version when shared data already + depends on the old shape. +- Properties are immutable semantic definitions. Datatype changes, `classtype` + changes, and semantic meaning changes require a new Property subject. +- Metadata-only edits can be allowed during development, but published + properties should be treated as content-locked. ## Sync and Trust @@ -302,6 +759,12 @@ Because Classes and Properties may be DID resources, validation cannot assume the owner is an HTTP origin. The proof is the signed resource history plus the trust context that introduced the schema. +The trusted entry point is usually the Ontology resource: it may be bundled by +the app, set as a Drive's `defaultOntology`, referenced by a plugin/app config, +or synced with a Drive. The Ontology then points to individual Class and +Property resources. Those linked resources must still be present locally, +bundled, or resolvable through the same Drive/app trust context. + ## Interaction with Sign-at-Drain Schema registration creates a small set of important resources. It should not be @@ -320,49 +783,180 @@ Rules: ## Implementation Plan -- [ ] Decide whether the grouping resource reuses `Ontology` or gets a new - `SchemaBundle` class. -- [ ] Add a TypeScript schema module with `defineSchema` and type definitions +### Phase 0: Decide contracts + +- [x] Decide whether the grouping resource reuses `Ontology` or gets a new + `SchemaBundle` class: reuse `Ontology`. +- [x] Decide the initial immutability contract for Property resources: + genesis-DID with tooling enforcement first; content-derived IDs remain a + later option before broad public release. +- [x] Decide the initial import expression in code: named `imports` map with + Ontology `subject` plus optional `expectedHash`, and imported Properties + referenced as `$ref: "importAlias.properties.shortname"`. +- [ ] Decide where schema discovery is attached first: Drive `defaultOntology`, + app/plugin config, explicit `registerSchema`, or all of them. + +### Phase 1: TypeScript code-first creation + +- [x] Add a TypeScript schema module with `defineSchema` and type definitions for the supported JSON Schema subset plus `atomic:*` extensions. -- [ ] Implement JSON Schema -> in-memory ontology conversion. -- [ ] Implement local DID resource creation for generated Classes and +- [x] Implement JSON Schema/code schema -> in-memory Ontology/Class/Property + model conversion. +- [x] Implement schema hashing over canonical normalized schema package JSON. +- [x] Implement local DID resource creation for the generated Ontology bundle. +- [x] Implement local DID resource creation for generated Classes and Properties. -- [ ] Add `store.registerSchema(schema, options)` in `@tomic/lib`. +- [x] Add `store.registerSchema(schema, options)` in `@tomic/lib`. +- [x] Add an explicit `registerSchema(schema, { save: true })` path that saves + generated schema resources through the normal Commit/outbox flow. +- [x] Verify registered schema resources reload from local DB after save. +- [x] Add a local/app schema index for `schemaHash -> ontology subject`. +- [x] Ensure `store.getProperty` resolves generated DID Properties from + in-memory registered schemas without requiring HTTP. +- [x] Reject explicit Property subject reuse when immutable machine fields do + not match the already-loaded Property definition. - [ ] Ensure `store.getProperty` and class loading resolve DID schema resources - from local storage without HTTP. -- [ ] Persist the schema bundle and generated resources before returning from - `registerSchema`. -- [ ] Add JSON Schema export from Atomic Class/Property resources. + from local DB, registered app schemas, and Drive context without + requiring HTTP. + +### Phase 1.5: Content-addressed frozen identity + +- [x] RFC 8785 JCS canonicalization (`jcs.ts`). +- [x] Generic `freezeResources()` content-addressing primitive with one-unit-per- + cycle (`freeze.ts`). +- [x] `freezeSchema()` building frozen JSON-AD bodies from a defined schema. +- [x] Per-ontology shortname uniqueness enforcement (content-aware). +- [ ] `did:ad:frozen` server support (storage, serve, resolve, materialize) — see + [did-ad-frozen-server.md](./did-ad-frozen-server.md). +- [ ] Switch `Store.registerSchema` from signed genesis DIDs to `freezeSchema` + (behind a flag during migration). +- [ ] Browser Store resolution of `did:ad:frozen:` (fetch -> verify-by-rehash -> + materialize read-only Resource; expand `urn:atomic-freeze:unit` objects). +- [ ] Signed "latest version" pointer / overlay layer on the author's drive. + +### Phase 2: CLI integration + +- [x] Extend `@tomic/cli` so the existing Ontology -> TypeScript path can also + consume locally registered/generated Ontologies. +- [x] Add a CLI command or option for code schema -> Ontology resources, using + the same conversion code as `store.registerSchema` where possible. +- [x] Add SDK import/hash pin checks: if an external Ontology is referenced + with an expected hash and the resolved resource does not match, + `registerSchema()` fails. +- [ ] Add CLI import/hash pin checks: if an external Ontology or Property + subject is referenced with an expected hash/version and the resolved + resource does not match, generation fails. +- [ ] Keep existing `ontologies` config working for HTTP and DID Ontology + subjects. +- [x] Generate native bindings from materialized Class and Property resources, + not from anonymous JSON-only schema members. +- [x] Let `ad-generate schema` optionally update `atomic.config.json` with the + published Ontology subject for a follow-up `ad-generate ontologies` run. +- [x] Let `ad-generate schema --generate` write bindings immediately from the + materialized Ontology/Class/Property resources in the current Store. +- [x] Emit a committed `*.schema.lock.json` (frozen objects + `@index` + + `presentation`) via `ad-generate schema --lock`, using + `buildSchemaLock()`/`verifySchemaLock()` in `@tomic/lib`. The emitter + refuses to write an unverifiable lock. +- [x] Load + register a lockfile at app startup for offline availability: + `Store.loadSchemaLock(lock)` verifies every frozen object by re-hash and + materializes them locally, so a bundled `*.schema.lock.json` resolves with + no server. Tested (`frozen-resolve.test.ts`). +- [ ] Add a CI "regenerate and diff" guard so the lockfile cannot drift from the + source schema (`verifySchemaLock` + a re-emit check are the building blocks). + +### Phase 2.5: End-to-end target + +- [x] Add a minimal JS producer schema fixture. +- [x] Add a minimal JS consumer schema fixture that imports the producer + Ontology with an expected hash and reuses one Property. +- [x] Add an integration test where producer publishes to a real server and + consumer fetches/reuses that schema through a separate Store. +- [x] Add a CLI test where a minimal JS project defines a schema and + `ad-generate schema --local --generate` produces generated bindings. +- [ ] Add a CLI-side producer/consumer import e2e where one generated schema + imports another published/local Ontology with an expected hash. + +### Phase 3: Editor and table compatibility / UI + +- [x] Keep Ontology editor creation/editing as a first-class schema authoring + path (untouched; verified a code-first ontology renders + is editable in + `OntologyPage`). +- [x] A generic **"Freeze" UI** — resource context-menu action + `FreezeDialog` + (copy/download/publish, JSON-AD mode, Loro coming-soon, closure toggle) over + `Store.freezeStructure`. Works on any resource, not just ontologies. +- [x] **Content-locked / immutable UI**: a frozen (`did:ad:frozen`) resource + hides all write actions and shows a "❄ Frozen" badge. (UI *warnings* when a + datatype-changing edit would mint a new id — for the signed-DID editor path + — remain a follow-up.) +- [x] Table columns and forms continue to resolve DID/frozen Properties through + `store.getProperty()` (description made optional for frozen). +- [ ] Round-trip: export table/editor-created Classes & Properties to code schema + / freeze them and regenerate without changing meaning. + +### Phase 4: Import/export and validation + +- [ ] Add JSON Schema export from Atomic Ontology/Class/Property resources. - [ ] Add browser SDK tests: - create schema from code - - generated Class and Property resources are local DID resources + - schema package hashing is stable under object key reordering + - generated Ontology, Class, and Property resources are local DID resources - create an instance using returned ontology subjects - reload store and resolve the generated Property without HTTP + - import an external schema with an expected hash and fail on mismatch + - reuse one imported Property in another Class + - table-created schema exports and re-imports - required/datatype validation still works -- [ ] Add Rust-side import/export structs after the TypeScript API shape is - stable. +- [x] Cross-language **authoring** in Rust: `lib/src/frozen.rs#freeze_resources` + (the content-addressing core — Tarjan SCC + color refinement + + one-unit-per-cycle) and `freeze_schema` (the order-preserving schema DSL → + frozen Ontology/Class/Property ids). Both byte-for-byte identical to TS, + pinned by `test-vectors/freeze-resources.json` and `freeze-schema.json`. A + Rust app can now author a schema and get the same `did:ad:frozen` ids as the + TS producer. (Key sorting uses byte order; TS `localeCompare` coincides for + lowercase-ASCII shortnames — the convention. Codegen of native Rust structs + from frozen schemas is a further, optional step.) +- [ ] Add optional JSON Schema validation in the browser SDK and Rust/server + (richer keywords beyond the Atomic subset). - [ ] Add optional JSON Schema validation in the browser SDK. - [ ] Add optional JSON Schema validation in Rust/server. + +### Phase 5: Docs + +- [ ] Update `docs/src/schema/intro.md` so schema resources can resolve through + HTTP, local store, Drive sync, or app-bundled schema registry. +- [ ] Update `docs/src/schema/classes.md` with immutability/versioning guidance + for Properties, Classes, and Ontologies. +- [ ] Update `docs/src/schema/compare.md` with the JSON Schema compatible + boundary story. +- [ ] Update `docs/src/schema/migrations.md` with Property immutability and + version-link examples. - [ ] Write public docs and a tutorial once the API has survived tests. ## Non-goals for the First Pass - Full JSON Schema 2020-12 coverage. - Replacing Atomic Class and Property resources with raw JSON Schema documents. +- Embedding all schema meaning in a single Ontology-only JSON document without + materialized Class and Property resources. - Requiring all schema resources to be public HTTP URLs. - Solving global package-manager style schema discovery. - Automatic migrations of existing instance data. ## Open Questions -- Should schema bundles be Resources of existing `Ontology`, or a new - `SchemaBundle` class? -- Should generated Property subjects be reused across schema versions when only - display metadata changes? - Where should schema bundles be attached for discoverability: drive `defaultOntology`, app config, plugin resource, or all of them? - Should JSON Schema validation be strict by default, or opt-in per app/class? - How should custom JSON Schema formats map to Atomic datatypes? -- Do we want a future content-derived schema DID, or are signed genesis DIDs - enough? - +- ~~Do we want a future content-derived schema DID, or are signed genesis DIDs + enough?~~ **Decided: content-derived `did:ad:frozen` identifiers** (see Identity + Model). Only the machine-contract identity is hashed (descriptions and other + presentation are excluded so cosmetic edits don't churn ids); cycles are hashed + as a unit. +- ~~For a cycle (strongly-connected group), should members keep individually + derived ids or share one group id?~~ **Decided: one unit per cycle** — members + share the unit id and resolve together, keeping every stored object + verifiable by re-hash. Per-member addressing inside a cycle is deferred. +- Should canonical schema JSON also be stored as a blob by default, or only + when callers ask for reproducible byte-level pinning? diff --git a/server/build.rs b/server/build.rs index 84487eee9..ddcc5ddd8 100644 --- a/server/build.rs +++ b/server/build.rs @@ -178,10 +178,23 @@ fn should_build(dirs: &Dirs) -> bool { fn build_js(dirs: &Dirs) { let pkg_manager = "pnpm"; + // The JS build runs `build:wasm` -> `wasm-pack` -> a *nested* cargo. That + // child cargo would otherwise block forever on the workspace + // `target/.cargo-lock` that the outer cargo running THIS build script + // already holds (a recursive-cargo deadlock — e.g. when an editor's + // `cargo check` triggers build.rs while it also runs the JS build). + // Pointing the nested build at its own CARGO_TARGET_DIR gives it a separate + // lock, so it can't deadlock against us. wasm artifacts are copied out via + // `--out-dir`, so this dir holds only intermediates and is safe to isolate. + let nested_target_dir = std::env::var("CARGO_MANIFEST_DIR") + .map(|m| Path::new(&m).join("../target/frontend-build")) + .unwrap_or_else(|_| PathBuf::from("../target/frontend-build")); + p!("install js packages..."); std::process::Command::new(pkg_manager) .current_dir(&dirs.browser_root) + .env("CARGO_TARGET_DIR", &nested_target_dir) .args(["install"]) .output() .unwrap_or_else(|_| { @@ -193,6 +206,7 @@ fn build_js(dirs: &Dirs) { p!("build js assets..."); let out = std::process::Command::new(pkg_manager) .current_dir(&dirs.browser_root) + .env("CARGO_TARGET_DIR", &nested_target_dir) .args(["run", "build"]) .output() .expect("Failed to build js bundle"); diff --git a/server/src/handlers/frozen.rs b/server/src/handlers/frozen.rs new file mode 100644 index 000000000..305e9f896 --- /dev/null +++ b/server/src/handlers/frozen.rs @@ -0,0 +1,71 @@ +use actix_web::{web, HttpResponse}; + +use crate::{appstate::AppState, errors::AtomicServerResult}; + +/// Content type for materializable JSON-AD frozen bodies. +const AD_JSON: &str = "application/ad+json"; + +fn validate_hash_hex(hash_hex: &str) -> AtomicServerResult<()> { + if hash_hex.len() != 64 || !hash_hex.bytes().all(|b| b.is_ascii_hexdigit()) { + return Err("Frozen hash must be 64 hex chars (BLAKE3)".into()); + } + Ok(()) +} + +/// HTTP fallback for pushing a `did:ad:frozen` body to the server. Unlike a blob +/// (opaque bytes hashed directly), a frozen body is JSON-AD and its id is +/// `blake3(JCS(body))` — so we parse, canonicalize, and check the content hash +/// matches the URL hash. A mismatch is rejected. +/// +/// Public on purpose: the hash is the capability, and storage is content- +/// addressed and immutable — re-posting the same hash is a no-op. We store the +/// canonical JCS bytes so reads round-trip and re-verify. +#[tracing::instrument(skip(appstate, body))] +pub async fn put_frozen( + path: web::Path, + appstate: web::Data, + body: web::Bytes, +) -> AtomicServerResult { + let hash_hex = path.into_inner(); + validate_hash_hex(&hash_hex)?; + + let parsed: serde_json::Value = + serde_json::from_slice(&body).map_err(|e| format!("Frozen body must be JSON: {e}"))?; + + let id = atomic_lib::frozen::frozen_id(&parsed)?; + let expected = format!("{}{}", atomic_lib::subject::DID_AD_FROZEN_PREFIX, hash_hex); + if id != expected { + return Err( + format!("Frozen body hashes to {id}, does not match URL hash {hash_hex}").into(), + ); + } + + let canonical = serde_jcs::to_string(&parsed).map_err(|e| e.to_string())?; + appstate.store.kv.insert( + atomic_lib::db::trees::Tree::Frozen, + hash_hex.as_bytes(), + canonical.as_bytes(), + )?; + + Ok(HttpResponse::NoContent().finish()) +} + +/// Serves the raw JSON-AD bytes of a frozen resource by hash. Content-addressed, +/// so the client re-verifies by re-hashing; the server is just a cache. +#[tracing::instrument(skip(appstate))] +pub async fn get_frozen( + path: web::Path, + appstate: web::Data, +) -> AtomicServerResult { + let hash_hex = path.into_inner(); + validate_hash_hex(&hash_hex)?; + + match appstate + .store + .kv + .get(atomic_lib::db::trees::Tree::Frozen, hash_hex.as_bytes())? + { + Some(bytes) => Ok(HttpResponse::Ok().content_type(AD_JSON).body(bytes)), + None => Ok(HttpResponse::NotFound().finish()), + } +} diff --git a/server/src/handlers/mod.rs b/server/src/handlers/mod.rs index 656047ded..56f55c334 100644 --- a/server/src/handlers/mod.rs +++ b/server/src/handlers/mod.rs @@ -9,6 +9,7 @@ pub mod blob; pub mod commit; pub mod download; pub mod export; +pub mod frozen; pub mod get_resource; #[cfg(feature = "image")] pub mod image; diff --git a/server/src/routes.rs b/server/src/routes.rs index 8c427146c..b66bbcb7a 100644 --- a/server/src/routes.rs +++ b/server/src/routes.rs @@ -207,6 +207,16 @@ pub fn config_routes(app: &mut actix_web::web::ServiceConfig) { .guard(guard::Method(Method::PUT)) .to(handlers::blob::put_blob), ) + .service( + web::resource("/frozen/{hash}") + .guard(guard::Method(Method::PUT)) + .to(handlers::frozen::put_frozen), + ) + .service( + web::resource("/frozen/{hash}") + .guard(guard::Method(Method::GET)) + .to(handlers::frozen::get_frozen), + ) .service( web::resource("/setup") .guard(guard::Method(Method::POST)) diff --git a/server/src/tests.rs b/server/src/tests.rs index 35f0487e9..b86224566 100644 --- a/server/src/tests.rs +++ b/server/src/tests.rs @@ -627,3 +627,75 @@ async fn upload_download_test() { let downloaded_bytes = test::read_body(resp).await; assert_eq!(downloaded_bytes, test_content.as_slice()); } + +#[actix_rt::test] +async fn frozen_endpoint_roundtrip() { + let unique_string = atomic_lib::utils::random_string(10); + use clap::Parser; + let opts = Opts::parse_from([ + "atomic-server", + "--initialize", + "--data-dir", + &format!("./.temp/{}/db", unique_string), + "--config-dir", + &format!("./.temp/{}/config", unique_string), + ]); + let mut config = config::build_config(opts).expect("failed init config"); + config.search_index_path = format!("./.temp/{}/search_index", unique_string).into(); + let appstate = crate::appstate::AppState::init(config.clone()) + .await + .expect("failed init appstate"); + atomic_lib::test_utils::setup_test_env(&appstate.store) + .await + .unwrap(); + let data = Data::new(appstate.clone()); + let app = test::init_service( + App::new() + .app_data(data) + .configure(crate::routes::config_routes), + ) + .await; + + // An identity-only frozen Property body. + let body = serde_json::json!({ + "https://atomicdata.dev/properties/isA": ["https://atomicdata.dev/classes/Property"], + "https://atomicdata.dev/properties/shortname": "title", + "https://atomicdata.dev/properties/datatype": "https://atomicdata.dev/datatypes/string" + }); + let id = atomic_lib::frozen::frozen_id(&body).unwrap(); + let hash = id.strip_prefix("did:ad:frozen:").unwrap(); + let canonical = serde_jcs::to_string(&body).unwrap(); + + // PUT stores it (content-addressed, public). + let put = TestRequest::put() + .uri(&format!("/frozen/{}", hash)) + .set_payload(canonical.clone()) + .to_request(); + let resp = test::call_service(&app, put).await; + assert!(resp.status().is_success(), "PUT status: {}", resp.status()); + + // GET returns the same canonical bytes. + let get = TestRequest::get() + .uri(&format!("/frozen/{}", hash)) + .to_request(); + let resp = test::call_service(&app, get).await; + assert!(resp.status().is_success()); + let returned = test::read_body(resp).await; + assert_eq!(returned, canonical.as_bytes()); + + // A body that doesn't hash to the URL id is rejected. + let bad = TestRequest::put() + .uri(&format!("/frozen/{}", "00".repeat(32))) + .set_payload(canonical.clone()) + .to_request(); + let resp = test::call_service(&app, bad).await; + assert!(!resp.status().is_success()); + + // And it now resolves through the normal get_resource path. + let subject = atomic_lib::Subject::from_raw(&id, None); + let resource = appstate.store.get_resource(&subject).await.unwrap(); + assert_eq!( + resource.get(urls::SHORTNAME).unwrap().to_string(), + "title" + ); +} diff --git a/test-vectors/README.md b/test-vectors/README.md new file mode 100644 index 000000000..204252c2b --- /dev/null +++ b/test-vectors/README.md @@ -0,0 +1,18 @@ +# Cross-language test vectors + +Shared fixtures that pin behavior across the TypeScript and Rust implementations. + +## `frozen.json` + +The `did:ad:frozen` content-addressing contract. Each vector is a JSON-AD `body` +and its expected frozen `id` = `did:ad:frozen:` + `blake3(JCS(body))`, where JCS +is RFC 8785. + +Both sides must reproduce the same `id` from the same `body`: + +- TypeScript: `browser/lib/src/frozen-vectors.test.ts` asserts `frozenIdFor(body) === id`. +- Rust: `lib/src/frozen.rs` tests assert `frozen_id(body) == id`. + +If you change the canonicalization or hashing on either side, regenerate the +vectors and confirm both suites pass — a diff here is a cross-language identity +break. diff --git a/test-vectors/freeze-resources.json b/test-vectors/freeze-resources.json new file mode 100644 index 000000000..4591e56ac --- /dev/null +++ b/test-vectors/freeze-resources.json @@ -0,0 +1,68 @@ +{ + "cases": [ + { + "name": "acyclic", + "input": [ + { + "localId": "p:title", + "content": { + "https://atomicdata.dev/properties/shortname": "title", + "https://atomicdata.dev/properties/datatype": "https://atomicdata.dev/datatypes/string" + } + }, + { + "localId": "c:todo", + "content": { + "https://atomicdata.dev/properties/shortname": "todo", + "https://atomicdata.dev/properties/requires": [ + "p:title" + ], + "https://atomicdata.dev/properties/recommends": [] + } + }, + { + "localId": "o:app", + "content": { + "https://atomicdata.dev/properties/shortname": "todoApp", + "https://atomicdata.dev/properties/classes": [ + "c:todo" + ], + "https://atomicdata.dev/properties/properties": [ + "p:title" + ] + } + } + ], + "expected": { + "p:title": "did:ad:frozen:d8499b11374d5d211421ebe66b4c7f1eb84c39dc7b10ac41df89d56e37e98cc0", + "c:todo": "did:ad:frozen:90b6416aa4e40216c4484d993be0c5556fc611c373f96afbfa1cf0409dbc0f7d", + "o:app": "did:ad:frozen:4b87a1fd6c43770d8a6b7b08fda884f95e6901baf5d5f0d2c5e13d747704e043" + } + }, + { + "name": "cyclic", + "input": [ + { + "localId": "p:friend", + "content": { + "https://atomicdata.dev/properties/shortname": "friend", + "https://atomicdata.dev/properties/classtype": "c:person" + } + }, + { + "localId": "c:person", + "content": { + "https://atomicdata.dev/properties/shortname": "person", + "https://atomicdata.dev/properties/requires": [ + "p:friend" + ] + } + } + ], + "expected": { + "p:friend": "did:ad:frozen:651b32175d89aa5d234215056c37d88effc9b1636c15300e2b0df67586d38ba0", + "c:person": "did:ad:frozen:651b32175d89aa5d234215056c37d88effc9b1636c15300e2b0df67586d38ba0" + } + } + ] +} diff --git a/test-vectors/freeze-schema.json b/test-vectors/freeze-schema.json new file mode 100644 index 000000000..7f26cbae5 --- /dev/null +++ b/test-vectors/freeze-schema.json @@ -0,0 +1,38 @@ +{ + "schema": { + "name": "FrozenTodoApp", + "version": "1.0.0", + "classes": { + "todo": { + "type": "object", + "required": [ + "title" + ], + "properties": { + "title": { + "type": "string", + "description": "Task title" + }, + "done": { + "type": "boolean" + }, + "dueAt": { + "type": "string", + "format": "date" + } + } + } + } + }, + "expected": { + "ontology": "did:ad:frozen:5359f2dbb9d46cc3188be1d76f27d0a8b5a89add18defcd9b26ca538c3b1d4da", + "classes": { + "todo": "did:ad:frozen:cf44ff3df002db21831e487bc930b0e03b4a3adbac4223445d4eaa1cb283c9e7" + }, + "properties": { + "todo.done": "did:ad:frozen:29a60b4ed180a3f1c0fc60c38ed35bd57c35487fc782b6477d195f5b400defd5", + "todo.dueAt": "did:ad:frozen:03171f4e8940ec0661c37504748c5a20d213d498cd583137c139cd226d44707c", + "todo.title": "did:ad:frozen:9c227a3e5d127dfdeeecf7fc93ae8bf8c97f31083e44e2440cf70a2d30c4b702" + } + } +} diff --git a/test-vectors/frozen.json b/test-vectors/frozen.json new file mode 100644 index 000000000..f7ece5f03 --- /dev/null +++ b/test-vectors/frozen.json @@ -0,0 +1,67 @@ +{ + "vectors": [ + { + "name": "property", + "body": { + "https://atomicdata.dev/properties/isA": [ + "https://atomicdata.dev/classes/Property" + ], + "https://atomicdata.dev/properties/shortname": "title", + "https://atomicdata.dev/properties/datatype": "https://atomicdata.dev/datatypes/string" + }, + "id": "did:ad:frozen:9c227a3e5d127dfdeeecf7fc93ae8bf8c97f31083e44e2440cf70a2d30c4b702" + }, + { + "name": "class-with-ref", + "body": { + "https://atomicdata.dev/properties/isA": [ + "https://atomicdata.dev/classes/Class" + ], + "https://atomicdata.dev/properties/shortname": "todo", + "https://atomicdata.dev/properties/requires": [ + "did:ad:frozen:0000000000000000000000000000000000000000000000000000000000000000" + ], + "https://atomicdata.dev/properties/recommends": [] + }, + "id": "did:ad:frozen:625c076abddc8a830468a5c8ad352150a9d4e81e839de1b84b7b1dc843b14d5a" + }, + { + "name": "unicode-and-number", + "body": { + "é": 1, + "a": 1.5, + "z": [ + 3, + 1, + 2 + ], + "b": true + }, + "id": "did:ad:frozen:9d588cb60acc4ab171d45789223f9d86a15ea5acfe19552bbf7b974fd71f8d11" + }, + { + "name": "cycle-unit", + "body": { + "urn:atomic-freeze:unit": [ + { + "https://atomicdata.dev/properties/isA": [ + "https://atomicdata.dev/classes/Property" + ], + "https://atomicdata.dev/properties/shortname": "friend", + "https://atomicdata.dev/properties/classtype": "did:ad:frozen:self:1" + }, + { + "https://atomicdata.dev/properties/isA": [ + "https://atomicdata.dev/classes/Class" + ], + "https://atomicdata.dev/properties/shortname": "person", + "https://atomicdata.dev/properties/requires": [ + "did:ad:frozen:self:0" + ] + } + ] + }, + "id": "did:ad:frozen:edc1f1b79e78ef17556e05c564d9a69c21ba0eb19aa13d628eafb1e8689d30b8" + } + ] +}