From 4c6bb6dc0153b695573d90e8f98a3ccb41ea6c70 Mon Sep 17 00:00:00 2001 From: Joep Meindertsma Date: Tue, 2 Jun 2026 11:02:23 +0200 Subject: [PATCH 01/20] WIP Schema in code #1207 + did:ad:frozen #1208 --- browser/cli/src/commands/schema.test.ts | 85 ++ browser/cli/src/commands/schema.ts | 249 ++++++ browser/cli/src/config.ts | 2 + browser/cli/src/generateBaseObject.ts | 15 +- browser/cli/src/generateClassExports.ts | 5 +- browser/cli/src/generateClasses.ts | 8 +- browser/cli/src/generateExternals.ts | 9 +- browser/cli/src/generateIndex.ts | 5 +- browser/cli/src/generateOntology.ts | 24 +- browser/cli/src/generatePropTypeMapping.ts | 13 +- browser/cli/src/index.ts | 6 + browser/cli/src/store.ts | 21 +- browser/cli/src/usage.ts | 6 + browser/lib/package.json | 7 + browser/lib/src/freeze.test.ts | 276 ++++++ browser/lib/src/freeze.ts | 425 ++++++++++ browser/lib/src/index.ts | 2 + browser/lib/src/jcs.test.ts | 42 + browser/lib/src/jcs.ts | 58 ++ browser/lib/src/schema-lock.test.ts | 100 +++ browser/lib/src/schema-lock.ts | 126 +++ browser/lib/src/schema.test.ts | 784 ++++++++++++++++++ browser/lib/src/schema.ts | 608 ++++++++++++++ browser/lib/src/store.ts | 279 ++++++- .../schema-code-first.integration.test.ts | 89 ++ .../schema-code-first/consumer-schema.ts | 31 + .../schema-code-first/producer-schema.ts | 26 + browser/lib/tsup.config.ts | 1 + planning/did-ad-frozen-server.md | 169 ++++ planning/json-schema-code-first.md | 633 ++++++++++++-- 30 files changed, 4025 insertions(+), 79 deletions(-) create mode 100644 browser/cli/src/commands/schema.test.ts create mode 100644 browser/cli/src/commands/schema.ts create mode 100644 browser/lib/src/freeze.test.ts create mode 100644 browser/lib/src/freeze.ts create mode 100644 browser/lib/src/jcs.test.ts create mode 100644 browser/lib/src/jcs.ts create mode 100644 browser/lib/src/schema-lock.test.ts create mode 100644 browser/lib/src/schema-lock.ts create mode 100644 browser/lib/src/schema.test.ts create mode 100644 browser/lib/src/schema.ts create mode 100644 browser/lib/tests/schema-code-first.integration.test.ts create mode 100644 browser/lib/tests/schema-code-first/consumer-schema.ts create mode 100644 browser/lib/tests/schema-code-first/producer-schema.ts create mode 100644 planning/did-ad-frozen-server.md diff --git a/browser/cli/src/commands/schema.test.ts b/browser/cli/src/commands/schema.test.ts new file mode 100644 index 000000000..fa53384c4 --- /dev/null +++ b/browser/cli/src/commands/schema.test.ts @@ -0,0 +1,85 @@ +import { mkdtemp, readFile, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { Agent } from '@tomic/lib'; + +const originalCwd = process.cwd(); + +describe('schema command', () => { + afterEach(() => { + process.chdir(originalCwd); + vi.restoreAllMocks(); + }); + + it('registers a code schema and generates ontology bindings from the same Store', async () => { + vi.spyOn(console, 'log').mockImplementation(() => undefined); + + const dir = await mkdtemp(path.join(tmpdir(), 'atomic-schema-cli-')); + const keys = await Agent.generateKeyPair(); + const agentSecret = Agent.buildSecret( + keys.privateKey, + `did:ad:agent:${keys.publicKey}`, + ); + + await writeFile( + path.join(dir, 'atomic.config.json'), + JSON.stringify( + { + outputFolder: './src/ontologies', + ontologies: [], + agentSecret, + _ISLIB_: false, + }, + null, + 2, + ), + ); + await writeFile( + path.join(dir, 'todo-schema.mjs'), + `export default { + name: 'TodoApp', + version: '1.0.0', + classes: { + todo: { + type: 'object', + required: ['title'], + properties: { + title: { type: 'string', description: 'Task title' }, + done: { type: 'boolean' } + } + } + } + };`, + ); + + process.chdir(dir); + + const { schemaCommand } = await import('./schema.js'); + await schemaCommand(['./todo-schema.mjs', '--local', '--generate', '--lock']); + + const generated = await readFile( + path.join(dir, 'src/ontologies/todoApp.ts'), + 'utf8', + ); + const index = await readFile( + path.join(dir, 'src/ontologies/index.ts'), + 'utf8', + ); + + expect(generated).toContain('export const todoapp'); + expect(generated).toContain('classes:'); + expect(generated).toContain('properties:'); + expect(generated).toContain('interface PropTypeMapping'); + expect(generated).toContain('did:ad:'); + expect(index).toContain("from './todoapp.js'"); + + // The committed, self-verifying lockfile is written and verifies. + const { verifySchemaLock } = await import('@tomic/lib'); + const lock = JSON.parse( + await readFile(path.join(dir, 'src/ontologies/TodoApp.schema.lock.json'), 'utf8'), + ); + expect(verifySchemaLock(lock)).toEqual({ ok: true, errors: [] }); + expect(lock.ontology).toMatch(/^did:ad:frozen:[0-9a-f]{64}$/); + }); +}); diff --git a/browser/cli/src/commands/schema.ts b/browser/cli/src/commands/schema.ts new file mode 100644 index 000000000..9b3b23312 --- /dev/null +++ b/browser/cli/src/commands/schema.ts @@ -0,0 +1,249 @@ +/* eslint-disable no-console */ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; +import chalk from 'chalk'; +import { + resolveConfig as prettierResolveConfig, + format as prettierFormat, +} from 'prettier'; +import { + buildSchemaLock, + verifySchemaLock, + type DefinedSchema, + type AtomicSchemaPackage, + type RegisteredSchema, +} from '@tomic/lib'; +import { createConfiguredStore } from '../store.js'; +import { atomicConfig } from '../config.js'; +import { generateOntology } from '../generateOntology.js'; +import { PropertyRecord } from '../PropertyRecord.js'; +import { generateExternals } from '../generateExternals.js'; +import { generateIndex } from '../generateIndex.js'; + +type SchemaModule = Record; +type SchemaInput = DefinedSchema | AtomicSchemaPackage; + +interface SchemaCommandOptions { + exportName: string; + save: boolean; + addToConfig: boolean; + generateBindings: boolean; + lock: boolean; +} + +export const schemaCommand = async (args: string[]) => { + const [schemaPath, ...rest] = args; + + if (!schemaPath) { + console.error( + chalk.red('ERROR: Missing schema module path.'), + '\nUsage: ad-generate schema ./schema.js [--export todoSchema] [--local] [--generate]', + ); + process.exit(1); + } + + const opts = parseOptions(rest); + const schema = await loadSchema(schemaPath, opts.exportName); + const store = await createConfiguredStore(); + + if (opts.save) { + store.setServerConnected(true); + } + + const registered = await store.registerSchema(schema, { save: opts.save }); + + if (opts.addToConfig) { + await addOntologyToConfig(registered.ontology.subject); + } + + if (opts.generateBindings) { + await writeOntologyBindings(registered, store); + } + + if (opts.lock) { + await writeSchemaLock(schema); + } + + printRegisteredSchema(registered, opts.save); +}; + +async function writeSchemaLock(schema: SchemaInput): Promise { + const lock = buildSchemaLock(schema); + const verification = verifySchemaLock(lock); + + if (!verification.ok) { + throw new Error( + `Refusing to write an invalid schema lock:\n${verification.errors.join('\n')}`, + ); + } + + const outputFolder = path.resolve(process.cwd(), atomicConfig.outputFolder); + const filePath = path.join(outputFolder, `${lock.name}.schema.lock.json`); + await fs.mkdir(outputFolder, { recursive: true }); + await fs.writeFile(filePath, `${JSON.stringify(lock, null, 2)}\n`); + + console.log(chalk.blue('Wrote lockfile'), chalk.cyan(filePath)); +} + +function parseOptions(args: string[]): SchemaCommandOptions { + let exportName = 'default'; + let save = true; + let addToConfig = false; + let generateBindings = false; + let lock = false; + + for (let i = 0; i < args.length; i++) { + const arg = args[i]; + + if (arg === '--export' || arg === '-e') { + exportName = args[i + 1]; + i++; + continue; + } + + if (arg === '--local') { + save = false; + continue; + } + + if (arg === '--add-to-config') { + addToConfig = true; + continue; + } + + if (arg === '--generate') { + generateBindings = true; + continue; + } + + if (arg === '--lock') { + lock = true; + continue; + } + + throw new Error(`Unknown schema command option: ${arg}`); + } + + if (!exportName) { + throw new Error('Missing value for --export'); + } + + return { exportName, save, addToConfig, generateBindings, lock }; +} + +async function loadSchema( + schemaPath: string, + exportName: string, +): Promise { + const absolutePath = path.resolve(process.cwd(), schemaPath); + const moduleUrl = pathToFileURL(absolutePath).href; + const module = (await import(moduleUrl)) as SchemaModule; + const value = module[exportName]; + + if (!isSchemaInput(value)) { + throw new Error( + `Export ${exportName} from ${schemaPath} is not a defineSchema() result or schema package.`, + ); + } + + return value; +} + +function isSchemaInput(value: unknown): value is SchemaInput { + if (typeof value !== 'object' || value === null) { + return false; + } + + if ('schemaHash' in value && 'normalized' in value) { + return true; + } + + return 'name' in value && 'classes' in value; +} + +function printRegisteredSchema( + registered: RegisteredSchema, + saved: boolean, +): void { + console.log(chalk.green(saved ? 'Published schema' : 'Registered schema')); + console.log(`${chalk.blue('Ontology:')} ${registered.ontology.subject}`); + console.log(`${chalk.blue('Hash:')} ${registered.model.ontology.schemaHash}`); + console.log( + `${chalk.blue('Classes:')} ${Object.keys(registered.classes).length}`, + ); + console.log( + `${chalk.blue('Properties:')} ${Object.keys(registered.properties).length}`, + ); +} + +async function addOntologyToConfig(subject: string): Promise { + const configPath = path.resolve(process.cwd(), './atomic.config.json'); + const raw = await fs.readFile(configPath, 'utf8'); + const config = JSON.parse(raw) as { ontologies?: string[] }; + const ontologies = config.ontologies ?? []; + + if (!ontologies.includes(subject)) { + config.ontologies = [...ontologies, subject]; + await fs.writeFile(configPath, `${JSON.stringify(config, null, 2)}\n`); + } + + console.log(`${chalk.blue('Config:')} added ${subject}`); +} + +async function writeOntologyBindings( + registered: RegisteredSchema, + store: Awaited>, +): Promise { + const propertyRecord = new PropertyRecord(); + const ontology = await generateOntology( + registered.ontology.subject, + propertyRecord, + store, + ); + + await writeGeneratedFile(ontology); + + const missingProps = propertyRecord.getMissingProperties(); + + if (missingProps.length > 0) { + const externalsContent = await generateExternals(missingProps, store); + await writeGeneratedFile({ + filename: 'externals.ts', + content: externalsContent, + }); + } + + await writeGeneratedFile( + generateIndex( + [registered.ontology.subject], + missingProps.length > 0, + store, + ), + ); +} + +async function writeGeneratedFile({ + filename, + content, +}: { + filename: string; + content: string; +}): Promise { + const outputFolder = path.resolve(process.cwd(), atomicConfig.outputFolder); + const filePath = path.join(outputFolder, filename); + await fs.mkdir(outputFolder, { recursive: true }); + + let formatted = content; + const prettierConfig = await prettierResolveConfig(filePath); + + if (prettierConfig) { + formatted = await prettierFormat(content, { + ...prettierConfig, + parser: 'typescript', + }); + } + + await fs.writeFile(filePath, formatted); + console.log(chalk.blue('Wrote to'), chalk.cyan(filePath)); +} diff --git a/browser/cli/src/config.ts b/browser/cli/src/config.ts index 22115b44a..919354c3f 100644 --- a/browser/cli/src/config.ts +++ b/browser/cli/src/config.ts @@ -20,6 +20,8 @@ export interface AtomicConfig { * If left empty the public agent is used. */ agentSecret?: string; + /** Server used for relative subjects and publishing code-first schemas. */ + serverUrl?: string; /** The list of subjects of your ontologies */ ontologies: string[]; diff --git a/browser/cli/src/generateBaseObject.ts b/browser/cli/src/generateBaseObject.ts index eabca06fe..7692aa17a 100644 --- a/browser/cli/src/generateBaseObject.ts +++ b/browser/cli/src/generateBaseObject.ts @@ -1,4 +1,4 @@ -import { Resource, type Core, core } from '@tomic/lib'; +import { Resource, type Core, core, type Store } from '@tomic/lib'; import { store } from './store.js'; import { camelCaseify, dedupe } from './utils.js'; import chalk from 'chalk'; @@ -13,6 +13,7 @@ type BaseObject = { export const generateBaseObject = async ( ontology: Resource, + activeStore: Store = store, ): Promise<[string, ReverseMapping]> => { if (ontology.error) { throw ontology.error; @@ -23,9 +24,9 @@ export const generateBaseObject = async ( const name = camelCaseify(ontology.props.shortname); const baseObj = { - classes: await listToObj(classes, 'classes'), - properties: await listToObj(properties, 'properties'), - __classDefs: await createClassDefs(classes), + classes: await listToObj(classes, 'classes', activeStore), + properties: await listToObj(properties, 'properties', activeStore), + __classDefs: await createClassDefs(classes, activeStore), }; const objStr = `export const ${name} = { @@ -40,10 +41,11 @@ export const generateBaseObject = async ( const listToObj = async ( list: string[], type: string, + activeStore: Store, ): Promise> => { const entries = await Promise.all( list.map(async subject => { - const resource = await store.getResource(subject); + const resource = await activeStore.getResource(subject); return [camelCaseify(resource.get(core.properties.shortname)), subject]; }), @@ -76,9 +78,10 @@ const listToObj = async ( const createClassDefs = async ( classes: string[], + activeStore: Store, ): Promise> => { const classResources = await Promise.all( - classes.map(async c => await store.getResource(c)), + classes.map(async c => await activeStore.getResource(c)), ); const entries = classResources.map(resource => { diff --git a/browser/cli/src/generateClassExports.ts b/browser/cli/src/generateClassExports.ts index 2d375f748..ea9967b81 100644 --- a/browser/cli/src/generateClassExports.ts +++ b/browser/cli/src/generateClassExports.ts @@ -1,4 +1,4 @@ -import { Resource, urls, type Core } from '@tomic/lib'; +import { Resource, urls, type Core, type Store } from '@tomic/lib'; import { atomicConfig } from './config.js'; import { ReverseMapping } from './generateBaseObject.js'; import { store } from './store.js'; @@ -19,12 +19,13 @@ const NAMESPACE_TEMPLATE = ` export const generateClassExports = ( ontology: Resource, reverseMapping: ReverseMapping, + activeStore: Store = store, ): string => { const classes = ontology.getArray(urls.properties.classes) as string[]; const body = classes .map(subject => { - const res = store.getResourceLoading(subject); + const res = activeStore.getResourceLoading(subject); const objectPath = reverseMapping[subject]; return createExportLine(res.props.shortname, objectPath); diff --git a/browser/cli/src/generateClasses.ts b/browser/cli/src/generateClasses.ts index 00af3d767..5a43fff0e 100644 --- a/browser/cli/src/generateClasses.ts +++ b/browser/cli/src/generateClasses.ts @@ -1,4 +1,4 @@ -import { type Core, type Resource } from '@tomic/lib'; +import { type Core, type Resource, type Store } from '@tomic/lib'; import { store } from './store.js'; import { ReverseMapping } from './generateBaseObject.js'; import { PropertyRecord } from './PropertyRecord.js'; @@ -8,11 +8,12 @@ export const generateClasses = ( ontology: Resource, reverseMapping: ReverseMapping, propertyRecord: PropertyRecord, + activeStore: Store = store, ): string => { const classes = dedupe(ontology.props.classes ?? []); const classStringList = classes.map(subject => { - return generateClass(subject, reverseMapping, propertyRecord); + return generateClass(subject, reverseMapping, propertyRecord, activeStore); }); const innerStr = classStringList.join('\n'); @@ -26,8 +27,9 @@ const generateClass = ( subject: string, reverseMapping: ReverseMapping, propertyRecord: PropertyRecord, + activeStore: Store, ): string => { - const resource = store.getResourceLoading(subject); + const resource = activeStore.getResourceLoading(subject); const transformSubject = (str: string) => { const name = reverseMapping[str]; diff --git a/browser/cli/src/generateExternals.ts b/browser/cli/src/generateExternals.ts index 4a80f56f7..57f2da83f 100644 --- a/browser/cli/src/generateExternals.ts +++ b/browser/cli/src/generateExternals.ts @@ -1,4 +1,4 @@ -import { Core, Datatype, Resource } from '@tomic/lib'; +import { Core, Datatype, Resource, type Store } from '@tomic/lib'; import { atomicConfig } from './config.js'; import { DatatypeToTSTypeMap } from './DatatypeToTSTypeMap.js'; import { store } from './store.js'; @@ -65,9 +65,12 @@ const generateBaseObjectProperties = ( return lines.join('\n'); }; -export const generateExternals = async (props: string[]) => { +export const generateExternals = async ( + props: string[], + activeStore: Store = store, +) => { const properties: Resource[] = await Promise.all( - props.map(p => store.getResource(p)), + props.map(p => activeStore.getResource(p)), ); const baseOjbectProperties = generateBaseObjectProperties(properties); diff --git a/browser/cli/src/generateIndex.ts b/browser/cli/src/generateIndex.ts index 1cb9119fd..e05f4e2ab 100644 --- a/browser/cli/src/generateIndex.ts +++ b/browser/cli/src/generateIndex.ts @@ -1,7 +1,7 @@ import { store } from './store.js'; import { camelCaseify, getExtension } from './utils.js'; import { atomicConfig } from './config.js'; -import type { Core } from '@tomic/lib'; +import type { Core, Store } from '@tomic/lib'; enum Inserts { MODULE_ALIAS = '{{1}}', @@ -30,9 +30,10 @@ export function initOntologies(): void { export const generateIndex = ( ontologies: string[], inludeExternals: boolean, + activeStore: Store = store, ) => { const names = ontologies.map(x => { - const res = store.getResourceLoading(x); + const res = activeStore.getResourceLoading(x); return camelCaseify(res.props.shortname); }); diff --git a/browser/cli/src/generateOntology.ts b/browser/cli/src/generateOntology.ts index 9fc7c5549..789a2794d 100644 --- a/browser/cli/src/generateOntology.ts +++ b/browser/cli/src/generateOntology.ts @@ -8,7 +8,7 @@ import { generateClassExports } from './generateClassExports.js'; import { atomicConfig } from './config.js'; import { PropertyRecord } from './PropertyRecord.js'; -import { Core } from '@tomic/lib'; +import { Core, type Store } from '@tomic/lib'; enum Inserts { MODULE_ALIAS = '{{1}}', @@ -44,11 +44,12 @@ declare module '${Inserts.MODULE_ALIAS}' { export const generateOntology = async ( subject: string, propertyRecord: PropertyRecord, + activeStore: Store = store, ): Promise<{ filename: string; content: string; }> => { - const ontology = await store.getResource(subject); + const ontology = await activeStore.getResource(subject); const properties = dedupe(ontology.props.properties ?? []); @@ -56,14 +57,27 @@ export const generateOntology = async ( propertyRecord.reportPropertyDefined(prop); } - const [baseObjStr, reverseMapping] = await generateBaseObject(ontology); - const classesStr = generateClasses(ontology, reverseMapping, propertyRecord); + const [baseObjStr, reverseMapping] = await generateBaseObject( + ontology, + activeStore, + ); + const classesStr = generateClasses( + ontology, + reverseMapping, + propertyRecord, + activeStore, + ); const [propertiesStr, propertiesImports] = generatePropTypeMapping( ontology, reverseMapping, + activeStore, ); const subToNameStr = generateSubjectToNameMapping(ontology, reverseMapping); - const classExportsStr = generateClassExports(ontology, reverseMapping); + const classExportsStr = generateClassExports( + ontology, + reverseMapping, + activeStore, + ); const content = TEMPLATE.replaceAll( Inserts.MODULE_ALIAS, diff --git a/browser/cli/src/generatePropTypeMapping.ts b/browser/cli/src/generatePropTypeMapping.ts index 87cbf8278..5154e8bdd 100644 --- a/browser/cli/src/generatePropTypeMapping.ts +++ b/browser/cli/src/generatePropTypeMapping.ts @@ -1,4 +1,4 @@ -import { Datatype, Resource, type Core } from '@tomic/lib'; +import { Datatype, Resource, type Core, type Store } from '@tomic/lib'; import { store } from './store.js'; import { ReverseMapping } from './generateBaseObject.js'; import { DatatypeToTSTypeMap } from './DatatypeToTSTypeMap.js'; @@ -7,11 +7,12 @@ import { dedupe } from './utils.js'; export const generatePropTypeMapping = ( ontology: Resource, reverseMapping: ReverseMapping, + activeStore: Store = store, ): [mappingString: string, usedImports: string[]] => { const properties = dedupe(ontology.props.properties ?? []); const lines = properties - .map(subject => generateLine(subject, reverseMapping)) + .map(subject => generateLine(subject, reverseMapping, activeStore)) .join('\n'); const mappingString = `interface PropTypeMapping { @@ -23,8 +24,12 @@ export const generatePropTypeMapping = ( return [mappingString, imports]; }; -const generateLine = (subject: string, reverseMapping: ReverseMapping) => { - const resource = store.getResourceLoading(subject); +const generateLine = ( + subject: string, + reverseMapping: ReverseMapping, + activeStore: Store, +) => { + const resource = activeStore.getResourceLoading(subject); const datatype = resource.props.datatype as Datatype; const type = DatatypeToTSTypeMap[datatype]; diff --git a/browser/cli/src/index.ts b/browser/cli/src/index.ts index 2ca1e9547..0c97c16b5 100644 --- a/browser/cli/src/index.ts +++ b/browser/cli/src/index.ts @@ -18,6 +18,12 @@ commands.set('init', () => import('./commands/init.js').then(m => m.initCommand(process.argv.slice(3))), ); +commands.set('schema', () => + import('./commands/schema.js').then(m => + m.schemaCommand(process.argv.slice(3)), + ), +); + if (commands.has(command)) { commands.get(command)?.(); } else { diff --git a/browser/cli/src/store.ts b/browser/cli/src/store.ts index ca122cbdc..fd8da68f8 100644 --- a/browser/cli/src/store.ts +++ b/browser/cli/src/store.ts @@ -1,6 +1,8 @@ import { Agent, Store } from '@tomic/lib'; import { atomicConfig } from './config.js'; +const DEFAULT_SERVER_URL = 'http://localhost:9883'; + const getCommandIndex = (): number | undefined => { const agentIndex = process.argv.indexOf('--agent'); if (agentIndex !== -1) return agentIndex; @@ -11,7 +13,7 @@ const getCommandIndex = (): number | undefined => { return undefined; }; -const getAgent = async (): Promise => { +export const getAgent = async (): Promise => { let secret; const agentCommandIndex = getCommandIndex(); @@ -26,10 +28,25 @@ const getAgent = async (): Promise => { return Agent.fromSecret(secret, 'js'); }; -export const store = new Store(); +export const store = new Store({ + serverUrl: atomicConfig.serverUrl ?? DEFAULT_SERVER_URL, +}); getAgent().then(agent => { if (agent) { store.setAgent(agent); } }); + +export const createConfiguredStore = async (): Promise => { + const configuredStore = new Store({ + serverUrl: atomicConfig.serverUrl ?? DEFAULT_SERVER_URL, + }); + const agent = await getAgent(); + + if (agent) { + configuredStore.setAgent(agent); + } + + return configuredStore; +}; diff --git a/browser/cli/src/usage.ts b/browser/cli/src/usage.ts index 0169269ee..a4721f660 100644 --- a/browser/cli/src/usage.ts +++ b/browser/cli/src/usage.ts @@ -3,5 +3,11 @@ ad-generate Commands: ontologies Generates typescript files for ontologies specified in the config file. + schema Registers or publishes a code-first schema module. init Creates a template config file. + +Schema command: + ad-generate schema ./schema.js [--export schemaName] [--local] [--add-to-config] [--generate] [--lock] + + --lock Write a committed *.schema.lock.json (the self-verifying, shareable frozen artifact). `; diff --git a/browser/lib/package.json b/browser/lib/package.json index 866cdfbe0..812c1cbdc 100644 --- a/browser/lib/package.json +++ b/browser/lib/package.json @@ -62,6 +62,13 @@ }, "require": "./dist/ontologies/*.cjs" }, + "./schema.js": { + "import": { + "types": "./dist/src/schema.d.ts", + "default": "./dist/schema.js" + }, + "require": "./dist/schema.cjs" + }, "./client-db.worker.js": "./dist/client-db.worker.js" }, "main-dev": "src/index.ts", diff --git a/browser/lib/src/freeze.test.ts b/browser/lib/src/freeze.test.ts new file mode 100644 index 000000000..a04f9a0ac --- /dev/null +++ b/browser/lib/src/freeze.test.ts @@ -0,0 +1,276 @@ +import { describe, it } from 'vitest'; + +import { + freezeResources, + SELF_PREFIX, + UNIT_MEMBERS_KEY, + type FreezableResource, + type JsonValue, +} from './freeze.js'; + +const CORE_DATATYPE = 'https://atomicdata.dev/properties/datatype'; +const CORE_SHORTNAME = 'https://atomicdata.dev/properties/shortname'; +const CORE_DESCRIPTION = 'https://atomicdata.dev/properties/description'; +const CORE_REQUIRES = 'https://atomicdata.dev/properties/requires'; +const CORE_CLASSTYPE = 'https://atomicdata.dev/properties/classtype'; +const CORE_CLASSES = 'https://atomicdata.dev/properties/classes'; +const CORE_PROPERTIES = 'https://atomicdata.dev/properties/properties'; + +/** prop -> class -> ontology, the common acyclic schema shape. */ +function acyclicSchema(): FreezableResource[] { + return [ + { + localId: 'p:title', + content: { + [CORE_SHORTNAME]: 'title', + [CORE_DATATYPE]: 'string', + [CORE_DESCRIPTION]: 'Task title', + }, + }, + { + localId: 'c:todo', + content: { + [CORE_SHORTNAME]: 'todo', + [CORE_REQUIRES]: ['p:title'], + }, + }, + { + localId: 'o:todoApp', + content: { + [CORE_SHORTNAME]: 'todoApp', + [CORE_CLASSES]: ['c:todo'], + [CORE_PROPERTIES]: ['p:title'], + }, + }, + ]; +} + +describe('freezeResources — acyclic', () => { + it('assigns each resource a distinct did:ad:frozen id', ({ expect }) => { + const { byLocalId } = freezeResources(acyclicSchema()); + + for (const id of byLocalId.values()) { + expect(id).toMatch(/^did:ad:frozen:[0-9a-f]{64}$/); + } + + expect(new Set(byLocalId.values()).size).toBe(3); + }); + + it('rewrites references to the referent frozen id', ({ expect }) => { + const { resources, byLocalId } = freezeResources(acyclicSchema()); + const byId = (localId: string) => + resources.find(r => r.frozenId === byLocalId.get(localId))!; + const todoClass = byId('c:todo'); + const ontology = byId('o:todoApp'); + + expect((todoClass.content as Record)[CORE_REQUIRES]).toEqual( + [byLocalId.get('p:title')], + ); + expect((ontology.content as Record)[CORE_CLASSES]).toEqual([ + byLocalId.get('c:todo'), + ]); + expect((ontology.content as Record)[CORE_PROPERTIES]).toEqual( + [byLocalId.get('p:title')], + ); + }); + + it('is independent of input order', ({ expect }) => { + const forward = freezeResources(acyclicSchema()); + const reversed = freezeResources([...acyclicSchema()].reverse()); + + for (const localId of ['p:title', 'c:todo', 'o:todoApp']) { + expect(reversed.byLocalId.get(localId)).toBe( + forward.byLocalId.get(localId), + ); + } + }); + + it('is independent of property key order', ({ expect }) => { + const reordered: FreezableResource[] = [ + { + localId: 'p:title', + content: { + [CORE_DESCRIPTION]: 'Task title', + [CORE_DATATYPE]: 'string', + [CORE_SHORTNAME]: 'title', + }, + }, + ]; + + expect(freezeResources(reordered).byLocalId.get('p:title')).toBe( + freezeResources([ + { + localId: 'p:title', + content: { + [CORE_SHORTNAME]: 'title', + [CORE_DATATYPE]: 'string', + [CORE_DESCRIPTION]: 'Task title', + }, + }, + ]).byLocalId.get('p:title'), + ); + }); + + it('dedupes identical content across separate runs', ({ expect }) => { + const make = (localId: string): FreezableResource => ({ + localId, + content: { [CORE_SHORTNAME]: 'title', [CORE_DATATYPE]: 'string' }, + }); + + expect(freezeResources([make('a')]).byLocalId.get('a')).toBe( + freezeResources([make('b')]).byLocalId.get('b'), + ); + }); + + it('changes the id when any content (incl. description) changes', ({ + expect, + }) => { + const base = freezeResources(acyclicSchema()).byLocalId.get('p:title'); + const edited = acyclicSchema(); + (edited[0].content as Record)[CORE_DESCRIPTION] = 'Changed'; + + expect(freezeResources(edited).byLocalId.get('p:title')).not.toBe(base); + }); + + it('leaves external (non-localId) references untouched', ({ expect }) => { + const externalProp = 'https://atomicdata.dev/properties/parent'; + const { resources } = freezeResources([ + { + localId: 'c:todo', + content: { [CORE_SHORTNAME]: 'todo', [externalProp]: 'did:ad:someDrive' }, + }, + ]); + + expect( + (resources[0].content as Record)[externalProp], + ).toBe('did:ad:someDrive'); + }); +}); + +/** Person.friend (classtype Person) <-> Person.requires friend: a 2-cycle. */ +function cyclicSchema(): FreezableResource[] { + return [ + { + localId: 'p:friend', + content: { + [CORE_SHORTNAME]: 'friend', + [CORE_DATATYPE]: 'atomicURL', + [CORE_CLASSTYPE]: 'c:person', + }, + }, + { + localId: 'c:person', + content: { + [CORE_SHORTNAME]: 'person', + [CORE_REQUIRES]: ['p:friend'], + }, + }, + ]; +} + +describe('freezeResources — cycles', () => { + it('freezes a cycle as one unit whose members share its id', ({ expect }) => { + const { resources, byLocalId } = freezeResources(cyclicSchema()); + + expect(byLocalId.get('p:friend')).toMatch(/^did:ad:frozen:[0-9a-f]{64}$/); + // Both members resolve to the same unit id; the unit is one frozen object. + expect(byLocalId.get('p:friend')).toBe(byLocalId.get('c:person')); + expect(resources).toHaveLength(1); + expect([...resources[0].unit].sort()).toEqual(['c:person', 'p:friend']); + }); + + it('wraps members under the unit key with intra-cycle self tokens', ({ + expect, + }) => { + const { resources } = freezeResources(cyclicSchema()); + const members = (resources[0].content as Record)[ + UNIT_MEMBERS_KEY + ] as Array>; + + expect(members).toHaveLength(2); + + const friend = members.find(m => m[CORE_SHORTNAME] === 'friend')!; + const person = members.find(m => m[CORE_SHORTNAME] === 'person')!; + + expect(friend[CORE_CLASSTYPE]).toMatch( + new RegExp(`^${SELF_PREFIX}\\d+$`), + ); + expect((person[CORE_REQUIRES] as string[])[0]).toMatch( + new RegExp(`^${SELF_PREFIX}\\d+$`), + ); + }); + + it('is independent of input order', ({ expect }) => { + const forward = freezeResources(cyclicSchema()); + const reversed = freezeResources([...cyclicSchema()].reverse()); + + expect(reversed.byLocalId.get('p:friend')).toBe( + forward.byLocalId.get('p:friend'), + ); + expect(reversed.byLocalId.get('c:person')).toBe( + forward.byLocalId.get('c:person'), + ); + }); + + it('re-hashes the whole unit when one member changes', ({ expect }) => { + const before = freezeResources(cyclicSchema()); + const edited = cyclicSchema(); + (edited[0].content as Record)[CORE_DESCRIPTION] = 'A friend'; + const after = freezeResources(edited); + + expect(after.byLocalId.get('p:friend')).not.toBe( + before.byLocalId.get('p:friend'), + ); + }); + + it('verifies by re-hash: the unit id is blake3(JCS(content))', async ({ + expect, + }) => { + const { blake3 } = await import('@noble/hashes/blake3.js'); + const { bytesToHex, utf8ToBytes } = await import( + '@noble/hashes/utils.js' + ); + const { jcsCanonicalize } = await import('./jcs.js'); + const { resources } = freezeResources(cyclicSchema()); + const unit = resources[0]; + + const recomputed = `did:ad:frozen:${bytesToHex( + blake3(utf8ToBytes(jcsCanonicalize(unit.content))), + )}`; + + expect(recomputed).toBe(unit.frozenId); + }); + + it('handles a self-referential resource as a unit', ({ expect }) => { + const { resources, byLocalId } = freezeResources([ + { + localId: 'c:node', + content: { + [CORE_SHORTNAME]: 'node', + [CORE_CLASSTYPE]: 'c:node', + }, + }, + ]); + + expect(byLocalId.get('c:node')).toMatch(/^did:ad:frozen:[0-9a-f]{64}$/); + expect(resources[0].unit).toEqual(['c:node']); + + const members = (resources[0].content as Record)[ + UNIT_MEMBERS_KEY + ] as Array>; + expect(members[0][CORE_CLASSTYPE]).toMatch( + new RegExp(`^${SELF_PREFIX}\\d+$`), + ); + }); +}); + +describe('freezeResources — validation', () => { + it('rejects duplicate localIds', ({ expect }) => { + expect(() => + freezeResources([ + { localId: 'dup', content: {} }, + { localId: 'dup', content: {} }, + ]), + ).toThrow('unique'); + }); +}); diff --git a/browser/lib/src/freeze.ts b/browser/lib/src/freeze.ts new file mode 100644 index 000000000..8381850e5 --- /dev/null +++ b/browser/lib/src/freeze.ts @@ -0,0 +1,425 @@ +import { blake3 } from '@noble/hashes/blake3.js'; +import { bytesToHex, utf8ToBytes } from '@noble/hashes/utils.js'; + +import { jcsCanonicalize } from './jcs.js'; + +export type JsonValue = + | string + | number + | boolean + | null + | JsonValue[] + | { [key: string]: JsonValue }; + +export type FrozenId = `did:ad:frozen:${string}`; + +const FROZEN_PREFIX = 'did:ad:frozen:'; + +/** + * Placeholder for an intra-cycle reference, by the referent's canonical index. + * Part of the frozen-unit format: a materializer rewires `did:ad:frozen:self:{i}` + * to the i-th member of the unit. + */ +export const SELF_PREFIX = 'did:ad:frozen:self:'; + +/** Reserved key carrying the ordered members of a cycle's frozen unit object. */ +export const UNIT_MEMBERS_KEY = 'urn:atomic-freeze:unit'; + +export interface FreezableResource { + /** + * Temporary, unique id. It is used both as this resource's bookkeeping key and + * as the token other resources put inside their `content` to reference it. Any + * string value anywhere in another resource's `content` that equals this + * `localId` is treated as a reference and rewritten to the computed + * {@link FrozenId}. `content` must NOT contain a self-identifier — the subject + * IS the hash, so it is excluded from the hashed bytes. + */ + localId: string; + /** The resource body to freeze. References to other resources are localIds. */ + content: JsonValue; +} + +export interface FrozenResource { + frozenId: FrozenId; + /** + * JSON-AD body with internal references resolved to FrozenIds. For a cycle + * unit this is a `{ [UNIT_MEMBERS_KEY]: [...members] }` wrapper whose members + * reference each other by canonical index. + */ + content: JsonValue; + /** + * The localIds this frozen object covers: one for an ordinary resource, the + * whole cycle for a unit. Members of a cycle are not individually addressable + * — they share the unit's FrozenId and resolve together. + */ + unit: string[]; +} + +export interface FreezeResult { + /** One entry per distinct frozen object (ordinary resource or cycle unit). */ + resources: FrozenResource[]; + /** localId -> FrozenId for every input resource (cycle members map to the unit). */ + byLocalId: Map; +} + +/** + * Content-addresses a set of resources that may reference each other, producing + * a Merkle DAG of `did:ad:frozen:{blake3}` identifiers. A reference is rewritten + * to the referent's hash before hashing, so a parent's id depends on its + * children's ids (the whole content is hashed, including descriptions). + * + * Cyclic references (e.g. a `Person` class with a `friend` property whose + * classtype is `Person`) have no leaf to start from. Each strongly-connected + * group is therefore frozen as a single **unit** object so the id stays + * `blake3(canonical bytes)` and remains verifiable by re-hashing. The cycle's + * members share that unit id and resolve together. + */ +export function freezeResources(input: FreezableResource[]): FreezeResult { + const ids = new Set(input.map(r => r.localId)); + + if (ids.size !== input.length) { + throw new Error('freezeResources: localId values must be unique'); + } + + const byId = new Map(input.map(r => [r.localId, r] as const)); + const edges = new Map>( + input.map(r => [r.localId, collectRefs(r.content, ids)] as const), + ); + + // Tarjan emits SCCs in reverse topological order (sinks first), which is + // exactly the bottom-up order we need: every out-edge of a component points + // at an already-frozen component. + const sccs = stronglyConnectedComponents([...ids], edges); + const frozenIdByLocal = new Map(); + const resources = new Map(); + + for (const scc of sccs) { + const isCycle = + scc.length > 1 || (edges.get(scc[0])?.has(scc[0]) ?? false); + + if (isCycle) { + freezeCycle(scc, byId, edges, frozenIdByLocal, resources); + } else { + freezeSingleton(scc[0], byId, edges, frozenIdByLocal, resources); + } + } + + return { resources: [...resources.values()], byLocalId: frozenIdByLocal }; +} + +function freezeSingleton( + localId: string, + byId: Map, + edges: Map>, + frozenIdByLocal: Map, + out: Map, +): void { + // All references point at earlier (already-frozen) components. + const content = substitute( + byId.get(localId)!.content, + resolvedRefMap(edges.get(localId), frozenIdByLocal), + ); + const frozenId = frozenIdFor(content); + + frozenIdByLocal.set(localId, frozenId); + + const existing = out.get(frozenId); + + if (existing) { + existing.unit.push(localId); + } else { + out.set(frozenId, { frozenId, content, unit: [localId] }); + } +} + +function freezeCycle( + scc: string[], + byId: Map, + edges: Map>, + frozenIdByLocal: Map, + out: Map, +): void { + const sccSet = new Set(scc); + const order = canonicalOrder(scc, sccSet, byId, edges, frozenIdByLocal); + const indexOf = new Map(order.map((id, i) => [id, i] as const)); + + // The unit wraps its members in canonical order; intra-cycle refs become a + // self token (by index), refs that leave the cycle become their FrozenId. + const members = order.map(localId => + substitute( + byId.get(localId)!.content, + cycleRefMap(edges.get(localId), sccSet, indexOf, frozenIdByLocal), + ), + ); + const content: JsonValue = { [UNIT_MEMBERS_KEY]: members }; + const frozenId = frozenIdFor(content); + + for (const localId of order) { + frozenIdByLocal.set(localId, frozenId); + } + + out.set(frozenId, { frozenId, content, unit: [...order] }); +} + +/** + * Deterministic ordering of a cycle's members, independent of input order, via + * color refinement: start each member colored by its content (intra-cycle refs + * blanked), then repeatedly recolor using neighbors' colors until the partition + * stabilizes. Ties (true structural automorphisms — vanishingly rare for + * schemas) are broken by localId, which makes those — and only those — cases + * input-dependent. + */ +function canonicalOrder( + scc: string[], + sccSet: Set, + byId: Map, + edges: Map>, + frozenIdByLocal: Map, +): string[] { + let color = new Map( + scc.map(localId => [ + localId, + hashCanonical( + substitute( + byId.get(localId)!.content, + cycleRefMap(edges.get(localId), sccSet, undefined, frozenIdByLocal), + ), + ), + ]), + ); + + for (let round = 0; round < scc.length; round++) { + const next = new Map( + scc.map(localId => [ + localId, + hashCanonical( + substitute( + byId.get(localId)!.content, + neighborColorRefMap( + edges.get(localId), + sccSet, + color, + frozenIdByLocal, + ), + ), + ), + ]), + ); + + if (partitionSignature(scc, next) === partitionSignature(scc, color)) { + color = next; + break; + } + + color = next; + } + + return [...scc].sort((a, b) => { + const ca = color.get(a)!; + const cb = color.get(b)!; + + if (ca !== cb) { + return ca < cb ? -1 : 1; + } + + return a < b ? -1 : a > b ? 1 : 0; + }); +} + +/** Maps every reference to its already-computed FrozenId (drops unknowns). */ +function resolvedRefMap( + refs: Set | undefined, + frozenIdByLocal: Map, +): Map { + const map = new Map(); + + for (const ref of refs ?? []) { + const frozenId = frozenIdByLocal.get(ref); + + if (frozenId) { + map.set(ref, frozenId); + } + } + + return map; +} + +/** + * For hashing a cycle: intra-cycle refs become a self token (by canonical index, + * or a constant when `indexOf` is omitted during initial coloring); refs that + * leave the cycle become their FrozenId. + */ +function cycleRefMap( + refs: Set | undefined, + sccSet: Set, + indexOf: Map | undefined, + frozenIdByLocal: Map, +): Map { + const map = new Map(); + + for (const ref of refs ?? []) { + if (sccSet.has(ref)) { + map.set(ref, indexOf ? `${SELF_PREFIX}${indexOf.get(ref)}` : SELF_PREFIX); + } else { + const frozenId = frozenIdByLocal.get(ref); + + if (frozenId) { + map.set(ref, frozenId); + } + } + } + + return map; +} + +/** Like {@link cycleRefMap} but intra-cycle refs carry the neighbor's color. */ +function neighborColorRefMap( + refs: Set | undefined, + sccSet: Set, + color: Map, + frozenIdByLocal: Map, +): Map { + const map = new Map(); + + for (const ref of refs ?? []) { + if (sccSet.has(ref)) { + map.set(ref, `${SELF_PREFIX}${color.get(ref)}`); + } else { + const frozenId = frozenIdByLocal.get(ref); + + if (frozenId) { + map.set(ref, frozenId); + } + } + } + + return map; +} + +/** Canonical signature of the equivalence classes induced by `color`. */ +function partitionSignature( + scc: string[], + color: Map, +): string { + const groups = new Map(); + + for (const id of scc) { + const key = color.get(id)!; + const group = groups.get(key); + + if (group) { + group.push(id); + } else { + groups.set(key, [id]); + } + } + + return [...groups.values()] + .map(group => [...group].sort().join(',')) + .sort() + .join('|'); +} + +function collectRefs(value: JsonValue, ids: Set): Set { + const out = new Set(); + + const walk = (node: JsonValue): void => { + if (typeof node === 'string') { + if (ids.has(node)) { + out.add(node); + } + } else if (Array.isArray(node)) { + node.forEach(walk); + } else if (node !== null && typeof node === 'object') { + Object.values(node).forEach(walk); + } + }; + + walk(value); + + return out; +} + +function substitute(value: JsonValue, map: Map): JsonValue { + if (typeof value === 'string') { + return map.get(value) ?? value; + } + + if (Array.isArray(value)) { + return value.map(item => substitute(item, map)); + } + + if (value !== null && typeof value === 'object') { + return Object.fromEntries( + Object.entries(value).map(([key, child]) => [ + key, + substitute(child, map), + ]), + ); + } + + return value; +} + +function hashCanonical(value: JsonValue): string { + return bytesToHex(blake3(utf8ToBytes(jcsCanonicalize(value)))); +} + +/** + * The canonical frozen id for a JSON-AD body: `did:ad:frozen:{blake3(JCS(content))}`. + * The single source of truth shared by production and verification — re-hashing a + * stored frozen object with this must reproduce its id. + */ +export function frozenIdFor(content: JsonValue): FrozenId { + return (FROZEN_PREFIX + hashCanonical(content)) as FrozenId; +} + +function stronglyConnectedComponents( + nodes: string[], + edges: Map>, +): string[][] { + let counter = 0; + const index = new Map(); + const low = new Map(); + const onStack = new Set(); + const stack: string[] = []; + const result: string[][] = []; + + const connect = (v: string): void => { + index.set(v, counter); + low.set(v, counter); + counter++; + stack.push(v); + onStack.add(v); + + for (const w of edges.get(v) ?? []) { + if (!index.has(w)) { + connect(w); + low.set(v, Math.min(low.get(v)!, low.get(w)!)); + } else if (onStack.has(w)) { + low.set(v, Math.min(low.get(v)!, index.get(w)!)); + } + } + + if (low.get(v) === index.get(v)) { + const component: string[] = []; + let w: string; + + do { + w = stack.pop()!; + onStack.delete(w); + component.push(w); + } while (w !== v); + + result.push(component); + } + }; + + for (const v of nodes) { + if (!index.has(v)) { + connect(v); + } + } + + return result; +} diff --git a/browser/lib/src/index.ts b/browser/lib/src/index.ts index cf9e96294..1e06b4e2a 100644 --- a/browser/lib/src/index.ts +++ b/browser/lib/src/index.ts @@ -45,6 +45,8 @@ export * from './error.js'; export * from './datatypes.js'; export * from './parse.js'; export * from './search.js'; +export * from './schema.js'; +export * from './schema-lock.js'; export * from './resource.js'; export * from './store.js'; export * from './subject.js'; diff --git a/browser/lib/src/jcs.test.ts b/browser/lib/src/jcs.test.ts new file mode 100644 index 000000000..0080eb5d3 --- /dev/null +++ b/browser/lib/src/jcs.test.ts @@ -0,0 +1,42 @@ +import { describe, it } from 'vitest'; + +import { jcsCanonicalize } from './jcs.js'; + +describe('jcsCanonicalize (RFC 8785)', () => { + it('sorts object keys by UTF-16 code unit', ({ expect }) => { + expect(jcsCanonicalize({ b: 1, a: 2 })).toBe('{"a":2,"b":1}'); + }); + + it('sorts nested objects and preserves array order', ({ expect }) => { + expect(jcsCanonicalize({ z: [3, 1, 2], a: { d: 1, c: 2 } })).toBe( + '{"a":{"c":2,"d":1},"z":[3,1,2]}', + ); + }); + + it('orders ascii before higher code points', ({ expect }) => { + expect(jcsCanonicalize({ 'é': 1, a: 2 })).toBe('{"a":2,"é":1}'); + }); + + it('serializes numbers with ECMAScript semantics', ({ expect }) => { + expect(jcsCanonicalize(1.0)).toBe('1'); + expect(jcsCanonicalize(1.5)).toBe('1.5'); + expect(jcsCanonicalize(-0)).toBe('0'); + }); + + it('escapes strings minimally like JSON.stringify', ({ expect }) => { + expect(jcsCanonicalize('a"b\\c')).toBe('"a\\"b\\\\c"'); + }); + + it('serializes primitives', ({ expect }) => { + expect(jcsCanonicalize(null)).toBe('null'); + expect(jcsCanonicalize(true)).toBe('true'); + expect(jcsCanonicalize(false)).toBe('false'); + }); + + it('rejects non-finite numbers', ({ expect }) => { + expect(() => jcsCanonicalize(Number.NaN)).toThrow('non-finite'); + expect(() => jcsCanonicalize(Number.POSITIVE_INFINITY)).toThrow( + 'non-finite', + ); + }); +}); diff --git a/browser/lib/src/jcs.ts b/browser/lib/src/jcs.ts new file mode 100644 index 000000000..f048b6359 --- /dev/null +++ b/browser/lib/src/jcs.ts @@ -0,0 +1,58 @@ +/** + * RFC 8785 JSON Canonicalization Scheme (JCS). + * + * Produces the canonical serialization used for all content-addressed + * `did:ad:frozen` hashing, so the same value hashes identically here and in any + * other conformant implementation — notably the `serde_jcs` crate on the Rust + * side. Using a named standard (rather than an ad-hoc stable stringify) is what + * makes cross-language frozen ids byte-for-byte reproducible. + * + * Covers the JSON value space we hash: + * - objects: keys sorted by UTF-16 code unit (`Array.prototype.sort` default, + * which RFC 8785 §3.2.3 mandates) + * - arrays: order preserved + * - strings and finite numbers: ECMAScript `JSON.stringify` semantics, which + * RFC 8785 §3.2.2.2/§3.2.2.3 reference directly + * - booleans and null + * + * Non-finite numbers are rejected (JSON has no representation for them). + */ +export type JcsValue = + | string + | number + | boolean + | null + | JcsValue[] + | { [key: string]: JcsValue }; + +export function jcsCanonicalize(value: JcsValue): string { + if (value === null) { + return 'null'; + } + + if (typeof value === 'boolean') { + return value ? 'true' : 'false'; + } + + if (typeof value === 'number') { + if (!Number.isFinite(value)) { + throw new Error(`JCS cannot serialize a non-finite number: ${value}`); + } + + return JSON.stringify(value); + } + + if (typeof value === 'string') { + return JSON.stringify(value); + } + + if (Array.isArray(value)) { + return `[${value.map(jcsCanonicalize).join(',')}]`; + } + + const keys = Object.keys(value).sort(); + + return `{${keys + .map(key => `${JSON.stringify(key)}:${jcsCanonicalize(value[key])}`) + .join(',')}}`; +} diff --git a/browser/lib/src/schema-lock.test.ts b/browser/lib/src/schema-lock.test.ts new file mode 100644 index 000000000..f8dfb6a74 --- /dev/null +++ b/browser/lib/src/schema-lock.test.ts @@ -0,0 +1,100 @@ +import { describe, it } from 'vitest'; + +import { buildSchemaLock, verifySchemaLock } from './schema-lock.js'; +import { type AtomicSchemaPackage } from './schema.js'; + +const todoSchema: AtomicSchemaPackage = { + name: 'TodoApp', + version: '1.0.0', + classes: { + todo: { + title: 'Todo', + description: 'A task', + type: 'object', + required: ['title'], + properties: { + title: { type: 'string', description: 'Task title' }, + done: { type: 'boolean' }, + }, + }, + }, +}; + +describe('buildSchemaLock', () => { + it('produces a self-verifying lockfile', ({ expect }) => { + const lock = buildSchemaLock(todoSchema); + + expect(verifySchemaLock(lock)).toEqual({ ok: true, errors: [] }); + }); + + it('decodes every frozen id in @index', ({ expect }) => { + const lock = buildSchemaLock(todoSchema); + + for (const id of Object.keys(lock.frozen)) { + expect(lock['@index'][id]).toBeDefined(); + } + + expect(lock['@index'][lock.ontology]).toBe('TodoApp'); + expect( + lock['@index'][lock.presentation.properties['todo.title'].id], + ).toBe('TodoApp.title'); + }); + + it('keeps descriptions in presentation, not in the hashed objects', ({ + expect, + }) => { + const lock = buildSchemaLock(todoSchema); + const titleId = lock.presentation.properties['todo.title'].id; + + expect(lock.presentation.properties['todo.title'].description).toBe( + 'Task title', + ); + expect( + JSON.stringify(lock.frozen[titleId]).includes('Task title'), + ).toBe(false); + }); + + it('is deterministic across runs', ({ expect }) => { + expect(JSON.stringify(buildSchemaLock(todoSchema))).toBe( + JSON.stringify(buildSchemaLock(todoSchema)), + ); + }); + + it('keeps frozen ids stable when only a description changes', ({ expect }) => { + const base = buildSchemaLock(todoSchema); + const reworded = buildSchemaLock({ + ...todoSchema, + classes: { + todo: { + ...todoSchema.classes.todo, + properties: { + ...todoSchema.classes.todo.properties, + title: { type: 'string', description: 'Different' }, + }, + }, + }, + }); + + expect(Object.keys(reworded.frozen).sort()).toEqual( + Object.keys(base.frozen).sort(), + ); + }); +}); + +describe('verifySchemaLock', () => { + it('fails when a frozen object is tampered with', ({ expect }) => { + const lock = buildSchemaLock(todoSchema); + const [firstId] = Object.keys(lock.frozen); + const tampered = { + ...lock, + frozen: { + ...lock.frozen, + [firstId]: { ...(lock.frozen[firstId] as object), tampered: true }, + }, + }; + + const result = verifySchemaLock(tampered as typeof lock); + expect(result.ok).toBe(false); + expect(result.errors[0]).toContain(firstId); + }); +}); diff --git a/browser/lib/src/schema-lock.ts b/browser/lib/src/schema-lock.ts new file mode 100644 index 000000000..b01ea7100 --- /dev/null +++ b/browser/lib/src/schema-lock.ts @@ -0,0 +1,126 @@ +import { frozenIdFor, type FrozenId, type JsonValue } from './freeze.js'; +import { + freezeSchema, + type AtomicSchemaPackage, + type DefinedSchema, + type SchemaHash, +} from './schema.js'; + +/** + * The committed, shareable schema artifact. A frozen id is a deterministic + * function of the schema source, so this file makes a schema *available* without + * any server: any implementation re-hashes the `frozen` objects to verify, and + * registers them locally. See `verifySchemaLock`. + * + * Only `frozen` is hashed (identity-only JSON-AD). `@index` and `presentation` + * are non-hashed human/metadata aids — editing them never moves an id. + */ +export interface SchemaLock { + readonly name: string; + readonly version?: string; + readonly ontology: FrozenId; + /** Frozen id -> `Ontology.shortname`, a human decoder. Not hashed. */ + readonly '@index': Record; + /** Frozen id -> identity-only canonical JSON-AD body. The hashed objects. */ + readonly frozen: Record; + readonly presentation: SchemaLockPresentation; +} + +export interface SchemaLockPresentationEntry { + readonly id: FrozenId; + readonly description: string; +} + +export interface SchemaLockPresentation { + readonly ontology: { + readonly description: string; + readonly version?: string; + readonly schemaHash: SchemaHash; + readonly jsonSchema: AtomicSchemaPackage; + }; + readonly classes: Record; + readonly properties: Record; +} + +/** Builds the committed lockfile for a schema (deterministic, server-free). */ +export function buildSchemaLock( + input: AtomicSchemaPackage | DefinedSchema, +): SchemaLock { + const frozen = freezeSchema(input); + const name = frozen.model.ontology.shortname; + + const index: Record = {}; + + const addIndex = (id: FrozenId, label: string): void => { + // Cycle members share a unit id; join their labels rather than clobber. + index[id] = index[id] ? `${index[id]} + ${label}` : label; + }; + + addIndex(frozen.ontology, name); + + for (const klass of frozen.model.classes) { + addIndex(frozen.classes[klass.key], `${name}.${klass.shortname}`); + } + + for (const property of frozen.model.properties) { + addIndex(frozen.properties[property.key], `${name}.${property.shortname}`); + } + + const frozenObjects: Record = {}; + + for (const resource of frozen.resources) { + frozenObjects[resource.frozenId] = resource.content; + } + + return { + name, + version: frozen.model.ontology.version, + ontology: frozen.ontology, + '@index': index, + frozen: frozenObjects, + presentation: { + ontology: frozen.presentation.ontology, + classes: mapPresentation(frozen.presentation.classes), + properties: mapPresentation(frozen.presentation.properties), + }, + }; +} + +export interface SchemaLockVerification { + readonly ok: boolean; + readonly errors: string[]; +} + +/** + * Re-hashes every frozen object and checks it matches its id. This is the + * language-neutral verification a consumer (or a CI stale-lockfile guard) runs; + * it depends only on JCS + blake3, never on the freeze algorithm. + */ +export function verifySchemaLock(lock: SchemaLock): SchemaLockVerification { + const errors: string[] = []; + + for (const [id, object] of Object.entries(lock.frozen)) { + const actual = frozenIdFor(object); + + if (actual !== id) { + errors.push(`Frozen object "${id}" hashes to "${actual}"`); + } + } + + if (!(lock.ontology in lock.frozen)) { + errors.push(`Ontology id "${lock.ontology}" is missing from "frozen"`); + } + + return { ok: errors.length === 0, errors }; +} + +function mapPresentation( + entries: Record, +): Record { + return Object.fromEntries( + Object.entries(entries).map(([key, value]) => [ + key, + { id: value.frozenId, description: value.description }, + ]), + ); +} diff --git a/browser/lib/src/schema.test.ts b/browser/lib/src/schema.test.ts new file mode 100644 index 000000000..7301938f9 --- /dev/null +++ b/browser/lib/src/schema.test.ts @@ -0,0 +1,784 @@ +import { describe, it, vi } from 'vitest'; + +import { Agent } from './agent.js'; +import type { Commit } from './commit.js'; +import { JSCryptoProvider } from './CryptoProvider.js'; +import { + canonicalizeSchemaPackage, + defineSchema, + freezeSchema, + hashSchemaPackage, + SCHEMA_HASH_PROPERTY, + schemaToOntologyModel, + type AtomicSchemaPackage, +} from './schema.js'; +import { Datatype } from './datatypes.js'; +import { core } from './ontologies/core.js'; +import { Store } from './store.js'; + +interface MockClientDbEntry { + json: string; + snapshot?: Uint8Array; +} + +function attachMockClientDb( + store: Store, + dbState = new Map(), +): Map { + store.setClientDb({ + isReady: true, + isInitialized: true, + initError: undefined, + putResourceWithSnapshot: vi.fn( + async (subject: string, json: string, snapshot?: Uint8Array) => { + dbState.set(subject, { json, snapshot }); + }, + ), + getResource: async (subject: string) => dbState.get(subject)?.json ?? null, + getResourceWithSnapshot: async (subject: string) => { + const entry = dbState.get(subject); + + return { jsonAd: entry?.json ?? null, snapshot: entry?.snapshot }; + }, + getLoroSnapshot: async (subject: string) => dbState.get(subject)?.snapshot, + waitForInit: async () => true, + waitForReady: async () => true, + } as unknown as Parameters[0]); + + return dbState; +} + +const todoSchema: AtomicSchemaPackage = { + name: 'TodoApp', + version: '1.0.0', + classes: { + todo: { + title: 'Todo', + description: 'A task', + type: 'object', + required: ['title'], + properties: { + title: { + type: 'string', + description: 'Task title', + }, + done: { + type: 'boolean', + default: false, + }, + }, + }, + }, +}; + +describe('schema package hashing', () => { + it('hashes equivalent schemas the same regardless of object key order', ({ + expect, + }) => { + const differentlyOrdered: AtomicSchemaPackage = { + classes: { + todo: { + properties: { + done: { + default: false, + type: 'boolean', + }, + title: { + description: 'Task title', + type: 'string', + }, + }, + required: ['title'], + type: 'object', + description: 'A task', + title: 'Todo', + }, + }, + version: '1.0.0', + name: 'TodoApp', + }; + + expect(hashSchemaPackage(differentlyOrdered)).toBe( + hashSchemaPackage(todoSchema), + ); + }); + + it('changes the hash when a property datatype changes', ({ expect }) => { + const changed: AtomicSchemaPackage = { + ...todoSchema, + classes: { + todo: { + ...todoSchema.classes.todo, + properties: { + ...todoSchema.classes.todo.properties, + title: { + type: 'integer', + description: 'Task title', + }, + }, + }, + }, + }; + + expect(hashSchemaPackage(changed)).not.toBe(hashSchemaPackage(todoSchema)); + }); + + it('normalizes away undefined fields before canonicalization', ({ + expect, + }) => { + const withUndefined = { + ...todoSchema, + description: undefined, + } as AtomicSchemaPackage; + + expect(canonicalizeSchemaPackage(withUndefined)).toBe( + canonicalizeSchemaPackage(todoSchema), + ); + }); + + it('rejects non-finite numbers', ({ expect }) => { + const invalid = { + ...todoSchema, + classes: { + todo: { + ...todoSchema.classes.todo, + properties: { + ratio: { + type: 'number', + default: Number.NaN, + }, + }, + }, + }, + } as AtomicSchemaPackage; + + expect(() => hashSchemaPackage(invalid)).toThrow('non-finite number'); + }); +}); + +describe('defineSchema', () => { + it('returns the original schema, normalized schema, and schema hash', ({ + expect, + }) => { + const defined = defineSchema(todoSchema); + + expect(defined.schema).toBe(todoSchema); + expect(defined.normalized).toEqual(todoSchema); + expect(defined.schemaHash).toMatch(/^blake3:[0-9a-f]{64}$/); + }); + + it('supports pinned external ontology imports', ({ expect }) => { + const external = defineSchema({ + name: 'ProjectApp', + imports: { + todo: { + subject: 'did:ad:todoOntology', + expectedHash: defineSchema(todoSchema).schemaHash, + }, + }, + classes: { + project: { + type: 'object', + properties: { + title: { + $ref: 'todo.properties.title', + }, + }, + }, + }, + }); + + expect(external.normalized.imports?.todo.subject).toBe( + 'did:ad:todoOntology', + ); + expect(external.normalized.imports?.todo.expectedHash).toMatch( + /^blake3:[0-9a-f]{64}$/, + ); + }); +}); + +describe('schemaToOntologyModel', () => { + it('converts schema classes and properties to Atomic ontology model entries', ({ + expect, + }) => { + const model = schemaToOntologyModel(todoSchema); + + expect(model.ontology.shortname).toBe('TodoApp'); + expect(model.ontology.schemaHash).toMatch(/^blake3:[0-9a-f]{64}$/); + expect(model.classes).toEqual([ + { + key: 'todo', + subject: undefined, + shortname: 'todo', + description: 'A task', + requires: ['todo.title'], + recommends: ['todo.done'], + }, + ]); + expect( + model.properties.toSorted((a, b) => + a.propertyKey.localeCompare(b.propertyKey), + ), + ).toMatchObject([ + { + key: 'todo.done', + classKey: 'todo', + propertyKey: 'done', + shortname: 'done', + description: 'done', + datatype: Datatype.BOOLEAN, + }, + { + key: 'todo.title', + classKey: 'todo', + propertyKey: 'title', + shortname: 'title', + description: 'Task title', + datatype: Datatype.STRING, + }, + ]); + }); + + it('maps common JSON Schema datatypes to Atomic datatypes', ({ expect }) => { + const model = schemaToOntologyModel({ + name: 'Types', + classes: { + item: { + type: 'object', + properties: { + dueAt: { type: 'string', format: 'date' }, + url: { type: 'string', format: 'uri' }, + count: { type: 'integer' }, + price: { type: 'number' }, + payload: { type: 'object' }, + tags: { type: 'array', items: { type: 'string' } }, + owner: { $ref: '#/$defs/Agent' }, + files: { type: 'array', items: { $ref: '#/$defs/File' } }, + }, + }, + }, + }); + + expect( + Object.fromEntries( + model.properties.map(prop => [prop.propertyKey, prop.datatype]), + ), + ).toEqual({ + dueAt: Datatype.DATE, + url: Datatype.URI, + count: Datatype.INTEGER, + price: Datatype.FLOAT, + payload: Datatype.JSON, + tags: Datatype.JSON, + owner: Datatype.ATOMIC_URL, + files: Datatype.RESOURCEARRAY, + }); + }); + + it('keeps atomic extension metadata for generated properties and classes', ({ + expect, + }) => { + const model = schemaToOntologyModel({ + name: 'People', + classes: { + person: { + type: 'object', + 'atomic:subject': 'did:ad:personClass', + 'atomic:shortname': 'person', + required: ['friend'], + properties: { + friend: { + type: 'string', + 'atomic:subject': 'did:ad:friendProperty', + 'atomic:datatype': Datatype.ATOMIC_URL, + 'atomic:classType': 'did:ad:personClass', + 'atomic:isLocked': true, + }, + }, + }, + }, + }); + + expect(model.classes[0].subject).toBe('did:ad:personClass'); + expect(model.properties[0]).toMatchObject({ + subject: 'did:ad:friendProperty', + datatype: Datatype.ATOMIC_URL, + classType: 'did:ad:personClass', + isLocked: true, + }); + }); +}); + +describe('freezeSchema', () => { + it('produces did:ad:frozen ids for the ontology, classes, and properties', ({ + expect, + }) => { + const frozen = freezeSchema(todoSchema); + + expect(frozen.ontology).toMatch(/^did:ad:frozen:[0-9a-f]{64}$/); + expect(frozen.classes.todo).toMatch(/^did:ad:frozen:[0-9a-f]{64}$/); + expect(frozen.properties['todo.title']).toMatch( + /^did:ad:frozen:[0-9a-f]{64}$/, + ); + expect(frozen.properties['todo.done']).toMatch( + /^did:ad:frozen:[0-9a-f]{64}$/, + ); + }); + + it('rewrites class requires to the frozen property id', ({ expect }) => { + const frozen = freezeSchema(todoSchema); + const todoClass = frozen.resources.find( + resource => resource.frozenId === frozen.classes.todo, + )!; + + expect( + (todoClass.content as Record)[core.properties.requires], + ).toEqual([frozen.properties['todo.title']]); + }); + + it('rewrites ontology members to frozen ids and keeps the schema hash', ({ + expect, + }) => { + const frozen = freezeSchema(todoSchema); + const ontology = frozen.resources.find( + resource => resource.frozenId === frozen.ontology, + )!; + const content = ontology.content as Record; + + expect(content[core.properties.classes]).toEqual([frozen.classes.todo]); + expect(content[core.properties.properties]).toEqual( + expect.arrayContaining([ + frozen.properties['todo.title'], + frozen.properties['todo.done'], + ]), + ); + // Presentation (schemaHash, descriptions) is NOT part of frozen identity. + expect(content[SCHEMA_HASH_PROPERTY]).toBeUndefined(); + expect(content[core.properties.description]).toBeUndefined(); + expect(frozen.presentation.ontology.schemaHash).toBe( + frozen.model.ontology.schemaHash, + ); + }); + + it('is deterministic and content-addressed across runs', ({ expect }) => { + expect(freezeSchema(todoSchema).properties['todo.title']).toBe( + freezeSchema(defineSchema(todoSchema)).properties['todo.title'], + ); + }); + + it('keeps the property id stable when only its description changes', ({ + expect, + }) => { + const base = freezeSchema(todoSchema).properties['todo.title']; + const reworded = freezeSchema({ + ...todoSchema, + classes: { + todo: { + ...todoSchema.classes.todo, + properties: { + ...todoSchema.classes.todo.properties, + title: { type: 'string', description: 'A different title' }, + }, + }, + }, + }); + + // Description is presentation, not identity — the id must not move... + expect(reworded.properties['todo.title']).toBe(base); + // ...but the new wording is captured in the presentation layer. + expect(reworded.presentation.properties['todo.title'].description).toBe( + 'A different title', + ); + }); + + it('changes the property id when its datatype changes', ({ expect }) => { + const base = freezeSchema(todoSchema).properties['todo.title']; + const changed = freezeSchema({ + ...todoSchema, + classes: { + todo: { + ...todoSchema.classes.todo, + properties: { + ...todoSchema.classes.todo.properties, + title: { type: 'integer', description: 'Task title' }, + }, + }, + }, + }); + + expect(changed.properties['todo.title']).not.toBe(base); + }); + + it('keeps a class id stable when a property description changes', ({ + expect, + }) => { + const base = freezeSchema(todoSchema).classes.todo; + const reworded = freezeSchema({ + ...todoSchema, + classes: { + todo: { + ...todoSchema.classes.todo, + properties: { + ...todoSchema.classes.todo.properties, + title: { type: 'string', description: 'Reworded' }, + }, + }, + }, + }); + + // No cascade: a cosmetic edit deep in the tree moves nothing. + expect(reworded.classes.todo).toBe(base); + expect(reworded.ontology).toBe(freezeSchema(todoSchema).ontology); + }); + + it('allows the same shortname across classes when definitions are identical', ({ + expect, + }) => { + const frozen = freezeSchema({ + name: 'Notes', + classes: { + note: { + type: 'object', + properties: { title: { type: 'string', description: 'Title' } }, + }, + memo: { + type: 'object', + properties: { title: { type: 'string', description: 'Title' } }, + }, + }, + }); + + // Identical "title" definitions dedupe to a single frozen id. + expect(frozen.properties['note.title']).toBe(frozen.properties['memo.title']); + }); + + it('rejects different definitions that share a shortname', ({ expect }) => { + expect(() => + freezeSchema({ + name: 'Notes', + classes: { + note: { + type: 'object', + properties: { title: { type: 'string', description: 'A' } }, + }, + memo: { + type: 'object', + properties: { title: { type: 'integer', description: 'B' } }, + }, + }, + }), + ).toThrow('shortname "title"'); + }); + + it('throws on imported property references it cannot resolve', ({ + expect, + }) => { + expect(() => + freezeSchema({ + name: 'ProjectApp', + imports: { + todo: { subject: 'did:ad:todoOntology' }, + }, + classes: { + project: { + type: 'object', + required: ['title'], + properties: { + title: { $ref: 'todo.properties.title' }, + }, + }, + }, + }), + ).toThrow('Imported properties require store-backed registration'); + }); +}); + +describe('Store.registerSchema', () => { + it('creates local DID ontology, class, and property resources that resolve', async ({ + expect, + }) => { + const keys = await Agent.generateKeyPair(); + const store = new Store({ serverUrl: 'https://example.com' }); + store.setAgent( + new Agent( + new JSCryptoProvider(keys.privateKey), + `did:ad:agent:${keys.publicKey}`, + ), + ); + + const registered = await store.registerSchema(defineSchema(todoSchema)); + const titleProperty = registered.properties['todo.title']; + const todoClass = registered.classes.todo; + + expect(registered.ontology.subject).toMatch(/^did:ad:/); + expect(titleProperty.subject).toMatch(/^did:ad:/); + expect(todoClass.subject).toMatch(/^did:ad:/); + expect(registered.ontology.props.classes).toEqual([todoClass.subject]); + expect(registered.ontology.props.properties).toEqual( + expect.arrayContaining([ + titleProperty.subject, + registered.properties['todo.done'].subject, + ]), + ); + expect(todoClass.props.requires).toEqual([titleProperty.subject]); + + const resolvedProperty = await store.getProperty(titleProperty.subject); + + expect(resolvedProperty.shortname).toBe('title'); + expect(resolvedProperty.datatype).toBe(Datatype.STRING); + expect( + store.getRegisteredSchemaSubject(registered.model.ontology.schemaHash), + ).toBe(registered.ontology.subject); + expect(registered.ontology.hasClasses(core.classes.ontology)).toBe(true); + }); + + it('can save registered schemas through the normal commit path', async ({ + expect, + }) => { + const keys = await Agent.generateKeyPair(); + const store = new Store({ serverUrl: 'https://example.com' }); + store.setServerConnected(true); + store.setAgent( + new Agent( + new JSCryptoProvider(keys.privateKey), + `did:ad:agent:${keys.publicKey}`, + ), + ); + const posted: Commit[] = []; + const postCommit = vi.fn(async (commit: Commit) => { + const created = { + ...commit, + id: `https://example.com/commits/${commit.signature}`, + } as Commit; + posted.push(created); + + return created; + }); + ( + store as unknown as { client: { postCommit: typeof postCommit } } + ).client.postCommit = postCommit; + + const registered = await store.registerSchema(defineSchema(todoSchema), { + save: true, + }); + + expect(postCommit).toHaveBeenCalled(); + expect(posted.map(commit => commit.subject)).toEqual( + expect.arrayContaining([ + registered.ontology.subject, + registered.classes.todo.subject, + registered.properties['todo.title'].subject, + registered.properties['todo.done'].subject, + ]), + ); + }); + + it('reloads saved schema resources from the local DB in a fresh Store', async ({ + expect, + }) => { + const keys = await Agent.generateKeyPair(); + const agent = new Agent( + new JSCryptoProvider(keys.privateKey), + `did:ad:agent:${keys.publicKey}`, + ); + const producer = new Store({ serverUrl: 'https://example.com', agent }); + producer.setServerConnected(false); + const dbState = attachMockClientDb(producer); + + const registered = await producer.registerSchema(defineSchema(todoSchema), { + save: true, + }); + const ontologySubject = registered.ontology.subject; + const classSubject = registered.classes.todo.subject; + const titlePropertySubject = registered.properties['todo.title'].subject; + + expect(dbState.has(ontologySubject)).toBe(true); + expect(dbState.has(classSubject)).toBe(true); + expect(dbState.has(titlePropertySubject)).toBe(true); + + const consumer = new Store({ serverUrl: 'https://example.com', agent }); + consumer.setServerConnected(false); + attachMockClientDb(consumer, dbState); + + const ontology = await consumer.getResource(ontologySubject); + expect(ontology.get(SCHEMA_HASH_PROPERTY)).toBe( + registered.model.ontology.schemaHash, + ); + expect(ontology.props.classes).toEqual([classSubject]); + expect(ontology.props.properties).toEqual( + expect.arrayContaining([titlePropertySubject]), + ); + + const todoClass = await consumer.getResource(classSubject); + expect(todoClass.props.requires).toEqual([titlePropertySubject]); + + const titleProperty = await consumer.getProperty(titlePropertySubject); + expect(titleProperty.shortname).toBe('title'); + expect(titleProperty.datatype).toBe(Datatype.STRING); + }); + + it('checks expected hashes for imported ontologies', async ({ expect }) => { + const keys = await Agent.generateKeyPair(); + const store = new Store({ serverUrl: 'https://example.com' }); + store.setAgent( + new Agent( + new JSCryptoProvider(keys.privateKey), + `did:ad:agent:${keys.publicKey}`, + ), + ); + + const todo = await store.registerSchema(defineSchema(todoSchema)); + const projectSchema = defineSchema({ + name: 'ProjectApp', + imports: { + todo: { + subject: todo.ontology.subject, + expectedHash: todo.model.ontology.schemaHash, + }, + }, + classes: { + project: { + type: 'object', + properties: { + title: { type: 'string' }, + }, + }, + }, + }); + + await expect(store.registerSchema(projectSchema)).resolves.toMatchObject({ + ontology: expect.objectContaining({ subject: expect.any(String) }), + }); + + const mismatched = defineSchema({ + ...projectSchema.schema, + imports: { + todo: { + subject: todo.ontology.subject, + expectedHash: + 'blake3:0000000000000000000000000000000000000000000000000000000000000000', + }, + }, + }); + + await expect(store.registerSchema(mismatched)).rejects.toThrow( + 'Schema import todo expected', + ); + }); + + it('can create an instance using returned schema subjects', async ({ + expect, + }) => { + const keys = await Agent.generateKeyPair(); + const store = new Store({ serverUrl: 'https://example.com' }); + store.setAgent( + new Agent( + new JSCryptoProvider(keys.privateKey), + `did:ad:agent:${keys.publicKey}`, + ), + ); + const registered = await store.registerSchema(defineSchema(todoSchema)); + const resource = await store.newResource({ + isA: registered.classes.todo.subject, + propVals: { + [registered.properties['todo.title'].subject]: 'Buy milk', + [registered.properties['todo.done'].subject]: false, + }, + }); + + expect(resource.getClasses()).toContain(registered.classes.todo.subject); + expect(resource.get(registered.properties['todo.title'].subject)).toBe( + 'Buy milk', + ); + expect(resource.get(registered.properties['todo.done'].subject)).toBe( + false, + ); + }); + + it('rejects datatype changes for an explicitly reused Property subject', async ({ + expect, + }) => { + const keys = await Agent.generateKeyPair(); + const store = new Store({ serverUrl: 'https://example.com' }); + store.setAgent( + new Agent( + new JSCryptoProvider(keys.privateKey), + `did:ad:agent:${keys.publicKey}`, + ), + ); + const propertySubject = 'did:ad:sharedTitleProperty'; + + await store.registerSchema( + defineSchema({ + name: 'TodoApp', + classes: { + todo: { + type: 'object', + properties: { + title: { + type: 'string', + 'atomic:subject': propertySubject, + }, + }, + }, + }, + }), + ); + + await expect( + store.registerSchema( + defineSchema({ + name: 'ChangedTodoApp', + classes: { + todo: { + type: 'object', + properties: { + title: { + type: 'integer', + 'atomic:subject': propertySubject, + }, + }, + }, + }, + }), + ), + ).rejects.toThrow('already registered with a different'); + }); + + it('can reuse one imported Property in another Class', async ({ expect }) => { + const keys = await Agent.generateKeyPair(); + const store = new Store({ serverUrl: 'https://example.com' }); + store.setAgent( + new Agent( + new JSCryptoProvider(keys.privateKey), + `did:ad:agent:${keys.publicKey}`, + ), + ); + const todo = await store.registerSchema(defineSchema(todoSchema)); + const project = await store.registerSchema( + defineSchema({ + name: 'ProjectApp', + imports: { + todo: { + subject: todo.ontology.subject, + expectedHash: todo.model.ontology.schemaHash, + }, + }, + classes: { + project: { + type: 'object', + required: ['title'], + properties: { + title: { $ref: 'todo.properties.title' }, + }, + }, + }, + }), + ); + + expect(project.properties['project.title']).toBeUndefined(); + expect(project.classes.project.props.requires).toEqual([ + todo.properties['todo.title'].subject, + ]); + }); +}); diff --git a/browser/lib/src/schema.ts b/browser/lib/src/schema.ts new file mode 100644 index 000000000..8f8e92bf9 --- /dev/null +++ b/browser/lib/src/schema.ts @@ -0,0 +1,608 @@ +import { blake3 } from '@noble/hashes/blake3.js'; +import { bytesToHex, utf8ToBytes } from '@noble/hashes/utils.js'; + +import { Datatype } from './datatypes.js'; +import { + freezeResources, + type FreezableResource, + type FrozenId, + type JsonValue as FreezeJsonValue, +} from './freeze.js'; +import { jcsCanonicalize } from './jcs.js'; +import { core } from './ontologies/core.js'; +import { server } from './ontologies/server.js'; + +type JsonPrimitive = string | number | boolean | null; + +export type JsonValue = + | JsonPrimitive + | JsonValue[] + | { [key: string]: JsonValue }; + +// Re-export the content-addressing primitive (but not its `JsonValue`, which +// would collide with the one above). +export { + freezeResources, + type FrozenId, + type FreezableResource, + type FrozenResource, + type FreezeResult, +} from './freeze.js'; + +export type SchemaHash = `blake3:${string}`; + +export const SCHEMA_HASH_PROPERTY = + 'https://atomicdata.dev/properties/schemaHash'; + +export interface SchemaImportReference { + /** Ontology subject, usually `did:ad:{genesis}` or an HTTP URL. */ + subject: string; + /** Expected Ontology package hash. Generation/registration must fail on mismatch. */ + expectedHash?: SchemaHash; +} + +export interface AtomicSchemaPropertyDefinition { + type?: 'string' | 'integer' | 'number' | 'boolean' | 'object' | 'array'; + format?: string; + description?: string; + default?: JsonValue; + enum?: JsonValue[]; + items?: AtomicSchemaPropertyDefinition | SchemaRef; + properties?: Record; + required?: string[]; + additionalProperties?: boolean | AtomicSchemaPropertyDefinition | SchemaRef; + $ref?: string; + 'atomic:subject'?: string; + 'atomic:shortname'?: string; + 'atomic:datatype'?: Datatype | string; + 'atomic:classType'?: string; + 'atomic:recommends'?: string[]; + 'atomic:allowsOnly'?: JsonValue[]; + 'atomic:isDynamic'?: boolean; + 'atomic:isLocked'?: boolean; +} + +export interface SchemaRef { + $ref: string; +} + +export interface AtomicSchemaClassDefinition { + title?: string; + description?: string; + type: 'object'; + required?: string[]; + properties: Record; + $defs?: Record; + 'atomic:subject'?: string; + 'atomic:shortname'?: string; + 'atomic:recommends'?: string[]; +} + +export interface AtomicSchemaPackage { + name: string; + version?: string; + description?: string; + imports?: Record; + classes: Record; + $defs?: Record; +} + +export interface DefinedSchema< + Schema extends AtomicSchemaPackage = AtomicSchemaPackage, +> { + readonly schema: Schema; + readonly normalized: AtomicSchemaPackage; + readonly schemaHash: SchemaHash; +} + +export interface ConvertedSchemaProperty { + readonly key: string; + readonly classKey: string; + readonly propertyKey: string; + readonly subject?: string; + readonly shortname: string; + readonly description: string; + readonly datatype: Datatype | string; + readonly classType?: string; + readonly allowsOnly?: JsonValue[]; + readonly isDynamic?: boolean; + readonly isLocked?: boolean; +} + +export interface ConvertedSchemaClass { + readonly key: string; + readonly subject?: string; + readonly shortname: string; + readonly description: string; + readonly requires: string[]; + readonly recommends: string[]; +} + +export interface ConvertedSchemaOntology { + readonly shortname: string; + readonly description: string; + readonly version?: string; + readonly schemaHash: SchemaHash; + readonly jsonSchema: AtomicSchemaPackage; +} + +export interface ConvertedSchemaPackage { + readonly ontology: ConvertedSchemaOntology; + readonly classes: ConvertedSchemaClass[]; + readonly properties: ConvertedSchemaProperty[]; +} + +export function defineSchema( + schema: Schema, +): DefinedSchema { + const normalized = normalizeSchemaPackage(schema); + + return Object.freeze({ + schema, + normalized, + schemaHash: hashSchemaPackage(normalized), + }); +} + +export function schemaToOntologyModel( + input: AtomicSchemaPackage | DefinedSchema, +): ConvertedSchemaPackage { + const defined = isDefinedSchema(input) ? input : defineSchema(input); + const schema = defined.normalized; + const properties: ConvertedSchemaProperty[] = []; + const classes = Object.entries(schema.classes).map(([classKey, klass]) => { + const required = new Set(klass.required ?? []); + const classPropertyKeys: Array<{ modelKey: string; propertyKey: string }> = + []; + + for (const [propertyKey, propertyDefinition] of Object.entries( + klass.properties, + )) { + if ( + isSchemaRef(propertyDefinition) && + isImportedPropertyRef(propertyDefinition.$ref) + ) { + classPropertyKeys.push({ + modelKey: `ref:${propertyDefinition.$ref}`, + propertyKey, + }); + continue; + } + + const convertedProperty = convertPropertyDefinition( + classKey, + propertyKey, + propertyDefinition, + ); + + properties.push(convertedProperty); + classPropertyKeys.push({ + modelKey: convertedProperty.key, + propertyKey, + }); + } + + const requiredKeys = classPropertyKeys.filter(key => + required.has(key.propertyKey), + ); + const recommendedKeys = classPropertyKeys.filter( + key => !required.has(key.propertyKey), + ); + + return { + key: classKey, + subject: klass['atomic:subject'], + shortname: klass['atomic:shortname'] ?? classKey, + description: klass.description ?? klass.title ?? classKey, + requires: requiredKeys.map(key => key.modelKey), + recommends: recommendedKeys.map(key => key.modelKey), + } satisfies ConvertedSchemaClass; + }); + + return { + ontology: { + shortname: schema.name, + description: schema.description ?? schema.name, + version: schema.version, + schemaHash: defined.schemaHash, + jsonSchema: schema, + }, + classes, + properties, + }; +} + +export interface FrozenSchemaResource { + readonly frozenId: FrozenId; + /** Identity-only JSON-AD body with internal references resolved to FrozenIds. */ + readonly content: JsonValue; +} + +export interface FrozenSchemaMetadata { + readonly frozenId: FrozenId; + /** Human description — presentation, not part of the frozen identity. */ + readonly description: string; +} + +/** + * Mutable, human-facing metadata that is deliberately NOT hashed into frozen + * ids, so editing it (typo fixes, rewording, translations) never churns an id. + * Keyed by the developer-facing model key so each usage keeps its own text even + * when identical definitions dedupe to one frozen id. + */ +export interface FrozenSchemaPresentation { + readonly ontology: { + readonly description: string; + readonly version?: string; + readonly schemaHash: SchemaHash; + readonly jsonSchema: AtomicSchemaPackage; + }; + readonly classes: Record; + readonly properties: Record; +} + +export interface FrozenSchema { + readonly ontology: FrozenId; + readonly classes: Record; + readonly properties: Record; + readonly resources: FrozenSchemaResource[]; + readonly presentation: FrozenSchemaPresentation; + readonly model: ConvertedSchemaPackage; +} + +const FREEZE_NS = 'urn:atomic-freeze:'; +const ONTOLOGY_LOCAL_ID = `${FREEZE_NS}ontology`; +const classLocalId = (key: string): string => `${FREEZE_NS}class:${key}`; +const propLocalId = (key: string): string => `${FREEZE_NS}prop:${key}`; + +/** + * Converts a schema package into content-addressed `did:ad:frozen` resources: + * one materialized Ontology, Class, and Property JSON-AD body each, with every + * cross-reference resolved to the referent's frozen hash. The whole body is + * hashed (descriptions included), so the same definition always yields the same + * id and any edit yields a new one. + * + * Self-contained schemas only: an imported property (`$ref: + * "alias.properties.x"`) cannot be resolved to a frozen id without the + * referenced Ontology, so callers needing imports must go through the + * store-backed registration path. + */ +export function freezeSchema( + input: AtomicSchemaPackage | DefinedSchema, +): FrozenSchema { + const model = schemaToOntologyModel(input); + const propertyKeys = new Set(model.properties.map(property => property.key)); + + const resolveModelKey = (key: string): string => { + if (propertyKeys.has(key)) { + return propLocalId(key); + } + + throw new Error( + `freezeSchema cannot resolve property reference "${key}". Imported properties require store-backed registration.`, + ); + }; + + const freezable: FreezableResource[] = []; + + // Frozen bodies hold IDENTITY only — the machine contract that decides how + // data is validated/interpreted. Presentation (descriptions, labels, + // translations) is excluded so cosmetic edits never churn a frozen id; it + // rides in the mutable package layer (`presentation`, below). + for (const property of model.properties) { + freezable.push({ + localId: propLocalId(property.key), + content: compactContent({ + [core.properties.isA]: [core.classes.property], + [core.properties.shortname]: property.shortname, + [core.properties.datatype]: property.datatype, + [core.properties.classtype]: property.classType, + [core.properties.allowsOnly]: property.allowsOnly as + | FreezeJsonValue + | undefined, + [core.properties.isDynamic]: property.isDynamic, + [core.properties.isLocked]: property.isLocked, + }), + }); + } + + for (const klass of model.classes) { + freezable.push({ + localId: classLocalId(klass.key), + content: compactContent({ + [core.properties.isA]: [core.classes.class], + [core.properties.shortname]: klass.shortname, + [core.properties.requires]: klass.requires.map(resolveModelKey), + [core.properties.recommends]: klass.recommends.map(resolveModelKey), + }), + }); + } + + freezable.push({ + localId: ONTOLOGY_LOCAL_ID, + content: compactContent({ + [core.properties.isA]: [core.classes.ontology], + [core.properties.shortname]: model.ontology.shortname, + [core.properties.classes]: model.classes.map(klass => + classLocalId(klass.key), + ), + [core.properties.properties]: model.properties.map(property => + propLocalId(property.key), + ), + [server.properties.version]: model.ontology.version, + }), + }); + + const { resources, byLocalId } = freezeResources(freezable); + + const requireId = (localId: string): FrozenId => { + const frozenId = byLocalId.get(localId); + + if (!frozenId) { + throw new Error(`freezeSchema did not produce an id for ${localId}`); + } + + return frozenId; + }; + + const classes: Record = {}; + const properties: Record = {}; + + for (const klass of model.classes) { + classes[klass.key] = requireId(classLocalId(klass.key)); + } + + for (const property of model.properties) { + properties[property.key] = requireId(propLocalId(property.key)); + } + + // A shortname must map to a single definition per ontology, so the lockfile + // `@index` and generated bindings can key on it. Content-addressing makes this + // forgiving: identical definitions across classes dedupe to one frozen id and + // pass; only genuinely different definitions sharing a shortname are rejected. + assertUniqueShortnames( + model.ontology.shortname, + 'property', + model.properties.map(p => [p.shortname, properties[p.key]] as const), + ); + assertUniqueShortnames( + model.ontology.shortname, + 'class', + model.classes.map(k => [k.shortname, classes[k.key]] as const), + ); + + const presentation: FrozenSchemaPresentation = { + ontology: { + description: model.ontology.description, + version: model.ontology.version, + schemaHash: model.ontology.schemaHash, + jsonSchema: model.ontology.jsonSchema, + }, + classes: Object.fromEntries( + model.classes.map(klass => [ + klass.key, + { frozenId: classes[klass.key], description: klass.description }, + ]), + ), + properties: Object.fromEntries( + model.properties.map(property => [ + property.key, + { frozenId: properties[property.key], description: property.description }, + ]), + ), + }; + + return { + ontology: requireId(ONTOLOGY_LOCAL_ID), + classes, + properties, + resources: resources.map(resource => ({ + frozenId: resource.frozenId, + content: resource.content as JsonValue, + })), + presentation, + model, + }; +} + +function assertUniqueShortnames( + ontologyShortname: string, + kind: 'class' | 'property', + entries: ReadonlyArray, +): void { + const idsByShortname = new Map>(); + + for (const [shortname, frozenId] of entries) { + const ids = idsByShortname.get(shortname) ?? new Set(); + ids.add(frozenId); + idsByShortname.set(shortname, ids); + } + + for (const [shortname, ids] of idsByShortname) { + if (ids.size > 1) { + throw new Error( + `Schema "${ontologyShortname}" defines ${ids.size} different ${kind} resources with shortname "${shortname}". A shortname must map to one ${kind} per ontology — make the definitions identical to share, or rename.`, + ); + } + } +} + +function compactContent( + entries: Record, +): FreezeJsonValue { + return Object.fromEntries( + Object.entries(entries).filter(([, value]) => value !== undefined), + ) as FreezeJsonValue; +} + +export function hashSchemaPackage(schema: AtomicSchemaPackage): SchemaHash { + const canonical = canonicalizeSchemaPackage(schema); + const hash = blake3(utf8ToBytes(canonical)); + + return `blake3:${bytesToHex(hash)}`; +} + +export function canonicalizeSchemaPackage(schema: AtomicSchemaPackage): string { + return jcsCanonicalize( + normalizeSchemaPackage(schema) as unknown as FreezeJsonValue, + ); +} + +export function normalizeSchemaPackage( + schema: Schema, +): Schema { + return normalizeValue(schema) as unknown as Schema; +} + +function normalizeValue(value: unknown): JsonValue | undefined { + if (value === undefined) { + return undefined; + } + + if (value === null) { + return null; + } + + if (typeof value === 'string' || typeof value === 'boolean') { + return value; + } + + if (typeof value === 'number') { + if (!Number.isFinite(value)) { + throw new Error(`Schema contains a non-finite number: ${value}`); + } + + return value; + } + + if (Array.isArray(value)) { + return value.map(item => { + const normalized = normalizeValue(item); + + return normalized === undefined ? null : normalized; + }); + } + + if (typeof value === 'object') { + const normalizedEntries = Object.entries( + value as Record, + ).flatMap(([key, child]) => { + const normalized = normalizeValue(child); + + return normalized === undefined ? [] : [[key, normalized] as const]; + }); + + return Object.fromEntries( + normalizedEntries.sort(([a], [b]) => a.localeCompare(b)), + ); + } + + throw new Error(`Schema contains an unsupported value: ${String(value)}`); +} + +function convertPropertyDefinition( + classKey: string, + propertyKey: string, + propertyDefinition: AtomicSchemaPropertyDefinition | SchemaRef, +): ConvertedSchemaProperty { + return { + key: `${classKey}.${propertyKey}`, + classKey, + propertyKey, + subject: + 'atomic:subject' in propertyDefinition + ? propertyDefinition['atomic:subject'] + : undefined, + shortname: + ('atomic:shortname' in propertyDefinition + ? propertyDefinition['atomic:shortname'] + : undefined) ?? propertyKey, + description: + 'description' in propertyDefinition + ? (propertyDefinition.description ?? propertyKey) + : propertyKey, + datatype: + ('atomic:datatype' in propertyDefinition + ? propertyDefinition['atomic:datatype'] + : undefined) ?? datatypeFromJsonSchema(propertyDefinition), + classType: + 'atomic:classType' in propertyDefinition + ? propertyDefinition['atomic:classType'] + : undefined, + allowsOnly: + ('atomic:allowsOnly' in propertyDefinition + ? propertyDefinition['atomic:allowsOnly'] + : undefined) ?? + ('enum' in propertyDefinition ? propertyDefinition.enum : undefined), + isDynamic: + 'atomic:isDynamic' in propertyDefinition + ? propertyDefinition['atomic:isDynamic'] + : undefined, + isLocked: + 'atomic:isLocked' in propertyDefinition + ? propertyDefinition['atomic:isLocked'] + : undefined, + }; +} + +function datatypeFromJsonSchema( + propertyDefinition: AtomicSchemaPropertyDefinition | SchemaRef, +): Datatype { + if (isSchemaRef(propertyDefinition)) { + return Datatype.ATOMIC_URL; + } + + switch (propertyDefinition.type) { + case 'string': + if (propertyDefinition.format === 'date') { + return Datatype.DATE; + } + + if (propertyDefinition.format === 'uri') { + return Datatype.URI; + } + + return Datatype.STRING; + case 'integer': + return Datatype.INTEGER; + case 'number': + return Datatype.FLOAT; + case 'boolean': + return Datatype.BOOLEAN; + case 'array': + return isSchemaRef(propertyDefinition.items) + ? Datatype.RESOURCEARRAY + : Datatype.JSON; + case 'object': + return Datatype.JSON; + default: + if (propertyDefinition.$ref) { + return Datatype.ATOMIC_URL; + } + + return Datatype.JSON; + } +} + +function isDefinedSchema(value: unknown): value is DefinedSchema { + return ( + typeof value === 'object' && + value !== null && + 'schema' in value && + 'normalized' in value && + 'schemaHash' in value + ); +} + +function isSchemaRef(value: unknown): value is SchemaRef { + return ( + typeof value === 'object' && + value !== null && + '$ref' in value && + typeof (value as SchemaRef).$ref === 'string' + ); +} + +function isImportedPropertyRef(ref: string): boolean { + return /^[^.]+\.properties\.[^.]+$/.test(ref); +} diff --git a/browser/lib/src/store.ts b/browser/lib/src/store.ts index 6404b4df1..6217f4551 100644 --- a/browser/lib/src/store.ts +++ b/browser/lib/src/store.ts @@ -16,12 +16,19 @@ import { EventManager } from './EventManager.js'; import { hasBrowserAPI } from './hasBrowserAPI.js'; import { collections } from './ontologies/collections.js'; import { commits } from './ontologies/commits.js'; -import { core } from './ontologies/core.js'; +import { core, type Core } from './ontologies/core.js'; import { server, type Server } from './ontologies/server.js'; import type { OptionalClass, UnknownClass } from './ontology.js'; import { JSONADParser } from './parse.js'; import { Resource, unknownSubject } from './resource.js'; import { type SearchOpts, buildSearchSubject } from './search.js'; +import { + SCHEMA_HASH_PROPERTY, + schemaToOntologyModel, + type AtomicSchemaPackage, + type ConvertedSchemaPackage, + type DefinedSchema, +} from './schema.js'; import { stringToSlug } from './stringToSlug.js'; import { bytesToHex, hexToBytes, type JSONValue } from './value.js'; import { WSClient } from './websockets.js'; @@ -72,10 +79,24 @@ type CreateResourceOptions = { isA?: string | string[]; /** Any additional properties the resource should have */ propVals?: Record; + /** Validate `propVals` while creating the resource. Defaults to true. */ + validatePropVals?: boolean; /** Set to true if the resource should have a DID as subject. Defaults to `true` for `did:ad` agents, otherwise `false`. */ did?: boolean; }; +export interface RegisteredSchema { + ontology: Resource; + classes: Record>; + properties: Record>; + model: ConvertedSchemaPackage; +} + +export interface RegisterSchemaOptions { + /** Save generated resources through the normal Commit/outbox path. */ + save?: boolean; +} + export interface StoreOpts { /** The default store URL, where to send commits and where to create new instances */ serverUrl?: string; @@ -268,7 +289,6 @@ function yieldToEventLoop(): Promise { }); } - /** * Cheap equality for commit-log property values. Strict `===` would always * report arrays/objects as different even when their contents match, so the @@ -318,6 +338,7 @@ export class Store { private _resources: Map; /** Mapping from HTTP aliases to primary subjects (e.g. DIDs) */ private aliases: Map = new Map(); + private schemaHashIndex: Map = new Map(); /** List of resources that have parents that are not saved to the server, when a parent is saved it should also save its children */ private batchedResources: Map> = new Map(); @@ -1570,6 +1591,7 @@ export class Store { parent, isA, propVals, + validatePropVals = true, noParent, did, }: CreateResourceOptions = {}): Promise> { @@ -1603,7 +1625,7 @@ export class Store { if (propVals) { for (const [key, value] of Object.entries(propVals)) { - await resource.set(key, value); + await resource.set(key, value, validatePropVals); } } @@ -1636,6 +1658,257 @@ export class Store { return resource; } + public async registerSchema( + schema: AtomicSchemaPackage | DefinedSchema, + opts: RegisterSchemaOptions = {}, + ): Promise { + const model = schemaToOntologyModel(schema); + + await this.validateSchemaImports(model.ontology.jsonSchema); + + const ontology = await this.newResource({ + isA: core.classes.ontology, + validatePropVals: false, + propVals: { + [core.properties.shortname]: stringToSlug(model.ontology.shortname), + [core.properties.description]: model.ontology.description, + [core.properties.classes]: [], + [core.properties.properties]: [], + [core.properties.instances]: [], + }, + }); + + const properties: Record> = {}; + + for (const property of model.properties) { + this.assertCompatibleExistingSchemaProperty(property); + + const resource = await this.newResource({ + subject: property.subject, + parent: ontology.subject, + isA: core.classes.property, + validatePropVals: false, + propVals: { + [core.properties.shortname]: property.shortname, + [core.properties.description]: property.description, + [core.properties.datatype]: property.datatype, + ...(property.classType + ? { [core.properties.classtype]: property.classType } + : {}), + ...(property.allowsOnly + ? { [core.properties.allowsOnly]: property.allowsOnly as string[] } + : {}), + ...(property.isDynamic !== undefined + ? { [core.properties.isDynamic]: property.isDynamic } + : {}), + ...(property.isLocked !== undefined + ? { [core.properties.isLocked]: property.isLocked } + : {}), + }, + }); + + properties[property.key] = resource; + } + + const classes: Record> = {}; + + for (const klass of model.classes) { + const requires = await Promise.all( + klass.requires.map(key => + this.resolveRegisteredSchemaPropertyKey( + key, + model.ontology.jsonSchema, + properties, + ), + ), + ); + const recommends = await Promise.all( + klass.recommends.map(key => + this.resolveRegisteredSchemaPropertyKey( + key, + model.ontology.jsonSchema, + properties, + ), + ), + ); + const resource = await this.newResource({ + subject: klass.subject, + parent: ontology.subject, + isA: core.classes.class, + validatePropVals: false, + propVals: { + [core.properties.shortname]: klass.shortname, + [core.properties.description]: klass.description, + [core.properties.requires]: requires, + [core.properties.recommends]: recommends, + }, + }); + + classes[klass.key] = resource; + } + + await ontology.set( + core.properties.classes, + Object.values(classes).map(resource => resource.subject), + false, + ); + await ontology.set( + core.properties.properties, + Object.values(properties).map(resource => resource.subject), + false, + ); + await ontology.set( + server.properties.jsonSchema, + model.ontology.jsonSchema as unknown as JSONValue, + false, + ); + await ontology.set(SCHEMA_HASH_PROPERTY, model.ontology.schemaHash, false); + + if (model.ontology.version) { + await ontology.set( + server.properties.version, + model.ontology.version, + false, + ); + } + + this.schemaHashIndex.set(model.ontology.schemaHash, ontology.subject); + + if (opts.save) { + await ontology.save(); + + for (const property of Object.values(properties)) { + await property.save(); + } + + for (const klass of Object.values(classes)) { + await klass.save(); + } + } + + return { + ontology, + classes, + properties, + model, + }; + } + + public getRegisteredSchemaSubject(schemaHash: string): string | undefined { + return this.schemaHashIndex.get(schemaHash); + } + + private assertCompatibleExistingSchemaProperty( + property: ConvertedSchemaPackage['properties'][number], + ): void { + if (!property.subject) { + return; + } + + const existing = this.resources.get(property.subject); + + if (!existing?.isReady() || !existing.hasClasses(core.classes.property)) { + return; + } + + const checks: Array< + [string, JSONValue | undefined, JSONValue | undefined] + > = [ + [ + core.properties.datatype, + existing.get(core.properties.datatype), + property.datatype, + ], + [ + core.properties.classtype, + existing.get(core.properties.classtype), + property.classType, + ], + [ + core.properties.allowsOnly, + existing.get(core.properties.allowsOnly), + property.allowsOnly as JSONValue | undefined, + ], + ]; + + for (const [field, actual, expected] of checks) { + if (JSON.stringify(actual ?? null) !== JSON.stringify(expected ?? null)) { + throw new Error( + `Schema property ${property.subject} is already registered with a different ${field}. Publish a new Property subject for datatype or semantic changes.`, + ); + } + } + } + + private async validateSchemaImports( + schema: AtomicSchemaPackage, + ): Promise { + for (const [name, schemaImport] of Object.entries(schema.imports ?? {})) { + if (!schemaImport.expectedHash) { + continue; + } + + const resource = await this.getResource(schemaImport.subject); + const actualHash = resource.get(SCHEMA_HASH_PROPERTY)?.toString(); + + if (actualHash !== schemaImport.expectedHash) { + throw new Error( + `Schema import ${name} expected ${schemaImport.expectedHash} but resolved ${actualHash ?? 'no schema hash'} from ${schemaImport.subject}`, + ); + } + } + } + + private async resolveRegisteredSchemaPropertyKey( + key: string, + schema: AtomicSchemaPackage, + generatedProperties: Record>, + ): Promise { + const generated = generatedProperties[key]; + + if (generated) { + return generated.subject; + } + + if (!key.startsWith('ref:')) { + throw new Error(`Could not resolve generated schema property key ${key}`); + } + + const ref = key.slice('ref:'.length); + const [importName, collectionName, propertyShortname] = ref.split('.'); + + if (collectionName !== 'properties' || !propertyShortname) { + throw new Error(`Unsupported schema property reference ${ref}`); + } + + const schemaImport = schema.imports?.[importName]; + + if (!schemaImport) { + throw new Error( + `Schema property reference ${ref} uses unknown import ${importName}`, + ); + } + + const importedOntology = await this.getResource( + schemaImport.subject, + ); + const propertySubjects = importedOntology.getArray( + core.properties.properties, + ) as string[]; + + for (const subject of propertySubjects) { + const property = await this.getResource(subject); + + if (property.get(core.properties.shortname) === propertyShortname) { + return property.subject; + } + } + + throw new Error( + `Schema property reference ${ref} did not match any Property in ${schemaImport.subject}`, + ); + } + /** * Creates a new personal Drive for the current Agent, saves it, and links * it to the Agent resource. Returns the Drive's Resource (already saved). diff --git a/browser/lib/tests/schema-code-first.integration.test.ts b/browser/lib/tests/schema-code-first.integration.test.ts new file mode 100644 index 000000000..4a7d33ffb --- /dev/null +++ b/browser/lib/tests/schema-code-first.integration.test.ts @@ -0,0 +1,89 @@ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { setTimeout as delay } from 'node:timers/promises'; + +import { Agent } from '../src/agent.js'; +import { Store } from '../src/store.js'; +import { SCHEMA_HASH_PROPERTY } from '../src/schema.js'; +import { startServer, type ServerHandle } from './server-fixture.js'; +import { todoSchema } from './schema-code-first/producer-schema.js'; +import { defineProjectSchema } from './schema-code-first/consumer-schema.js'; + +async function waitForResource(store: Store, subject: string) { + const deadline = Date.now() + 15_000; + let lastError: unknown; + + while (Date.now() < deadline) { + try { + return await store.getResource(subject); + } catch (e) { + lastError = e; + await delay(250); + } + } + + throw lastError instanceof Error + ? lastError + : new Error(`Timed out fetching ${subject}`); +} + +describe('code-first schema publish and reuse', () => { + let server: ServerHandle; + + beforeAll(async () => { + server = await startServer(); + }, 60_000); + + afterAll(async () => { + await server?.stop(); + }); + + it('publishes a JS-defined schema, then another Store fetches and reuses one Property', async () => { + const agent = await Agent.fromSecret(server.agentSecret); + const producer = new Store({ serverUrl: server.serverUrl, agent }); + producer.setServerConnected(true); + + const published = await producer.registerSchema(todoSchema, { save: true }); + + const consumer = new Store({ serverUrl: server.serverUrl, agent }); + consumer.setServerConnected(true); + + const fetchedOntology = await waitForResource( + consumer, + published.ontology.subject, + ); + + expect(fetchedOntology.get(SCHEMA_HASH_PROPERTY)).toBe( + published.model.ontology.schemaHash, + ); + + const consumerSchema = defineProjectSchema( + published.ontology.subject, + published.model.ontology.schemaHash, + ); + const registeredConsumerSchema = + await consumer.registerSchema(consumerSchema); + + expect(registeredConsumerSchema.classes.project.props.requires).toEqual([ + published.properties['todo.title'].subject, + ]); + + const reusedProperty = await consumer.getProperty( + published.properties['todo.title'].subject, + ); + expect(reusedProperty.shortname).toBe('title'); + + const project = await consumer.newResource({ + isA: registeredConsumerSchema.classes.project.subject, + propVals: { + [published.properties['todo.title'].subject]: 'Shared title', + }, + }); + + expect(project.get(published.properties['todo.title'].subject)).toBe( + 'Shared title', + ); + + producer.disconnect(); + consumer.disconnect(); + }, 60_000); +}); diff --git a/browser/lib/tests/schema-code-first/consumer-schema.ts b/browser/lib/tests/schema-code-first/consumer-schema.ts new file mode 100644 index 000000000..76fd496d3 --- /dev/null +++ b/browser/lib/tests/schema-code-first/consumer-schema.ts @@ -0,0 +1,31 @@ +import { defineSchema, type SchemaHash } from '../../src/schema.js'; + +export function defineProjectSchema( + todoOntologySubject: string, + expectedHash: SchemaHash, +) { + return defineSchema({ + name: 'ProjectConsumer', + version: '1.0.0', + description: 'A consumer schema that reuses a producer property', + imports: { + todo: { + subject: todoOntologySubject, + expectedHash, + }, + }, + classes: { + project: { + title: 'Project', + description: 'A project that reuses todo.title', + type: 'object', + required: ['title'], + properties: { + title: { + $ref: 'todo.properties.title', + }, + }, + }, + }, + }); +} diff --git a/browser/lib/tests/schema-code-first/producer-schema.ts b/browser/lib/tests/schema-code-first/producer-schema.ts new file mode 100644 index 000000000..8fa8287a9 --- /dev/null +++ b/browser/lib/tests/schema-code-first/producer-schema.ts @@ -0,0 +1,26 @@ +import { defineSchema } from '../../src/schema.js'; + +export const todoSchema = defineSchema({ + name: 'TodoProject', + version: '1.0.0', + description: 'A tiny producer-owned todo schema', + classes: { + todo: { + title: 'Todo', + description: 'A task in a todo list', + type: 'object', + required: ['title'], + properties: { + title: { + type: 'string', + description: 'Task title', + }, + done: { + type: 'boolean', + description: 'Whether the task is complete', + default: false, + }, + }, + }, + }, +}); diff --git a/browser/lib/tsup.config.ts b/browser/lib/tsup.config.ts index 7ef0a56d8..86ee4934b 100644 --- a/browser/lib/tsup.config.ts +++ b/browser/lib/tsup.config.ts @@ -7,6 +7,7 @@ export default defineConfig(options => ({ minify: !options.watch, entry: { index: 'src/index.ts', + schema: 'src/schema.ts', // The DedicatedWorker that hosts the WASM ClientDb. Bundled as its own // entry so consumers can load it via `new Worker(new URL(..., // import.meta.url))`. Without this, the previous setup relied on a diff --git a/planning/did-ad-frozen-server.md b/planning/did-ad-frozen-server.md new file mode 100644 index 000000000..afc66d6f7 --- /dev/null +++ b/planning/did-ad-frozen-server.md @@ -0,0 +1,169 @@ +# Server-side `did:ad:frozen` plan + +Companion to [json-schema-code-first.md](./json-schema-code-first.md). That doc +decided the **identity model**: schema definitions are immutable, so they are +content-addressed as `did:ad:frozen:{blake3-hex}` instead of signed genesis DIDs. +The TypeScript producer side is built (`browser/lib/src/freeze.ts`, +`schema.ts#freezeSchema`). This doc plans the Rust/server and Store work needed +to **store, serve, resolve, and sync** frozen objects. + +## Core decision: frozen objects are blob-like, not resource-like + +A frozen object must NOT go through the Commit/Resource pipeline. The commit +validator hard-codes that a genesis resource's subject equals its signature +(`lib/src/commit.rs:318-341`); a signatureless content-addressed object cannot +satisfy that. Rather than bend the genesis path, model frozen objects on the +existing **blob** mechanism, which is already a commit-free, content-addressed +byte store: + +| Concern | Blob (today) | Frozen (new, parallel) | +| --- | --- | --- | +| Storage | `Tree::Blobs`, key = 32-byte BLAKE3, value = raw bytes (`lib/src/db/trees.rs:29`) | `Tree::Frozen`, key = 32-byte BLAKE3, value = canonical JSON-AD bytes | +| Write | `PUT /blob/{hash}`, verify-by-rehash (`server/src/handlers/blob.rs:14-41`) | `PUT /frozen/{hash}`, verify-by-rehash | +| Read | raw bytes (`/download/...`) | **materialized read-only Resource** (parse JSON-AD) | +| Identity parse | `subject.rs#is_blob_did/blob_hash_hex/from_blob_hash` | mirror as `is_frozen_did/frozen_hash_hex/from_frozen_hash` | +| Sync | `BLOB_REQUEST/RESPONSE` frames | `FROZEN_REQUEST/RESPONSE` (or reuse the blob "fetch-if-missing" path) | + +**The invariant that makes this simple:** every stored frozen object satisfies +`blake3(its canonical bytes) == its hash`. The server verifies on write and can +re-verify on read; the client always re-verifies. No host is ever trusted — a +frozen object is self-authenticating by content. Rust needs only BLAKE3 + a +JSON-AD parser; it never needs the freeze/SCC algorithm. + +The one difference from blobs: a frozen object **resolves to a parsed Resource**, +not opaque bytes, so `get_resource`, datatype/required validation, forms, and +`getProperty` work against frozen schema resources with no special-casing at +those call sites. + +## How resolution stays decentralized (and atomicdata.dev stays optional) + +Because every fetch is verify-by-rehash, the source is irrelevant to +correctness. A well-known default server (e.g. `atomicdata.dev`) is therefore a +**cache/CDN, never a dependency**. Resolution order for a `did:ad:frozen` subject: + +1. local `Tree::Frozen` / client ClientDb (OPFS) cache +2. app-bundled frozen objects (JSON-AD shipped with the app) +3. `?drive=` routing hint → pkarr (`discovery.rs`) → iroh peer → `FROZEN_REQUEST` + by hash (the working p2p path today) +4. a **configurable** default-server list (defaults to `atomicdata.dev`) → + `GET /frozen/{hash}` → verify-by-rehash. Removable; the spec never names it. +5. _(future)_ content-hash discovery: announce frozen hashes directly on a DHT so + holders can be found without a drive hint. The dormant `mainline` crate + (`server/Cargo.toml:65`, currently unused) or a pkarr record are the + candidates. Not needed for v1. + +"Not formally relying on atomicdata.dev" = the default-server entry is config, +documented as optional, and the protocol/spec never references it. If it is down +and the bytes exist in a bundle, on the author's drive, or on any peer, resolution +still succeeds and verifies. + +## Decided + implemented: cyclic members freeze as one unit + +This was the one place the producer and server models disagreed; it is now +resolved in `freeze.ts` so every stored frozen object satisfies +`blake3(canonical bytes) == hash`. + +- **Acyclic** resources (the common case: ontology → classes → properties) each + freeze to one self-verifying blob, `id = blake3(JCS(content))`. +- A **cycle** (e.g. `Person` class with a `friend` property whose classtype is + `Person`) freezes to a single **unit** object, + `{ "urn:atomic-freeze:unit": [ ...members ] }` (`UNIT_MEMBERS_KEY`), members in + a deterministic canonical order with intra-cycle references rewritten to + `did:ad:frozen:self:{index}` self tokens (`SELF_PREFIX`). The unit id is + `blake3(JCS(unit))` and all members share it; they resolve together. This keeps + the server invariant exact (one blob, verify-by-rehash) and respects "DIDs have + no subpaths" (`docs/src/did.md`). + +Consequence for the **server materializer**: when it parses a frozen object whose +top-level key is `urn:atomic-freeze:unit`, it must expand the array into multiple +read-only Resources and rewire each `did:ad:frozen:self:{index}` token to the +i-th member. The constants are exported from `browser/lib/src/freeze.ts` as the +frozen-format contract. (The Rust side still only ever verifies a hash; it does +not run the freeze algorithm.) + +Deferred: independent addressing/reuse of a single member *inside* a cycle (would +need fragment addressing like `did:ad:frozen:{unit}#{shortname}`, which the DID +spec currently disallows). Co-dependent members travel together, so this is rarely +needed. + +## Work breakdown + +### Phase A — lib + server storage, parsing, materialization + +- [ ] `lib/src/subject.rs`: add `is_frozen_did`, `frozen_hash_hex`, + `from_frozen_hash`, mirroring the blob helpers. `pure_id()` already strips + `?drive=`; subject equality is already string-based — reuse both. +- [ ] `lib/src/db/trees.rs`: add `Tree::Frozen` (key = 32-byte BLAKE3, value = + canonical JSON-AD bytes). +- [ ] `lib/src/db.rs#get_resource`: if `is_frozen_did(subject)`, look up + `Tree::Frozen`, verify hash, parse JSON-AD into a **read-only** Resource, + return it — bypassing `Tree::Resources`/Loro. Cache in memory like any + fetched resource. +- [ ] Reject any Commit whose subject is `did:ad:frozen:` (immutable; no edits). +- [ ] Use **RFC 8785 JCS** for the canonical JSON-AD bytes on the Rust side + (`serde_jcs` / `serde_json_canonicalizer`) so it byte-matches the TS + producer (`browser/lib/src/jcs.ts#jcsCanonicalize`, already used by + `freeze.ts`). Lock it with shared test vectors (fixed schema → fixed hash, + asserted in both Rust and TS). **This is the sharpest correctness risk** — + a one-byte canonicalization difference makes ids diverge across languages, + so JCS (a named standard with conformant impls on both sides) is required. + +### Phase B — server endpoints + +- [ ] `PUT /frozen/{hash}`: accept bytes, verify `blake3 == hash`, store + idempotently (re-post is a no-op, like `/blob`). Auth: require an + authenticated agent (forgery is impossible since content is hash-checked, so + the only risk is storage spam — gate with auth + a size/rate limit). +- [ ] `GET /frozen/{hash}`: return canonical JSON-AD bytes + (`content-type: application/ad+json`). +- [ ] Wire `did:ad:frozen:` subjects into the normal resource GET path so they + resolve through the same `/` endpoint and the WebSocket `GET` frame, not + only the dedicated route. + +### Phase C — browser Store + registerSchema switch-over + +- [ ] Store: detect `did:ad:frozen:` in `getResource`/`fetchResourceFromServer`, + fetch (local cache → `GET /frozen` → sync), **re-hash to verify**, parse + JSON-AD into a read-only Resource, cache. `getProperty` then works + unchanged. +- [ ] Persist frozen objects in ClientDb (OPFS) keyed by hash for offline use. +- [ ] `Store.registerSchema`: replace genesis-DID minting with `freezeSchema`; + write frozen objects locally and optionally `PUT /frozen` to the author's + server; return the frozen ids. Keep the old path behind a flag during + migration so existing tests/producers keep working until parity is proven. +- [ ] The signed **"latest version" pointer** stays a normal genesis-DID resource + on the author's drive (`name → latest frozen ontology id`), plus any + mutable display overlay the app chooses to keep editable. + +### Phase D — sync (frozen travels with drives) + +- [ ] Add `FROZEN_REQUEST/FROZEN_RESPONSE` frames (or generalize the blob + fetch-if-missing path) in `lib/src/sync/protocol.rs` + + `sync/engine.rs#import_sync_push`: when an imported resource references a + `did:ad:frozen:` subject the receiver lacks, request it by hash — exactly + how missing blobs are pulled today. Frozen objects are immutable and + self-verifying, so they do **not** join version vectors or Loro merge. +- [ ] Route frozen fetches over iroh via the `?drive=` hint → pkarr → peer. + +### Phase E — spec, docs, discovery polish + +- [ ] `docs/src/did.md`: add `did:ad:frozen` as the next `did:ad` form — + content-addressed, resolves to canonical JSON-AD (vs. `blob`'s opaque + bytes), immutable, verify-by-rehash, optional `?drive=` hint. +- [ ] Update `docs/src/schema/*` per the companion doc's Phase 5. +- [ ] _(optional/future)_ content-hash DHT announce using the dormant `mainline` + crate or pkarr, for hint-free discovery. + +## Risks / open questions + +- **Cross-language canonicalization** must be byte-identical (Phase A): RFC 8785 + JCS on both sides (`jcs.ts` ↔ `serde_jcs`). Lock it with shared test vectors + before anything else depends on the hash. +- ~~**Cyclic addressing**~~ — decided: one unit per cycle (see above), implemented + in `freeze.ts`. +- **Garbage collection / retention.** Frozen objects are immutable and + accumulate. Refcount from referencing resources, or keep schema frozens + forever? Open. +- **PUT /frozen auth + abuse.** Open vs. authenticated vs. quota'd. +- **`mainline` is dead weight today** — either wire it for content-hash discovery + or drop the dependency; don't leave it implying capability that isn't there. diff --git a/planning/json-schema-code-first.md b/planning/json-schema-code-first.md index 6f421e6f1..5d24dbdaa 100644 --- a/planning/json-schema-code-first.md +++ b/planning/json-schema-code-first.md @@ -8,7 +8,8 @@ code, without first publishing Classes and Properties at HTTP URLs. The desired workflow: 1. An app declares a JSON Schema-like model in TypeScript, Rust, or another SDK. -2. Atomic turns that declaration into local Atomic Class and Property resources. +2. Atomic turns that declaration into one local Ontology resource plus local + Atomic Class and Property resources. 3. Those schema resources get `did:ad` subjects and are signed like normal data. 4. The app can immediately create, validate, render, query, and sync resources using those Classes and Properties. @@ -50,9 +51,194 @@ This means import/export must be first-class: plugin config schemas. - `planning/SDK-API-design.md` already names "Schema creation in-code" as a future SDK capability. +- `@tomic/cli` already generates TypeScript ontology bindings from existing + Ontology resources. +- The data browser already creates Ontologies, Classes, and Properties through + the Ontology editor and table editor. + +The missing piece is a coherent code-first Ontology model and SDK API that +produces locally available DID-backed schema resources from code. + +## Current Status + +This is not full-stack working yet. + +Implemented so far: + +- `browser/lib/src/schema.ts` exports `defineSchema()`, schema package types, + canonical normalization, BLAKE3 `schemaHash`, and a pinned import shape. +- `schemaToOntologyModel()` converts the supported schema subset to an + in-memory Ontology/Class/Property model. +- `Store.registerSchema()` creates local DID Ontology, Class, and Property + resources in memory and registers a `schemaHash -> ontology subject` index. +- `Store.registerSchema(schema, { save: true })` pushes generated schema + resources through the normal Commit/outbox path. +- `Store.registerSchema()` validates pinned imports with `expectedHash` and + fails when the resolved Ontology hash does not match. +- Imported individual Properties can be reused with `$ref: + "importAlias.properties.shortname"`; generated Classes point at the imported + Property subject instead of creating a duplicate Property. +- Generated local Properties resolve through `store.getProperty()` after + registration. +- `browser/lib/tests/schema-code-first.integration.test.ts` verifies the + end-to-end target with a real server: one JS producer module defines and + publishes a schema, another JS consumer module fetches the Ontology by DID, + validates the expected hash, reuses one imported Property, and creates an + instance with that Property. +- `@tomic/cli` has an initial `schema` command that loads a JS schema module + and calls `Store.registerSchema()`, publishing by default or registering + locally with `--local`. It can append the published Ontology subject to + `atomic.config.json` with `--add-to-config`. +- `ad-generate schema --generate` now writes TypeScript ontology bindings + directly from the just-registered Ontology resources, using the same Store so + local code-first schemas do not need a server round-trip before codegen. +- The CLI accepts an optional `serverUrl` in `atomic.config.json` and falls back + to `http://localhost:9883` for relative resource creation/publishing. +- `browser/cli/src/commands/schema.test.ts` verifies a minimal JS project can + define a raw schema package, run `ad-generate schema --local --generate`, and + produce TypeScript bindings with generated DID subjects. +- `browser/lib/src/schema.test.ts` verifies saved generated Ontology, Class, + and Property DID resources can be resolved by a fresh offline `Store` from + local DB state. +- `Store.registerSchema()` now rejects an explicit `atomic:subject` Property + reuse when immutable fields such as datatype, classtype, or `allowsOnly` + differ from an already-loaded Property resource. +- Focused SDK tests cover stable schema hashing, datatype-change hash changes, + undefined normalization, invalid numeric values, pinned import metadata, + schema conversion, local DID schema registration, and local property + resolution, instance creation using returned schema subjects, import hash + mismatch failures, and single imported Property reuse. +- `browser/lib/src/jcs.ts` implements RFC 8785 JCS canonicalization, used for all + frozen hashing (and the schema-package hash), so ids are byte-reproducible + across languages. Covered by `browser/lib/src/jcs.test.ts`. +- `browser/lib/src/freeze.ts` implements `freezeResources()`: a generic, + schema-agnostic primitive that content-addresses a set of mutually-referencing + resources into a `did:ad:frozen` Merkle DAG over JCS bytes, freezing each + strongly-connected cycle as one self-verifying unit object (Tarjan SCC + + color-refinement canonical ordering). Covered by + `browser/lib/src/freeze.test.ts` (acyclic determinism/dedup/order-independence, + reference rewriting, external refs, cycle-as-unit, verify-by-rehash, + self-reference, validation). +- `browser/lib/src/schema.ts#freezeSchema()` builds **identity-only** frozen + Ontology/Class/Property JSON-AD bodies (machine contract: shortname, datatype, + classtype, allowsOnly, requires/recommends) with cross-references resolved to + frozen ids, and returns descriptions + ontology version/schemaHash/jsonSchema + separately as `presentation`. So editing a description does not churn any id. + It enforces per-ontology shortname uniqueness (content-aware: identical + definitions dedupe and pass, genuinely different ones sharing a shortname are + rejected). Tested for id format, reference rewriting, identity/presentation + split (description edit keeps ids stable, datatype edit changes them, no + cascade), determinism, shortname dedupe/conflict, and the imported-property + guard. + +Not implemented yet: + +- `did:ad:frozen` is designed and produced by `freezeSchema`, but + `Store.registerSchema` still mints signed genesis DIDs; switching it over needs + server-side store/serve/resolve support for `did:ad:frozen` (verify-by-rehash, + read-only) plus a `did:ad:frozen` resolution path in the browser Store. +- The signed "latest version" pointer / overlay layer on the author's drive. +- Schema package resolution from Drive context, app registry, CLI config, synced + Ontologies, or schema hash before the schema is explicitly registered. +- `store.getProperty()` fallback to unresolved schema packages. +- CLI-side consumer/type-generation e2e for imported schemas. +- Data browser/table/editor handling for content-locked Properties. +- Rust/server import/export or JSON Schema validation. + +## Existing Touch Points + +### TypeScript SDK + +- `browser/lib/src/ontology.ts` defines `OntologyBaseObject`, global + `registerOntologies()`, quick prop name lookup, and class/property type + inference. +- `browser/lib/src/store.ts#getProperty` fetches a Property resource by subject + and converts it to the lightweight `Property` interface used by forms, table + cells, and validation. +- `browser/lib/src/resource.ts#set` calls `store.getProperty()` before datatype + validation. If a generated DID property is not locally registered or synced, + edits can only skip client-side validation and rely on server rejection. +- `browser/lib/src/store.ts#newResource` already supports locally signed + genesis DID resources, which is the primitive needed for code-first + Ontology/Class/Property creation. + +### CLI + +- `browser/cli/src/commands/ontologies.ts` validates configured Ontology + subjects and writes generated TypeScript files. +- `browser/cli/src/generateOntology.ts` is the main Ontology resource -> + generated code pipeline. +- `browser/cli/src/generateBaseObject.ts` reads Ontology `classes` and + `properties`, creates the exported `OntologyBaseObject`, and builds + `__classDefs`. +- `browser/cli/src/generateClasses.ts` and + `browser/cli/src/generatePropTypeMapping.ts` generate native TypeScript + class/property typings from materialized Class and Property resources. +- `browser/cli/src/validateOntologies.ts` currently requires every configured + subject to resolve to `core.classes.ontology`. + +The CLI is therefore not obsolete. It should become part of the code-first loop: + +```text +code schema -> Ontology/Class/Property resources -> @tomic/cli bindings +existing Ontology resources -> @tomic/cli bindings +``` -The missing piece is a coherent schema-bundle model and SDK API that produces -locally available DID-backed schema resources from code. +### Data Browser + +- `browser/data-browser/src/views/OntologyPage` is the existing editor for + Ontology resources. `OntologyContext` mutates the Ontology `classes` and + `properties` arrays. +- `browser/data-browser/src/components/forms/.../NewOntologyDialog.tsx` + creates Ontology resources with empty `classes`, `properties`, and + `instances` arrays. +- `browser/data-browser/src/chunks/TablePage/PropertyForm/NewPropertyDialog.tsx` + creates Property resources from table columns and adds them to the parent + Ontology when the table class is inside an Ontology. +- `browser/data-browser/src/chunks/TablePage/useTableColumns.tsx` resolves table + columns through `store.getProperty()`, so table rendering needs generated DID + properties to be materialized or locally indexed. +- `browser/data-browser/src/components/forms/ResourceForm.tsx` renders required + and recommended fields from a Class resource's `requires` and `recommends` + arrays. + +Code-first schemas must not bypass these paths. Ontology editor and table +editor output should remain valid input to codegen/export, and code-first +output should remain editable and inspectable in these views. + +### Rust / Server + +- `lib/src/schema.rs` has the canonical Rust `Property` and `Class` structs for + materializing schema resources. +- `lib/src/validate.rs` and `Resource` validation fetch Properties and Classes + by subject to check datatypes and required properties. +- `lib/src/populate.rs` defines the bootstrap schema resources and currently + describes the `Property` and `Class` contracts. + +Rust validation should continue to validate against materialized Class and +Property resources. JSON Schema validation can be layered on later for extra +constraints; it should not be required for basic Atomic datatype and required +property validation. + +### Docs / Spec + +- `docs/src/schema/intro.md` currently says Classes and Properties are resolved + using HTTP and that Property URLs should resolve. +- `docs/src/schema/classes.md` documents Property, Datatype, and Class as + first-class resources. +- `docs/src/schema/compare.md` explicitly contrasts Atomic Schema with JSON + Schema by noting that JSON Schema scopes properties to a schema, while Atomic + Properties are reusable. +- `docs/src/schema/migrations.md` already recommends adding new properties + instead of changing existing relationships in place. +- `docs/src/did.md` defines DID resource resolution through Drive context and + says `did:ad:` identifiers have no subpaths. Code-first schema references + should therefore use normal DID resource subjects, not path-like property + members inside one DID. + +The public docs need a coordinated update after the API shape stabilizes: +resolvability should become "resolvable through URL, local store, Drive sync, or +app-bundled schema registry" rather than "must be HTTP-resolvable". ## Proposed Developer API @@ -116,49 +302,154 @@ const todo = await store.newResource({ ## Identity Model -Schema resources need stable local identity without HTTP. - -Recommended first step: signed genesis DID resources. +Schema resources need stable identity without HTTP. -- Each generated Class is a normal Resource with `isA = core.classes.class`. -- Each generated Property is a normal Resource with - `isA = core.classes.property`. -- The first registration signs each resource as a DID genesis resource. -- The returned ontology object stores those DID subjects. -- The schema bundle stores a mapping from developer keys to DID subjects: - `todo -> did:ad:...`, `title -> did:ad:...`. +**Decided: content-addressed `did:ad:frozen` identifiers.** A schema definition is +immutable by intent, so it does not need a signature (provenance) or an owner +who can edit it (authoritative mutability). Both of those are what signing buys; +neither applies to a frozen definition. So schema resources are identified by a +content hash, not a genesis signature: -This is compatible with the current DID model and avoids inventing a new -content-addressed DID form before the rest of the stack is ready. +```text +did:ad:frozen:{blake3-hex} +did:ad:frozen:{blake3-hex}?drive=did:ad:{your_drive} // optional routing hint +``` -Open question for later: content-derived schema IDs. A future version may add a -canonical `did:ad:schema:` or a signed statement that binds a content hash -to the schema. Do not block the first implementation on that. +- A `did:ad:frozen` subject resolves to **canonical JSON-AD** (not opaque bytes, + which is what `did:ad:blob` is for). The resolver fetches, **re-hashes to + verify**, parses, and materializes a read-only Resource. +- `id = blake3(JCS(content))`, where `content` is the **identity** of the + resource — the machine contract only. For a Property that is `shortname`, + `datatype`, `classtype`, `allowsOnly`, `isDynamic`, `isLocked`; for a Class, + `shortname` + `requires`/`recommends`. **Presentation is excluded** — + description, label, translations, icon, ordering live in the mutable package + layer (see Identity vs Presentation below), so cosmetic edits never churn an + id. Same identity -> same id (global dedup); a change to the machine contract + (datatype, rename, add/remove a required property) -> a new id, which is a real + new version to link. +- No commit, no signature, no history. Immutable by construction. No keypair is + needed to mint one; ids can be computed offline and deterministically. +- Each Property, Class, and Ontology is its own frozen resource. The Ontology + references its members by their frozen ids, so single-property reuse still + works by pointing at a property's frozen id. Efficient ontology-level + resolution comes from **shipping the members bundled together**, not from + collapsing them into one blob. + +Canonical bytes use **RFC 8785 JCS** (`browser/lib/src/jcs.ts`) so a frozen id is +reproducible byte-for-byte across languages (the Rust side uses a conformant JCS +crate). `id = blake3(JCS(content))`. + +Because frozen resources reference each other by hash, the ids are +interdependent (an Ontology's id depends on its Classes' ids, which depend on +their Properties' ids) — a Merkle DAG built by topological hashing. Mutually +referencing definitions (e.g. a `Person` class with a `friend` property whose +classtype is `Person`) form a cycle with no leaf to start from; each +strongly-connected group is frozen **as a single unit object** +(`{ "urn:atomic-freeze:unit": [...members] }`, members in canonical order with +intra-cycle refs as `did:ad:frozen:self:{index}` tokens). All members share the +unit's id, so `blake3(JCS(bytes)) == id` holds for every stored object and stays +verifiable by re-hashing. + +This is implemented as a generic, schema-agnostic primitive: +`browser/lib/src/freeze.ts#freezeResources(resources)` content-addresses any set +of mutually-referencing resources (Tarjan SCC + color-refinement canonical +ordering for cycles, one unit per cycle), and +`browser/lib/src/schema.ts#freezeSchema(schema)` builds the frozen +Ontology/Class/Property JSON-AD bodies from a defined schema. + +### Identity vs presentation + +Hashing the *whole* body — descriptions included — was the original plan, but it +makes identity churn on every cosmetic edit: a reworded description changes a +property's id, which cascades up through the class and ontology, producing piles +of near-duplicate versions. Descriptions change constantly; identity must not. + +So the boundary is set by one test: **does it change how data is validated or +interpreted?** + +- **Identity (hashed):** `shortname`, `datatype`, `classtype`, `allowsOnly`, + `isDynamic`, `isLocked`; for a Class, `shortname` + `requires`/`recommends`. +- **Presentation (not hashed):** description, label, translations, icon, + ordering, examples — plus the Ontology's `jsonSchema` source and `schemaHash` + (both encode descriptions). These ride in the **mutable package layer**: the + Ontology resource and the lockfile, keyed by frozen id / model key. + +Consequences: + +- Cosmetic edits cause **zero** id churn — no cascade. New ids appear only on a + real machine-contract change (datatype, rename, add/remove a required + property), which is exactly when a new version is warranted; the mutable + name-pointer absorbs that churn for consumers. +- A frozen property is **not self-describing**: its human text comes from the + package you got it through. This is arguably correct — meaning is universal, + wording is contextual and localizable. +- The **index dedupes on meaning**, not wording, so "title: string" is one entry + no matter how differently apps describe it. + +`freezeSchema` returns this split: `resources` (identity-only frozen bodies) and +`presentation` (descriptions + ontology `version`/`schemaHash`/`jsonSchema`, +keyed by model key so each usage keeps its own text even when identical +definitions dedupe to one id). `browser/lib/src/schema-lock.ts#buildSchemaLock()` +assembles both into the committed lockfile, and `verifySchemaLock()` re-hashes +every frozen object to confirm it matches its id — the language-neutral +verification a consumer or CI guard runs, depending only on JCS + blake3. + +### Two layers: frozen definitions + signed pointers + +Not everything can be frozen. Three things are inherently mutable and stay +**normal signed genesis-DID resources on the author's own drive**: + +1. The "latest version" pointer — `name -> latest frozen ontology id` — so an app + can mean "the current TodoApp" even as it evolves. +2. Editable display metadata that should not be frozen into identity, when an app + chooses to keep it mutable (labels, translations, ordering, icons). +3. Endorsement — "this is the official Ontola schema" is provenance, i.e. a + signature, attached at the pointer layer. + +There is no central schema host. Authors publish frozen definitions and signed +pointers on **their own drive**; the `?drive=` hint routes resolution to it over +HTTP, Mainline DHT, or Reticulum, and any node can replicate that drive. + +### Migration from the current implementation + +The shipped code currently mints **signed genesis DID** resources for Ontologies, +Classes, and Properties (`Store.registerSchema`). Moving to `did:ad:frozen` keeps +the conversion and validation code but replaces identity minting with +`freezeSchema`, and requires server-side support for storing, serving, and +resolving `did:ad:frozen` resources (verify-by-rehash, read-only). That server +work is the main remaining gap before `registerSchema` can switch over, and is +planned in detail in [did-ad-frozen-server.md](./did-ad-frozen-server.md) +(model frozen as blob-like, not resource-like; resolve over local cache / +bundle / iroh+pkarr / optional default server; resolve the cyclic-addressing +question first). ## Local Availability The main behavior change is that Property and Class URLs no longer imply HTTP availability. -Resolution order should be: +Resolution order for `did:ad` schema resources should be: 1. in-memory store 2. local persistent store / OPFS / native DB -3. synced drive schema registry +3. the currently opened Drive, when the subject carries `?drive=` or the caller + has an active Drive context 4. bundled app schema registry -5. network fetch, only if the subject is fetchable +5. synced Ontology resources referenced by the Drive, for example the Drive's + `defaultOntology` or app/plugin configuration +6. network / peer discovery, only when there is enough routing context to find + a Drive replica For `did:ad` schema resources, network fetch is not the primary mechanism. The resource must travel with the app, the drive, or sync. -## Schema Bundle Resource +## Ontology as Bundle -Add an Atomic resource that groups generated schema resources. This could reuse -the existing Ontology class or introduce a narrower `SchemaBundle` class. +Use the existing Ontology class as the schema bundle resource. Minimum useful fields: -- `isA`: Ontology or SchemaBundle +- `isA`: Ontology - `shortname` / name - `version` - `classes`: ResourceArray of generated Class resources @@ -167,10 +458,118 @@ Minimum useful fields: - `schemaHash`: canonical hash of the JSON Schema document - `replaces` / `previousVersion`: optional pointer to older bundle -Using a bundle solves two problems: +Using an Ontology as the bundle solves three problems: - app startup can register one thing and get every generated subject - sync can discover the schema resources needed to interpret app data +- the existing OntologyPage, `defaultOntology`, code generation, and generated + `OntologyBaseObject` shape keep working + +The Ontology should not embed Properties and Classes as anonymous JSON-only +children in the first implementation. Keeping them materialized as standalone +resources preserves Atomic's semantic reuse model and avoids creating a second +schema runtime for forms, table columns, validation, graph views, usage views, +and query building. + +## Schema Hash + +`schemaHash` is a version fingerprint, not a resolver. + +Define it as the hash of a canonical normalized schema document, for example: + +```text +schemaHash = blake3(canonical_json(normalized_json_schema)) +``` + +It answers "is this exactly the schema version I expected?", not "where do I +fetch this from?" Resolution still starts from the Ontology subject, app-bundled +registry, Drive context, or local/synced store. + +The hash should be stored as a normal Atomic property on the Ontology resource. +Because the Ontology is backed by Loro like any other Resource, the hash is +already part of the resource state and signed history. It should not be stored +as a special resolver entry in the Loro oplog. + +If all a caller has is a `schemaHash`, lookup requires an index: + +```text +schemaHash -> ontology subject +``` + +That index can be derived from app-bundled schemas, local storage, and synced +Drive Ontologies. It is a cache/discovery aid, not authoritative state. + +Canonical schema bytes may optionally be stored as a `did:ad:blob:{blake3}` for +package-lock style pinning. That blob is immutable byte content and cannot +replace the Ontology resource, because blobs have no class, parent, ACL, +history, or links to materialized Class and Property resources. + +## Frozen lockfile (the shareable artifact) + +A frozen id is a pure, deterministic function of the schema source (JCS → +blake3). So the schema does not need to be *hosted* to be *available*: ship a +self-verifying copy of the canonical bytes alongside the code. `ad-generate` +emits a `*.schema.lock.json` next to the generated bindings; it is committed and +guarded by a CI "regenerate and diff" check (stale-lockfile guard). This is +resolution step 2 ("app-bundled frozen objects") made concrete, and it makes any +default server (atomicdata.dev or a drive) a pure cache, never a dependency. + +### Format + +```jsonc +{ + "name": "TodoApp", + "version": "1.0.0", + "ontology": "did:ad:frozen:7c1…", + "@index": { // human aid, NOT hashed + "did:ad:frozen:7c1…": "TodoApp.todo", // class + "did:ad:frozen:9f2…": "TodoApp.title", // property + "did:ad:frozen:a3b…": "TodoApp.done" + }, + "frozen": { // the verbatim, hashed objects (identity only) + "did:ad:frozen:9f2…": { + "https://atomicdata.dev/properties/isA": ["https://atomicdata.dev/classes/Property"], + "https://atomicdata.dev/properties/shortname": "title", + "https://atomicdata.dev/properties/datatype": "https://atomicdata.dev/datatypes/string" + // no description here — that is presentation, below + } + // …one entry per frozen object; a cycle is one `urn:atomic-freeze:unit` entry + }, + "presentation": { // mutable, NOT hashed + "TodoApp.title": { "id": "did:ad:frozen:9f2…", "description": "Task title" } + // …plus ontology-level description / version / jsonSchema + } +} +``` + +### The hashing invariant + +Only the values of `frozen` are hashed, and they hold **identity only** (the +machine contract), serialized as **verbatim canonical JSON-AD** (full property +URLs, `did:ad:frozen:` refs, JCS order). Descriptions/labels live in +`presentation`, which is never hashed — so re-wording text changes the lockfile +diff but not a single id. Verification is trivial and language-neutral: for each +`id -> object` in `frozen`, assert `"did:ad:frozen:" + blake3(JCS(object)) == id`; +everything else (`name`, `version`, `@index`, `presentation`) is a non-hashed +aid. We deliberately do **not** make references shortnames in the hashed form — +that would reopen the cross-language canonicalization surface JCS just closed. +Shortnames live only in `@index`. + +### Why this shape + +- **Verbose, not gibberish.** The "noise" is the frozen ids (hashes), which are + intrinsic to content-addressing; `@index` decodes every one to + `Ontology.shortname`, so a reviewer reads the index, and a changed hash in a + diff is the visible "this identity moved" signal. +- **Consume without reimplementing freeze.** Any language embeds the file (Rust + `include_str!`, etc.), iterates `frozen`, re-hashes to verify, and materializes + read-only Resources. Reimplementing `freezeResources` is only needed to + *author* schemas in another language — and the committed `frozen` bytes are then + the cross-implementation conformance check. +- **Unambiguous index.** `freezeSchema` enforces per-ontology shortname + uniqueness (content-aware: identical definitions across classes dedupe to one + frozen id and pass; only genuinely different definitions sharing a shortname are + rejected), so each `@index` value is a clean `Ontology.shortname`. ## JSON Schema Mapping @@ -198,6 +597,10 @@ JSON object properties map to Atomic Property resources. - scalar enum -> store in JSON Schema first; map to `allowsOnly` only when the values are valid Atomic values for the Property +Property subjects should normally be generated once and then reused across +schema versions when the property's machine meaning is unchanged. Changes that +alter datatype or semantics should produce a new Property subject. + ### Datatypes Initial mapping: @@ -263,10 +666,10 @@ JSON Schema validation can be added incrementally: ## Versioning -Default to immutable schema versions. +Default to immutable schema versions and immutable property meaning. Editing code should not silently mutate the meaning of existing data on another -device. A changed schema should normally produce a new schema bundle version and +device. A changed schema should normally produce a new Ontology version and possibly new Class/Property DID subjects. Developer API: @@ -283,9 +686,21 @@ Supported policies: - `update-in-place`: edit existing DID resources, useful during development - `fail-if-changed`: production-safe mode for apps that expect exact schemas -Open question: Property identity. If a developer changes only a description, the -Property subject can probably remain stable. If datatype or meaning changes, use -a new Property subject. +If a developer changes only display metadata, such as description, label, +translation, icon, or ordering, the Property subject can remain stable. If +datatype or meaning changes, use a new Property subject. + +Working rule: + +- Ontologies are versioned packages. They may link to previous versions and can + change by publishing a new version. +- Classes are versioned shapes over Properties. Changing required/recommended + structure should generally create a new Class version when shared data already + depends on the old shape. +- Properties are immutable semantic definitions. Datatype changes, `classtype` + changes, and semantic meaning changes require a new Property subject. +- Metadata-only edits can be allowed during development, but published + properties should be treated as content-locked. ## Sync and Trust @@ -302,6 +717,12 @@ Because Classes and Properties may be DID resources, validation cannot assume the owner is an HTTP origin. The proof is the signed resource history plus the trust context that introduced the schema. +The trusted entry point is usually the Ontology resource: it may be bundled by +the app, set as a Drive's `defaultOntology`, referenced by a plugin/app config, +or synced with a Drive. The Ontology then points to individual Class and +Property resources. Those linked resources must still be present locally, +bundled, or resolvable through the same Drive/app trust context. + ## Interaction with Sign-at-Drain Schema registration creates a small set of important resources. It should not be @@ -320,49 +741,163 @@ Rules: ## Implementation Plan -- [ ] Decide whether the grouping resource reuses `Ontology` or gets a new - `SchemaBundle` class. -- [ ] Add a TypeScript schema module with `defineSchema` and type definitions +### Phase 0: Decide contracts + +- [x] Decide whether the grouping resource reuses `Ontology` or gets a new + `SchemaBundle` class: reuse `Ontology`. +- [x] Decide the initial immutability contract for Property resources: + genesis-DID with tooling enforcement first; content-derived IDs remain a + later option before broad public release. +- [x] Decide the initial import expression in code: named `imports` map with + Ontology `subject` plus optional `expectedHash`, and imported Properties + referenced as `$ref: "importAlias.properties.shortname"`. +- [ ] Decide where schema discovery is attached first: Drive `defaultOntology`, + app/plugin config, explicit `registerSchema`, or all of them. + +### Phase 1: TypeScript code-first creation + +- [x] Add a TypeScript schema module with `defineSchema` and type definitions for the supported JSON Schema subset plus `atomic:*` extensions. -- [ ] Implement JSON Schema -> in-memory ontology conversion. -- [ ] Implement local DID resource creation for generated Classes and +- [x] Implement JSON Schema/code schema -> in-memory Ontology/Class/Property + model conversion. +- [x] Implement schema hashing over canonical normalized schema package JSON. +- [x] Implement local DID resource creation for the generated Ontology bundle. +- [x] Implement local DID resource creation for generated Classes and Properties. -- [ ] Add `store.registerSchema(schema, options)` in `@tomic/lib`. +- [x] Add `store.registerSchema(schema, options)` in `@tomic/lib`. +- [x] Add an explicit `registerSchema(schema, { save: true })` path that saves + generated schema resources through the normal Commit/outbox flow. +- [x] Verify registered schema resources reload from local DB after save. +- [x] Add a local/app schema index for `schemaHash -> ontology subject`. +- [x] Ensure `store.getProperty` resolves generated DID Properties from + in-memory registered schemas without requiring HTTP. +- [x] Reject explicit Property subject reuse when immutable machine fields do + not match the already-loaded Property definition. - [ ] Ensure `store.getProperty` and class loading resolve DID schema resources - from local storage without HTTP. -- [ ] Persist the schema bundle and generated resources before returning from - `registerSchema`. -- [ ] Add JSON Schema export from Atomic Class/Property resources. + from local DB, registered app schemas, and Drive context without + requiring HTTP. + +### Phase 1.5: Content-addressed frozen identity + +- [x] RFC 8785 JCS canonicalization (`jcs.ts`). +- [x] Generic `freezeResources()` content-addressing primitive with one-unit-per- + cycle (`freeze.ts`). +- [x] `freezeSchema()` building frozen JSON-AD bodies from a defined schema. +- [x] Per-ontology shortname uniqueness enforcement (content-aware). +- [ ] `did:ad:frozen` server support (storage, serve, resolve, materialize) — see + [did-ad-frozen-server.md](./did-ad-frozen-server.md). +- [ ] Switch `Store.registerSchema` from signed genesis DIDs to `freezeSchema` + (behind a flag during migration). +- [ ] Browser Store resolution of `did:ad:frozen:` (fetch -> verify-by-rehash -> + materialize read-only Resource; expand `urn:atomic-freeze:unit` objects). +- [ ] Signed "latest version" pointer / overlay layer on the author's drive. + +### Phase 2: CLI integration + +- [x] Extend `@tomic/cli` so the existing Ontology -> TypeScript path can also + consume locally registered/generated Ontologies. +- [x] Add a CLI command or option for code schema -> Ontology resources, using + the same conversion code as `store.registerSchema` where possible. +- [x] Add SDK import/hash pin checks: if an external Ontology is referenced + with an expected hash and the resolved resource does not match, + `registerSchema()` fails. +- [ ] Add CLI import/hash pin checks: if an external Ontology or Property + subject is referenced with an expected hash/version and the resolved + resource does not match, generation fails. +- [ ] Keep existing `ontologies` config working for HTTP and DID Ontology + subjects. +- [x] Generate native bindings from materialized Class and Property resources, + not from anonymous JSON-only schema members. +- [x] Let `ad-generate schema` optionally update `atomic.config.json` with the + published Ontology subject for a follow-up `ad-generate ontologies` run. +- [x] Let `ad-generate schema --generate` write bindings immediately from the + materialized Ontology/Class/Property resources in the current Store. +- [x] Emit a committed `*.schema.lock.json` (frozen objects + `@index` + + `presentation`) via `ad-generate schema --lock`, using + `buildSchemaLock()`/`verifySchemaLock()` in `@tomic/lib`. The emitter + refuses to write an unverifiable lock. +- [ ] Load + register a lockfile at app startup for offline availability (needs + the Store `did:ad:frozen` resolution path). +- [ ] Add a CI "regenerate and diff" guard so the lockfile cannot drift from the + source schema (`verifySchemaLock` + a re-emit check are the building blocks). + +### Phase 2.5: End-to-end target + +- [x] Add a minimal JS producer schema fixture. +- [x] Add a minimal JS consumer schema fixture that imports the producer + Ontology with an expected hash and reuses one Property. +- [x] Add an integration test where producer publishes to a real server and + consumer fetches/reuses that schema through a separate Store. +- [x] Add a CLI test where a minimal JS project defines a schema and + `ad-generate schema --local --generate` produces generated bindings. +- [ ] Add a CLI-side producer/consumer import e2e where one generated schema + imports another published/local Ontology with an expected hash. + +### Phase 3: Editor and table compatibility + +- [ ] Keep Ontology editor creation/editing as a first-class schema authoring + path. +- [ ] Ensure table-created Properties and Classes can be exported to JSON + Schema/code schema and then regenerated without changing meaning. +- [ ] Add UI indicators or warnings when editing a published/content-locked + Property would change datatype, `classtype`, or semantic meaning. +- [ ] Ensure table columns and forms continue to resolve generated DID + Properties through `store.getProperty()`. + +### Phase 4: Import/export and validation + +- [ ] Add JSON Schema export from Atomic Ontology/Class/Property resources. - [ ] Add browser SDK tests: - create schema from code - - generated Class and Property resources are local DID resources + - schema package hashing is stable under object key reordering + - generated Ontology, Class, and Property resources are local DID resources - create an instance using returned ontology subjects - reload store and resolve the generated Property without HTTP + - import an external schema with an expected hash and fail on mismatch + - reuse one imported Property in another Class + - table-created schema exports and re-imports - required/datatype validation still works - [ ] Add Rust-side import/export structs after the TypeScript API shape is stable. - [ ] Add optional JSON Schema validation in the browser SDK. - [ ] Add optional JSON Schema validation in Rust/server. + +### Phase 5: Docs + +- [ ] Update `docs/src/schema/intro.md` so schema resources can resolve through + HTTP, local store, Drive sync, or app-bundled schema registry. +- [ ] Update `docs/src/schema/classes.md` with immutability/versioning guidance + for Properties, Classes, and Ontologies. +- [ ] Update `docs/src/schema/compare.md` with the JSON Schema compatible + boundary story. +- [ ] Update `docs/src/schema/migrations.md` with Property immutability and + version-link examples. - [ ] Write public docs and a tutorial once the API has survived tests. ## Non-goals for the First Pass - Full JSON Schema 2020-12 coverage. - Replacing Atomic Class and Property resources with raw JSON Schema documents. +- Embedding all schema meaning in a single Ontology-only JSON document without + materialized Class and Property resources. - Requiring all schema resources to be public HTTP URLs. - Solving global package-manager style schema discovery. - Automatic migrations of existing instance data. ## Open Questions -- Should schema bundles be Resources of existing `Ontology`, or a new - `SchemaBundle` class? -- Should generated Property subjects be reused across schema versions when only - display metadata changes? - Where should schema bundles be attached for discoverability: drive `defaultOntology`, app config, plugin resource, or all of them? - Should JSON Schema validation be strict by default, or opt-in per app/class? - How should custom JSON Schema formats map to Atomic datatypes? -- Do we want a future content-derived schema DID, or are signed genesis DIDs - enough? - +- ~~Do we want a future content-derived schema DID, or are signed genesis DIDs + enough?~~ **Decided: content-derived `did:ad:frozen` identifiers** (see Identity + Model). Only the machine-contract identity is hashed (descriptions and other + presentation are excluded so cosmetic edits don't churn ids); cycles are hashed + as a unit. +- ~~For a cycle (strongly-connected group), should members keep individually + derived ids or share one group id?~~ **Decided: one unit per cycle** — members + share the unit id and resolve together, keeping every stored object + verifiable by re-hash. Per-member addressing inside a cycle is deferred. +- Should canonical schema JSON also be stored as a blob by default, or only + when callers ask for reproducible byte-level pinning? From ce650ad447b89a3d5b3081b47930426eab81c598 Mon Sep 17 00:00:00 2001 From: Joep Meindertsma Date: Tue, 2 Jun 2026 12:52:27 +0200 Subject: [PATCH 02/20] Fix recursive-cargo deadlock in build.rs atomic-server's build.rs runs `pnpm run build`, which runs `build:wasm` -> `wasm-pack` -> a nested cargo. That child cargo blocked forever on the workspace `target/.cargo-lock` held by the outer cargo running the build script (e.g. an editor's `cargo check`), parking everything at 0% CPU. Give the nested frontend/wasm build its own CARGO_TARGET_DIR so it takes a separate lock and can't deadlock against the outer build. wasm artifacts are copied out via --out-dir, so the isolated dir holds only intermediates. --- server/build.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/server/build.rs b/server/build.rs index 84487eee9..ddcc5ddd8 100644 --- a/server/build.rs +++ b/server/build.rs @@ -178,10 +178,23 @@ fn should_build(dirs: &Dirs) -> bool { fn build_js(dirs: &Dirs) { let pkg_manager = "pnpm"; + // The JS build runs `build:wasm` -> `wasm-pack` -> a *nested* cargo. That + // child cargo would otherwise block forever on the workspace + // `target/.cargo-lock` that the outer cargo running THIS build script + // already holds (a recursive-cargo deadlock — e.g. when an editor's + // `cargo check` triggers build.rs while it also runs the JS build). + // Pointing the nested build at its own CARGO_TARGET_DIR gives it a separate + // lock, so it can't deadlock against us. wasm artifacts are copied out via + // `--out-dir`, so this dir holds only intermediates and is safe to isolate. + let nested_target_dir = std::env::var("CARGO_MANIFEST_DIR") + .map(|m| Path::new(&m).join("../target/frontend-build")) + .unwrap_or_else(|_| PathBuf::from("../target/frontend-build")); + p!("install js packages..."); std::process::Command::new(pkg_manager) .current_dir(&dirs.browser_root) + .env("CARGO_TARGET_DIR", &nested_target_dir) .args(["install"]) .output() .unwrap_or_else(|_| { @@ -193,6 +206,7 @@ fn build_js(dirs: &Dirs) { p!("build js assets..."); let out = std::process::Command::new(pkg_manager) .current_dir(&dirs.browser_root) + .env("CARGO_TARGET_DIR", &nested_target_dir) .args(["run", "build"]) .output() .expect("Failed to build js bundle"); From 84ba418e2086022d16f0e2b471a6019a2ce86ef5 Mon Sep 17 00:00:00 2001 From: Joep Meindertsma Date: Tue, 2 Jun 2026 12:52:40 +0200 Subject: [PATCH 03/20] Implement did:ad:frozen content-addressed schemas (server + client) Immutable, signatureless, content-addressed schema resources, end to end. A frozen id is the BLAKE3 of the RFC 8785 (JCS) canonicalization of the machine-meaning only; descriptions/labels are presentation, excluded from the hash, so cosmetic edits never churn ids. Resources are verified by re-hash, never by signature. lib: - frozen.rs: frozen_id / verify_frozen (serde_jcs + blake3). - subject.rs: is_frozen_did / frozen_hash_hex / from_frozen_hash. - Tree::Frozen storage wired through both backends. - db.rs#materialize_frozen: get_resource resolves did:ad:frozen by re-hash + read-only parse (SaveOpts::DontSave, no class-requires validation). - commit.rs: reject commits to frozen subjects. server: - GET/PUT /frozen/{hash} (PUT verifies frozen_id(body) == url hash). browser/lib: - Store.fetchFrozenResource resolves did:ad:frozen (GET /frozen -> verify -> materialize); registerFrozenSchema materializes locally + optionally PUTs. - getProperty treats description as optional (presentation, not identity). Cross-language contract pinned by test-vectors/frozen.json (TS jcs.ts and Rust serde_jcs produce identical ids). Capstone integration test publishes from a producer and resolves from a fresh consumer against a real server. --- browser/lib/src/frozen-resolve.test.ts | 140 ++++++++++++++++++ browser/lib/src/frozen-vectors.test.ts | 31 ++++ browser/lib/src/schema.ts | 1 + browser/lib/src/store.ts | 115 +++++++++++++- .../lib/tests/frozen-e2e.integration.test.ts | 76 ++++++++++ lib/src/commit.rs | 46 ++++++ lib/src/db.rs | 50 +++++++ lib/src/db/redb_store.rs | 5 + lib/src/db/sled_store.rs | 10 ++ lib/src/db/test.rs | 64 ++++++++ lib/src/db/trees.rs | 7 + lib/src/frozen.rs | 100 +++++++++++++ lib/src/lib.rs | 1 + lib/src/parse.rs | 6 +- lib/src/subject.rs | 84 +++++++++++ planning/did-ad-frozen-server.md | 97 +++++++----- server/src/handlers/frozen.rs | 71 +++++++++ server/src/handlers/mod.rs | 1 + server/src/routes.rs | 10 ++ server/src/tests.rs | 72 +++++++++ test-vectors/README.md | 18 +++ test-vectors/frozen.json | 67 +++++++++ 22 files changed, 1026 insertions(+), 46 deletions(-) create mode 100644 browser/lib/src/frozen-resolve.test.ts create mode 100644 browser/lib/src/frozen-vectors.test.ts create mode 100644 browser/lib/tests/frozen-e2e.integration.test.ts create mode 100644 lib/src/frozen.rs create mode 100644 server/src/handlers/frozen.rs create mode 100644 test-vectors/README.md create mode 100644 test-vectors/frozen.json diff --git a/browser/lib/src/frozen-resolve.test.ts b/browser/lib/src/frozen-resolve.test.ts new file mode 100644 index 000000000..40f38dccc --- /dev/null +++ b/browser/lib/src/frozen-resolve.test.ts @@ -0,0 +1,140 @@ +import { afterEach, describe, it, vi } from 'vitest'; + +import { Datatype } from './datatypes.js'; +import { frozenIdFor } from './freeze.js'; +import { core } from './ontologies/core.js'; +import { Store } from './store.js'; + +const body = { + [core.properties.isA]: [core.classes.property], + [core.properties.shortname]: 'title', + [core.properties.datatype]: Datatype.STRING, +}; +const id = frozenIdFor(body); +const hash = id.replace('did:ad:frozen:', ''); + +const ok = (obj: unknown) => ({ + ok: true, + status: 200, + text: async () => JSON.stringify(obj), +}); + +function connectedStore(): Store { + const store = new Store({ serverUrl: 'https://example.com' }); + store.setServerConnected(true); + + return store; +} + +describe('Store frozen resolution', () => { + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it('fetches, verifies, and materializes a frozen resource', async ({ + expect, + }) => { + const fetchMock = vi.fn(async () => ok(body)); + vi.stubGlobal('fetch', fetchMock); + + const resource = await connectedStore().getResource(id); + + expect(resource.subject).toBe(id); + expect(resource.get(core.properties.shortname)).toBe('title'); + expect(resource.get(core.properties.datatype)).toBe(Datatype.STRING); + expect(fetchMock).toHaveBeenCalledWith( + `https://example.com/frozen/${hash}`, + expect.objectContaining({ headers: { Accept: 'application/ad+json' } }), + ); + }); + + it('rejects a body that fails hash verification', async ({ expect }) => { + vi.stubGlobal('fetch', vi.fn(async () => ok({ tampered: true }))); + + await expect(connectedStore().getResource(id)).rejects.toThrow( + /hash verification/i, + ); + }); + + it('errors when the frozen resource is absent', async ({ expect }) => { + vi.stubGlobal( + 'fetch', + vi.fn(async () => ({ ok: false, status: 404, text: async () => '' })), + ); + + await expect(connectedStore().getResource(id)).rejects.toThrow(/404/); + }); +}); + +const todoPackage = { + name: 'TodoApp', + classes: { + todo: { + type: 'object' as const, + required: ['title'], + properties: { + title: { type: 'string' as const, description: 'Task title' }, + done: { type: 'boolean' as const }, + }, + }, + }, +}; + +describe('Store.registerFrozenSchema', () => { + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it('materializes frozen resources locally so a property resolves offline', async ({ + expect, + }) => { + // Any network call would throw — local materialization must need none. + vi.stubGlobal( + 'fetch', + vi.fn(async () => { + throw new Error('should not hit the network'); + }), + ); + + const store = new Store({ serverUrl: 'https://example.com' }); + const frozen = await store.registerFrozenSchema(todoPackage); + const titleId = frozen.properties['todo.title']; + + expect(titleId).toMatch(/^did:ad:frozen:[0-9a-f]{64}$/); + + const prop = await store.getProperty(titleId); + expect(prop.shortname).toBe('title'); + expect(prop.datatype).toBe(Datatype.STRING); + // Description is presentation — excluded from the frozen body. + expect(prop.description).toBe(''); + expect(frozen.presentation.properties['todo.title'].description).toBe( + 'Task title', + ); + }); + + it('publishes every frozen resource to /frozen with { save: true }', async ({ + expect, + }) => { + const calls: string[] = []; + vi.stubGlobal( + 'fetch', + vi.fn(async (url: string, init: { method: string }) => { + calls.push(`${init.method} ${url}`); + + return { ok: true, status: 204, text: async () => '' }; + }), + ); + + const store = new Store({ serverUrl: 'https://example.com' }); + const frozen = await store.registerFrozenSchema(todoPackage, { + save: true, + }); + + expect(calls.length).toBe(frozen.resources.length); + expect( + calls.every(call => + call.startsWith('PUT https://example.com/frozen/'), + ), + ).toBe(true); + }); +}); diff --git a/browser/lib/src/frozen-vectors.test.ts b/browser/lib/src/frozen-vectors.test.ts new file mode 100644 index 000000000..26ed255fb --- /dev/null +++ b/browser/lib/src/frozen-vectors.test.ts @@ -0,0 +1,31 @@ +import { readFileSync } from 'node:fs'; +import path from 'node:path'; +import { describe, it } from 'vitest'; + +import { frozenIdFor, type JsonValue } from './freeze.js'; + +interface Vector { + name: string; + body: JsonValue; + id: string; +} + +const fixturePath = path.resolve( + process.cwd(), + '../../test-vectors/frozen.json', +); +const { vectors } = JSON.parse(readFileSync(fixturePath, 'utf8')) as { + vectors: Vector[]; +}; + +describe('cross-language frozen vectors', () => { + it('has vectors to check', ({ expect }) => { + expect(vectors.length).toBeGreaterThan(0); + }); + + for (const vector of vectors) { + it(`reproduces the id for "${vector.name}"`, ({ expect }) => { + expect(frozenIdFor(vector.body)).toBe(vector.id); + }); + } +}); diff --git a/browser/lib/src/schema.ts b/browser/lib/src/schema.ts index 8f8e92bf9..f615a5586 100644 --- a/browser/lib/src/schema.ts +++ b/browser/lib/src/schema.ts @@ -23,6 +23,7 @@ export type JsonValue = // would collide with the one above). export { freezeResources, + frozenIdFor, type FrozenId, type FreezableResource, type FrozenResource, diff --git a/browser/lib/src/store.ts b/browser/lib/src/store.ts index 6217f4551..0c93a9c0b 100644 --- a/browser/lib/src/store.ts +++ b/browser/lib/src/store.ts @@ -23,12 +23,16 @@ import { JSONADParser } from './parse.js'; import { Resource, unknownSubject } from './resource.js'; import { type SearchOpts, buildSearchSubject } from './search.js'; import { + freezeSchema, SCHEMA_HASH_PROPERTY, schemaToOntologyModel, type AtomicSchemaPackage, type ConvertedSchemaPackage, type DefinedSchema, + type FrozenSchema, } from './schema.js'; +import { frozenIdFor, UNIT_MEMBERS_KEY, type FrozenId } from './freeze.js'; +import { jcsCanonicalize } from './jcs.js'; import { stringToSlug } from './stringToSlug.js'; import { bytesToHex, hexToBytes, type JSONValue } from './value.js'; import { WSClient } from './websockets.js'; @@ -1798,6 +1802,57 @@ export class Store { return this.schemaHashIndex.get(schemaHash); } + /** + * Content-addressed schema registration: freezes the schema into immutable + * `did:ad:frozen` resources, materializes them into the local store so + * `getResource`/`getProperty` resolve them immediately (offline), and — with + * `{ save: true }` — publishes the canonical bytes to `/frozen/{hash}` so other + * stores can resolve them. Returns the {@link FrozenSchema} (frozen ids per + * developer key, plus the mutable `presentation` layer). Unlike + * {@link registerSchema}, identity is the content hash, not a signed DID. + */ + public async registerFrozenSchema( + schema: AtomicSchemaPackage | DefinedSchema, + opts: RegisterSchemaOptions = {}, + ): Promise { + const frozen = freezeSchema(schema); + + for (const { frozenId, content } of frozen.resources) { + const [resource] = new JSONADParser().parse(content, frozenId); + resource.loading = false; + this.addResource(resource, { skipCommitCompare: true }); + } + + if (opts.save) { + await Promise.all( + frozen.resources.map(({ frozenId, content }) => + this.publishFrozenResource(frozenId, content), + ), + ); + } + + return frozen; + } + + private async publishFrozenResource( + frozenId: FrozenId, + content: unknown, + ): Promise { + const hash = frozenId.replace('did:ad:frozen:', ''); + const base = this.getServerUrl().replace(/\/$/, ''); + const response = await fetch(`${base}/frozen/${hash}`, { + method: 'PUT', + headers: { 'Content-Type': 'application/ad+json' }, + body: jcsCanonicalize(content as Parameters[0]), + }); + + if (!response.ok) { + throw new Error( + `Failed to publish frozen resource ${frozenId} (HTTP ${response.status})`, + ); + } + } + private assertCompatibleExistingSchemaProperty( property: ConvertedSchemaPackage['properties'][number], ): void { @@ -2390,6 +2445,48 @@ export class Store { return work; } + /** + * Resolves a `did:ad:frozen:` subject: fetches the JSON-AD bytes from + * `/frozen/{hash}`, verifies they hash to the id (trustless — the server is + * just a cache), and materializes a read-only Resource. Cycle "unit" objects + * are not yet materializable individually. + */ + private async fetchFrozenResource( + subject: string, + ): Promise> { + const pureId = subject.split('?')[0].split('#')[0]; + const hash = pureId.replace('did:ad:frozen:', ''); + const base = this.getServerUrl().replace(/\/$/, ''); + const response = await fetch(`${base}/frozen/${hash}`, { + headers: { Accept: 'application/ad+json' }, + }); + + if (!response.ok) { + throw new Error( + `Could not resolve frozen resource ${pureId} (HTTP ${response.status})`, + ); + } + + const body = JSON.parse(await response.text()); + + // Trustless: the bytes must hash to the requested id. + if (frozenIdFor(body) !== pureId) { + throw new Error(`Frozen resource ${pureId} failed hash verification`); + } + + if (body && typeof body === 'object' && UNIT_MEMBERS_KEY in body) { + throw new Error( + `Frozen resource ${pureId} is a reference-cycle unit; individual materialization is not yet supported.`, + ); + } + + const [resource] = new JSONADParser().parse(body, pureId); + resource.loading = false; + this.addResource(resource as Resource, { skipCommitCompare: true }); + + return resource as Resource; + } + private async _fetchResourceFromServerImpl< C extends OptionalClass = UnknownClass, >( @@ -2425,6 +2522,13 @@ export class Store { return local; } + // Frozen resources are content-addressed and immutable: fetch their bytes + // from `/frozen/{hash}`, verify by re-hash, and materialize locally — no + // commits, no auth, no trust in the source. + if (normalizedSubject.startsWith('did:ad:frozen:')) { + return this.fetchFrozenResource(normalizedSubject); + } + if (opts.setLoading) { const newR = new Resource(subject); newR.loading = true; @@ -2722,21 +2826,18 @@ export class Store { ); } + // Description is presentation, not identity — content-addressed `did:ad:frozen` + // properties omit it deliberately. Default to empty rather than rejecting, + // so a frozen property still resolves and validates. const description = resource.get(core.properties.description); - if (description === undefined) { - throw Error( - `Property ${subject} has no description: ${resource.debugValueSummary()}`, - ); - } - const classTypeURL = resource.get(core.properties.classtype)?.toString(); const propery: Property = { subject, classType: classTypeURL, shortname: shortname.toString(), - description: description.toString(), + description: description?.toString() ?? '', datatype: datatypeFromUrl(datatypeUrl.toString()), allowsOnly: resource.get(core.properties.allowsOnly), }; diff --git a/browser/lib/tests/frozen-e2e.integration.test.ts b/browser/lib/tests/frozen-e2e.integration.test.ts new file mode 100644 index 000000000..315d17ae9 --- /dev/null +++ b/browser/lib/tests/frozen-e2e.integration.test.ts @@ -0,0 +1,76 @@ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { setTimeout as delay } from 'node:timers/promises'; + +import { Agent } from '../src/agent.js'; +import { Store } from '../src/store.js'; +import { Datatype } from '../src/datatypes.js'; +import { core } from '../src/ontologies/core.js'; +import { startServer, type ServerHandle } from './server-fixture.js'; + +const todoSchema = { + name: 'FrozenTodoApp', + classes: { + todo: { + type: 'object' as const, + required: ['title'], + properties: { + title: { type: 'string' as const, description: 'Task title' }, + done: { type: 'boolean' as const }, + }, + }, + }, +}; + +describe('frozen schema: publish + resolve through a real server', () => { + let server: ServerHandle; + + beforeAll(async () => { + server = await startServer(); + }, 60_000); + + afterAll(async () => { + await server?.stop(); + }); + + it('producer publishes frozen resources; a fresh consumer resolves and uses them', async () => { + const agent = await Agent.fromSecret(server.agentSecret); + const producer = new Store({ serverUrl: server.serverUrl, agent }); + producer.setServerConnected(true); + + // Freeze + PUT every frozen body to /frozen on the real server. + const frozen = await producer.registerFrozenSchema(todoSchema, { + save: true, + }); + const titleId = frozen.properties['todo.title']; + const classId = frozen.classes.todo; + + expect(titleId).toMatch(/^did:ad:frozen:[0-9a-f]{64}$/); + + // A brand-new consumer with no shared memory and no agent. + const consumer = new Store({ serverUrl: server.serverUrl }); + consumer.setServerConnected(true); + + // Give the server a beat to have the bytes durably available. + await delay(250); + + // Resolves over HTTP: GET /frozen -> re-hash verify -> materialize. + const prop = await consumer.getProperty(titleId); + expect(prop.shortname).toBe('title'); + expect(prop.datatype).toBe(Datatype.STRING); + + const todoClass = await consumer.getResource(classId); + expect(todoClass.get(core.properties.shortname)).toBe('todo'); + expect(todoClass.get(core.properties.requires)).toEqual([titleId]); + + // The consumer can build an instance against the frozen class + property. + const todo = await consumer.newResource({ + isA: classId, + propVals: { [titleId]: 'Buy milk' }, + }); + expect(todo.getClasses()).toContain(classId); + expect(todo.get(titleId)).toBe('Buy milk'); + + producer.disconnect(); + consumer.disconnect(); + }, 60_000); +}); diff --git a/lib/src/commit.rs b/lib/src/commit.rs index b6090f2c6..5f283b0e1 100644 --- a/lib/src/commit.rs +++ b/lib/src/commit.rs @@ -353,6 +353,16 @@ impl Commit { let commit = self; let subject = commit.subject.clone(); + // Frozen resources are immutable, content-addressed, and signatureless — + // they live in Tree::Frozen and are verified by re-hash, never edited. + if subject.is_frozen_did() { + return Err(format!( + "Cannot commit to {}: `did:ad:frozen` resources are immutable and content-addressed.", + subject + ) + .into()); + } + if subject.is_did() && subject.as_str().starts_with("did:ad:") { let pure_id = subject.pure_id(); let b64_part = if subject.is_agent_did() { @@ -1870,6 +1880,42 @@ mod test { ); } + #[tokio::test] + async fn commit_to_frozen_subject_is_rejected() { + let (store, _agent) = store_with_known_agent().await; + let commit = Commit { + subject: + "did:ad:frozen:0000000000000000000000000000000000000000000000000000000000000000" + .into(), + created_at: 0, + signer: "did:ad:agent:placeholder".into(), + loro_update: None, + destroy: None, + signature: Some("placeholder".to_string()), + previous_commit: None, + is_genesis: None, + url: None, + }; + let opts = CommitOpts { + validate_signature: false, + validate_timestamp: false, + validate_previous_commit: false, + validate_rights: false, + ..CommitOpts::no_validations_no_index() + }; + // The frozen check fires before signature validation, so an invalid + // signature here is irrelevant — immutability is enforced first. + let err = commit + .validate_and_build_response(&opts, &store) + .await + .unwrap_err(); + assert!( + err.to_string().to_lowercase().contains("immutable"), + "expected an immutability error, got: {}", + err + ); + } + /// Signing a commit with agent B but writing to a resource that agent A /// created (and owns) must fail signature validation. #[tokio::test] diff --git a/lib/src/db.rs b/lib/src/db.rs index d4cfae23b..91dffbac0 100644 --- a/lib/src/db.rs +++ b/lib/src/db.rs @@ -850,6 +850,51 @@ impl Db { store.get_resource_extended(subject, false, for_agent).await } + /// Resolves a `did:ad:frozen:` subject from [`Tree::Frozen`]: loads the + /// stored JSON-AD bytes, **verifies they hash to the requested id** + /// (trustless — no signature, no trust in the source), and parses them into + /// a read-only Resource. Immutability is enforced elsewhere by rejecting + /// commits to frozen subjects. Cycle "unit" objects are not yet + /// materializable as individual resources. + async fn materialize_frozen(&self, subject: &Subject) -> AtomicResult { + let id = subject.pure_id(); + let hash_hex = subject + .frozen_hash_hex() + .ok_or_else(|| AtomicError::not_found(format!("Invalid frozen subject: {}", id)))?; + let bytes = self + .kv + .get(Tree::Frozen, hash_hex.as_bytes())? + .ok_or_else(|| AtomicError::not_found(format!("Frozen resource not found: {}", id)))?; + let body: serde_json::Value = serde_json::from_slice(&bytes) + .map_err(|e| format!("Stored frozen body for {} is not valid JSON: {}", id, e))?; + + // Trustless verification: the stored bytes must hash to the requested id. + crate::frozen::verify_frozen(&id, &body)?; + + if crate::frozen::is_unit(&body) { + return Err(format!( + "Frozen subject {} is a reference-cycle unit; materializing individual members is not yet supported.", + id + ) + .into()); + } + + let serde_json::Value::Object(map) = body else { + return Err(format!("Frozen body for {} must be a JSON object", id).into()); + }; + + // DontSave: a frozen resource is read-only — never write it to + // Tree::Resources and never run class-`requires` validation (a frozen + // definition deliberately omits presentation like `description`; its + // validity is the hash, not class completeness). + let parse_opts = crate::parse::ParseOpts { + save: crate::parse::SaveOpts::DontSave, + skip_unknown_props: true, + ..Default::default() + }; + crate::parse::parse_json_ad_map_to_resource(map, self, Some(id), &parse_opts).await + } + pub fn add_class_extender(&self, class_extender: ClassExtender) -> AtomicResult<()> { let mut extenders = self .class_extenders @@ -2027,6 +2072,11 @@ impl Storelike for Db { #[instrument(skip_all)] async fn get_resource(&self, subject: &Subject) -> AtomicResult { let normalized = self.normalize_subject(subject); + // Frozen resources are content-addressed and immutable; they live in + // Tree::Frozen, not Tree::Resources, and materialize by re-hash + parse. + if normalized.is_frozen_did() { + return self.materialize_frozen(&normalized).await; + } let subject_str = normalized.pure_id(); if let Ok(propvals) = self.get_propvals(&subject_str) { let mut res_subject = normalized.clone(); diff --git a/lib/src/db/redb_store.rs b/lib/src/db/redb_store.rs index eeaef3508..127fbd4ca 100644 --- a/lib/src/db/redb_store.rs +++ b/lib/src/db/redb_store.rs @@ -31,6 +31,7 @@ const TABLE_DRIVE_MAPPING: TableDefinition<&[u8], &[u8]> = TableDefinition::new( const TABLE_DID_MAPPING: TableDefinition<&[u8], &[u8]> = TableDefinition::new("did_mapping"); const TABLE_LORO_SNAPSHOTS: TableDefinition<&[u8], &[u8]> = TableDefinition::new("loro_snapshots"); const TABLE_BLOBS: TableDefinition<&[u8], &[u8]> = TableDefinition::new("blobs"); +const TABLE_FROZEN: TableDefinition<&[u8], &[u8]> = TableDefinition::new("frozen_v1"); fn table_def(tree: Tree) -> TableDefinition<'static, &'static [u8], &'static [u8]> { match tree { @@ -44,6 +45,7 @@ fn table_def(tree: Tree) -> TableDefinition<'static, &'static [u8], &'static [u8 Tree::DidMapping => TABLE_DID_MAPPING, Tree::LoroSnapshots => TABLE_LORO_SNAPSHOTS, Tree::Blobs => TABLE_BLOBS, + Tree::Frozen => TABLE_FROZEN, } } @@ -151,6 +153,7 @@ impl RedbStore { let _ = tx.open_table(TABLE_DID_MAPPING); let _ = tx.open_table(TABLE_LORO_SNAPSHOTS); let _ = tx.open_table(TABLE_BLOBS); + let _ = tx.open_table(TABLE_FROZEN); tx.commit() .map_err(|e| format!("Failed to commit initial tables: {e}"))?; } @@ -186,6 +189,7 @@ impl RedbStore { let _ = tx.open_table(TABLE_DID_MAPPING); let _ = tx.open_table(TABLE_LORO_SNAPSHOTS); let _ = tx.open_table(TABLE_BLOBS); + let _ = tx.open_table(TABLE_FROZEN); tx.commit() .map_err(|e| format!("Failed to commit initial tables: {e}"))?; } @@ -224,6 +228,7 @@ impl RedbStore { let _ = tx.open_table(TABLE_DID_MAPPING); let _ = tx.open_table(TABLE_LORO_SNAPSHOTS); let _ = tx.open_table(TABLE_BLOBS); + let _ = tx.open_table(TABLE_FROZEN); tx.commit() .map_err(|e| format!("Failed to commit initial tables: {e}"))?; } diff --git a/lib/src/db/sled_store.rs b/lib/src/db/sled_store.rs index 283402217..b28e6e7c9 100644 --- a/lib/src/db/sled_store.rs +++ b/lib/src/db/sled_store.rs @@ -24,6 +24,7 @@ pub struct SledStore { did_mapping: sled::Tree, loro_snapshots: sled::Tree, blobs: sled::Tree, + frozen: sled::Tree, } impl SledStore { @@ -47,6 +48,7 @@ impl SledStore { let did_mapping = db.open_tree(Tree::DidMapping)?; let loro_snapshots = db.open_tree(Tree::LoroSnapshots)?; let blobs = db.open_tree(Tree::Blobs)?; + let frozen = db.open_tree(Tree::Frozen)?; Ok(SledStore { db, @@ -60,6 +62,7 @@ impl SledStore { did_mapping, loro_snapshots, blobs, + frozen, }) } @@ -80,6 +83,7 @@ impl SledStore { Tree::DidMapping => &self.did_mapping, Tree::LoroSnapshots => &self.loro_snapshots, Tree::Blobs => &self.blobs, + Tree::Frozen => &self.frozen, } } } @@ -159,6 +163,7 @@ impl KvStore for SledStore { let mut batch_did_mapping = sled::Batch::default(); let mut batch_loro_snapshots = sled::Batch::default(); let mut batch_blobs = sled::Batch::default(); + let mut batch_frozen = sled::Batch::default(); for op in operations { let batch = match op.tree { @@ -172,6 +177,7 @@ impl KvStore for SledStore { Tree::DidMapping => &mut batch_did_mapping, Tree::LoroSnapshots => &mut batch_loro_snapshots, Tree::Blobs => &mut batch_blobs, + Tree::Frozen => &mut batch_frozen, }; match op.method { Method::Insert => { @@ -226,6 +232,10 @@ impl KvStore for SledStore { .apply_batch(batch_blobs) .map_err(|e| format!("Failed to apply blobs batch: {}", e))?; + self.frozen + .apply_batch(batch_frozen) + .map_err(|e| format!("Failed to apply frozen batch: {}", e))?; + Ok(()) } diff --git a/lib/src/db/test.rs b/lib/src/db/test.rs index a73d08582..0c20f8f14 100644 --- a/lib/src/db/test.rs +++ b/lib/src/db/test.rs @@ -656,6 +656,70 @@ async fn blobs_storage() { assert_eq!(data.to_vec(), retrieved); } +#[tokio::test] +async fn frozen_storage() { + let store = Db::init_temp("frozen_storage").await.unwrap(); + // A frozen object is content-addressed by the BLAKE3 of its JCS bytes. + let body = serde_json::json!({ + "https://atomicdata.dev/properties/shortname": "title", + "https://atomicdata.dev/properties/datatype": "https://atomicdata.dev/datatypes/string" + }); + let id = crate::frozen::frozen_id(&body).unwrap(); + let hash_hex = id.strip_prefix("did:ad:frozen:").unwrap(); + let bytes = serde_jcs::to_string(&body).unwrap().into_bytes(); + + store + .kv + .insert(Tree::Frozen, hash_hex.as_bytes(), &bytes) + .unwrap(); + let retrieved = store + .kv + .get(Tree::Frozen, hash_hex.as_bytes()) + .unwrap() + .unwrap(); + + // Round-trips, and the stored bytes still verify against the id. + let parsed: serde_json::Value = serde_json::from_slice(&retrieved).unwrap(); + assert!(crate::frozen::verify_frozen(&id, &parsed).is_ok()); +} + +#[tokio::test] +async fn frozen_materialization() { + let store = Db::init_temp("frozen_materialization").await.unwrap(); + let body = serde_json::json!({ + "https://atomicdata.dev/properties/isA": ["https://atomicdata.dev/classes/Property"], + "https://atomicdata.dev/properties/shortname": "title", + "https://atomicdata.dev/properties/datatype": "https://atomicdata.dev/datatypes/string" + }); + let id = crate::frozen::frozen_id(&body).unwrap(); + let hash_hex = id.strip_prefix("did:ad:frozen:").unwrap(); + let bytes = serde_jcs::to_string(&body).unwrap().into_bytes(); + store + .kv + .insert(Tree::Frozen, hash_hex.as_bytes(), &bytes) + .unwrap(); + + // Resolving the frozen subject materializes a read-only Resource. + let subject = Subject::from_raw(&id, None); + let resource = store.get_resource(&subject).await.unwrap(); + + assert_eq!(resource.get_subject().as_str(), id); + assert_eq!(resource.get(urls::SHORTNAME).unwrap().to_string(), "title"); + assert_eq!( + resource.get(urls::DATATYPE_PROP).unwrap().to_string(), + "https://atomicdata.dev/datatypes/string" + ); + + // Bytes that don't hash to the addressed id are rejected on resolve. + let bogus_hex = "00".repeat(32); + store + .kv + .insert(Tree::Frozen, bogus_hex.as_bytes(), b"{\"x\":1}") + .unwrap(); + let bogus_subject = Subject::from_raw(&format!("did:ad:frozen:{}", bogus_hex), None); + assert!(store.get_resource(&bogus_subject).await.is_err()); +} + #[tokio::test] /// Changing these values actually correctly updates the index. async fn invalidate_cache() { diff --git a/lib/src/db/trees.rs b/lib/src/db/trees.rs index e505eb2d6..f717ec79b 100644 --- a/lib/src/db/trees.rs +++ b/lib/src/db/trees.rs @@ -27,6 +27,10 @@ pub enum Tree { LoroSnapshots, /// Content-addressed storage for binary files, keyed by BLAKE3 hash. Blobs, + /// Content-addressed storage for immutable `did:ad:frozen` JSON-AD bodies, + /// keyed by the 32-byte BLAKE3 hash of their JCS canonicalization. Unlike + /// `Blobs` (opaque bytes), these materialize into read-only Resources. + Frozen, } const RESOURCES: &str = "resources_v3"; @@ -44,6 +48,7 @@ const DRIVE_MAPPING: &str = "drive_mapping"; const DID_MAPPING: &str = "did_mapping"; const LORO_SNAPSHOTS: &str = "loro_snapshots"; const BLOBS: &str = "blobs"; +const FROZEN: &str = "frozen_v1"; impl std::fmt::Display for Tree { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { @@ -58,6 +63,7 @@ impl std::fmt::Display for Tree { Tree::DidMapping => f.write_str(DID_MAPPING), Tree::LoroSnapshots => f.write_str(LORO_SNAPSHOTS), Tree::Blobs => f.write_str(BLOBS), + Tree::Frozen => f.write_str(FROZEN), } } } @@ -76,6 +82,7 @@ impl AsRef<[u8]> for Tree { Tree::DidMapping => DID_MAPPING.as_bytes(), Tree::LoroSnapshots => LORO_SNAPSHOTS.as_bytes(), Tree::Blobs => BLOBS.as_bytes(), + Tree::Frozen => FROZEN.as_bytes(), } } } diff --git a/lib/src/frozen.rs b/lib/src/frozen.rs new file mode 100644 index 000000000..656da5597 --- /dev/null +++ b/lib/src/frozen.rs @@ -0,0 +1,100 @@ +//! Content-addressed `did:ad:frozen` resources. +//! +//! A frozen resource is identified by `did:ad:frozen:{blake3-hex}` over the +//! RFC 8785 (JCS) canonicalization of its JSON-AD body. This is the exact hash +//! the TypeScript producer computes +//! (`browser/lib/src/freeze.ts#frozenIdFor`), so ids are byte-for-byte +//! reproducible across languages. The shared contract is pinned by +//! `test-vectors/frozen.json`. +//! +//! Frozen objects are immutable and signatureless: they are verified by +//! re-hashing, never by a commit signature. See `planning/did-ad-frozen-server.md`. + +use crate::errors::AtomicResult; +use crate::subject::DID_AD_FROZEN_PREFIX; + +/// Reserved top-level key marking a frozen **unit** object: the materialized form +/// of a reference cycle, whose value is an ordered array of member bodies that +/// reference each other by `did:ad:frozen:self:{index}`. Matches +/// `browser/lib/src/freeze.ts#UNIT_MEMBERS_KEY`. +pub const FROZEN_UNIT_KEY: &str = "urn:atomic-freeze:unit"; + +/// True if a frozen body is a cycle unit (vs. a single resource body). +pub fn is_unit(body: &serde_json::Value) -> bool { + body.get(FROZEN_UNIT_KEY).is_some() +} + +/// Computes the `did:ad:frozen:{blake3-hex}` id for a JSON-AD body. +pub fn frozen_id(body: &serde_json::Value) -> AtomicResult { + let canonical = serde_jcs::to_string(body) + .map_err(|e| format!("Failed to JCS-canonicalize frozen body: {}", e))?; + let hash = blake3::hash(canonical.as_bytes()); + + Ok(format!("{}{}", DID_AD_FROZEN_PREFIX, hash.to_hex())) +} + +/// Returns `Ok(())` when `body` hashes to `id`, otherwise an error. This is the +/// verify-by-rehash check the server runs on store and serve; no signature or +/// trust in the source is required. +pub fn verify_frozen(id: &str, body: &serde_json::Value) -> AtomicResult<()> { + let actual = frozen_id(body)?; + + if actual == id { + Ok(()) + } else { + Err(format!( + "Frozen body hashes to {} but was addressed as {}", + actual, id + ) + .into()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[derive(serde::Deserialize)] + struct Vector { + name: String, + body: serde_json::Value, + id: String, + } + + #[derive(serde::Deserialize)] + struct Vectors { + vectors: Vec, + } + + /// Proves the Rust frozen id matches the TypeScript producer for every + /// shared vector. A failure here is a cross-language identity break. + #[test] + fn matches_cross_language_vectors() { + let raw = include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../test-vectors/frozen.json" + )); + let parsed: Vectors = serde_json::from_str(raw).expect("valid fixture"); + + assert!(!parsed.vectors.is_empty(), "fixture has no vectors"); + + for vector in parsed.vectors { + assert_eq!( + frozen_id(&vector.body).unwrap(), + vector.id, + "frozen id mismatch for vector {}", + vector.name + ); + } + } + + #[test] + fn verify_frozen_rejects_a_mismatch() { + let body = serde_json::json!({ "a": 1 }); + let wrong = + "did:ad:frozen:0000000000000000000000000000000000000000000000000000000000000000"; + + assert!(verify_frozen(&frozen_id(&body).unwrap(), &body).is_ok()); + assert!(verify_frozen(wrong, &body).is_err()); + } +} diff --git a/lib/src/lib.rs b/lib/src/lib.rs index 5ccc5e70a..1c75e23c9 100644 --- a/lib/src/lib.rs +++ b/lib/src/lib.rs @@ -80,6 +80,7 @@ pub mod discovery; #[cfg(feature = "db")] pub mod endpoints; pub mod errors; +pub mod frozen; pub mod hierarchy; /// Resource version history (time-travel reads). Prefer this over `loro` in app code. pub mod history { diff --git a/lib/src/parse.rs b/lib/src/parse.rs index 23c11954d..9b32c4319 100644 --- a/lib/src/parse.rs +++ b/lib/src/parse.rs @@ -568,7 +568,11 @@ pub fn parse_propval<'a>( /// Parse a single Json AD string, convert to Atoms /// Adds to the store if `add` is true. #[tracing::instrument(skip_all)] -async fn parse_json_ad_map_to_resource( +/// Parse a JSON-AD object (property-URL -> value map) into a [Resource]. +/// `overwrite_subject` forces the resource's subject (and forbids an inline +/// `@id`), which is what frozen materialization uses — the subject is the +/// content hash, never stored in the body. +pub async fn parse_json_ad_map_to_resource( json: Map, store: &impl crate::Storelike, overwrite_subject: Option, diff --git a/lib/src/subject.rs b/lib/src/subject.rs index 957b2e6cc..6137f683d 100644 --- a/lib/src/subject.rs +++ b/lib/src/subject.rs @@ -11,6 +11,13 @@ pub const DID_AD_COMMIT_PREFIX: &str = "did:ad:commit:"; /// 32-byte BLAKE3 hash of the bytes, hex-encoded (64 chars). pub const DID_AD_BLOB_PREFIX: &str = "did:ad:blob:"; +/// The prefix for Frozen DIDs: `did:ad:frozen:`. The remainder is the +/// 32-byte BLAKE3 hash, hex-encoded (64 chars), of the RFC 8785 (JCS) +/// canonicalization of an immutable JSON-AD body. Unlike a blob, a frozen +/// subject resolves to structured JSON-AD that materializes into a read-only +/// Resource. See `crate::frozen`. +pub const DID_AD_FROZEN_PREFIX: &str = "did:ad:frozen:"; + /// The Subject of a Resource. /// /// In Atomic Data, every subject is a URI. @@ -236,6 +243,45 @@ impl Subject { } } + /// Returns true if this is a DID Frozen subject (did:ad:frozen:). + pub fn is_frozen_did(&self) -> bool { + match self { + Subject::Did { url, .. } => url.as_str().starts_with(DID_AD_FROZEN_PREFIX), + _ => false, + } + } + + /// If this is a `did:ad:frozen:` subject, returns the hex-encoded BLAKE3 + /// hash (the part after the prefix, with any `?drive=` hint stripped). + /// Returns `None` for any other variant. + pub fn frozen_hash_hex(&self) -> Option<&str> { + match self { + Subject::Did { url, .. } => { + let rest = url.as_str().strip_prefix(DID_AD_FROZEN_PREFIX)?; + // Drop query (`?drive=...`) / fragment if present. + let end = rest.find(['?', '#']).unwrap_or(rest.len()); + Some(&rest[..end]) + } + _ => None, + } + } + + /// Construct a `did:ad:frozen:` subject from a 32-byte BLAKE3 hash. + pub fn from_frozen_hash(hash: &[u8; 32]) -> Self { + let mut hex = String::with_capacity(DID_AD_FROZEN_PREFIX.len() + 64); + hex.push_str(DID_AD_FROZEN_PREFIX); + for byte in hash { + // Inline lowercase-hex; avoids pulling in the `hex` crate just for this. + hex.push(char::from_digit((byte >> 4) as u32, 16).unwrap()); + hex.push(char::from_digit((byte & 0xf) as u32, 16).unwrap()); + } + // Url::parse on a `did:ad:frozen:` always succeeds (hex is RFC-3986 safe). + Subject::Did { + url: Url::parse(&hex).expect("valid did:ad:frozen: URL"), + drive_hint: None, + } + } + /// Returns true if this is an internal subject (mapped to the server's base domain). pub fn is_internal(&self) -> bool { matches!(self, Subject::Internal { .. }) @@ -585,6 +631,44 @@ mod tests { ); } + #[test] + fn test_frozen_did_parsing() { + let frozen_did = + "did:ad:frozen:af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262"; + let subject = Subject::from_raw(frozen_did, None); + + assert!(matches!(subject, Subject::Did { .. })); + assert!(subject.is_frozen_did()); + assert!(!subject.is_blob_did()); + assert!(!subject.is_commit_did()); + assert_eq!( + subject.frozen_hash_hex(), + Some("af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262") + ); + + // Roundtrip via raw bytes. + let mut bytes = [0u8; 32]; + for (i, byte) in bytes.iter_mut().enumerate() { + *byte = i as u8; + } + let from_bytes = Subject::from_frozen_hash(&bytes); + assert!(from_bytes.is_frozen_did()); + assert_eq!( + from_bytes.frozen_hash_hex(), + Some("000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f") + ); + + // Drive hint is preserved, hash extraction strips it. + let with_drive = format!("{}?drive=did:ad:abc", frozen_did); + let routed = Subject::from_raw(&with_drive, None); + assert!(routed.is_frozen_did()); + assert_eq!(routed.drive_hint(), Some("did:ad:abc")); + assert_eq!( + routed.frozen_hash_hex(), + Some("af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262") + ); + } + #[test] fn test_did_drive_hint_parsing() { let did_with_drive = "did:ad:123?drive=abc"; diff --git a/planning/did-ad-frozen-server.md b/planning/did-ad-frozen-server.md index afc66d6f7..a0d936e9c 100644 --- a/planning/did-ad-frozen-server.md +++ b/planning/did-ad-frozen-server.md @@ -90,50 +90,71 @@ needed. ### Phase A — lib + server storage, parsing, materialization -- [ ] `lib/src/subject.rs`: add `is_frozen_did`, `frozen_hash_hex`, - `from_frozen_hash`, mirroring the blob helpers. `pure_id()` already strips - `?drive=`; subject equality is already string-based — reuse both. -- [ ] `lib/src/db/trees.rs`: add `Tree::Frozen` (key = 32-byte BLAKE3, value = - canonical JSON-AD bytes). -- [ ] `lib/src/db.rs#get_resource`: if `is_frozen_did(subject)`, look up - `Tree::Frozen`, verify hash, parse JSON-AD into a **read-only** Resource, - return it — bypassing `Tree::Resources`/Loro. Cache in memory like any - fetched resource. -- [ ] Reject any Commit whose subject is `did:ad:frozen:` (immutable; no edits). -- [ ] Use **RFC 8785 JCS** for the canonical JSON-AD bytes on the Rust side - (`serde_jcs` / `serde_json_canonicalizer`) so it byte-matches the TS - producer (`browser/lib/src/jcs.ts#jcsCanonicalize`, already used by - `freeze.ts`). Lock it with shared test vectors (fixed schema → fixed hash, - asserted in both Rust and TS). **This is the sharpest correctness risk** — - a one-byte canonicalization difference makes ids diverge across languages, - so JCS (a named standard with conformant impls on both sides) is required. +- [x] `lib/src/frozen.rs`: `frozen_id` / `verify_frozen`. Done + cross-language + tested. +- [x] `lib/src/subject.rs`: `is_frozen_did`, `frozen_hash_hex`, + `from_frozen_hash` (+ `DID_AD_FROZEN_PREFIX`), mirroring the blob helpers. + Done + tested (`test_frozen_did_parsing`, incl. `?drive=` hint stripping). +- [x] `lib/src/db/trees.rs`: `Tree::Frozen` (key = BLAKE3 hex, value = canonical + JSON-AD bytes), wired through both backends (`sled_store`, `redb_store`). + Done + storage round-trip test (`db::test::frozen_storage`). +- [x] `lib/src/db.rs#get_resource`: if `is_frozen_did(subject)`, + `materialize_frozen` looks up `Tree::Frozen`, **verifies by re-hash** + (trustless), and parses the JSON-AD into a read-only Resource via + `parse_json_ad_map_to_resource` with `SaveOpts::DontSave` — bypassing + `Tree::Resources`/Loro and, crucially, class-`requires` validation (a frozen + definition is valid by its hash, not by completeness, so omitting + `description` is fine). Cycle "unit" objects error for now. Done + tested + (`db::test::frozen_materialization`, incl. re-hash rejection of a mismatched + body). +- [x] Reject any Commit whose subject is `did:ad:frozen:` (immutable). Done: + `commit.rs#validate_and_build_response` rejects frozen subjects up front + (before signature checks). Tested (`commit_to_frozen_subject_is_rejected`). +- [x] Use **RFC 8785 JCS** for the canonical JSON-AD bytes on the Rust side. + Done: `lib/src/frozen.rs#frozen_id` uses the already-present `serde_jcs` + + `blake3`, and `test-vectors/frozen.json` pins the contract. Both + `browser/lib/src/frozen-vectors.test.ts` (TS) and the `frozen.rs` test + assert identical ids; verified byte-for-byte across all vectors (incl. + unicode key ordering and a cycle unit). **The sharpest correctness risk is + retired.** ### Phase B — server endpoints -- [ ] `PUT /frozen/{hash}`: accept bytes, verify `blake3 == hash`, store - idempotently (re-post is a no-op, like `/blob`). Auth: require an - authenticated agent (forgery is impossible since content is hash-checked, so - the only risk is storage spam — gate with auth + a size/rate limit). -- [ ] `GET /frozen/{hash}`: return canonical JSON-AD bytes - (`content-type: application/ad+json`). -- [ ] Wire `did:ad:frozen:` subjects into the normal resource GET path so they - resolve through the same `/` endpoint and the WebSocket `GET` frame, not - only the dedicated route. +- [x] `PUT /frozen/{hash}`: parses the JSON-AD body, verifies + `frozen_id(body) == did:ad:frozen:{hash}` (blake3 of JCS, not raw bytes), + stores the canonical bytes in `Tree::Frozen` idempotently. Public, like + `/blob` (the hash is the capability). Done: `server/src/handlers/frozen.rs`. +- [x] `GET /frozen/{hash}`: returns the stored JSON-AD bytes + (`content-type: application/ad+json`), 404 if absent. +- [x] End-to-end HTTP test (`server::tests::frozen_endpoint_roundtrip`): + PUT -> GET round-trips, a wrong-hash PUT is rejected, and the stored body + resolves through the normal `get_resource` materialization path. +- [ ] _(optional)_ Wire `did:ad:frozen:` into the WebSocket `GET` frame / sync so + frozen objects travel with drives (Phase D), not only the dedicated route. + `get_resource` already resolves them for the HTTP `/` path. ### Phase C — browser Store + registerSchema switch-over -- [ ] Store: detect `did:ad:frozen:` in `getResource`/`fetchResourceFromServer`, - fetch (local cache → `GET /frozen` → sync), **re-hash to verify**, parse - JSON-AD into a read-only Resource, cache. `getProperty` then works - unchanged. -- [ ] Persist frozen objects in ClientDb (OPFS) keyed by hash for offline use. -- [ ] `Store.registerSchema`: replace genesis-DID minting with `freezeSchema`; - write frozen objects locally and optionally `PUT /frozen` to the author's - server; return the frozen ids. Keep the old path behind a flag during - migration so existing tests/producers keep working until parity is proven. -- [ ] The signed **"latest version" pointer** stays a normal genesis-DID resource - on the author's drive (`name → latest frozen ontology id`), plus any - mutable display overlay the app chooses to keep editable. +- [x] Store: `fetchFrozenResource` detects `did:ad:frozen:` in the fetch path, + fetches `GET /frozen/{hash}`, **re-hashes to verify** (`frozenIdFor`), and + materializes a read-only Resource via `JSONADParser`. `getProperty` was + relaxed to treat `description` as optional (presentation, not identity), so + frozen properties resolve. Tested (`frozen-resolve.test.ts`). +- [x] `Store.registerFrozenSchema` (additive — leaves the signed-DID + `registerSchema` intact): freezes the schema, materializes the frozen bodies + into the local store (offline-resolvable immediately), and with + `{ save: true }` PUTs each to `/frozen/{hash}`. Returns the `FrozenSchema` + (frozen ids per key + `presentation`). Tested. +- [x] **Capstone e2e** (`tests/frozen-e2e.integration.test.ts`, real server): + producer `registerFrozenSchema(..., { save: true })` PUTs frozen bodies; a + fresh agent-less consumer `getProperty`/`getResource` resolves them over + `GET /frozen` (verify-by-rehash), refs intact, and builds an instance + against the frozen class + property. Proves the whole producer→server→ + consumer loop live. +- [ ] Persist frozen objects in ClientDb (OPFS) keyed by hash for offline reload + (currently in-memory only). +- [ ] The signed **"latest version" pointer** on the author's drive + (`name → latest frozen ontology id`) + any mutable display overlay. ### Phase D — sync (frozen travels with drives) diff --git a/server/src/handlers/frozen.rs b/server/src/handlers/frozen.rs new file mode 100644 index 000000000..305e9f896 --- /dev/null +++ b/server/src/handlers/frozen.rs @@ -0,0 +1,71 @@ +use actix_web::{web, HttpResponse}; + +use crate::{appstate::AppState, errors::AtomicServerResult}; + +/// Content type for materializable JSON-AD frozen bodies. +const AD_JSON: &str = "application/ad+json"; + +fn validate_hash_hex(hash_hex: &str) -> AtomicServerResult<()> { + if hash_hex.len() != 64 || !hash_hex.bytes().all(|b| b.is_ascii_hexdigit()) { + return Err("Frozen hash must be 64 hex chars (BLAKE3)".into()); + } + Ok(()) +} + +/// HTTP fallback for pushing a `did:ad:frozen` body to the server. Unlike a blob +/// (opaque bytes hashed directly), a frozen body is JSON-AD and its id is +/// `blake3(JCS(body))` — so we parse, canonicalize, and check the content hash +/// matches the URL hash. A mismatch is rejected. +/// +/// Public on purpose: the hash is the capability, and storage is content- +/// addressed and immutable — re-posting the same hash is a no-op. We store the +/// canonical JCS bytes so reads round-trip and re-verify. +#[tracing::instrument(skip(appstate, body))] +pub async fn put_frozen( + path: web::Path, + appstate: web::Data, + body: web::Bytes, +) -> AtomicServerResult { + let hash_hex = path.into_inner(); + validate_hash_hex(&hash_hex)?; + + let parsed: serde_json::Value = + serde_json::from_slice(&body).map_err(|e| format!("Frozen body must be JSON: {e}"))?; + + let id = atomic_lib::frozen::frozen_id(&parsed)?; + let expected = format!("{}{}", atomic_lib::subject::DID_AD_FROZEN_PREFIX, hash_hex); + if id != expected { + return Err( + format!("Frozen body hashes to {id}, does not match URL hash {hash_hex}").into(), + ); + } + + let canonical = serde_jcs::to_string(&parsed).map_err(|e| e.to_string())?; + appstate.store.kv.insert( + atomic_lib::db::trees::Tree::Frozen, + hash_hex.as_bytes(), + canonical.as_bytes(), + )?; + + Ok(HttpResponse::NoContent().finish()) +} + +/// Serves the raw JSON-AD bytes of a frozen resource by hash. Content-addressed, +/// so the client re-verifies by re-hashing; the server is just a cache. +#[tracing::instrument(skip(appstate))] +pub async fn get_frozen( + path: web::Path, + appstate: web::Data, +) -> AtomicServerResult { + let hash_hex = path.into_inner(); + validate_hash_hex(&hash_hex)?; + + match appstate + .store + .kv + .get(atomic_lib::db::trees::Tree::Frozen, hash_hex.as_bytes())? + { + Some(bytes) => Ok(HttpResponse::Ok().content_type(AD_JSON).body(bytes)), + None => Ok(HttpResponse::NotFound().finish()), + } +} diff --git a/server/src/handlers/mod.rs b/server/src/handlers/mod.rs index 656047ded..56f55c334 100644 --- a/server/src/handlers/mod.rs +++ b/server/src/handlers/mod.rs @@ -9,6 +9,7 @@ pub mod blob; pub mod commit; pub mod download; pub mod export; +pub mod frozen; pub mod get_resource; #[cfg(feature = "image")] pub mod image; diff --git a/server/src/routes.rs b/server/src/routes.rs index 8c427146c..b66bbcb7a 100644 --- a/server/src/routes.rs +++ b/server/src/routes.rs @@ -207,6 +207,16 @@ pub fn config_routes(app: &mut actix_web::web::ServiceConfig) { .guard(guard::Method(Method::PUT)) .to(handlers::blob::put_blob), ) + .service( + web::resource("/frozen/{hash}") + .guard(guard::Method(Method::PUT)) + .to(handlers::frozen::put_frozen), + ) + .service( + web::resource("/frozen/{hash}") + .guard(guard::Method(Method::GET)) + .to(handlers::frozen::get_frozen), + ) .service( web::resource("/setup") .guard(guard::Method(Method::POST)) diff --git a/server/src/tests.rs b/server/src/tests.rs index 35f0487e9..b86224566 100644 --- a/server/src/tests.rs +++ b/server/src/tests.rs @@ -627,3 +627,75 @@ async fn upload_download_test() { let downloaded_bytes = test::read_body(resp).await; assert_eq!(downloaded_bytes, test_content.as_slice()); } + +#[actix_rt::test] +async fn frozen_endpoint_roundtrip() { + let unique_string = atomic_lib::utils::random_string(10); + use clap::Parser; + let opts = Opts::parse_from([ + "atomic-server", + "--initialize", + "--data-dir", + &format!("./.temp/{}/db", unique_string), + "--config-dir", + &format!("./.temp/{}/config", unique_string), + ]); + let mut config = config::build_config(opts).expect("failed init config"); + config.search_index_path = format!("./.temp/{}/search_index", unique_string).into(); + let appstate = crate::appstate::AppState::init(config.clone()) + .await + .expect("failed init appstate"); + atomic_lib::test_utils::setup_test_env(&appstate.store) + .await + .unwrap(); + let data = Data::new(appstate.clone()); + let app = test::init_service( + App::new() + .app_data(data) + .configure(crate::routes::config_routes), + ) + .await; + + // An identity-only frozen Property body. + let body = serde_json::json!({ + "https://atomicdata.dev/properties/isA": ["https://atomicdata.dev/classes/Property"], + "https://atomicdata.dev/properties/shortname": "title", + "https://atomicdata.dev/properties/datatype": "https://atomicdata.dev/datatypes/string" + }); + let id = atomic_lib::frozen::frozen_id(&body).unwrap(); + let hash = id.strip_prefix("did:ad:frozen:").unwrap(); + let canonical = serde_jcs::to_string(&body).unwrap(); + + // PUT stores it (content-addressed, public). + let put = TestRequest::put() + .uri(&format!("/frozen/{}", hash)) + .set_payload(canonical.clone()) + .to_request(); + let resp = test::call_service(&app, put).await; + assert!(resp.status().is_success(), "PUT status: {}", resp.status()); + + // GET returns the same canonical bytes. + let get = TestRequest::get() + .uri(&format!("/frozen/{}", hash)) + .to_request(); + let resp = test::call_service(&app, get).await; + assert!(resp.status().is_success()); + let returned = test::read_body(resp).await; + assert_eq!(returned, canonical.as_bytes()); + + // A body that doesn't hash to the URL id is rejected. + let bad = TestRequest::put() + .uri(&format!("/frozen/{}", "00".repeat(32))) + .set_payload(canonical.clone()) + .to_request(); + let resp = test::call_service(&app, bad).await; + assert!(!resp.status().is_success()); + + // And it now resolves through the normal get_resource path. + let subject = atomic_lib::Subject::from_raw(&id, None); + let resource = appstate.store.get_resource(&subject).await.unwrap(); + assert_eq!( + resource.get(urls::SHORTNAME).unwrap().to_string(), + "title" + ); +} diff --git a/test-vectors/README.md b/test-vectors/README.md new file mode 100644 index 000000000..204252c2b --- /dev/null +++ b/test-vectors/README.md @@ -0,0 +1,18 @@ +# Cross-language test vectors + +Shared fixtures that pin behavior across the TypeScript and Rust implementations. + +## `frozen.json` + +The `did:ad:frozen` content-addressing contract. Each vector is a JSON-AD `body` +and its expected frozen `id` = `did:ad:frozen:` + `blake3(JCS(body))`, where JCS +is RFC 8785. + +Both sides must reproduce the same `id` from the same `body`: + +- TypeScript: `browser/lib/src/frozen-vectors.test.ts` asserts `frozenIdFor(body) === id`. +- Rust: `lib/src/frozen.rs` tests assert `frozen_id(body) == id`. + +If you change the canonicalization or hashing on either side, regenerate the +vectors and confirm both suites pass — a diff here is a cross-language identity +break. diff --git a/test-vectors/frozen.json b/test-vectors/frozen.json new file mode 100644 index 000000000..f7ece5f03 --- /dev/null +++ b/test-vectors/frozen.json @@ -0,0 +1,67 @@ +{ + "vectors": [ + { + "name": "property", + "body": { + "https://atomicdata.dev/properties/isA": [ + "https://atomicdata.dev/classes/Property" + ], + "https://atomicdata.dev/properties/shortname": "title", + "https://atomicdata.dev/properties/datatype": "https://atomicdata.dev/datatypes/string" + }, + "id": "did:ad:frozen:9c227a3e5d127dfdeeecf7fc93ae8bf8c97f31083e44e2440cf70a2d30c4b702" + }, + { + "name": "class-with-ref", + "body": { + "https://atomicdata.dev/properties/isA": [ + "https://atomicdata.dev/classes/Class" + ], + "https://atomicdata.dev/properties/shortname": "todo", + "https://atomicdata.dev/properties/requires": [ + "did:ad:frozen:0000000000000000000000000000000000000000000000000000000000000000" + ], + "https://atomicdata.dev/properties/recommends": [] + }, + "id": "did:ad:frozen:625c076abddc8a830468a5c8ad352150a9d4e81e839de1b84b7b1dc843b14d5a" + }, + { + "name": "unicode-and-number", + "body": { + "é": 1, + "a": 1.5, + "z": [ + 3, + 1, + 2 + ], + "b": true + }, + "id": "did:ad:frozen:9d588cb60acc4ab171d45789223f9d86a15ea5acfe19552bbf7b974fd71f8d11" + }, + { + "name": "cycle-unit", + "body": { + "urn:atomic-freeze:unit": [ + { + "https://atomicdata.dev/properties/isA": [ + "https://atomicdata.dev/classes/Property" + ], + "https://atomicdata.dev/properties/shortname": "friend", + "https://atomicdata.dev/properties/classtype": "did:ad:frozen:self:1" + }, + { + "https://atomicdata.dev/properties/isA": [ + "https://atomicdata.dev/classes/Class" + ], + "https://atomicdata.dev/properties/shortname": "person", + "https://atomicdata.dev/properties/requires": [ + "did:ad:frozen:self:0" + ] + } + ] + }, + "id": "did:ad:frozen:edc1f1b79e78ef17556e05c564d9a69c21ba0eb19aa13d628eafb1e8689d30b8" + } + ] +} From 71eaba16b5e7c05238d489b5c68e955ddfad5666 Mon Sep 17 00:00:00 2001 From: Joep Meindertsma Date: Tue, 2 Jun 2026 12:54:10 +0200 Subject: [PATCH 04/20] Add Store.loadSchemaLock for offline, server-free schema availability MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Register an app-bundled *.schema.lock.json into the store: verify every frozen object by re-hash, then materialize each as a read-only Resource. A schema shipped with the code (the lockfile) now resolves with zero network — the "available without a host" path. Symmetric with buildSchemaLock on the producer. --- browser/lib/src/frozen-resolve.test.ts | 49 ++++++++++++++++++++++++++ browser/lib/src/store.ts | 35 ++++++++++++++++++ planning/json-schema-code-first.md | 6 ++-- 3 files changed, 88 insertions(+), 2 deletions(-) diff --git a/browser/lib/src/frozen-resolve.test.ts b/browser/lib/src/frozen-resolve.test.ts index 40f38dccc..519741e1c 100644 --- a/browser/lib/src/frozen-resolve.test.ts +++ b/browser/lib/src/frozen-resolve.test.ts @@ -3,6 +3,7 @@ import { afterEach, describe, it, vi } from 'vitest'; import { Datatype } from './datatypes.js'; import { frozenIdFor } from './freeze.js'; import { core } from './ontologies/core.js'; +import { buildSchemaLock } from './schema-lock.js'; import { Store } from './store.js'; const body = { @@ -138,3 +139,51 @@ describe('Store.registerFrozenSchema', () => { ).toBe(true); }); }); + +describe('Store.loadSchemaLock', () => { + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it('makes a bundled lockfile resolve offline with no server', async ({ + expect, + }) => { + // The lockfile is the only input — any network call would throw. + vi.stubGlobal( + 'fetch', + vi.fn(async () => { + throw new Error('should not hit the network'); + }), + ); + + const lock = buildSchemaLock(todoPackage); + const store = new Store({ serverUrl: 'https://example.com' }); + store.loadSchemaLock(lock); + + const titleId = lock.presentation.properties['todo.title'].id; + const prop = await store.getProperty(titleId); + + expect(prop.shortname).toBe('title'); + expect(prop.datatype).toBe(Datatype.STRING); + + const todoClass = await store.getResource(lock.presentation.classes.todo.id); + expect(todoClass.get(core.properties.shortname)).toBe('todo'); + }); + + it('rejects a tampered lockfile', ({ expect }) => { + const lock = buildSchemaLock(todoPackage); + const [firstId] = Object.keys(lock.frozen); + const tampered = { + ...lock, + frozen: { + ...lock.frozen, + [firstId]: { ...(lock.frozen[firstId] as object), tampered: true }, + }, + }; + + const store = new Store({ serverUrl: 'https://example.com' }); + expect(() => store.loadSchemaLock(tampered as typeof lock)).toThrow( + /invalid schema lock/i, + ); + }); +}); diff --git a/browser/lib/src/store.ts b/browser/lib/src/store.ts index 0c93a9c0b..f4489fafa 100644 --- a/browser/lib/src/store.ts +++ b/browser/lib/src/store.ts @@ -33,6 +33,7 @@ import { } from './schema.js'; import { frozenIdFor, UNIT_MEMBERS_KEY, type FrozenId } from './freeze.js'; import { jcsCanonicalize } from './jcs.js'; +import { verifySchemaLock, type SchemaLock } from './schema-lock.js'; import { stringToSlug } from './stringToSlug.js'; import { bytesToHex, hexToBytes, type JSONValue } from './value.js'; import { WSClient } from './websockets.js'; @@ -1834,6 +1835,40 @@ export class Store { return frozen; } + /** + * Registers an app-bundled `*.schema.lock.json` into the store: verifies every + * frozen object by re-hash, then materializes each as a read-only Resource so + * the schema resolves offline with no server. This is "available without a + * host" — the lockfile travels with the code, and a frozen id is reproducible + * from it. Returns the lock so callers can read its id maps. Cycle "unit" + * objects are skipped (not yet individually materializable). + */ + public loadSchemaLock(lock: SchemaLock): SchemaLock { + const verification = verifySchemaLock(lock); + + if (!verification.ok) { + throw new Error( + `Refusing to load an invalid schema lock: ${verification.errors.join('; ')}`, + ); + } + + for (const [frozenId, content] of Object.entries(lock.frozen)) { + if ( + content && + typeof content === 'object' && + UNIT_MEMBERS_KEY in content + ) { + continue; + } + + const [resource] = new JSONADParser().parse(content, frozenId); + resource.loading = false; + this.addResource(resource, { skipCommitCompare: true }); + } + + return lock; + } + private async publishFrozenResource( frozenId: FrozenId, content: unknown, diff --git a/planning/json-schema-code-first.md b/planning/json-schema-code-first.md index 5d24dbdaa..af957a1ee 100644 --- a/planning/json-schema-code-first.md +++ b/planning/json-schema-code-first.md @@ -816,8 +816,10 @@ Rules: `presentation`) via `ad-generate schema --lock`, using `buildSchemaLock()`/`verifySchemaLock()` in `@tomic/lib`. The emitter refuses to write an unverifiable lock. -- [ ] Load + register a lockfile at app startup for offline availability (needs - the Store `did:ad:frozen` resolution path). +- [x] Load + register a lockfile at app startup for offline availability: + `Store.loadSchemaLock(lock)` verifies every frozen object by re-hash and + materializes them locally, so a bundled `*.schema.lock.json` resolves with + no server. Tested (`frozen-resolve.test.ts`). - [ ] Add a CI "regenerate and diff" guard so the lockfile cannot drift from the source schema (`verifySchemaLock` + a re-emit check are the building blocks). From edaeca3ed01ec820d55f12e88022d0153e69e827 Mon Sep 17 00:00:00 2001 From: Joep Meindertsma Date: Tue, 2 Jun 2026 13:08:06 +0200 Subject: [PATCH 05/20] Add Store.createSchemaPointer (signed latest-version pointer) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The mutable half of the two-layer model: a normal signed Ontology (genesis DID) on the author's drive whose classes/properties reference the immutable frozen ids. Its stable subject is the durable "name" for the latest version, and its signed commit history is the version log — re-running it on a schema change records a new version while old frozen ids stay permanently resolvable. With { save: true } it is signed and committed. --- browser/lib/src/frozen-resolve.test.ts | 35 +++++++++++++++++++++++ browser/lib/src/store.ts | 39 ++++++++++++++++++++++++++ planning/did-ad-frozen-server.md | 8 ++++-- 3 files changed, 80 insertions(+), 2 deletions(-) diff --git a/browser/lib/src/frozen-resolve.test.ts b/browser/lib/src/frozen-resolve.test.ts index 519741e1c..63f2f6730 100644 --- a/browser/lib/src/frozen-resolve.test.ts +++ b/browser/lib/src/frozen-resolve.test.ts @@ -1,5 +1,7 @@ import { afterEach, describe, it, vi } from 'vitest'; +import { Agent } from './agent.js'; +import { JSCryptoProvider } from './CryptoProvider.js'; import { Datatype } from './datatypes.js'; import { frozenIdFor } from './freeze.js'; import { core } from './ontologies/core.js'; @@ -187,3 +189,36 @@ describe('Store.loadSchemaLock', () => { ); }); }); + +describe('Store.createSchemaPointer', () => { + it('creates a signed Ontology pointing at the frozen ids', async ({ + expect, + }) => { + const keys = await Agent.generateKeyPair(); + const store = new Store({ serverUrl: 'https://example.com' }); + store.setAgent( + new Agent( + new JSCryptoProvider(keys.privateKey), + `did:ad:agent:${keys.publicKey}`, + ), + ); + + const frozen = await store.registerFrozenSchema({ + ...todoPackage, + version: '1.0.0', + }); + const pointer = await store.createSchemaPointer(frozen); + + // Stable, signed DID — the durable "name" for the latest version. + expect(pointer.subject).toMatch(/^did:ad:/); + expect(pointer.hasClasses(core.classes.ontology)).toBe(true); + // Its members point at the immutable frozen ids. + expect(pointer.props.classes).toEqual([frozen.classes.todo]); + expect(pointer.props.properties).toEqual( + expect.arrayContaining([ + frozen.properties['todo.title'], + frozen.properties['todo.done'], + ]), + ); + }); +}); diff --git a/browser/lib/src/store.ts b/browser/lib/src/store.ts index f4489fafa..33b39c144 100644 --- a/browser/lib/src/store.ts +++ b/browser/lib/src/store.ts @@ -1835,6 +1835,45 @@ export class Store { return frozen; } + /** + * Creates the mutable, signed "latest version" pointer for a frozen schema: a + * normal Ontology resource (genesis DID) on the author's drive whose + * `classes`/`properties` point at the immutable frozen ids. Its stable subject + * is the durable name ("the current TodoApp"), and its signed commit history is + * the version log — re-running this when the schema changes records a new + * version while old frozen ids stay permanently resolvable. With + * `{ save: true }` it is signed and committed. + */ + public async createSchemaPointer( + frozen: FrozenSchema, + opts: { parent?: string; save?: boolean } = {}, + ): Promise> { + const version = frozen.presentation.ontology.version; + const ontology = await this.newResource({ + parent: opts.parent, + isA: core.classes.ontology, + validatePropVals: false, + propVals: { + [core.properties.shortname]: stringToSlug( + frozen.model.ontology.shortname, + ), + [core.properties.description]: frozen.presentation.ontology.description, + [core.properties.classes]: Object.values(frozen.classes) as string[], + [core.properties.properties]: Object.values( + frozen.properties, + ) as string[], + [core.properties.instances]: [], + ...(version ? { [server.properties.version]: version } : {}), + }, + }); + + if (opts.save) { + await ontology.save(); + } + + return ontology; + } + /** * Registers an app-bundled `*.schema.lock.json` into the store: verifies every * frozen object by re-hash, then materializes each as a read-only Resource so diff --git a/planning/did-ad-frozen-server.md b/planning/did-ad-frozen-server.md index a0d936e9c..fab13f4ef 100644 --- a/planning/did-ad-frozen-server.md +++ b/planning/did-ad-frozen-server.md @@ -153,8 +153,12 @@ needed. consumer loop live. - [ ] Persist frozen objects in ClientDb (OPFS) keyed by hash for offline reload (currently in-memory only). -- [ ] The signed **"latest version" pointer** on the author's drive - (`name → latest frozen ontology id`) + any mutable display overlay. +- [x] The signed **"latest version" pointer**: `Store.createSchemaPointer(frozen)` + builds a mutable, signed Ontology (genesis DID) on the author's drive whose + `classes`/`properties` reference the immutable frozen ids. Its stable subject + is the durable name and its commit history is the version log; old frozen + ids stay permanently resolvable. Tested (construction). Explicit `replaces` + links would need a new bootstrapped property — deferred. ### Phase D — sync (frozen travels with drives) From 703cb627b3f3e2d82c62a20128b1fd81a780f2fc Mon Sep 17 00:00:00 2001 From: Joep Meindertsma Date: Tue, 2 Jun 2026 13:15:23 +0200 Subject: [PATCH 06/20] Port freeze_resources to Rust for cross-language schema authoring MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rust can now author frozen resource graphs that produce ids byte-for-byte identical to the TypeScript producer. Ports the full content-addressing primitive — Tarjan SCC, color-refinement canonical ordering, topological hashing, and one-unit-per-cycle — mirroring browser/lib/src/freeze.ts. Validated by a new cross-language fixture (test-vectors/freeze-resources.json): a Rust test freezes the same acyclic DAG and 2-cycle the TS side produced and asserts identical local_id -> frozen_id maps. --- lib/src/frozen.rs | 420 +++++++++++++++++++++++++++++ test-vectors/freeze-resources.json | 68 +++++ 2 files changed, 488 insertions(+) create mode 100644 test-vectors/freeze-resources.json diff --git a/lib/src/frozen.rs b/lib/src/frozen.rs index 656da5597..dfcf8ae5e 100644 --- a/lib/src/frozen.rs +++ b/lib/src/frozen.rs @@ -10,9 +10,17 @@ //! Frozen objects are immutable and signatureless: they are verified by //! re-hashing, never by a commit signature. See `planning/did-ad-frozen-server.md`. +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; + +use serde_json::Value; + use crate::errors::AtomicResult; use crate::subject::DID_AD_FROZEN_PREFIX; +/// Placeholder for an intra-cycle reference, by the referent's canonical index. +/// Matches `browser/lib/src/freeze.ts#SELF_PREFIX`. +const SELF_PREFIX: &str = "did:ad:frozen:self:"; + /// Reserved top-level key marking a frozen **unit** object: the materialized form /// of a reference cycle, whose value is an ordered array of member bodies that /// reference each other by `did:ad:frozen:self:{index}`. Matches @@ -50,6 +58,367 @@ pub fn verify_frozen(id: &str, body: &serde_json::Value) -> AtomicResult<()> { } } +/// A resource to be frozen, identified by a temporary `local_id`. Any string +/// value inside another resource's `content` that equals this `local_id` is a +/// reference and is rewritten to the computed frozen id. +pub struct FreezableResource { + pub local_id: String, + pub content: Value, +} + +/// A content-addressed result: one per distinct frozen object (ordinary resource +/// or cycle unit). `unit` lists the local_ids it covers. +pub struct FrozenResource { + pub frozen_id: String, + pub content: Value, + pub unit: Vec, +} + +pub struct FreezeResult { + pub resources: Vec, + pub by_local_id: HashMap, +} + +/// Content-addresses a set of mutually-referencing resources into a +/// `did:ad:frozen` Merkle DAG, byte-for-byte identical to +/// `browser/lib/src/freeze.ts#freezeResources`. References are rewritten to the +/// referent's hash before hashing; each reference cycle is frozen as one unit +/// object so every stored object stays verifiable by re-hash. +pub fn freeze_resources(input: Vec) -> AtomicResult { + let ids: HashSet = input.iter().map(|r| r.local_id.clone()).collect(); + + if ids.len() != input.len() { + return Err("freeze_resources: local_id values must be unique".into()); + } + + let by_id: HashMap = + input.into_iter().map(|r| (r.local_id, r.content)).collect(); + let nodes: Vec = by_id.keys().cloned().collect(); + let edges: HashMap> = by_id + .iter() + .map(|(id, content)| (id.clone(), collect_refs(content, &ids))) + .collect(); + + // Tarjan emits SCCs sinks-first (reverse topological), so every out-edge + // points at an already-frozen component. + let sccs = strongly_connected_components(&nodes, &edges); + let mut frozen_by_local: HashMap = HashMap::new(); + let mut out: Vec = Vec::new(); + + for scc in sccs { + let is_cycle = scc.len() > 1 + || edges + .get(&scc[0]) + .map(|e| e.contains(&scc[0])) + .unwrap_or(false); + + if is_cycle { + freeze_cycle(&scc, &by_id, &edges, &mut frozen_by_local, &mut out)?; + } else { + freeze_singleton(&scc[0], &by_id, &edges, &mut frozen_by_local, &mut out)?; + } + } + + Ok(FreezeResult { + resources: out, + by_local_id: frozen_by_local, + }) +} + +fn freeze_singleton( + local_id: &str, + by_id: &HashMap, + edges: &HashMap>, + frozen_by_local: &mut HashMap, + out: &mut Vec, +) -> AtomicResult<()> { + let map = resolved_ref_map(edges.get(local_id), frozen_by_local); + let content = substitute(&by_id[local_id], &map); + let frozen_id = frozen_id(&content)?; + + frozen_by_local.insert(local_id.to_string(), frozen_id.clone()); + + if let Some(existing) = out.iter_mut().find(|r| r.frozen_id == frozen_id) { + existing.unit.push(local_id.to_string()); + } else { + out.push(FrozenResource { + frozen_id, + content, + unit: vec![local_id.to_string()], + }); + } + + Ok(()) +} + +fn freeze_cycle( + scc: &[String], + by_id: &HashMap, + edges: &HashMap>, + frozen_by_local: &mut HashMap, + out: &mut Vec, +) -> AtomicResult<()> { + let scc_set: HashSet = scc.iter().cloned().collect(); + let order = canonical_order(scc, &scc_set, by_id, edges, frozen_by_local)?; + let index_of: HashMap = order + .iter() + .enumerate() + .map(|(i, id)| (id.clone(), i)) + .collect(); + + let mut members: Vec = Vec::with_capacity(order.len()); + for local_id in &order { + let map = cycle_ref_map( + edges.get(local_id), + &scc_set, + Some(&index_of), + frozen_by_local, + ); + members.push(substitute(&by_id[local_id], &map)); + } + let content = serde_json::json!({ FROZEN_UNIT_KEY: members }); + let frozen_id = frozen_id(&content)?; + + for local_id in &order { + frozen_by_local.insert(local_id.clone(), frozen_id.clone()); + } + + out.push(FrozenResource { + frozen_id, + content, + unit: order, + }); + + Ok(()) +} + +/// Deterministic, input-order-independent ordering of a cycle's members via +/// color refinement. Mirrors `freeze.ts#canonicalOrder`. +fn canonical_order( + scc: &[String], + scc_set: &HashSet, + by_id: &HashMap, + edges: &HashMap>, + frozen_by_local: &HashMap, +) -> AtomicResult> { + let mut color: HashMap = HashMap::new(); + for local_id in scc { + let map = cycle_ref_map(edges.get(local_id), scc_set, None, frozen_by_local); + color.insert( + local_id.clone(), + hash_canonical(&substitute(&by_id[local_id], &map))?, + ); + } + + for _ in 0..scc.len() { + let mut next: HashMap = HashMap::new(); + for local_id in scc { + let map = neighbor_color_ref_map(edges.get(local_id), scc_set, &color, frozen_by_local); + next.insert( + local_id.clone(), + hash_canonical(&substitute(&by_id[local_id], &map))?, + ); + } + + if partition_signature(scc, &next) == partition_signature(scc, &color) { + color = next; + break; + } + + color = next; + } + + let mut order: Vec = scc.to_vec(); + order.sort_by(|a, b| color[a].cmp(&color[b]).then_with(|| a.cmp(b))); + + Ok(order) +} + +fn resolved_ref_map( + refs: Option<&BTreeSet>, + frozen_by_local: &HashMap, +) -> HashMap { + let mut map = HashMap::new(); + for r in refs.into_iter().flatten() { + if let Some(fid) = frozen_by_local.get(r) { + map.insert(r.clone(), fid.clone()); + } + } + map +} + +fn cycle_ref_map( + refs: Option<&BTreeSet>, + scc_set: &HashSet, + index_of: Option<&HashMap>, + frozen_by_local: &HashMap, +) -> HashMap { + let mut map = HashMap::new(); + for r in refs.into_iter().flatten() { + if scc_set.contains(r) { + let token = match index_of { + Some(idx) => format!("{}{}", SELF_PREFIX, idx[r]), + None => SELF_PREFIX.to_string(), + }; + map.insert(r.clone(), token); + } else if let Some(fid) = frozen_by_local.get(r) { + map.insert(r.clone(), fid.clone()); + } + } + map +} + +fn neighbor_color_ref_map( + refs: Option<&BTreeSet>, + scc_set: &HashSet, + color: &HashMap, + frozen_by_local: &HashMap, +) -> HashMap { + let mut map = HashMap::new(); + for r in refs.into_iter().flatten() { + if scc_set.contains(r) { + map.insert(r.clone(), format!("{}{}", SELF_PREFIX, color[r])); + } else if let Some(fid) = frozen_by_local.get(r) { + map.insert(r.clone(), fid.clone()); + } + } + map +} + +fn partition_signature(scc: &[String], color: &HashMap) -> String { + let mut groups: BTreeMap> = BTreeMap::new(); + for id in scc { + groups + .entry(color[id].clone()) + .or_default() + .push(id.clone()); + } + let mut parts: Vec = groups + .into_values() + .map(|mut g| { + g.sort(); + g.join(",") + }) + .collect(); + parts.sort(); + parts.join("|") +} + +fn collect_refs(value: &Value, ids: &HashSet) -> BTreeSet { + let mut out = BTreeSet::new(); + walk_refs(value, ids, &mut out); + out +} + +fn walk_refs(value: &Value, ids: &HashSet, out: &mut BTreeSet) { + match value { + Value::String(s) => { + if ids.contains(s) { + out.insert(s.clone()); + } + } + Value::Array(a) => a.iter().for_each(|v| walk_refs(v, ids, out)), + Value::Object(o) => o.values().for_each(|v| walk_refs(v, ids, out)), + _ => {} + } +} + +fn substitute(value: &Value, map: &HashMap) -> Value { + match value { + Value::String(s) => map + .get(s) + .map(|r| Value::String(r.clone())) + .unwrap_or_else(|| value.clone()), + Value::Array(a) => Value::Array(a.iter().map(|v| substitute(v, map)).collect()), + Value::Object(o) => Value::Object( + o.iter() + .map(|(k, v)| (k.clone(), substitute(v, map))) + .collect(), + ), + _ => value.clone(), + } +} + +fn hash_canonical(value: &Value) -> AtomicResult { + let canonical = + serde_jcs::to_string(value).map_err(|e| format!("Failed to JCS-canonicalize: {}", e))?; + Ok(blake3::hash(canonical.as_bytes()).to_hex().to_string()) +} + +fn strongly_connected_components( + nodes: &[String], + edges: &HashMap>, +) -> Vec> { + struct State<'a> { + edges: &'a HashMap>, + index: HashMap, + low: HashMap, + on_stack: HashSet, + stack: Vec, + counter: usize, + result: Vec>, + } + + fn connect(s: &mut State, v: &str) { + s.index.insert(v.to_string(), s.counter); + s.low.insert(v.to_string(), s.counter); + s.counter += 1; + s.stack.push(v.to_string()); + s.on_stack.insert(v.to_string()); + + let neighbors: Vec = s + .edges + .get(v) + .map(|e| e.iter().cloned().collect()) + .unwrap_or_default(); + + for w in neighbors { + if !s.index.contains_key(&w) { + connect(s, &w); + let lw = s.low[&w]; + let lv = s.low[v]; + s.low.insert(v.to_string(), lv.min(lw)); + } else if s.on_stack.contains(&w) { + let iw = s.index[&w]; + let lv = s.low[v]; + s.low.insert(v.to_string(), lv.min(iw)); + } + } + + if s.low[v] == s.index[v] { + let mut component = Vec::new(); + loop { + let w = s.stack.pop().unwrap(); + s.on_stack.remove(&w); + let is_root = w == v; + component.push(w); + if is_root { + break; + } + } + s.result.push(component); + } + } + + let mut state = State { + edges, + index: HashMap::new(), + low: HashMap::new(), + on_stack: HashSet::new(), + stack: Vec::new(), + counter: 0, + result: Vec::new(), + }; + + for v in nodes { + if !state.index.contains_key(v) { + connect(&mut state, v); + } + } + + state.result +} + #[cfg(test)] mod tests { use super::*; @@ -97,4 +466,55 @@ mod tests { assert!(verify_frozen(&frozen_id(&body).unwrap(), &body).is_ok()); assert!(verify_frozen(wrong, &body).is_err()); } + + #[derive(serde::Deserialize)] + struct FreezeInput { + #[serde(rename = "localId")] + local_id: String, + content: serde_json::Value, + } + + #[derive(serde::Deserialize)] + struct FreezeCase { + name: String, + input: Vec, + expected: HashMap, + } + + #[derive(serde::Deserialize)] + struct FreezeCases { + cases: Vec, + } + + /// Proves the Rust `freeze_resources` graph algorithm (topological hashing + + /// cycle units + color refinement) is byte-for-byte identical to the + /// TypeScript producer — the foundation for authoring schemas in Rust. + #[test] + fn freeze_resources_matches_cross_language_vectors() { + let raw = include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../test-vectors/freeze-resources.json" + )); + let parsed: FreezeCases = serde_json::from_str(raw).expect("valid fixture"); + + assert!(!parsed.cases.is_empty(), "fixture has no cases"); + + for case in parsed.cases { + let input: Vec = case + .input + .into_iter() + .map(|i| FreezableResource { + local_id: i.local_id, + content: i.content, + }) + .collect(); + let result = freeze_resources(input).unwrap(); + + assert_eq!( + result.by_local_id, case.expected, + "freeze_resources mismatch for case {}", + case.name + ); + } + } } diff --git a/test-vectors/freeze-resources.json b/test-vectors/freeze-resources.json new file mode 100644 index 000000000..4591e56ac --- /dev/null +++ b/test-vectors/freeze-resources.json @@ -0,0 +1,68 @@ +{ + "cases": [ + { + "name": "acyclic", + "input": [ + { + "localId": "p:title", + "content": { + "https://atomicdata.dev/properties/shortname": "title", + "https://atomicdata.dev/properties/datatype": "https://atomicdata.dev/datatypes/string" + } + }, + { + "localId": "c:todo", + "content": { + "https://atomicdata.dev/properties/shortname": "todo", + "https://atomicdata.dev/properties/requires": [ + "p:title" + ], + "https://atomicdata.dev/properties/recommends": [] + } + }, + { + "localId": "o:app", + "content": { + "https://atomicdata.dev/properties/shortname": "todoApp", + "https://atomicdata.dev/properties/classes": [ + "c:todo" + ], + "https://atomicdata.dev/properties/properties": [ + "p:title" + ] + } + } + ], + "expected": { + "p:title": "did:ad:frozen:d8499b11374d5d211421ebe66b4c7f1eb84c39dc7b10ac41df89d56e37e98cc0", + "c:todo": "did:ad:frozen:90b6416aa4e40216c4484d993be0c5556fc611c373f96afbfa1cf0409dbc0f7d", + "o:app": "did:ad:frozen:4b87a1fd6c43770d8a6b7b08fda884f95e6901baf5d5f0d2c5e13d747704e043" + } + }, + { + "name": "cyclic", + "input": [ + { + "localId": "p:friend", + "content": { + "https://atomicdata.dev/properties/shortname": "friend", + "https://atomicdata.dev/properties/classtype": "c:person" + } + }, + { + "localId": "c:person", + "content": { + "https://atomicdata.dev/properties/shortname": "person", + "https://atomicdata.dev/properties/requires": [ + "p:friend" + ] + } + } + ], + "expected": { + "p:friend": "did:ad:frozen:651b32175d89aa5d234215056c37d88effc9b1636c15300e2b0df67586d38ba0", + "c:person": "did:ad:frozen:651b32175d89aa5d234215056c37d88effc9b1636c15300e2b0df67586d38ba0" + } + } + ] +} From c2448344ea18137db418ae1237df270328d86857 Mon Sep 17 00:00:00 2001 From: Joep Meindertsma Date: Tue, 2 Jun 2026 13:15:51 +0200 Subject: [PATCH 07/20] Update plan: Rust freeze_resources port done --- planning/json-schema-code-first.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/planning/json-schema-code-first.md b/planning/json-schema-code-first.md index af957a1ee..299a9f668 100644 --- a/planning/json-schema-code-first.md +++ b/planning/json-schema-code-first.md @@ -859,8 +859,13 @@ Rules: - reuse one imported Property in another Class - table-created schema exports and re-imports - required/datatype validation still works -- [ ] Add Rust-side import/export structs after the TypeScript API shape is - stable. +- [x] Port the content-addressing core to Rust for cross-language **authoring**: + `lib/src/frozen.rs#freeze_resources` (Tarjan SCC + color refinement + + one-unit-per-cycle), byte-for-byte identical to TS, pinned by + `test-vectors/freeze-resources.json`. A Rust `freeze_schema` DSL (the + JSON-Schema-subset → frozen bodies layer) on top is the remaining piece. +- [ ] Add optional JSON Schema validation in the browser SDK and Rust/server + (richer keywords beyond the Atomic subset). - [ ] Add optional JSON Schema validation in the browser SDK. - [ ] Add optional JSON Schema validation in Rust/server. From 1301e9b45b3d20515aff4ce3d2d53c8318419897 Mon Sep 17 00:00:00 2001 From: Joep Meindertsma Date: Tue, 2 Jun 2026 13:35:56 +0200 Subject: [PATCH 08/20] Add Rust freeze_schema authoring DSL (cross-language verified) Schemas can now be *authored* in Rust, not just consumed: an order-preserving SchemaDef -> freeze_schema produces did:ad:frozen ids byte-for-byte identical to the TS freezeSchema, closing the multi-language-authoring goal. Builds identity- only Ontology/Class/Property bodies (presentation excluded) and content-addresses them via freeze_resources. Mirrors TS normalization (keys sorted) so the order-sensitive ontology id matches. Validated by test-vectors/freeze-schema.json: Rust freeze_schema of the same schema reproduces the TS ontology/class/property ids exactly. --- lib/src/frozen.rs | 280 ++++++++++++++++++++++++++++++++ test-vectors/freeze-schema.json | 38 +++++ 2 files changed, 318 insertions(+) create mode 100644 test-vectors/freeze-schema.json diff --git a/lib/src/frozen.rs b/lib/src/frozen.rs index dfcf8ae5e..d6545880d 100644 --- a/lib/src/frozen.rs +++ b/lib/src/frozen.rs @@ -419,6 +419,223 @@ fn strongly_connected_components( state.result } +// --- Schema authoring DSL --------------------------------------------------- +// +// `freeze_schema` is the Rust counterpart of `browser/lib/src/schema.ts# +// freezeSchema`: it builds identity-only JSON-AD bodies for an Ontology and its +// Classes/Properties and content-addresses them, producing frozen ids +// byte-for-byte identical to the TS producer. Descriptions and other +// presentation are deliberately excluded from identity. Input is order- +// preserving (Vec-based), because an Ontology's `classes`/`properties` array +// order is significant (JCS does not sort arrays). + +const P_ISA: &str = "https://atomicdata.dev/properties/isA"; +const P_SHORTNAME: &str = "https://atomicdata.dev/properties/shortname"; +const P_DATATYPE: &str = "https://atomicdata.dev/properties/datatype"; +const P_CLASSTYPE: &str = "https://atomicdata.dev/properties/classtype"; +const P_REQUIRES: &str = "https://atomicdata.dev/properties/requires"; +const P_RECOMMENDS: &str = "https://atomicdata.dev/properties/recommends"; +const P_CLASSES: &str = "https://atomicdata.dev/properties/classes"; +const P_PROPERTIES: &str = "https://atomicdata.dev/properties/properties"; +const P_VERSION: &str = "https://atomicdata.dev/properties/version"; +const C_PROPERTY: &str = "https://atomicdata.dev/classes/Property"; +const C_CLASS: &str = "https://atomicdata.dev/classes/Class"; +const C_ONTOLOGY: &str = "https://atomicdata.dev/class/ontology"; + +/// An Atomic datatype, the machine contract of a Property. +#[derive(Clone, Copy)] +pub enum SchemaDatatype { + String, + Integer, + Float, + Boolean, + Date, + Uri, + AtomicUrl, + ResourceArray, + Json, +} + +impl SchemaDatatype { + pub fn url(&self) -> &'static str { + match self { + SchemaDatatype::String => "https://atomicdata.dev/datatypes/string", + SchemaDatatype::Integer => "https://atomicdata.dev/datatypes/integer", + SchemaDatatype::Float => "https://atomicdata.dev/datatypes/float", + SchemaDatatype::Boolean => "https://atomicdata.dev/datatypes/boolean", + SchemaDatatype::Date => "https://atomicdata.dev/datatypes/date", + SchemaDatatype::Uri => "https://atomicdata.dev/datatypes/uri", + SchemaDatatype::AtomicUrl => "https://atomicdata.dev/datatypes/atomicURL", + SchemaDatatype::ResourceArray => "https://atomicdata.dev/datatypes/resourceArray", + SchemaDatatype::Json => "https://atomicdata.dev/datatypes/json", + } + } +} + +pub struct SchemaProperty { + /// Developer key, also the default shortname. + pub key: String, + pub datatype: SchemaDatatype, + pub shortname: Option, + pub class_type: Option, +} + +pub struct SchemaClass { + pub key: String, + pub shortname: Option, + /// Property keys that are required (the rest become `recommends`). + pub required: Vec, + pub properties: Vec, +} + +pub struct SchemaDef { + pub name: String, + pub version: Option, + pub classes: Vec, +} + +/// Frozen ids for a schema, keyed by developer key (`"classKey.propKey"` for +/// properties). +pub struct FrozenSchema { + pub ontology: String, + pub classes: HashMap, + pub properties: HashMap, +} + +/// Freezes a code-first schema into `did:ad:frozen` ids identical to the TS +/// producer. Acyclic by construction (Ontology -> Classes -> Properties). +pub fn freeze_schema(def: &SchemaDef) -> AtomicResult { + let prop_local = |class_key: &str, prop_key: &str| format!("prop:{}.{}", class_key, prop_key); + let class_local = |class_key: &str| format!("class:{}", class_key); + let ontology_local = "ontology".to_string(); + + // The TS producer normalizes the schema (sorts object keys) before freezing, + // so ids are independent of declaration order. Mirror that by processing + // classes and properties sorted by key. (Byte order; TS uses `localeCompare`, + // which coincides for the lowercase-ASCII shortnames that are the convention.) + let mut classes: Vec<&SchemaClass> = def.classes.iter().collect(); + classes.sort_by(|a, b| a.key.cmp(&b.key)); + let prepared: Vec<(&SchemaClass, Vec<&SchemaProperty>)> = classes + .iter() + .map(|c| { + let mut props: Vec<&SchemaProperty> = c.properties.iter().collect(); + props.sort_by(|a, b| a.key.cmp(&b.key)); + (*c, props) + }) + .collect(); + + let mut freezable: Vec = Vec::new(); + + for (class, props) in &prepared { + for property in props { + let shortname = property.shortname.clone().unwrap_or(property.key.clone()); + let mut body = serde_json::Map::new(); + body.insert( + P_ISA.to_string(), + Value::Array(vec![Value::String(C_PROPERTY.into())]), + ); + body.insert(P_SHORTNAME.to_string(), Value::String(shortname)); + body.insert( + P_DATATYPE.to_string(), + Value::String(property.datatype.url().into()), + ); + if let Some(ct) = &property.class_type { + body.insert(P_CLASSTYPE.to_string(), Value::String(ct.clone())); + } + + freezable.push(FreezableResource { + local_id: prop_local(&class.key, &property.key), + content: Value::Object(body), + }); + } + } + + for (class, props) in &prepared { + let required: HashSet<&str> = class.required.iter().map(|s| s.as_str()).collect(); + let mut requires = Vec::new(); + let mut recommends = Vec::new(); + for property in props { + let local = prop_local(&class.key, &property.key); + if required.contains(property.key.as_str()) { + requires.push(Value::String(local)); + } else { + recommends.push(Value::String(local)); + } + } + + let shortname = class.shortname.clone().unwrap_or(class.key.clone()); + let mut body = serde_json::Map::new(); + body.insert( + P_ISA.to_string(), + Value::Array(vec![Value::String(C_CLASS.into())]), + ); + body.insert(P_SHORTNAME.to_string(), Value::String(shortname)); + body.insert(P_REQUIRES.to_string(), Value::Array(requires)); + body.insert(P_RECOMMENDS.to_string(), Value::Array(recommends)); + + freezable.push(FreezableResource { + local_id: class_local(&class.key), + content: Value::Object(body), + }); + } + + let class_refs: Vec = prepared + .iter() + .map(|(c, _)| Value::String(class_local(&c.key))) + .collect(); + let property_refs: Vec = prepared + .iter() + .flat_map(|(c, props)| { + props + .iter() + .map(move |p| Value::String(prop_local(&c.key, &p.key))) + }) + .collect(); + + let mut ontology_body = serde_json::Map::new(); + ontology_body.insert( + P_ISA.to_string(), + Value::Array(vec![Value::String(C_ONTOLOGY.into())]), + ); + ontology_body.insert(P_SHORTNAME.to_string(), Value::String(def.name.clone())); + ontology_body.insert(P_CLASSES.to_string(), Value::Array(class_refs)); + ontology_body.insert(P_PROPERTIES.to_string(), Value::Array(property_refs)); + if let Some(version) = &def.version { + ontology_body.insert(P_VERSION.to_string(), Value::String(version.clone())); + } + freezable.push(FreezableResource { + local_id: ontology_local.clone(), + content: Value::Object(ontology_body), + }); + + let result = freeze_resources(freezable)?; + let require_id = |local: &str| -> AtomicResult { + result + .by_local_id + .get(local) + .cloned() + .ok_or_else(|| format!("freeze_schema produced no id for {}", local).into()) + }; + + let mut classes = HashMap::new(); + let mut properties = HashMap::new(); + for (class, props) in &prepared { + classes.insert(class.key.clone(), require_id(&class_local(&class.key))?); + for property in props { + properties.insert( + format!("{}.{}", class.key, property.key), + require_id(&prop_local(&class.key, &property.key))?, + ); + } + } + + Ok(FrozenSchema { + ontology: require_id(&ontology_local)?, + classes, + properties, + }) +} + #[cfg(test)] mod tests { use super::*; @@ -517,4 +734,67 @@ mod tests { ); } } + + #[derive(serde::Deserialize)] + struct SchemaExpected { + ontology: String, + classes: HashMap, + properties: HashMap, + } + + #[derive(serde::Deserialize)] + struct SchemaVector { + expected: SchemaExpected, + } + + /// Proves the Rust `freeze_schema` authoring DSL produces ids identical to + /// the TS `freezeSchema` for the same schema — the multi-language authoring + /// guarantee. + #[test] + fn freeze_schema_matches_cross_language_vector() { + let raw = include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../test-vectors/freeze-schema.json" + )); + let vector: SchemaVector = serde_json::from_str(raw).expect("valid fixture"); + + let def = SchemaDef { + name: "FrozenTodoApp".into(), + version: Some("1.0.0".into()), + classes: vec![SchemaClass { + key: "todo".into(), + shortname: None, + required: vec!["title".into()], + properties: vec![ + SchemaProperty { + key: "title".into(), + datatype: SchemaDatatype::String, + shortname: None, + class_type: None, + }, + SchemaProperty { + key: "done".into(), + datatype: SchemaDatatype::Boolean, + shortname: None, + class_type: None, + }, + SchemaProperty { + key: "dueAt".into(), + datatype: SchemaDatatype::Date, + shortname: None, + class_type: None, + }, + ], + }], + }; + + let frozen = freeze_schema(&def).unwrap(); + + assert_eq!(frozen.ontology, vector.expected.ontology, "ontology id"); + assert_eq!(frozen.classes, vector.expected.classes, "class ids"); + assert_eq!( + frozen.properties, vector.expected.properties, + "property ids" + ); + } } diff --git a/test-vectors/freeze-schema.json b/test-vectors/freeze-schema.json new file mode 100644 index 000000000..7f26cbae5 --- /dev/null +++ b/test-vectors/freeze-schema.json @@ -0,0 +1,38 @@ +{ + "schema": { + "name": "FrozenTodoApp", + "version": "1.0.0", + "classes": { + "todo": { + "type": "object", + "required": [ + "title" + ], + "properties": { + "title": { + "type": "string", + "description": "Task title" + }, + "done": { + "type": "boolean" + }, + "dueAt": { + "type": "string", + "format": "date" + } + } + } + } + }, + "expected": { + "ontology": "did:ad:frozen:5359f2dbb9d46cc3188be1d76f27d0a8b5a89add18defcd9b26ca538c3b1d4da", + "classes": { + "todo": "did:ad:frozen:cf44ff3df002db21831e487bc930b0e03b4a3adbac4223445d4eaa1cb283c9e7" + }, + "properties": { + "todo.done": "did:ad:frozen:29a60b4ed180a3f1c0fc60c38ed35bd57c35487fc782b6477d195f5b400defd5", + "todo.dueAt": "did:ad:frozen:03171f4e8940ec0661c37504748c5a20d213d498cd583137c139cd226d44707c", + "todo.title": "did:ad:frozen:9c227a3e5d127dfdeeecf7fc93ae8bf8c97f31083e44e2440cf70a2d30c4b702" + } + } +} From 112f3c68ebedd0b482604c7c1f497eace666d687 Mon Sep 17 00:00:00 2001 From: Joep Meindertsma Date: Tue, 2 Jun 2026 13:36:17 +0200 Subject: [PATCH 09/20] Update plan: Rust freeze_schema authoring done --- planning/json-schema-code-first.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/planning/json-schema-code-first.md b/planning/json-schema-code-first.md index 299a9f668..3f110ee38 100644 --- a/planning/json-schema-code-first.md +++ b/planning/json-schema-code-first.md @@ -859,11 +859,15 @@ Rules: - reuse one imported Property in another Class - table-created schema exports and re-imports - required/datatype validation still works -- [x] Port the content-addressing core to Rust for cross-language **authoring**: - `lib/src/frozen.rs#freeze_resources` (Tarjan SCC + color refinement + - one-unit-per-cycle), byte-for-byte identical to TS, pinned by - `test-vectors/freeze-resources.json`. A Rust `freeze_schema` DSL (the - JSON-Schema-subset → frozen bodies layer) on top is the remaining piece. +- [x] Cross-language **authoring** in Rust: `lib/src/frozen.rs#freeze_resources` + (the content-addressing core — Tarjan SCC + color refinement + + one-unit-per-cycle) and `freeze_schema` (the order-preserving schema DSL → + frozen Ontology/Class/Property ids). Both byte-for-byte identical to TS, + pinned by `test-vectors/freeze-resources.json` and `freeze-schema.json`. A + Rust app can now author a schema and get the same `did:ad:frozen` ids as the + TS producer. (Key sorting uses byte order; TS `localeCompare` coincides for + lowercase-ASCII shortnames — the convention. Codegen of native Rust structs + from frozen schemas is a further, optional step.) - [ ] Add optional JSON Schema validation in the browser SDK and Rust/server (richer keywords beyond the Atomic subset). - [ ] Add optional JSON Schema validation in the browser SDK. From fa05d5af0c66970e96bc8d3de16e2b521e30689c Mon Sep 17 00:00:00 2001 From: Joep Meindertsma Date: Tue, 2 Jun 2026 14:14:20 +0200 Subject: [PATCH 10/20] =?UTF-8?q?Add=20Store.freezeStructure=20=E2=80=94?= =?UTF-8?q?=20generic=20freeze=20of=20any=20resource=20graph?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Backs the data-browser "Freeze" action: freezes a resource and, by default, the structure it references (Ontology + Classes + Properties, Document + Elements, etc.) into content-addressed did:ad:frozen JSON-AD. References between included resources are rewritten to frozen ids; references outside (core schema, drives, agents) stay as subjects. Hierarchy/server metadata (parent, lastCommit) is stripped. { closure: false } freezes only the root; { save: true } publishes to /frozen. Returns { root, bySubject, frozen } for display/download. --- browser/lib/src/frozen-resolve.test.ts | 74 +++++++++++ browser/lib/src/store.ts | 169 ++++++++++++++++++++++++- 2 files changed, 242 insertions(+), 1 deletion(-) diff --git a/browser/lib/src/frozen-resolve.test.ts b/browser/lib/src/frozen-resolve.test.ts index 63f2f6730..5f5e2f668 100644 --- a/browser/lib/src/frozen-resolve.test.ts +++ b/browser/lib/src/frozen-resolve.test.ts @@ -5,6 +5,7 @@ import { JSCryptoProvider } from './CryptoProvider.js'; import { Datatype } from './datatypes.js'; import { frozenIdFor } from './freeze.js'; import { core } from './ontologies/core.js'; +import { JSONADParser } from './parse.js'; import { buildSchemaLock } from './schema-lock.js'; import { Store } from './store.js'; @@ -222,3 +223,76 @@ describe('Store.createSchemaPointer', () => { ); }); }); + +describe('Store.freezeStructure', () => { + const P = 'https://atomicdata.dev/properties/'; + const propSubject = 'https://my.drive/p/title'; + const classSubject = 'https://my.drive/c/todo'; + const ontSubject = 'https://my.drive/o/todoapp'; + + const seeded = (): Store => { + const store = new Store({ serverUrl: 'https://my.drive' }); + + const add = (obj: Record) => { + const [res] = new JSONADParser().parse(obj, obj['@id'] as string); + res.loading = false; + store.addResource(res, { skipCommitCompare: true }); + }; + + add({ + '@id': propSubject, + [`${P}isA`]: ['https://atomicdata.dev/classes/Property'], + [`${P}shortname`]: 'title', + [`${P}datatype`]: Datatype.STRING, + }); + add({ + '@id': classSubject, + [`${P}isA`]: ['https://atomicdata.dev/classes/Class'], + [`${P}shortname`]: 'todo', + [`${P}requires`]: [propSubject], + }); + add({ + '@id': ontSubject, + [`${P}isA`]: ['https://atomicdata.dev/class/ontology'], + [`${P}shortname`]: 'todoapp', + [`${P}classes`]: [classSubject], + [`${P}properties`]: [propSubject], + [`${P}parent`]: 'https://my.drive', + }); + + return store; + }; + + it('freezes a resource and the structure it references', async ({ + expect, + }) => { + const frozen = await seeded().freezeStructure(ontSubject); + + expect(Object.keys(frozen.frozen)).toHaveLength(3); + expect(frozen.root).toBe(frozen.bySubject[ontSubject]); + expect(frozen.root).toMatch(/^did:ad:frozen:[0-9a-f]{64}$/); + + // Cross-references rewritten to frozen ids; hierarchy stripped. + const ontBody = frozen.frozen[frozen.root] as Record; + expect(ontBody[`${P}classes`]).toEqual([frozen.bySubject[classSubject]]); + expect(ontBody[`${P}properties`]).toEqual([frozen.bySubject[propSubject]]); + expect(ontBody[`${P}parent`]).toBeUndefined(); + + const classBody = frozen.frozen[frozen.bySubject[classSubject]] as Record< + string, + unknown + >; + expect(classBody[`${P}requires`]).toEqual([frozen.bySubject[propSubject]]); + }); + + it('freezes only the root with { closure: false }', async ({ expect }) => { + const frozen = await seeded().freezeStructure(ontSubject, { + closure: false, + }); + + expect(Object.keys(frozen.frozen)).toHaveLength(1); + // The reference stays as the original subject, not rewritten. + const ontBody = frozen.frozen[frozen.root] as Record; + expect(ontBody[`${P}classes`]).toEqual([classSubject]); + }); +}); diff --git a/browser/lib/src/store.ts b/browser/lib/src/store.ts index 33b39c144..aa0a5ba57 100644 --- a/browser/lib/src/store.ts +++ b/browser/lib/src/store.ts @@ -31,7 +31,14 @@ import { type DefinedSchema, type FrozenSchema, } from './schema.js'; -import { frozenIdFor, UNIT_MEMBERS_KEY, type FrozenId } from './freeze.js'; +import { + freezeResources, + frozenIdFor, + UNIT_MEMBERS_KEY, + type FrozenId, + type FreezableResource, + type JsonValue as FrozenJsonValue, +} from './freeze.js'; import { jcsCanonicalize } from './jcs.js'; import { verifySchemaLock, type SchemaLock } from './schema-lock.js'; import { stringToSlug } from './stringToSlug.js'; @@ -102,6 +109,27 @@ export interface RegisterSchemaOptions { save?: boolean; } +export interface FreezeStructureOptions { + /** + * Follow value references to other already-loaded resources, freezing the + * whole structure (e.g. an Ontology + its Classes + Properties). Defaults to + * true; set false to freeze only the root resource. + */ + closure?: boolean; + /** Also publish each frozen body to `/frozen` on the server. Defaults to false. */ + save?: boolean; +} + +/** The result of freezing a resource (and the structure it references). */ +export interface FrozenStructure { + /** The root resource's frozen id. */ + root: FrozenId; + /** Original subject -> frozen id, for every resource included. */ + bySubject: Record; + /** Frozen id -> identity JSON-AD body (what a consumer re-hashes and materializes). */ + frozen: Record; +} + export interface StoreOpts { /** The default store URL, where to send commits and where to create new instances */ serverUrl?: string; @@ -1874,6 +1902,145 @@ export class Store { return ontology; } + /** + * Freezes a resource — and, by default, the structure it references — into + * immutable, content-addressed `did:ad:frozen` JSON-AD. Generic: works on any + * resource (Ontology, Document, Folder, …), not just schemas. References + * between included resources are rewritten to frozen ids; references outside + * the structure (core schema, drives, agents) stay as their normal subjects. + * Hierarchy/server metadata (`parent`, `lastCommit`, `localId`) is stripped. + * With `{ save: true }` each frozen body is published to `/frozen`. + */ + public async freezeStructure( + subject: string, + opts: FreezeStructureOptions = {}, + ): Promise { + const { closure = true, save = false } = opts; + const root = await this.getResource(subject); + + if (root.error) { + throw new Error(`Cannot freeze ${subject}: ${root.error}`); + } + + const bodies = new Map(); + const queue: string[] = [subject]; + + while (queue.length > 0) { + const current = queue.shift()!; + + if (bodies.has(current)) { + continue; + } + + const resource = + current === subject ? root : this.resources.get(current); + + if (!resource || !resource.isReady()) { + continue; + } + + const body = this.frozenBodyOf(resource); + bodies.set(current, body); + + if (closure) { + for (const ref of this.structureReferences(body)) { + if (!bodies.has(ref) && this.isFreezableResource(ref)) { + queue.push(ref); + } + } + } + } + + const freezable: FreezableResource[] = [...bodies].map( + ([localId, content]) => ({ localId, content }), + ); + const { resources, byLocalId } = freezeResources(freezable); + + const rootId = byLocalId.get(subject); + + if (!rootId) { + throw new Error(`Freezing produced no id for ${subject}`); + } + + if (save) { + await Promise.all( + resources.map(resource => + this.publishFrozenResource(resource.frozenId, resource.content), + ), + ); + } + + return { + root: rootId, + bySubject: Object.fromEntries(byLocalId), + frozen: Object.fromEntries( + resources.map(resource => [resource.frozenId, resource.content]), + ), + }; + } + + /** A resource's propvals as a frozen body — strips subject and mutable metadata. */ + private frozenBodyOf(resource: Resource): FrozenJsonValue { + const strip = new Set([ + core.properties.parent, + 'https://atomicdata.dev/properties/lastCommit', + core.properties.localId, + ]); + const body: Record = {}; + + for (const [key, value] of resource.getEntries()) { + if (strip.has(key) || value instanceof Uint8Array) { + continue; + } + + body[key] = value as FrozenJsonValue; + } + + return body; + } + + /** Subjects referenced by `body` that are already loaded resources. */ + private structureReferences(body: FrozenJsonValue): string[] { + const out: string[] = []; + + const walk = (value: FrozenJsonValue): void => { + if (typeof value === 'string') { + if (this.resources.has(value)) { + out.push(value); + } + } else if (Array.isArray(value)) { + value.forEach(walk); + } else if (value !== null && typeof value === 'object') { + Object.values(value).forEach(walk); + } + }; + + walk(body); + + return out; + } + + /** Whether a referenced subject should be pulled into a frozen structure. */ + private isFreezableResource(subject: string): boolean { + if ( + subject.startsWith('https://atomicdata.dev/') || + subject.startsWith('did:ad:agent:') || + subject.startsWith('did:ad:commit:') || + subject.startsWith('did:ad:frozen:') || + subject.startsWith('did:ad:blob:') + ) { + return false; + } + + const resource = this.resources.get(subject); + + return ( + !!resource && + resource.isReady() && + !resource.hasClasses(server.classes.drive) + ); + } + /** * Registers an app-bundled `*.schema.lock.json` into the store: verifies every * frozen object by re-hash, then materializes each as a read-only Resource so From 6bdf1bab40e37a56b3d312f72cd6be91d9e2e7b1 Mon Sep 17 00:00:00 2001 From: Joep Meindertsma Date: Tue, 2 Jun 2026 14:18:25 +0200 Subject: [PATCH 11/20] Add "Freeze" context-menu action + dialog to the data-browser Adds a Freeze item to the resource context menu (any resource) that opens a dialog showing the content-addressed did:ad:frozen JSON-AD for the resource and the structure it references (via Store.freezeStructure). Copy / Download the artifact, or Publish it to the server's /frozen store. Format toggle: JSON-AD (reproducible, no history) active; Loro (keeps CRDT history, binary) shown as coming-soon. An "Include referenced structure" checkbox toggles the closure. --- .../ResourceContextMenu/FreezeDialog.tsx | 205 ++++++++++++++++++ .../components/ResourceContextMenu/index.tsx | 17 ++ 2 files changed, 222 insertions(+) create mode 100644 browser/data-browser/src/components/ResourceContextMenu/FreezeDialog.tsx diff --git a/browser/data-browser/src/components/ResourceContextMenu/FreezeDialog.tsx b/browser/data-browser/src/components/ResourceContextMenu/FreezeDialog.tsx new file mode 100644 index 000000000..6bc890609 --- /dev/null +++ b/browser/data-browser/src/components/ResourceContextMenu/FreezeDialog.tsx @@ -0,0 +1,205 @@ +import { useEffect, useState } from 'react'; +import { useResource, useStore } from '@tomic/react'; +import toast from 'react-hot-toast'; +import { styled } from 'styled-components'; +import { + Dialog, + DialogActions, + DialogContent, + DialogTitle, + useDialog, +} from '../Dialog'; +import { Button } from '../Button'; + +interface FreezeDialogProps { + subject: string; + show: boolean; + bindShow: (open: boolean) => void; +} + +type FreezeMode = 'json-ad' | 'loro'; + +/** + * Freezes a resource (and, by default, the structure it references) into + * immutable, content-addressed `did:ad:frozen` JSON-AD. Shows the result with + * copy / download, and can publish it to the server's `/frozen` store. + */ +export function FreezeDialog({ + subject, + show: open, + bindShow, +}: FreezeDialogProps): React.JSX.Element { + const store = useStore(); + const resource = useResource(subject); + const [dialogProps, show, hide, isOpen] = useDialog({ bindShow }); + + const [mode, setMode] = useState('json-ad'); + const [closure, setClosure] = useState(true); + const [json, setJson] = useState(''); + const [error, setError] = useState(); + const [publishing, setPublishing] = useState(false); + + useEffect(() => { + if (open) { + show(); + } else { + hide(); + } + }, [open]); + + useEffect(() => { + if (!isOpen || mode !== 'json-ad') { + return; + } + + let active = true; + setError(undefined); + + store + .freezeStructure(subject, { closure }) + .then(result => { + if (active) { + setJson(JSON.stringify(result.frozen, null, 2)); + } + }) + .catch((e: Error) => { + if (active) { + setError(e.message); + } + }); + + return () => { + active = false; + }; + }, [isOpen, subject, closure, mode, store]); + + const handleCopy = async () => { + await navigator.clipboard.writeText(json); + toast.success('Copied frozen JSON-AD'); + }; + + const handleDownload = () => { + const blob = new Blob([json], { type: 'application/ad+json' }); + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = `${resource.title || 'resource'}.frozen.json`; + a.click(); + URL.revokeObjectURL(url); + }; + + const handlePublish = async () => { + setPublishing(true); + + try { + const result = await store.freezeStructure(subject, { + closure, + save: true, + }); + const count = Object.keys(result.frozen).length; + toast.success(`Published ${count} frozen resource${count === 1 ? '' : 's'}`); + } catch (e) { + toast.error(`Publish failed: ${(e as Error).message}`); + } finally { + setPublishing(false); + } + }; + + return ( + + {isOpen && ( + <> + +

Freeze {resource.title}

+ + + + + + setMode('json-ad')} + /> + JSON-AD reproducible, no history + + + + Loro coming soon + + + + setClosure(e.target.checked)} + /> + Include referenced structure + + + {error ? {error} : 
{json}
} + + + + + + + + )} + + ); +} + +const StyledContent = styled(DialogContent)` + max-height: 90vh; + overflow-x: hidden; +`; + +const Controls = styled.div` + display: flex; + flex-wrap: wrap; + gap: 1rem 2rem; + align-items: center; + margin-bottom: 1rem; +`; + +const ModeGroup = styled.div` + display: flex; + gap: 1.5rem; +`; + +const ModeOption = styled.label<{ $disabled?: boolean }>` + display: inline-flex; + align-items: center; + gap: 0.4rem; + cursor: ${p => (p.$disabled ? 'not-allowed' : 'pointer')}; + opacity: ${p => (p.$disabled ? 0.5 : 1)}; +`; + +const Hint = styled.small` + color: ${p => p.theme.colors.textLight}; +`; + +const Pre = styled.pre` + background: ${p => p.theme.colors.bg1}; + border: 1px solid ${p => p.theme.colors.bg2}; + border-radius: ${p => p.theme.radius}; + padding: 1rem; + overflow: auto; + max-height: 60vh; + font-size: 0.8rem; +`; + +const ErrorText = styled.p` + color: ${p => p.theme.colors.alert}; +`; diff --git a/browser/data-browser/src/components/ResourceContextMenu/index.tsx b/browser/data-browser/src/components/ResourceContextMenu/index.tsx index 3f5a6b7a1..ca8dd0eca 100644 --- a/browser/data-browser/src/components/ResourceContextMenu/index.tsx +++ b/browser/data-browser/src/components/ResourceContextMenu/index.tsx @@ -26,6 +26,7 @@ import { FaPlus, FaArrowUpRightFromSquare, FaMessage, + FaSnowflake, } from 'react-icons/fa6'; import { useQueryScopeHandler } from '../../hooks/useQueryScope'; import { @@ -36,6 +37,7 @@ import { ResourceInline } from '../../views/ResourceInline'; import { ResourceUsage } from '../ResourceUsage'; import { useCurrentSubject } from '../../helpers/useCurrentSubject'; import { ResourceCodeUsageDialog } from '../../views/CodeUsage/ResourceCodeUsageDialog'; +import { FreezeDialog } from './FreezeDialog'; import { useNewRoute } from '../../helpers/useNewRoute'; import { addIf } from '../../helpers/addIf'; import { useNavigateWithTransition } from '../../hooks/useNavigateWithTransition'; @@ -64,6 +66,7 @@ export const ContextMenuOptions = { Export: 'export', Open: 'open', AddToChat: 'addToChat', + Freeze: 'freeze', } as const; export type ContextMenuOptionsUnion = @@ -101,6 +104,7 @@ export function ResourceContextMenu({ const resource = useResource(subject); const [showDeleteDialog, setShowDeleteDialog] = useState(false); const [showCodeUsageDialog, setShowCodeUsageDialog] = useState(false); + const [showFreezeDialog, setShowFreezeDialog] = useState(false); const handleAddClick = useNewRoute(subject); const [currentSubject] = useCurrentSubject(); const canWrite = useCanWrite(resource); @@ -202,6 +206,14 @@ export function ResourceContextMenu({ icon: , onClick: () => setShowCodeUsageDialog(true), }, + { + id: ContextMenuOptions.Freeze, + label: 'Freeze', + helper: + 'Create an immutable, content-addressed (did:ad:frozen) copy of this resource and the structure it references.', + icon: , + onClick: () => setShowFreezeDialog(true), + }, { id: ContextMenuOptions.AddToChat, label: 'Add to chat', @@ -302,6 +314,11 @@ export function ResourceContextMenu({ bindShow={setShowCodeUsageDialog} /> )} + ); } From ec6149ed05313dfdb3199942db2086576861faab Mon Sep 17 00:00:00 2001 From: Joep Meindertsma Date: Tue, 2 Jun 2026 14:29:58 +0200 Subject: [PATCH 12/20] Extract Freeze dialog i18n strings into locale catalogs --- browser/data-browser/src/locales/de.po | 57 ++++++++++++++++++++++++++ browser/data-browser/src/locales/en.po | 57 ++++++++++++++++++++++++++ browser/data-browser/src/locales/es.po | 57 ++++++++++++++++++++++++++ browser/data-browser/src/locales/fr.po | 57 ++++++++++++++++++++++++++ 4 files changed, 228 insertions(+) diff --git a/browser/data-browser/src/locales/de.po b/browser/data-browser/src/locales/de.po index 9e92ad8ca..10d744d48 100644 --- a/browser/data-browser/src/locales/de.po +++ b/browser/data-browser/src/locales/de.po @@ -212,6 +212,7 @@ msgstr "Fehler in die Zwischenablage kopiert" msgid "Clear" msgstr "Leeren" +#: src/components/ResourceContextMenu/FreezeDialog.tsx #: src/components/Toaster.tsx #: src/routes/SyncRoute.tsx msgid "Copy" @@ -4376,3 +4377,59 @@ msgstr "Build-Commit" #: src/routes/AboutRoute.tsx msgid "built {0}" msgstr "erstellt {0}" + +#: src/components/ResourceContextMenu/index.tsx +msgid "Freeze" +msgstr "Einfrieren" + +#: src/components/ResourceContextMenu/index.tsx +msgid "Create an immutable, content-addressed (did:ad:frozen) copy of this resource and the structure it references." +msgstr "Erstellen Sie eine unveränderliche, inhaltsadressierte (did:ad:frozen) Kopie dieser Ressource und der von ihr referenzierten Struktur." + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Copied frozen JSON-AD" +msgstr "Gefrorenes JSON-AD kopiert" + +#. placeholder {0}: count +#. placeholder {1}: count === 1 ? '' : 's' +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Published {0} frozen resource{1}" +msgstr "{0} eingefrorene Ressource{1} veröffentlicht" + +#. placeholder {0}: (e as Error).message +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Publish failed: {0}" +msgstr "Veröffentlichung fehlgeschlagen: {0}" + +#. placeholder {0}: resource.title +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Freeze {0}" +msgstr "{0} einfrieren" + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "<0/> JSON-AD <1>reproducible, no history" +msgstr "<0/> JSON-AD <1>reproduzierbar, keine Historie" + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "<0/> Loro <1>coming soon" +msgstr "<0/> Loro <1>demnächst verfügbar" + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Coming soon — keeps CRDT history, binary, id not reproducible" +msgstr "Bald verfügbar – behält CRDT-Verlauf, binär, ID nicht reproduzierbar" + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Freeze format" +msgstr "Format einfrieren" + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "<0/> Include referenced structure" +msgstr "<0/> Referenzierte Struktur einschließen" + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Download" +msgstr "Herunterladen" + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Publish to server" +msgstr "Auf Server veröffentlichen" diff --git a/browser/data-browser/src/locales/en.po b/browser/data-browser/src/locales/en.po index 9c85bb5c1..cb0c74475 100644 --- a/browser/data-browser/src/locales/en.po +++ b/browser/data-browser/src/locales/en.po @@ -2236,6 +2236,7 @@ msgstr "Nothing to copy." msgid "Copied error to clipboard" msgstr "Copied error to clipboard" +#: src/components/ResourceContextMenu/FreezeDialog.tsx #: src/components/Toaster.tsx #: src/routes/SyncRoute.tsx msgid "Copy" @@ -4392,3 +4393,59 @@ msgstr "Build commit" #: src/routes/AboutRoute.tsx msgid "built {0}" msgstr "built {0}" + +#: src/components/ResourceContextMenu/index.tsx +msgid "Freeze" +msgstr "Freeze" + +#: src/components/ResourceContextMenu/index.tsx +msgid "Create an immutable, content-addressed (did:ad:frozen) copy of this resource and the structure it references." +msgstr "Create an immutable, content-addressed (did:ad:frozen) copy of this resource and the structure it references." + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Copied frozen JSON-AD" +msgstr "Copied frozen JSON-AD" + +#. placeholder {0}: count +#. placeholder {1}: count === 1 ? '' : 's' +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Published {0} frozen resource{1}" +msgstr "Published {0} frozen resource{1}" + +#. placeholder {0}: (e as Error).message +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Publish failed: {0}" +msgstr "Publish failed: {0}" + +#. placeholder {0}: resource.title +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Freeze {0}" +msgstr "Freeze {0}" + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "<0/> JSON-AD <1>reproducible, no history" +msgstr "<0/> JSON-AD <1>reproducible, no history" + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "<0/> Loro <1>coming soon" +msgstr "<0/> Loro <1>coming soon" + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Coming soon — keeps CRDT history, binary, id not reproducible" +msgstr "Coming soon — keeps CRDT history, binary, id not reproducible" + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Freeze format" +msgstr "Freeze format" + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "<0/> Include referenced structure" +msgstr "<0/> Include referenced structure" + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Download" +msgstr "Download" + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Publish to server" +msgstr "Publish to server" diff --git a/browser/data-browser/src/locales/es.po b/browser/data-browser/src/locales/es.po index b6dbb4578..1224d7894 100644 --- a/browser/data-browser/src/locales/es.po +++ b/browser/data-browser/src/locales/es.po @@ -197,6 +197,7 @@ msgstr "Error copiado al portapapeles" msgid "Clear" msgstr "Borrar" +#: src/components/ResourceContextMenu/FreezeDialog.tsx #: src/components/Toaster.tsx #: src/routes/SyncRoute.tsx msgid "Copy" @@ -4371,3 +4372,59 @@ msgstr "Compilación commit" #: src/routes/AboutRoute.tsx msgid "built {0}" msgstr "compilado el {0}" + +#: src/components/ResourceContextMenu/index.tsx +msgid "Freeze" +msgstr "Congelar" + +#: src/components/ResourceContextMenu/index.tsx +msgid "Create an immutable, content-addressed (did:ad:frozen) copy of this resource and the structure it references." +msgstr "Crea una copia inmutable, direccionada por contenido (did:ad:frozen) de este recurso y la estructura a la que hace referencia." + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Copied frozen JSON-AD" +msgstr "JSON-AD congelado copiado" + +#. placeholder {0}: count +#. placeholder {1}: count === 1 ? '' : 's' +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Published {0} frozen resource{1}" +msgstr "Publicado {0} recurso{1} congelado" + +#. placeholder {0}: (e as Error).message +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Publish failed: {0}" +msgstr "Error al publicar: {0}" + +#. placeholder {0}: resource.title +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Freeze {0}" +msgstr "Congelar {0}" + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "<0/> JSON-AD <1>reproducible, no history" +msgstr "<0/> JSON-AD <1>reproducible, sin historial" + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "<0/> Loro <1>coming soon" +msgstr "<0/> Loro <1>próximamente" + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Coming soon — keeps CRDT history, binary, id not reproducible" +msgstr "Próximamente — conserva el historial CRDT, binario, id no reproducible" + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Freeze format" +msgstr "Formato de congelación" + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "<0/> Include referenced structure" +msgstr "<0/> Incluir estructura referenciada" + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Download" +msgstr "Descargar" + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Publish to server" +msgstr "Publicar en el servidor" diff --git a/browser/data-browser/src/locales/fr.po b/browser/data-browser/src/locales/fr.po index 860aaf375..a09576633 100644 --- a/browser/data-browser/src/locales/fr.po +++ b/browser/data-browser/src/locales/fr.po @@ -197,6 +197,7 @@ msgstr "Erreur copiée dans le presse-papier" msgid "Clear" msgstr "Effacer" +#: src/components/ResourceContextMenu/FreezeDialog.tsx #: src/components/Toaster.tsx #: src/routes/SyncRoute.tsx msgid "Copy" @@ -4384,3 +4385,59 @@ msgstr "Construire le commit" #: src/routes/AboutRoute.tsx msgid "built {0}" msgstr "construit {0}" + +#: src/components/ResourceContextMenu/index.tsx +msgid "Freeze" +msgstr "Geler" + +#: src/components/ResourceContextMenu/index.tsx +msgid "Create an immutable, content-addressed (did:ad:frozen) copy of this resource and the structure it references." +msgstr "Créez une copie immuable et adressée par le contenu (did:ad:frozen) de cette ressource et de la structure qu'elle référence." + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Copied frozen JSON-AD" +msgstr "JSON-AD gelé copié" + +#. placeholder {0}: count +#. placeholder {1}: count === 1 ? '' : 's' +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Published {0} frozen resource{1}" +msgstr "{0} ressource{1} gelée{1} publiée{1}" + +#. placeholder {0}: (e as Error).message +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Publish failed: {0}" +msgstr "Échec de la publication : {0}" + +#. placeholder {0}: resource.title +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Freeze {0}" +msgstr "Geler {0}" + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "<0/> JSON-AD <1>reproducible, no history" +msgstr "<0/> JSON-AD <1>reproductible, pas d'historique" + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "<0/> Loro <1>coming soon" +msgstr "<0/> Loro <1>bientôt disponible" + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Coming soon — keeps CRDT history, binary, id not reproducible" +msgstr "Bientôt disponible — conserve l'historique CRDT, binaire, id non reproductible" + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Freeze format" +msgstr "Format de gel" + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "<0/> Include referenced structure" +msgstr "<0/> Inclure la structure référencée" + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Download" +msgstr "Télécharger" + +#: src/components/ResourceContextMenu/FreezeDialog.tsx +msgid "Publish to server" +msgstr "Publier sur le serveur" From 84aa4fb247b9cf29fb8f353bdc47ba6b0c3f8bcb Mon Sep 17 00:00:00 2001 From: Joep Meindertsma Date: Tue, 2 Jun 2026 14:36:49 +0200 Subject: [PATCH 13/20] Add frozen e2e spec + make frozen resources immutable in the UI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ResourceContextMenu: a did:ad:frozen resource is never writable (canWrite is forced false), so Edit / Add child / Import / Delete are hidden — frozen resources are content-addressed and immutable. browser/e2e/tests/frozen.spec.ts: drives the full browser flow against a real server — freeze the dev drive (dialog shows a did:ad:frozen JSON-AD body), Publish to the server, resolve the frozen id back, and assert the frozen resource has no Edit affordance. Passes end to end. --- .../components/ResourceContextMenu/index.tsx | 6 ++- browser/e2e/tests/frozen.spec.ts | 50 +++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 browser/e2e/tests/frozen.spec.ts diff --git a/browser/data-browser/src/components/ResourceContextMenu/index.tsx b/browser/data-browser/src/components/ResourceContextMenu/index.tsx index ca8dd0eca..44ef91663 100644 --- a/browser/data-browser/src/components/ResourceContextMenu/index.tsx +++ b/browser/data-browser/src/components/ResourceContextMenu/index.tsx @@ -107,7 +107,11 @@ export function ResourceContextMenu({ const [showFreezeDialog, setShowFreezeDialog] = useState(false); const handleAddClick = useNewRoute(subject); const [currentSubject] = useCurrentSubject(); - const canWrite = useCanWrite(resource); + const writeRight = useCanWrite(resource); + // Frozen resources are content-addressed and immutable — never editable, + // regardless of rights. Hide all write actions and show a frozen badge. + const isFrozen = subject.startsWith('did:ad:frozen:'); + const canWrite = writeRight && !isFrozen; const { enableScope } = useQueryScopeHandler(subject); const { setContextItems, isOpen, setIsOpen } = useAISidebar(); const { items: customItems } = useCustomContextItemsContext(); diff --git a/browser/e2e/tests/frozen.spec.ts b/browser/e2e/tests/frozen.spec.ts new file mode 100644 index 000000000..349f2c5a9 --- /dev/null +++ b/browser/e2e/tests/frozen.spec.ts @@ -0,0 +1,50 @@ +import { expect, test } from '@playwright/test'; +import { before, contextMenuClick, openSubject } from './test-utils'; + +/** + * The frozen (`did:ad:frozen`) feature, end to end through the data-browser: + * 1. Freeze a resource into immutable, content-addressed JSON-AD. + * 2. Publish it to the server and resolve it back by its frozen id. + * 3. A frozen resource is immutable — no edit affordance. + */ +test.describe('freeze', () => { + test.beforeEach(before); + + test('freezes a resource, publishes it, and resolves the frozen copy', async ({ + page, + }) => { + // The current resource is the dev drive. Freeze it. + await contextMenuClick('freeze', page); + + const dialog = page.locator('dialog[open]'); + await expect( + dialog.getByRole('heading', { name: /^Freeze/ }), + ).toBeVisible(); + + // The frozen JSON-AD is content-addressed: a did:ad:frozen id appears. + const body = dialog.locator('pre'); + await expect(body).toContainText(/did:ad:frozen:[0-9a-f]{64}/, { + timeout: 15000, + }); + + const frozenId = (await body.innerText()).match( + /did:ad:frozen:[0-9a-f]{64}/, + )?.[0]; + expect(frozenId).toBeTruthy(); + + // Publish to the server, then resolve the frozen resource by its id. + await dialog.getByRole('button', { name: 'Publish to server' }).click(); + await expect( + page.getByText(/Published \d+ frozen resource/), + ).toBeVisible({ timeout: 15000 }); + + await openSubject(page, frozenId!); + await expect( + page.locator(`main[about="${frozenId}"]`).first(), + ).toBeVisible({ timeout: 20000 }); + + // A frozen resource is immutable: the context menu offers no Edit. + await page.click('[data-test="context-menu"]'); + await expect(page.getByTestId('menu-item-edit')).toHaveCount(0); + }); +}); From 9a38c1062b1f766221a60bb1324128a87101720d Mon Sep 17 00:00:00 2001 From: Joep Meindertsma Date: Tue, 2 Jun 2026 15:19:10 +0200 Subject: [PATCH 14/20] Add a "Frozen" badge to the resource view for did:ad:frozen resources MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A content-addressed resource now shows a "❄ Frozen" badge in the top bar (title: "Content-addressed and immutable — verified by hash"), the positive counterpart to hiding the edit affordances. The e2e spec asserts the badge is visible on a resolved frozen resource. --- .../data-browser/src/components/NavBar.tsx | 25 +++++++++++++++++++ browser/e2e/tests/frozen.spec.ts | 3 ++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/browser/data-browser/src/components/NavBar.tsx b/browser/data-browser/src/components/NavBar.tsx index ba80cdcd8..9acee2d91 100644 --- a/browser/data-browser/src/components/NavBar.tsx +++ b/browser/data-browser/src/components/NavBar.tsx @@ -27,6 +27,7 @@ import { FaBars, FaMagnifyingGlass, FaShare, + FaSnowflake, FaTags, } from 'react-icons/fa6'; import * as RadixPopover from '@radix-ui/react-popover'; @@ -252,6 +253,15 @@ export function NavBar({ resource: resourceProp }: NavBarProps): JSX.Element { {parent && } + {resource.subject.startsWith('did:ad:frozen:') && ( + + + Frozen + + )} p.theme.radius}; + background-color: ${p => p.theme.colors.bg1}; + border: 1px solid ${p => p.theme.colors.bg2}; + color: ${p => p.theme.colors.textLight}; + font-size: 0.8rem; + white-space: nowrap; +`; + const NavBarWrapper = styled.nav` height: 100%; width: 100%; diff --git a/browser/e2e/tests/frozen.spec.ts b/browser/e2e/tests/frozen.spec.ts index 349f2c5a9..df4bd12cb 100644 --- a/browser/e2e/tests/frozen.spec.ts +++ b/browser/e2e/tests/frozen.spec.ts @@ -43,7 +43,8 @@ test.describe('freeze', () => { page.locator(`main[about="${frozenId}"]`).first(), ).toBeVisible({ timeout: 20000 }); - // A frozen resource is immutable: the context menu offers no Edit. + // A frozen resource is immutable: a Frozen badge, and no Edit affordance. + await expect(page.getByTestId('frozen-badge')).toBeVisible(); await page.click('[data-test="context-menu"]'); await expect(page.getByTestId('menu-item-edit')).toHaveCount(0); }); From 0a93e5eb48c72d8dc3b7de2c0ffb9bda687f10e3 Mon Sep 17 00:00:00 2001 From: Joep Meindertsma Date: Tue, 2 Jun 2026 15:19:19 +0200 Subject: [PATCH 15/20] Extract Frozen badge i18n strings --- browser/data-browser/src/locales/de.po | 8 ++++++++ browser/data-browser/src/locales/en.po | 8 ++++++++ browser/data-browser/src/locales/es.po | 8 ++++++++ browser/data-browser/src/locales/fr.po | 8 ++++++++ 4 files changed, 32 insertions(+) diff --git a/browser/data-browser/src/locales/de.po b/browser/data-browser/src/locales/de.po index 10d744d48..b60dc2afa 100644 --- a/browser/data-browser/src/locales/de.po +++ b/browser/data-browser/src/locales/de.po @@ -4433,3 +4433,11 @@ msgstr "Herunterladen" #: src/components/ResourceContextMenu/FreezeDialog.tsx msgid "Publish to server" msgstr "Auf Server veröffentlichen" + +#: src/components/NavBar.tsx +msgid "Frozen" +msgstr "Eingefroren" + +#: src/components/NavBar.tsx +msgid "Content-addressed and immutable — verified by hash" +msgstr "Inhaltsadressiert und unveränderlich – verifiziert durch Hash" diff --git a/browser/data-browser/src/locales/en.po b/browser/data-browser/src/locales/en.po index cb0c74475..34ebc65a6 100644 --- a/browser/data-browser/src/locales/en.po +++ b/browser/data-browser/src/locales/en.po @@ -4449,3 +4449,11 @@ msgstr "Download" #: src/components/ResourceContextMenu/FreezeDialog.tsx msgid "Publish to server" msgstr "Publish to server" + +#: src/components/NavBar.tsx +msgid "Frozen" +msgstr "Frozen" + +#: src/components/NavBar.tsx +msgid "Content-addressed and immutable — verified by hash" +msgstr "Content-addressed and immutable — verified by hash" diff --git a/browser/data-browser/src/locales/es.po b/browser/data-browser/src/locales/es.po index 1224d7894..151d5e28f 100644 --- a/browser/data-browser/src/locales/es.po +++ b/browser/data-browser/src/locales/es.po @@ -4428,3 +4428,11 @@ msgstr "Descargar" #: src/components/ResourceContextMenu/FreezeDialog.tsx msgid "Publish to server" msgstr "Publicar en el servidor" + +#: src/components/NavBar.tsx +msgid "Frozen" +msgstr "Congelado" + +#: src/components/NavBar.tsx +msgid "Content-addressed and immutable — verified by hash" +msgstr "Dirigido por contenido e inmutable — verificado por hash" diff --git a/browser/data-browser/src/locales/fr.po b/browser/data-browser/src/locales/fr.po index a09576633..ba75d32d4 100644 --- a/browser/data-browser/src/locales/fr.po +++ b/browser/data-browser/src/locales/fr.po @@ -4441,3 +4441,11 @@ msgstr "Télécharger" #: src/components/ResourceContextMenu/FreezeDialog.tsx msgid "Publish to server" msgstr "Publier sur le serveur" + +#: src/components/NavBar.tsx +msgid "Frozen" +msgstr "Frozen" + +#: src/components/NavBar.tsx +msgid "Content-addressed and immutable — verified by hash" +msgstr "Adressable par contenu et immuable — vérifié par hachage" From 4850cae3e5ab3ae173f03e64a98bdfd078055eaf Mon Sep 17 00:00:00 2001 From: Joep Meindertsma Date: Tue, 2 Jun 2026 16:14:46 +0200 Subject: [PATCH 16/20] Add code-first schema example (@tomic/lib publish-schema.mjs) --- examples/code-first-schema/publish-schema.mjs | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 examples/code-first-schema/publish-schema.mjs diff --git a/examples/code-first-schema/publish-schema.mjs b/examples/code-first-schema/publish-schema.mjs new file mode 100644 index 000000000..3d193cd35 --- /dev/null +++ b/examples/code-first-schema/publish-schema.mjs @@ -0,0 +1,53 @@ +// A tiny @tomic/lib "code-first schema" app. +// +// AGENT_SECRET= node publish-schema.mjs +// +// Defines a schema in code, publishes it to a running atomic-server, and prints +// the resulting Ontology subject — which you can then open in the data-browser. +import { Agent, Store } from '@tomic/lib'; + +const SERVER = process.env.SERVER_URL || 'http://localhost:9883'; +const SECRET = process.env.AGENT_SECRET; + +if (!SECRET) { + throw new Error('Set AGENT_SECRET (your agent secret from the data-browser).'); +} + +// Define the data model in code — JSON-Schema-like, mapped to Atomic. +const todoSchema = { + name: 'TodoApp', + version: '1.0.0', + classes: { + todo: { + title: 'Todo', + description: 'A task in a todo list', + type: 'object', + required: ['title'], + properties: { + title: { type: 'string', description: 'Task title' }, + done: { type: 'boolean', description: 'Whether the task is complete' }, + dueAt: { type: 'string', format: 'date', description: 'Due date' }, + }, + }, + }, +}; + +const agent = await Agent.fromSecret(SECRET); +const store = new Store({ serverUrl: SERVER, agent }); +store.setServerConnected(true); + +// Publish the schema as Atomic Class & Property resources under an Ontology. +const registered = await store.registerSchema(todoSchema, { save: true }); + +console.log('Published ontology:'); +console.log(' ONTOLOGY ' + registered.ontology.subject); + +for (const [key, resource] of Object.entries(registered.classes)) { + console.log(` CLASS ${key}: ${resource.subject}`); +} + +for (const [key, resource] of Object.entries(registered.properties)) { + console.log(` PROPERTY ${key}: ${resource.subject}`); +} + +process.exit(0); From b8b7784735d47126757937a4913342a049fdcba1 Mon Sep 17 00:00:00 2001 From: Joep Meindertsma Date: Tue, 2 Jun 2026 16:18:49 +0200 Subject: [PATCH 17/20] Update planning docs: did:ad:frozen full-stack + UI + e2e done --- planning/did-ad-frozen-server.md | 9 +++ planning/json-schema-code-first.md | 94 ++++++++++++++++++++++-------- 2 files changed, 78 insertions(+), 25 deletions(-) diff --git a/planning/did-ad-frozen-server.md b/planning/did-ad-frozen-server.md index fab13f4ef..f8a48515f 100644 --- a/planning/did-ad-frozen-server.md +++ b/planning/did-ad-frozen-server.md @@ -1,5 +1,14 @@ # Server-side `did:ad:frozen` plan +> **Status: Phases A, B, and C are done and verified.** Storage, parsing, +> materialization, commit-rejection, `GET/PUT /frozen`, browser resolution, +> `registerFrozenSchema`/`loadSchemaLock`/`createSchemaPointer`/`freezeStructure`, +> the data-browser "Freeze" UI (+ immutability badge), and a browser e2e all +> land. Cross-language hashing (TS `jcs.ts` ↔ Rust `serde_jcs`) is proven by +> shared `test-vectors/`. Remaining: **Phase D (sync over iroh)** and the +> optional polish in the companion doc (Loro mode, ClientDb persistence, CI +> lock-guard). + Companion to [json-schema-code-first.md](./json-schema-code-first.md). That doc decided the **identity model**: schema definitions are immutable, so they are content-addressed as `did:ad:frozen:{blake3-hex}` instead of signed genesis DIDs. diff --git a/planning/json-schema-code-first.md b/planning/json-schema-code-first.md index 3f110ee38..c82b43cfc 100644 --- a/planning/json-schema-code-first.md +++ b/planning/json-schema-code-first.md @@ -61,7 +61,13 @@ produces locally available DID-backed schema resources from code. ## Current Status -This is not full-stack working yet. +**Full-stack working and verified end to end.** The content-addressed +`did:ad:frozen` schema system runs producer → server → client across both +TypeScript and Rust, with a data-browser UI driving it. Verified live: a +code-first `@tomic/lib` app publishes a schema and it renders in the browser; the +"Freeze" action produces immutable content-addressed resources that resolve back +by hash. (`registerSchema` keeps the signed-DID path for editable ontologies; +`registerFrozenSchema`/`freezeStructure` are the content-addressed path.) Implemented so far: @@ -130,20 +136,52 @@ Implemented so far: split (description edit keeps ids stable, datatype edit changes them, no cascade), determinism, shortname dedupe/conflict, and the imported-property guard. - -Not implemented yet: - -- `did:ad:frozen` is designed and produced by `freezeSchema`, but - `Store.registerSchema` still mints signed genesis DIDs; switching it over needs - server-side store/serve/resolve support for `did:ad:frozen` (verify-by-rehash, - read-only) plus a `did:ad:frozen` resolution path in the browser Store. -- The signed "latest version" pointer / overlay layer on the author's drive. -- Schema package resolution from Drive context, app registry, CLI config, synced - Ontologies, or schema hash before the schema is explicitly registered. -- `store.getProperty()` fallback to unresolved schema packages. -- CLI-side consumer/type-generation e2e for imported schemas. -- Data browser/table/editor handling for content-locked Properties. -- Rust/server import/export or JSON Schema validation. +- `browser/lib/src/schema-lock.ts`: `buildSchemaLock()`/`verifySchemaLock()` — the + committed, self-verifying lockfile (`frozen` objects + `@index` + non-hashed + `presentation`). Verify = re-hash each frozen object via the shared + `frozenIdFor()`. +- **Rust producer (cross-language authoring)** — `lib/src/frozen.rs`: + `frozen_id`/`verify_frozen`, `freeze_resources` (the full content-addressing + core: Tarjan SCC + color refinement + one-unit-per-cycle), and `freeze_schema` + (the order-preserving schema DSL). All **byte-for-byte identical to TS**, pinned + by `test-vectors/{frozen,freeze-resources,freeze-schema}.json` and asserted in + both languages. +- **Server (`did:ad:frozen`)** — `Tree::Frozen` storage (both backends), + `subject.rs` parsing, `db.rs#materialize_frozen` (resolve via re-hash → read-only + Resource), commit-rejection for frozen subjects, and `GET/PUT /frozen/{hash}` + endpoints. Tested incl. a server round-trip + `frozen_endpoint_roundtrip`. +- **Browser client** — `Store.fetchFrozenResource` (resolve `did:ad:frozen` over + HTTP, verify-by-rehash, materialize), `registerFrozenSchema` (freeze + publish), + `loadSchemaLock` (offline, server-free availability from a bundled lockfile), + `createSchemaPointer` (signed mutable Ontology referencing frozen ids), and the + generic `freezeStructure` (freeze any resource + its reference closure). + `getProperty` treats `description` as optional (presentation). Covered by + `frozen-resolve.test.ts`. +- **Capstone integration** — `tests/frozen-e2e.integration.test.ts`: producer + publishes frozen resources to a real server, a fresh consumer resolves and uses + them. +- **Data-browser UI** — a "Freeze" resource context-menu action + + `FreezeDialog` (copy/download/publish, JSON-AD mode, Loro toggle as coming-soon, + closure checkbox); frozen resources are immutable (write actions hidden) with a + "❄ Frozen" badge. Browser e2e `browser/e2e/tests/frozen.spec.ts` drives + freeze → publish → resolve → immutable against a real server. +- `examples/code-first-schema/publish-schema.mjs` — a minimal `@tomic/lib` app + that defines and publishes a schema (verified rendering in the data-browser). + +Not implemented yet (all optional/follow-up — none block the core flows): + +- **Loro freeze mode** (the disabled dialog toggle): a separate feature — + `did:ad:blob` over the (non-reproducible) Loro snapshot, materialized by + Loro-load, keeping history. +- ClientDb/OPFS persistence of frozen objects (offline resolution currently + in-memory; survives via the bundled lockfile but not a page reload). +- `ad-generate` emitting the lockfile + a CI stale-lock guard; CLI-side + import/hash-pin checks and a CLI consumer e2e. +- Phase D sync (frozen objects travelling with drives over iroh). +- Native-Rust-struct codegen from frozen schemas; JSON Schema validation + (richer keywords beyond the Atomic subset) in browser/Rust. +- Drive-context / app-registry discovery of schemas by hash before explicit + registration; round-trip of editor/table-created schemas into frozen. ## Existing Touch Points @@ -835,16 +873,22 @@ Rules: - [ ] Add a CLI-side producer/consumer import e2e where one generated schema imports another published/local Ontology with an expected hash. -### Phase 3: Editor and table compatibility - -- [ ] Keep Ontology editor creation/editing as a first-class schema authoring - path. -- [ ] Ensure table-created Properties and Classes can be exported to JSON - Schema/code schema and then regenerated without changing meaning. -- [ ] Add UI indicators or warnings when editing a published/content-locked - Property would change datatype, `classtype`, or semantic meaning. -- [ ] Ensure table columns and forms continue to resolve generated DID - Properties through `store.getProperty()`. +### Phase 3: Editor and table compatibility / UI + +- [x] Keep Ontology editor creation/editing as a first-class schema authoring + path (untouched; verified a code-first ontology renders + is editable in + `OntologyPage`). +- [x] A generic **"Freeze" UI** — resource context-menu action + `FreezeDialog` + (copy/download/publish, JSON-AD mode, Loro coming-soon, closure toggle) over + `Store.freezeStructure`. Works on any resource, not just ontologies. +- [x] **Content-locked / immutable UI**: a frozen (`did:ad:frozen`) resource + hides all write actions and shows a "❄ Frozen" badge. (UI *warnings* when a + datatype-changing edit would mint a new id — for the signed-DID editor path + — remain a follow-up.) +- [x] Table columns and forms continue to resolve DID/frozen Properties through + `store.getProperty()` (description made optional for frozen). +- [ ] Round-trip: export table/editor-created Classes & Properties to code schema + / freeze them and regenerate without changing meaning. ### Phase 4: Import/export and validation From e97b2713905a866d929b6f4c36704d9fc8e19442 Mon Sep 17 00:00:00 2001 From: Joep Meindertsma Date: Tue, 2 Jun 2026 16:25:18 +0200 Subject: [PATCH 18/20] docs: add code-first schema tutorial for JS app builders --- docs/src/SUMMARY.md | 1 + docs/src/schema/code-first.md | 172 ++++++++++++++++++++++++++++++++++ 2 files changed, 173 insertions(+) create mode 100644 docs/src/schema/code-first.md diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index 2ac073acb..c53c23c3d 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -71,6 +71,7 @@ - [Schema](schema/intro.md) - [Classes](schema/classes.md) - [Datatypes](schema/datatypes.md) + - [Code-first schemas](schema/code-first.md) - [FAQ](schema/faq.md) - [Atomic Data Extended](extended.md) diff --git a/docs/src/schema/code-first.md b/docs/src/schema/code-first.md new file mode 100644 index 000000000..8a9a2bc7e --- /dev/null +++ b/docs/src/schema/code-first.md @@ -0,0 +1,172 @@ +{{#title Code-first schemas: define your data model in JavaScript}} + +# Code-first schemas + +Instead of clicking together Classes and Properties in the browser first, you can +declare your data model **in code**, publish it with one command, and get +type-safe TypeScript bindings back. Your schema lives in your repo, next to the +code that uses it. + +This tutorial walks a JavaScript/TypeScript app builder through the whole loop: + +1. Define a schema in code. +2. Publish it to your Atomic Server. +3. Generate typed bindings and use them. +4. **Evolve** the schema — add and change Classes and Properties safely. + +> You'll need a running [Atomic Server](../atomic-server.md) and an +> [Agent](../agents.md) (the data-browser creates one for you on first run — copy +> its secret from **User Settings**). + +## 1. Define a schema + +Create a `todo-schema.mjs`. The shape is a small, JSON-Schema-like subset that +maps onto Atomic [Classes](classes.md), [Properties](classes.md), and +[Datatypes](datatypes.md): + +```js +// todo-schema.mjs +export default { + name: 'TodoApp', + version: '1.0.0', + classes: { + todo: { + type: 'object', + description: 'A task in a todo list', + required: ['title'], + properties: { + title: { type: 'string', description: 'Task title' }, + done: { type: 'boolean', description: 'Whether the task is complete' }, + dueAt: { type: 'string', format: 'date', description: 'Due date' }, + }, + }, + }, +}; +``` + +Each `properties` entry becomes an Atomic **Property**, each class becomes a +**Class**, and the whole thing becomes one **Ontology**. `type`/`format` map to +Atomic datatypes (`string`→string, `boolean`→boolean, `string`+`format: date`→date, +`integer`→integer, `number`→float, …). Anything not in `required` becomes +*recommended*. + +## 2. Publish it + +The simplest way is `@tomic/cli`. Create a config once: + +```bash +npm install -D @tomic/cli +npx ad-generate init # writes atomic.config.json +``` + +Set your `serverUrl` and `agentSecret` in `atomic.config.json`, then publish the +schema and generate bindings in one step: + +```bash +npx ad-generate schema ./todo-schema.mjs --generate +``` + +This creates the Ontology, Classes, and Properties on your server and writes +TypeScript bindings into your output folder. Add `--lock` to also write a +committed, self-verifying `TodoApp.schema.lock.json`. + +> **Prefer doing it from code?** Use the SDK directly: +> +> ```js +> import { Agent, Store } from '@tomic/lib'; +> import todoSchema from './todo-schema.mjs'; +> +> const agent = await Agent.fromSecret(process.env.AGENT_SECRET); +> const store = new Store({ serverUrl: 'http://localhost:9883', agent }); +> store.setServerConnected(true); +> +> const registered = await store.registerSchema(todoSchema, { save: true }); +> console.log(registered.ontology.subject); // did:ad:… +> ``` +> +> This is exactly the [`examples/code-first-schema`](https://github.com/atomicdata-dev/atomic-server/tree/develop/examples/code-first-schema) +> app. + +## 3. See it and use it + +Open the printed Ontology subject in the data-browser — it renders as a normal, +editable Ontology with your `todo` Class and its Properties. + +In code, use the generated bindings for full type-safety: + +```ts +import { todoApp } from './ontologies/todoApp'; // generated by --generate + +const todo = await store.newResource({ + isA: todoApp.classes.todo, + propVals: { + [todoApp.properties.title]: 'Buy milk', + [todoApp.properties.done]: false, + }, +}); +await todo.save(); +``` + +See [@tomic/cli](../js-cli.md) for more on the generated bindings. + +## 4. Updating your schema + +Your schema file is the source of truth. To change the model, **edit the file and +re-run the same publish/generate command** — then commit the regenerated bindings +(and lockfile) like you would a `package-lock.json`. The git diff shows exactly +what changed. + +What happens to identity when you change something is the important part, and +it's designed to keep existing data safe: + +- **Add a Class or Property** → new resources are created; everything else is + untouched. +- **Change only a description, label, or translation** → identifiers stay the + same. Presentation is not part of a Property's identity, so cosmetic edits never + churn ids or invalidate data. +- **Change a Property's datatype** (or other machine meaning) → this is a + breaking change, so it produces a **new** Property. The old one stays valid + forever, so resources created with it keep working. You're never silently + reinterpreting existing data. + +This immutability is strongest with the **content-addressed** flow. Instead of +`registerSchema`, publish with `registerFrozenSchema` (or click **Freeze** on an +Ontology in the data-browser): + +```js +const frozen = await store.registerFrozenSchema(todoSchema, { save: true }); +// frozen.properties['todo.title'] === "did:ad:frozen:9c22…" (a content hash) +``` + +Here every Property/Class identifier is the hash of its machine meaning. So the +*same* definition always yields the *same* id (re-running is idempotent; identical +definitions across apps even dedupe), and any change yields a new id +deterministically. Pin a release with the committed `*.schema.lock.json` and +[`Store.loadSchemaLock(lock)`](../js-lib/store.md) to make the schema resolvable +offline, with no server. A frozen resource is read-only and shows a **❄ Frozen** +badge in the data-browser. + +### Versioning across releases + +- Bump `version` in your schema and re-publish for a new release; old frozen ids + remain resolvable, so old data is never orphaned. +- For a stable "latest" handle, keep a normal (editable) Ontology whose + `classes`/`properties` point at the current frozen ids — `createSchemaPointer` + builds one, and its commit history is your version log. + +## What you keep from the GUI + +Code-first does **not** replace the [Ontology editor](../atomicserver/gui.md) or +[table view](../atomicserver/gui/tables.md) — +they still create the same Classes and Properties, and a code-first Ontology +remains fully viewable and editable in the browser. Use whichever fits: define in +code for repeatable, reviewable, version-controlled schemas; use the GUI for quick +exploration. + +## Cross-language + +A schema's frozen identifiers are a pure function of its content (RFC 8785 JCS + +BLAKE3), so they are reproducible in any language. The Rust SDK +(`atomic_lib::frozen::freeze_schema`) produces byte-for-byte identical ids — so a +schema authored in JavaScript and one authored in Rust converge on the same +resources. From 424a9d931c6872ed5b2906b4e3aaf874ee4437c2 Mon Sep 17 00:00:00 2001 From: Joep Meindertsma Date: Tue, 2 Jun 2026 17:32:16 +0200 Subject: [PATCH 19/20] planning: add Availability & incentives section for frozen schemas --- planning/did-ad-frozen-server.md | 64 ++++++++++++++++++++++++++++++ planning/json-schema-code-first.md | 4 ++ 2 files changed, 68 insertions(+) diff --git a/planning/did-ad-frozen-server.md b/planning/did-ad-frozen-server.md index f8a48515f..d54ad4251 100644 --- a/planning/did-ad-frozen-server.md +++ b/planning/did-ad-frozen-server.md @@ -66,6 +66,70 @@ documented as optional, and the protocol/spec never references it. If it is down and the bytes exist in a bundle, on the author's drive, or on any peer, resolution still succeeds and verifies. +## Availability & incentives: who stores and serves schemas, and why + +A frozen id is **self-certifying but not self-locating** (like a git object hash +or an IPFS CID): it proves the content but doesn't say where to get the bytes. So +the open question isn't correctness — verify-by-rehash already gives us that — it's +**availability**: who keeps the bytes and serves them? The design answer is to make +storing/serving a *byproduct of self-interest* and *trustlessly cheap*, rather than +to bolt on a reward. In priority order: + +1. **Storing is a side effect of using your own data.** A server cannot validate + or serve a resource of class `did:ad:frozen:X` without holding `X` — so every + data-hosting server already holds the schemas its data depends on (the + browser-side publish-on-save in [json-schema-code-first.md](./json-schema-code-first.md), + `Store.publishReferencedFrozen`, pushes them there before the commit lands). + Data and its schema travel together; no altruism required. + +2. **Serving to others is trustless and ~free.** `GET /frozen/{hash}` is public and + re-hashable, so a holder *cannot* serve a wrong answer — closing the endpoint + buys nothing. Frozen objects are tiny, immutable (infinite cache TTL, CDN-able), + and deduplicated (a reused property is stored once). Cost ≈ 0 beats any reward. + _Action:_ set `Cache-Control: public, immutable, max-age=31536000` on + `GET /frozen` responses. + +3. **The reuse flywheel.** Identical definitions hash to the same id, so reuse *is* + convergence: the more an "email"/"name"/… property is reused, the more servers + independently hold it, the more available and canonical it becomes — a Schelling + point. Popular vocabularies get *more* available with use, with zero coordination; + private variants stay isolated on one server (self-punishing divergence). + +4. **A registry/index for discovery + the long tail** — the high-leverage build, and + the same "index all ontologies on atomicdata.dev" goal noted elsewhere. Think + crates.io / npm / schema.org for ontologies: pin-on-push + search over + shortnames/descriptions + a "used by N ontologies" usage signal that surfaces the + flywheel. Incentives align: publishers get discoverability + a durable home, + consumers get one reliable resolve/search endpoint, the operator becomes the + ecosystem's discovery layer. Because apps ship lockfiles (below), the registry is + a convenience and flywheel — **not** a dependency or single point of failure. + +5. **Lockfiles make clients self-sufficient seeders.** A committed + `*.schema.lock.json` travels with the code, so an app *runs* with no server at + all, and any client holding the bytes can re-seed any server (`publishFrozenResource`). + Availability stops being a hard dependency for *running* — it's only needed for + *discovery/interop*, which de-risks the whole scheme. + +6. **Mesh + (optional) paid permanence, later.** Phase D sync makes this + BitTorrent-like: any peer with the bytes serves them, verified by re-hash + (incentive = reciprocity from a shared pool). Pay-once-store-forever + (Arweave-style) is a natural fit for *guaranteed* permanence of critical + vocabularies but is over-engineering for objects this cheap until there's demand — + flag as future, don't build. + +**Net:** don't pay anyone to store schemas. Make storing a side effect of using +your own data, make serving trustless and free, ship lockfiles so nothing *depends* +on a host, and let reuse-convergence plus a discovery registry turn "available +because someone happens to host it" into "available because everyone who uses it +hosts it." + +### Follow-up tasks + +- [ ] `GET /frozen/{hash}`: add long-lived immutable `Cache-Control` headers. +- [ ] Registry/index on a default server: pin-on-push, search by + shortname/description, and a `used-by` count per frozen id. +- [ ] Promote lockfiles to the documented default for shipping a schema with an app. + ## Decided + implemented: cyclic members freeze as one unit This was the one place the producer and server models disagreed; it is now diff --git a/planning/json-schema-code-first.md b/planning/json-schema-code-first.md index c82b43cfc..09ae9f57e 100644 --- a/planning/json-schema-code-first.md +++ b/planning/json-schema-code-first.md @@ -182,6 +182,10 @@ Not implemented yet (all optional/follow-up — none block the core flows): (richer keywords beyond the Atomic subset) in browser/Rust. - Drive-context / app-registry discovery of schemas by hash before explicit registration; round-trip of editor/table-created schemas into frozen. +- **Availability & incentives** (who stores/serves schemas, and a discovery + registry / "index all ontologies" hub) — see the + [Availability & incentives](./did-ad-frozen-server.md#availability--incentives-who-stores-and-serves-schemas-and-why) + section of the server plan. ## Existing Touch Points From 2efcd9f92895369efd603c6008f96d45ae9fef43 Mon Sep 17 00:00:00 2001 From: Joep Meindertsma Date: Tue, 2 Jun 2026 17:35:09 +0200 Subject: [PATCH 20/20] Update schema in code --- browser/lib/src/freeze.ts | 26 +++ browser/lib/src/schema.test.ts | 129 ++++++++++++++ browser/lib/src/schema.ts | 102 ++++++++++- browser/lib/src/store.ts | 105 +++++++++++ docs/src/schema/code-first.md | 165 ++++++++---------- examples/code-first-schema/publish-schema.mjs | 44 +++-- 6 files changed, 458 insertions(+), 113 deletions(-) diff --git a/browser/lib/src/freeze.ts b/browser/lib/src/freeze.ts index 8381850e5..003011c60 100644 --- a/browser/lib/src/freeze.ts +++ b/browser/lib/src/freeze.ts @@ -374,6 +374,32 @@ export function frozenIdFor(content: JsonValue): FrozenId { return (FROZEN_PREFIX + hashCanonical(content)) as FrozenId; } +/** + * Process-wide registry of frozen bodies by id. Populated whenever frozen + * resources are computed locally (`defineSchema`, `registerFrozenSchema`, + * `loadSchemaLock`), so the Store can lazily PUT a referenced definition to the + * server on save without the caller ever publishing it explicitly. Bodies are + * immutable and content-addressed, so first-write-wins is always safe. + */ +const frozenBodyRegistry = new Map(); + +export function registerFrozenBodies( + resources: ReadonlyArray<{ frozenId: FrozenId; content: JsonValue }>, +): void { + for (const { frozenId, content } of resources) { + if (!frozenBodyRegistry.has(frozenId)) { + frozenBodyRegistry.set(frozenId, content); + } + } +} + +/** The locally-known body for a frozen id, if one has been registered. */ +export function getRegisteredFrozenBody( + frozenId: string, +): JsonValue | undefined { + return frozenBodyRegistry.get(frozenId as FrozenId); +} + function stronglyConnectedComponents( nodes: string[], edges: Map>, diff --git a/browser/lib/src/schema.test.ts b/browser/lib/src/schema.test.ts index 7301938f9..e4487f1f9 100644 --- a/browser/lib/src/schema.test.ts +++ b/browser/lib/src/schema.test.ts @@ -14,6 +14,7 @@ import { } from './schema.js'; import { Datatype } from './datatypes.js'; import { core } from './ontologies/core.js'; +import { getKnownNameBySubject } from './ontology.js'; import { Store } from './store.js'; interface MockClientDbEntry { @@ -195,6 +196,79 @@ describe('defineSchema', () => { /^blake3:[0-9a-f]{64}$/, ); }); + + it('exposes content-addressed frozen ids as typed class/property handles', ({ + expect, + }) => { + // Inline literal (not annotated) so the generic captures the literal keys — + // `.classes.todo` / `.properties.title` are typed, autocompleted handles. + const defined = defineSchema({ + name: 'TodoApp', + version: '1.0.0', + classes: { + todo: { + type: 'object', + required: ['title'], + properties: { + title: { type: 'string' }, + done: { type: 'boolean' }, + }, + }, + }, + }); + + expect(defined.classes.todo).toMatch(/^did:ad:frozen:[0-9a-f]{64}$/); + expect(defined.properties.title).toMatch(/^did:ad:frozen:[0-9a-f]{64}$/); + expect(defined.properties.done).toMatch(/^did:ad:frozen:[0-9a-f]{64}$/); + + // The handles are exactly the frozen ids, addressable with no server. + const frozen = freezeSchema(defined); + expect(defined.classes.todo).toBe(frozen.classes.todo); + expect(defined.properties.title).toBe(frozen.properties['todo.title']); + }); + + it('computes handles deterministically and locally (no server)', ({ + expect, + }) => { + const a = defineSchema(todoSchema).classes.todo; + const b = defineSchema(todoSchema).classes.todo; + + expect(a).toBe(b); + }); + + it('registers the schema at runtime so props resolve by shortname', ({ + expect, + }) => { + const defined = defineSchema(todoSchema); + // Accessing a handle triggers lazy freeze + runtime registration. + const titleId = defined.properties.title; + + expect(getKnownNameBySubject(titleId)).toBe('title'); + }); + + it('does not freeze on definition, and throws on handle access for imports', ({ + expect, + }) => { + // Defining an import-containing schema must not throw (no eager freeze)... + const external = defineSchema({ + name: 'ProjectApp', + imports: { + todo: { + subject: 'did:ad:todoOntology', + expectedHash: defineSchema(todoSchema).schemaHash, + }, + }, + classes: { + project: { + type: 'object', + properties: { title: { $ref: 'todo.properties.title' } }, + }, + }, + }); + + // ...but reaching for a local handle (which needs a store) throws clearly. + expect(() => external.classes.project).toThrow(/store-backed registration/); + }); }); describe('schemaToOntologyModel', () => { @@ -782,3 +856,58 @@ describe('Store.registerSchema', () => { ]); }); }); + +describe('Store lazy frozen publish on save', () => { + it('PUTs referenced frozen definitions to /frozen when an instance is saved', async ({ + expect, + }) => { + const keys = await Agent.generateKeyPair(); + const store = new Store({ serverUrl: 'https://example.com' }); + store.setServerConnected(true); + store.setAgent( + new Agent( + new JSCryptoProvider(keys.privateKey), + `did:ad:agent:${keys.publicKey}`, + ), + ); + + // Commits go to the mocked client; nothing hits the network there. + ( + store as unknown as { client: { postCommit: typeof vi.fn } } + ).client.postCommit = vi.fn(async (commit: Commit) => ({ + ...commit, + id: `https://example.com/commits/${commit.signature}`, + })) as never; + + // Capture the `/frozen` PUTs the store makes on save. + const puts: string[] = []; + const fetchMock = vi.fn(async (url: string, init?: { method?: string }) => { + if (init?.method === 'PUT') { + puts.push(url); + } + + return { ok: true, status: 200 } as Response; + }); + vi.stubGlobal('fetch', fetchMock); + + try { + // No registerSchema, no CLI — just define and use. + const schema = defineSchema(todoSchema); + + const todo = await store.newResource({ + isA: schema.classes.todo, + propVals: { [schema.properties.title]: 'Buy milk' }, + }); + await todo.save(); + + const classHash = schema.classes.todo.replace('did:ad:frozen:', ''); + const titleHash = schema.properties.title.replace('did:ad:frozen:', ''); + + // The class and its property definitions were published automatically. + expect(puts.some(url => url.endsWith(`/frozen/${classHash}`))).toBe(true); + expect(puts.some(url => url.endsWith(`/frozen/${titleHash}`))).toBe(true); + } finally { + vi.unstubAllGlobals(); + } + }); +}); diff --git a/browser/lib/src/schema.ts b/browser/lib/src/schema.ts index f615a5586..412724cab 100644 --- a/browser/lib/src/schema.ts +++ b/browser/lib/src/schema.ts @@ -4,6 +4,7 @@ import { bytesToHex, utf8ToBytes } from '@noble/hashes/utils.js'; import { Datatype } from './datatypes.js'; import { freezeResources, + registerFrozenBodies, type FreezableResource, type FrozenId, type JsonValue as FreezeJsonValue, @@ -11,6 +12,7 @@ import { import { jcsCanonicalize } from './jcs.js'; import { core } from './ontologies/core.js'; import { server } from './ontologies/server.js'; +import { registerOntologies } from './ontology.js'; type JsonPrimitive = string | number | boolean | null; @@ -88,12 +90,39 @@ export interface AtomicSchemaPackage { $defs?: Record; } +/** The class keys of a schema package, mapped to their `did:ad:frozen:` ids. */ +export type SchemaClassHandles = { + readonly [C in keyof S['classes']]: string; +}; + +/** Union of every property key across all classes of a schema package. */ +export type SchemaPropertyKey = { + [C in keyof S['classes']]: keyof S['classes'][C]['properties'] & string; +}[keyof S['classes']]; + +/** The property keys of a schema package, mapped to their `did:ad:frozen:` ids. */ +export type SchemaPropertyHandles = { + readonly [P in SchemaPropertyKey]: string; +}; + export interface DefinedSchema< Schema extends AtomicSchemaPackage = AtomicSchemaPackage, > { readonly schema: Schema; readonly normalized: AtomicSchemaPackage; readonly schemaHash: SchemaHash; + /** + * Content-addressed `did:ad:frozen:` id for each class, keyed by class key. + * Computed lazily and locally (no server) the first time it is read — use + * directly as `isA` when creating resources. Self-contained schemas only; + * accessing this on a schema with `$ref` imports throws. + */ + readonly classes: SchemaClassHandles; + /** + * Content-addressed `did:ad:frozen:` id for each property, keyed by property + * key. Use directly as `propVals` keys. + */ + readonly properties: SchemaPropertyHandles; } export interface ConvertedSchemaProperty { @@ -138,11 +167,80 @@ export function defineSchema( ): DefinedSchema { const normalized = normalizeSchemaPackage(schema); - return Object.freeze({ + // `.classes` / `.properties` are content-addressed frozen ids. They are + // computed lazily — on first access — so that (a) merely defining a schema + // (or converting one to a model internally) does no hashing work, and (b) a + // schema with `$ref` imports, which cannot be frozen without a store, only + // errors if a caller actually reaches for the local handles. + let handles: SchemaHandles | undefined; + const getHandles = (): SchemaHandles => (handles ??= computeSchemaHandles(defined)); + + const defined = Object.freeze({ schema, normalized, schemaHash: hashSchemaPackage(normalized), - }); + get classes() { + return getHandles().classes; + }, + get properties() { + return getHandles().properties; + }, + }) as unknown as DefinedSchema; + + return defined; +} + +interface SchemaHandles { + readonly classes: Record; + readonly properties: Record; +} + +/** + * Freezes a self-contained schema into `did:ad:frozen:` ids and exposes them as + * flat `{ classKey -> id }` / `{ propertyKey -> id }` handles. Also registers the + * schema in the global runtime mapping so `resource.props.` resolves + * without any generated bindings. + */ +function computeSchemaHandles(defined: DefinedSchema): SchemaHandles { + const frozen = freezeSchema(defined); + + const properties: Record = {}; + + for (const property of frozen.model.properties) { + properties[property.propertyKey] = frozen.properties[property.key]; + } + + registerFrozenSchemaRuntime(frozen); + // Make the frozen bodies publishable so the Store can lazily PUT them on save. + registerFrozenBodies(frozen.resources); + + return { classes: { ...frozen.classes }, properties }; +} + +/** + * Teaches `@tomic/lib`'s runtime about a frozen schema (subject -> shortname and + * the per-class property set), so quick-access `resource.props` works for + * code-first schemas the same way it does for generated ontologies. + */ +function registerFrozenSchemaRuntime(frozen: FrozenSchema): void { + const properties: Record = {}; + + for (const property of frozen.model.properties) { + properties[property.shortname] = frozen.properties[property.key]; + } + + const classes: Record = {}; + const classDefs: Record = {}; + + for (const klass of frozen.model.classes) { + const classId = frozen.classes[klass.key]; + classes[klass.shortname] = classId; + classDefs[classId] = [...klass.requires, ...klass.recommends] + .map(modelKey => frozen.properties[modelKey]) + .filter((id): id is FrozenId => Boolean(id)); + } + + registerOntologies({ classes, properties, __classDefs: classDefs }); } export function schemaToOntologyModel( diff --git a/browser/lib/src/store.ts b/browser/lib/src/store.ts index aa0a5ba57..7bb0cefd8 100644 --- a/browser/lib/src/store.ts +++ b/browser/lib/src/store.ts @@ -34,6 +34,8 @@ import { import { freezeResources, frozenIdFor, + getRegisteredFrozenBody, + registerFrozenBodies, UNIT_MEMBERS_KEY, type FrozenId, type FreezableResource, @@ -372,6 +374,8 @@ export class Store { /** Mapping from HTTP aliases to primary subjects (e.g. DIDs) */ private aliases: Map = new Map(); private schemaHashIndex: Map = new Map(); + /** Frozen ids already PUT to the server this session — skip redundant publishes. */ + private publishedFrozenIds: Set = new Set(); /** List of resources that have parents that are not saved to the server, when a parent is saved it should also save its children */ private batchedResources: Map> = new Map(); @@ -925,6 +929,19 @@ export class Store { const endpoint = new URL('/commit', this.serverUrl).toString(); + // Lazy publish-on-save: ensure any content-addressed `did:ad:frozen` + // definitions this resource references (its class, a code-first property) + // exist on the server before the commit lands — so code-first schemas need + // no separate build/publish step. Best-effort: a failure here must not block + // the user's save. + const draining = this.resources.get(subject); + + if (draining) { + await this.publishReferencedFrozen(draining).catch(err => + console.warn('[Store] auto-publish of frozen references failed:', err), + ); + } + // Step 1: POST the pre-signed genesis if present. The genesis // commit's `loroUpdate` was captured in `signChanges` at sign // time; `_loroVersionAtLastSave` was advanced THERE to the same @@ -1846,6 +1863,8 @@ export class Store { ): Promise { const frozen = freezeSchema(schema); + registerFrozenBodies(frozen.resources); + for (const { frozenId, content } of frozen.resources) { const [resource] = new JSONADParser().parse(content, frozenId); resource.loading = false; @@ -2067,6 +2086,10 @@ export class Store { continue; } + registerFrozenBodies([ + { frozenId: frozenId as FrozenId, content: content as FrozenJsonValue }, + ]); + const [resource] = new JSONADParser().parse(content, frozenId); resource.loading = false; this.addResource(resource, { skipCommitCompare: true }); @@ -2094,6 +2117,88 @@ export class Store { } } + /** + * Lazy publish-on-save: if a resource about to be committed references any + * `did:ad:frozen:` definitions (its class, a code-first property, an enum + * value…) whose bodies are locally known but not yet on the server, PUT them + * first — together with the transitive closure they reference (a class points + * at its property ids). This is what makes code-first schemas need no build or + * publish step: defining a schema registers its bodies, and the first save of + * an instance pushes the definitions. Publishing is idempotent (hash-keyed) and + * de-duplicated per session, so unchanged saves do no extra work. + */ + private async publishReferencedFrozen(resource: Resource): Promise { + const discovered = new Set(); + const queue: string[] = []; + + for (const [key, value] of resource.getEntries()) { + this.collectFrozenRefs(key, discovered, queue); + this.collectFrozenRefs(value as FrozenJsonValue, discovered, queue); + } + + if (queue.length === 0) { + return; + } + + const toPublish = new Map(); + + while (queue.length > 0) { + const id = queue.shift()!; + + if (toPublish.has(id) || this.publishedFrozenIds.has(id)) { + continue; + } + + const body = getRegisteredFrozenBody(id); + + if (body === undefined) { + // Not locally known — assume it is already hosted (or external). + continue; + } + + toPublish.set(id, body as FrozenJsonValue); + // Expand the closure: a class body references its property ids, etc. + this.collectFrozenRefs(body as FrozenJsonValue, discovered, queue); + } + + if (toPublish.size === 0) { + return; + } + + await Promise.all( + [...toPublish].map(([id, body]) => + this.publishFrozenResource(id as FrozenId, body).then(() => { + this.publishedFrozenIds.add(id); + }), + ), + ); + } + + /** + * Walks a JSON-AD value, collecting every `did:ad:frozen:` string into `into` + * and (optionally) appending newly-seen ids to `queue` for closure expansion. + */ + private collectFrozenRefs( + value: FrozenJsonValue, + into: Set, + queue?: string[], + ): void { + if (typeof value === 'string') { + if (value.startsWith('did:ad:frozen:') && !into.has(value)) { + into.add(value); + queue?.push(value); + } + } else if (Array.isArray(value)) { + for (const item of value) { + this.collectFrozenRefs(item, into, queue); + } + } else if (value !== null && typeof value === 'object') { + for (const item of Object.values(value)) { + this.collectFrozenRefs(item as FrozenJsonValue, into, queue); + } + } + } + private assertCompatibleExistingSchemaProperty( property: ConvertedSchemaPackage['properties'][number], ): void { diff --git a/docs/src/schema/code-first.md b/docs/src/schema/code-first.md index 8a9a2bc7e..3dd7dbbcd 100644 --- a/docs/src/schema/code-first.md +++ b/docs/src/schema/code-first.md @@ -3,16 +3,15 @@ # Code-first schemas Instead of clicking together Classes and Properties in the browser first, you can -declare your data model **in code**, publish it with one command, and get -type-safe TypeScript bindings back. Your schema lives in your repo, next to the -code that uses it. +declare your data model **in code** and just use it. There is **no build step, no +codegen, and no publish command** — your schema is a value in your repo, and the +Classes and Properties are published automatically the first time you save data +that uses them. -This tutorial walks a JavaScript/TypeScript app builder through the whole loop: - -1. Define a schema in code. -2. Publish it to your Atomic Server. -3. Generate typed bindings and use them. -4. **Evolve** the schema — add and change Classes and Properties safely. +This works because schemas are **content-addressed**: a Class or Property's +identifier is a hash of its definition (see [`did:ad:frozen`](../did.md)). The +identifier is computed locally, in-process, so your code — not a server — is the +source of truth. > You'll need a running [Atomic Server](../atomic-server.md) and an > [Agent](../agents.md) (the data-browser creates one for you on first run — copy @@ -20,13 +19,14 @@ This tutorial walks a JavaScript/TypeScript app builder through the whole loop: ## 1. Define a schema -Create a `todo-schema.mjs`. The shape is a small, JSON-Schema-like subset that -maps onto Atomic [Classes](classes.md), [Properties](classes.md), and -[Datatypes](datatypes.md): +`defineSchema` takes a small, JSON-Schema-like object that maps onto Atomic +[Classes](classes.md), [Properties](classes.md), and [Datatypes](datatypes.md): + +```ts +// schema.ts +import { defineSchema } from '@tomic/lib'; -```js -// todo-schema.mjs -export default { +export const todoSchema = defineSchema({ name: 'TodoApp', version: '1.0.0', classes: { @@ -41,80 +41,53 @@ export default { }, }, }, -}; -``` - -Each `properties` entry becomes an Atomic **Property**, each class becomes a -**Class**, and the whole thing becomes one **Ontology**. `type`/`format` map to -Atomic datatypes (`string`→string, `boolean`→boolean, `string`+`format: date`→date, -`integer`→integer, `number`→float, …). Anything not in `required` becomes -*recommended*. - -## 2. Publish it - -The simplest way is `@tomic/cli`. Create a config once: - -```bash -npm install -D @tomic/cli -npx ad-generate init # writes atomic.config.json +}); ``` -Set your `serverUrl` and `agentSecret` in `atomic.config.json`, then publish the -schema and generate bindings in one step: +Each `properties` entry becomes a **Property**, each class a **Class**, and the +whole thing an **Ontology**. `type`/`format` map to Atomic datatypes +(`string`→string, `boolean`→boolean, `string`+`format: date`→date, +`integer`→integer, `number`→float, …). Anything not in `required` is *recommended*. -```bash -npx ad-generate schema ./todo-schema.mjs --generate -``` +## 2. Use it — no build step -This creates the Ontology, Classes, and Properties on your server and writes -TypeScript bindings into your output folder. Add `--lock` to also write a -committed, self-verifying `TodoApp.schema.lock.json`. - -> **Prefer doing it from code?** Use the SDK directly: -> -> ```js -> import { Agent, Store } from '@tomic/lib'; -> import todoSchema from './todo-schema.mjs'; -> -> const agent = await Agent.fromSecret(process.env.AGENT_SECRET); -> const store = new Store({ serverUrl: 'http://localhost:9883', agent }); -> store.setServerConnected(true); -> -> const registered = await store.registerSchema(todoSchema, { save: true }); -> console.log(registered.ontology.subject); // did:ad:… -> ``` -> -> This is exactly the [`examples/code-first-schema`](https://github.com/atomicdata-dev/atomic-server/tree/develop/examples/code-first-schema) -> app. - -## 3. See it and use it - -Open the printed Ontology subject in the data-browser — it renders as a normal, -editable Ontology with your `todo` Class and its Properties. - -In code, use the generated bindings for full type-safety: +`todoSchema.classes` and `todoSchema.properties` are typed handles to the +content-addressed ids. Use them directly. The first time you `save()` a resource +that references them, the Store publishes the Class and Property definitions to +your server automatically: ```ts -import { todoApp } from './ontologies/todoApp'; // generated by --generate +import { Agent, Store } from '@tomic/lib'; +import { todoSchema } from './schema'; + +const agent = await Agent.fromSecret(process.env.AGENT_SECRET); +const store = new Store({ serverUrl: 'http://localhost:9883', agent }); +store.setServerConnected(true); const todo = await store.newResource({ - isA: todoApp.classes.todo, + isA: todoSchema.classes.todo, // typed; autocompletes 'todo' propVals: { - [todoApp.properties.title]: 'Buy milk', - [todoApp.properties.done]: false, + [todoSchema.properties.title]: 'Buy milk', + [todoSchema.properties.done]: false, }, }); -await todo.save(); + +await todo.save(); // ← publishes the todo Class + its Properties if the server lacks them ``` -See [@tomic/cli](../js-cli.md) for more on the generated bindings. +That's the whole loop: **define, use, save.** No `ad-generate`, no generated +files to commit, no separate publish step. Publishing is idempotent (hash-keyed), +so an unchanged schema re-saves with no extra work. + +The handles give you autocomplete and typo-safety on class and property keys. +(Per-field inference of `resource.props.title` still requires the generated +bindings below; without them, `props` is loosely typed but works at runtime.) -## 4. Updating your schema +## 3. Updating your schema -Your schema file is the source of truth. To change the model, **edit the file and -re-run the same publish/generate command** — then commit the regenerated bindings -(and lockfile) like you would a `package-lock.json`. The git diff shows exactly -what changed. +Your schema file is the source of truth. To change the model, **just edit the +file.** The next time your app saves a resource using the changed Class or +Property, the new definition publishes itself — there's nothing else to run. What happens to identity when you change something is the important part, and it's designed to keep existing data safe: @@ -129,31 +102,35 @@ it's designed to keep existing data safe: forever, so resources created with it keep working. You're never silently reinterpreting existing data. -This immutability is strongest with the **content-addressed** flow. Instead of -`registerSchema`, publish with `registerFrozenSchema` (or click **Freeze** on an -Ontology in the data-browser): - -```js -const frozen = await store.registerFrozenSchema(todoSchema, { save: true }); -// frozen.properties['todo.title'] === "did:ad:frozen:9c22…" (a content hash) -``` - -Here every Property/Class identifier is the hash of its machine meaning. So the -*same* definition always yields the *same* id (re-running is idempotent; identical -definitions across apps even dedupe), and any change yields a new id -deterministically. Pin a release with the committed `*.schema.lock.json` and -[`Store.loadSchemaLock(lock)`](../js-lib/store.md) to make the schema resolvable -offline, with no server. A frozen resource is read-only and shows a **❄ Frozen** -badge in the data-browser. +You get this for free, because `defineSchema` already gives you content-addressed +(`did:ad:frozen:`) ids: every identifier is the hash of its machine meaning. The +*same* definition always yields the *same* id (idempotent; identical definitions +across apps even dedupe), and any change yields a new id deterministically. A +frozen definition is immutable and read-only, and shows a **❄ Frozen** badge in +the data-browser. ### Versioning across releases -- Bump `version` in your schema and re-publish for a new release; old frozen ids - remain resolvable, so old data is never orphaned. -- For a stable "latest" handle, keep a normal (editable) Ontology whose - `classes`/`properties` point at the current frozen ids — `createSchemaPointer` +- Bump `version` in your schema for a new release; old ids remain resolvable, so + old data is never orphaned. +- To make a release resolvable **offline, with no server**, commit a + `*.schema.lock.json` and load it with + [`Store.loadSchemaLock(lock)`](../js-lib/store.md) — the lockfile travels with + your code. +- For a stable, *editable* "latest" handle that also renders in the GUI, keep a + normal Ontology pointing at the current frozen ids — `createSchemaPointer` builds one, and its commit history is your version log. +## Optional: `@tomic/cli` + +You never *need* the CLI, but [`@tomic/cli`](../js-cli.md) is handy for two things: + +- **Pre-publish** a schema so it's browsable in the data-browser before any app + has saved data: `npx ad-generate schema ./schema.ts`. +- **Generate committed `.ts` bindings** (and a `--lock` file) if you prefer those + over the inline `defineSchema` handles — this is also what enables fully-typed + `resource.props.title` access. + ## What you keep from the GUI Code-first does **not** replace the [Ontology editor](../atomicserver/gui.md) or diff --git a/examples/code-first-schema/publish-schema.mjs b/examples/code-first-schema/publish-schema.mjs index 3d193cd35..aa041871c 100644 --- a/examples/code-first-schema/publish-schema.mjs +++ b/examples/code-first-schema/publish-schema.mjs @@ -1,10 +1,11 @@ -// A tiny @tomic/lib "code-first schema" app. +// A tiny @tomic/lib "code-first schema" app — no build step, no codegen. // // AGENT_SECRET= node publish-schema.mjs // -// Defines a schema in code, publishes it to a running atomic-server, and prints -// the resulting Ontology subject — which you can then open in the data-browser. -import { Agent, Store } from '@tomic/lib'; +// Defines a schema in code, creates a resource that uses it, and saves it. +// The Class and Property definitions are content-addressed and publish +// themselves to the server on the first save — nothing else to run. +import { Agent, Store, defineSchema } from '@tomic/lib'; const SERVER = process.env.SERVER_URL || 'http://localhost:9883'; const SECRET = process.env.AGENT_SECRET; @@ -14,7 +15,9 @@ if (!SECRET) { } // Define the data model in code — JSON-Schema-like, mapped to Atomic. -const todoSchema = { +// `todoSchema.classes` / `.properties` are content-addressed `did:ad:frozen` ids, +// computed locally (no server needed to have an identity). +const todoSchema = defineSchema({ name: 'TodoApp', version: '1.0.0', classes: { @@ -30,24 +33,31 @@ const todoSchema = { }, }, }, -}; +}); + +console.log('Schema ids (computed locally, no server):'); +console.log(' CLASS todo ' + todoSchema.classes.todo); +console.log(' PROPERTY title ' + todoSchema.properties.title); +console.log(' PROPERTY done ' + todoSchema.properties.done); const agent = await Agent.fromSecret(SECRET); const store = new Store({ serverUrl: SERVER, agent }); store.setServerConnected(true); -// Publish the schema as Atomic Class & Property resources under an Ontology. -const registered = await store.registerSchema(todoSchema, { save: true }); +// Create and save a resource that uses the schema. Saving auto-publishes the +// Class + Property definitions to the server (idempotent, hash-keyed). +const todo = await store.newResource({ + isA: todoSchema.classes.todo, + propVals: { + [todoSchema.properties.title]: 'Buy milk', + [todoSchema.properties.done]: false, + }, +}); -console.log('Published ontology:'); -console.log(' ONTOLOGY ' + registered.ontology.subject); +await todo.save(); -for (const [key, resource] of Object.entries(registered.classes)) { - console.log(` CLASS ${key}: ${resource.subject}`); -} - -for (const [key, resource] of Object.entries(registered.properties)) { - console.log(` PROPERTY ${key}: ${resource.subject}`); -} +console.log('\nSaved a todo — its Class & Properties are now on the server:'); +console.log(' TODO ' + todo.subject); +console.log('Open the class id above in the data-browser to see the schema.'); process.exit(0);