From 9d291ed14230e6e00c640b67bef9d33f30aa1c78 Mon Sep 17 00:00:00 2001 From: NathanFlurry Date: Tue, 7 Oct 2025 23:39:00 +0000 Subject: [PATCH 1/2] fix(core): fix json encoding (#1336) --- packages/rivetkit/src/actor/protocol/serde.ts | 78 ++++++- .../rivetkit/src/driver-test-suite/mod.ts | 93 ++++---- .../rivetkit/src/driver-test-suite/utils.ts | 5 +- packages/rivetkit/src/serde.ts | 6 +- packages/rivetkit/tests/json-escaping.test.ts | 221 ++++++++++++++++++ 5 files changed, 357 insertions(+), 46 deletions(-) create mode 100644 packages/rivetkit/tests/json-escaping.test.ts diff --git a/packages/rivetkit/src/actor/protocol/serde.ts b/packages/rivetkit/src/actor/protocol/serde.ts index bda7cf419..1c84128f8 100644 --- a/packages/rivetkit/src/actor/protocol/serde.ts +++ b/packages/rivetkit/src/actor/protocol/serde.ts @@ -123,9 +123,81 @@ export function encodeDataToString(message: OutputData): string { } } +function base64DecodeToUint8Array(base64: string): Uint8Array { + // Check if Buffer is available (Node.js) + if (typeof Buffer !== "undefined") { + return new Uint8Array(Buffer.from(base64, "base64")); + } + + // Browser environment - use atob + const binary = atob(base64); + const len = binary.length; + const bytes = new Uint8Array(len); + for (let i = 0; i < len; i++) { + bytes[i] = binary.charCodeAt(i); + } + return bytes; +} + +function base64DecodeToArrayBuffer(base64: string): ArrayBuffer { + return base64DecodeToUint8Array(base64).buffer as ArrayBuffer; +} + /** Stringifies with compat for values that BARE & CBOR supports. */ export function jsonStringifyCompat(input: any): string { - return JSON.stringify(input, (_key, value) => - typeof value === "bigint" ? value.toString() : value, - ); + return JSON.stringify(input, (_key, value) => { + if (typeof value === "bigint") { + return ["$BigInt", value.toString()]; + } else if (value instanceof ArrayBuffer) { + return ["$ArrayBuffer", base64EncodeArrayBuffer(value)]; + } else if (value instanceof Uint8Array) { + return ["$Uint8Array", base64EncodeUint8Array(value)]; + } + + // Escape user arrays that start with $ by prepending another $ + if ( + Array.isArray(value) && + value.length === 2 && + typeof value[0] === "string" && + value[0].startsWith("$") + ) { + return ["$" + value[0], value[1]]; + } + + return value; + }); +} + +/** Parses JSON with compat for values that BARE & CBOR supports. */ +export function jsonParseCompat(input: string): any { + return JSON.parse(input, (_key, value) => { + // Handle arrays with $ prefix + if ( + Array.isArray(value) && + value.length === 2 && + typeof value[0] === "string" && + value[0].startsWith("$") + ) { + // Known special types + if (value[0] === "$BigInt") { + return BigInt(value[1]); + } else if (value[0] === "$ArrayBuffer") { + return base64DecodeToArrayBuffer(value[1]); + } else if (value[0] === "$Uint8Array") { + return base64DecodeToUint8Array(value[1]); + } + + // Unescape user arrays that started with $ ($$foo -> $foo) + if (value[0].startsWith("$$")) { + return [value[0].substring(1), value[1]]; + } + + // Unknown type starting with $ - this is an error + throw new Error( + `Unknown JSON encoding type: ${value[0]}. This may indicate corrupted data or a version mismatch.`, + ); + } + + return value; + }); } diff --git a/packages/rivetkit/src/driver-test-suite/mod.ts b/packages/rivetkit/src/driver-test-suite/mod.ts index dd26f8d89..5d09af5ff 100644 --- a/packages/rivetkit/src/driver-test-suite/mod.ts +++ b/packages/rivetkit/src/driver-test-suite/mod.ts @@ -3,7 +3,7 @@ import { createNodeWebSocket, type NodeWebSocket } from "@hono/node-ws"; import { bundleRequire } from "bundle-require"; import invariant from "invariant"; import { describe } from "vitest"; -import type { Transport } from "@/client/mod"; +import type { Encoding, Transport } from "@/client/mod"; import { configureInspectorAccessToken } from "@/inspector/utils"; import { createManagerRouter } from "@/manager/router"; import { @@ -60,6 +60,8 @@ export interface DriverTestConfig { transport?: Transport; + encoding?: Encoding; + clientType: ClientType; cleanup?: () => Promise; @@ -83,68 +85,81 @@ export interface DriverDeployOutput { /** Runs all Vitest tests against the provided drivers. */ export function runDriverTests( - driverTestConfigPartial: Omit, + driverTestConfigPartial: Omit< + DriverTestConfig, + "clientType" | "transport" | "encoding" + >, ) { const clientTypes: ClientType[] = driverTestConfigPartial.skip?.inline ? ["http"] : ["http", "inline"]; for (const clientType of clientTypes) { - const driverTestConfig: DriverTestConfig = { - ...driverTestConfigPartial, - clientType, - }; - describe(`client type (${clientType})`, () => { - runActorDriverTests(driverTestConfig); - runManagerDriverTests(driverTestConfig); + const encodings: Encoding[] = ["bare", "cbor", "json"]; - const transports: Transport[] = driverTestConfig.skip?.sse - ? ["websocket"] - : ["websocket", "sse"]; - for (const transport of transports) { - describe(`transport (${transport})`, () => { - runActorConnTests({ - ...driverTestConfig, - transport, - }); + for (const encoding of encodings) { + describe(`encoding (${encoding})`, () => { + const driverTestConfig: DriverTestConfig = { + ...driverTestConfigPartial, + clientType, + encoding, + }; - runActorConnStateTests({ ...driverTestConfig, transport }); + runActorDriverTests(driverTestConfig); + runManagerDriverTests(driverTestConfig); - runActorReconnectTests({ ...driverTestConfig, transport }); + const transports: Transport[] = driverTestConfig.skip?.sse + ? ["websocket"] + : ["websocket", "sse"]; + for (const transport of transports) { + describe(`transport (${transport})`, () => { + runActorConnTests({ + ...driverTestConfig, + transport, + }); - runRequestAccessTests({ ...driverTestConfig, transport }); + runActorConnStateTests({ ...driverTestConfig, transport }); - runActorDriverTestsWithTransport({ ...driverTestConfig, transport }); - }); - } + runActorReconnectTests({ ...driverTestConfig, transport }); - runActorHandleTests(driverTestConfig); + runRequestAccessTests({ ...driverTestConfig, transport }); - runActionFeaturesTests(driverTestConfig); + runActorDriverTestsWithTransport({ + ...driverTestConfig, + transport, + }); + }); + } - runActorVarsTests(driverTestConfig); + runActorHandleTests(driverTestConfig); - runActorMetadataTests(driverTestConfig); + runActionFeaturesTests(driverTestConfig); - runActorOnStateChangeTests(driverTestConfig); + runActorVarsTests(driverTestConfig); - runActorErrorHandlingTests(driverTestConfig); + runActorMetadataTests(driverTestConfig); - runActorInlineClientTests(driverTestConfig); + runActorOnStateChangeTests(driverTestConfig); - runRawHttpTests(driverTestConfig); + runActorErrorHandlingTests(driverTestConfig); - runRawHttpRequestPropertiesTests(driverTestConfig); + runActorInlineClientTests(driverTestConfig); - runRawWebSocketTests(driverTestConfig); + runRawHttpTests(driverTestConfig); - // TODO: re-expose this once we can have actor queries on the gateway - // runRawHttpDirectRegistryTests(driverTestConfig); + runRawHttpRequestPropertiesTests(driverTestConfig); - // TODO: re-expose this once we can have actor queries on the gateway - // runRawWebSocketDirectRegistryTests(driverTestConfig); + runRawWebSocketTests(driverTestConfig); - runActorInspectorTests(driverTestConfig); + // TODO: re-expose this once we can have actor queries on the gateway + // runRawHttpDirectRegistryTests(driverTestConfig); + + // TODO: re-expose this once we can have actor queries on the gateway + // runRawWebSocketDirectRegistryTests(driverTestConfig); + + runActorInspectorTests(driverTestConfig); + }); + } }); } } diff --git a/packages/rivetkit/src/driver-test-suite/utils.ts b/packages/rivetkit/src/driver-test-suite/utils.ts index 5763977d8..bdf81a9a9 100644 --- a/packages/rivetkit/src/driver-test-suite/utils.ts +++ b/packages/rivetkit/src/driver-test-suite/utils.ts @@ -36,17 +36,20 @@ export async function setupDriverTest( namespace, runnerName, transport: driverTestConfig.transport, + encoding: driverTestConfig.encoding, }); } else if (driverTestConfig.clientType === "inline") { // Use inline client from driver const transport = driverTestConfig.transport ?? "websocket"; + const encoding = driverTestConfig.encoding ?? "bare"; const managerDriver = createTestInlineClientDriver( endpoint, - "bare", + encoding, transport, ); const runConfig = RunConfigSchema.parse({ transport: transport, + encoding: encoding, }); client = createClientWithDriver(managerDriver, runConfig); } else { diff --git a/packages/rivetkit/src/serde.ts b/packages/rivetkit/src/serde.ts index 785abcb79..f4d700e8f 100644 --- a/packages/rivetkit/src/serde.ts +++ b/packages/rivetkit/src/serde.ts @@ -3,7 +3,7 @@ import invariant from "invariant"; import { assertUnreachable } from "@/common/utils"; import type { VersionedDataHandler } from "@/common/versioned-data"; import type { Encoding } from "@/mod"; -import { jsonStringifyCompat } from "./actor/protocol/serde"; +import { jsonParseCompat, jsonStringifyCompat } from "./actor/protocol/serde"; export function uint8ArrayToBase64(uint8Array: Uint8Array): string { // Check if Buffer is available (Node.js) @@ -78,11 +78,11 @@ export function deserializeWithEncoding( ): T { if (encoding === "json") { if (typeof buffer === "string") { - return JSON.parse(buffer); + return jsonParseCompat(buffer); } else { const decoder = new TextDecoder("utf-8"); const jsonString = decoder.decode(buffer); - return JSON.parse(jsonString); + return jsonParseCompat(jsonString); } } else if (encoding === "cbor") { invariant( diff --git a/packages/rivetkit/tests/json-escaping.test.ts b/packages/rivetkit/tests/json-escaping.test.ts new file mode 100644 index 000000000..69586b4a4 --- /dev/null +++ b/packages/rivetkit/tests/json-escaping.test.ts @@ -0,0 +1,221 @@ +import { describe, expect, test } from "vitest"; +import { jsonParseCompat, jsonStringifyCompat } from "@/actor/protocol/serde"; + +describe("JSON Escaping", () => { + describe("BigInt", () => { + test("should serialize and deserialize BigInt", () => { + const input = { num: BigInt(123) }; + const json = jsonStringifyCompat(input); + const result = jsonParseCompat(json); + + expect(result.num).toBe(BigInt(123)); + }); + + test("should handle negative BigInt", () => { + const input = { num: BigInt(-456) }; + const json = jsonStringifyCompat(input); + const result = jsonParseCompat(json); + + expect(result.num).toBe(BigInt(-456)); + }); + + test("should handle very large BigInt", () => { + const input = { num: BigInt("9007199254740991999") }; + const json = jsonStringifyCompat(input); + const result = jsonParseCompat(json); + + expect(result.num).toBe(BigInt("9007199254740991999")); + }); + }); + + describe("ArrayBuffer", () => { + test("should serialize and deserialize ArrayBuffer", () => { + const buffer = new ArrayBuffer(4); + const view = new Uint8Array(buffer); + view[0] = 1; + view[1] = 2; + view[2] = 3; + view[3] = 4; + + const input = { buf: buffer }; + const json = jsonStringifyCompat(input); + const result = jsonParseCompat(json); + + const resultView = new Uint8Array(result.buf); + expect(resultView[0]).toBe(1); + expect(resultView[1]).toBe(2); + expect(resultView[2]).toBe(3); + expect(resultView[3]).toBe(4); + }); + }); + + describe("Uint8Array", () => { + test("should serialize and deserialize Uint8Array", () => { + const input = { arr: new Uint8Array([1, 2, 3, 4, 5]) }; + const json = jsonStringifyCompat(input); + const result = jsonParseCompat(json); + + expect(result.arr).toBeInstanceOf(Uint8Array); + expect(result.arr[0]).toBe(1); + expect(result.arr[1]).toBe(2); + expect(result.arr[2]).toBe(3); + expect(result.arr[3]).toBe(4); + expect(result.arr[4]).toBe(5); + }); + }); + + describe("User data with $-prefixed properties", () => { + test("should preserve user $type property", () => { + const input = { $type: "MyCustomType", value: 456 }; + const json = jsonStringifyCompat(input); + const result = jsonParseCompat(json); + + // User properties are preserved as-is + expect(result.$type).toBe("MyCustomType"); + expect(result.value).toBe(456); + }); + + test("should preserve user $$type property", () => { + const input = { $$type: "NestedType", id: 1 }; + const json = jsonStringifyCompat(input); + const result = jsonParseCompat(json); + + expect(result.$$type).toBe("NestedType"); + expect(result.id).toBe(1); + }); + + test("should escape user arrays starting with $", () => { + const input = { + userArray: ["$NotAType", "someValue"], + userType: { $type: "UserType1", id: 1 }, + }; + const json = jsonStringifyCompat(input); + + // User array should be escaped to $$NotAType + expect(json).toContain('"$$NotAType"'); + + const result = jsonParseCompat(json); + + // Should be unescaped back to original + expect(result.userArray).toEqual(["$NotAType", "someValue"]); + expect(result.userType.$type).toBe("UserType1"); + }); + + test("should handle user arrays already starting with $$", () => { + const input = { arr: ["$$AlreadyEscaped", "value"] }; + const json = jsonStringifyCompat(input); + + // Should add another $ to escape + expect(json).toContain('"$$$AlreadyEscaped"'); + + const result = jsonParseCompat(json); + + // Should be unescaped back to original + expect(result.arr).toEqual(["$$AlreadyEscaped", "value"]); + }); + }); + + describe("Complex scenarios", () => { + test("should handle mix of special types and user properties", () => { + const input = { + bigNum: BigInt(999), + buffer: new Uint8Array([1, 2, 3]), + metadata: { $type: "UserType", id: 1 }, + }; + const json = jsonStringifyCompat(input); + const result = jsonParseCompat(json); + + expect(result.bigNum).toBe(BigInt(999)); + expect(result.buffer).toBeInstanceOf(Uint8Array); + expect(result.buffer[0]).toBe(1); + expect(result.metadata.$type).toBe("UserType"); + }); + + test("should handle deeply nested structures", () => { + const input = { + level1: { + level2: { + level3: { + bigNum: BigInt(123), + metadata: { $type: "Deep", value: 456 }, + }, + }, + }, + }; + const json = jsonStringifyCompat(input); + const result = jsonParseCompat(json); + + expect(result.level1.level2.level3.bigNum).toBe(BigInt(123)); + expect(result.level1.level2.level3.metadata.$type).toBe("Deep"); + expect(result.level1.level2.level3.metadata.value).toBe(456); + }); + + test("should handle arrays containing special types", () => { + const input = { + items: [ + { num: BigInt(1) }, + { metadata: { $type: "ArrayItem", index: 0 } }, + { arr: new Uint8Array([255]) }, + ], + }; + const json = jsonStringifyCompat(input); + const result = jsonParseCompat(json); + + expect(result.items[0].num).toBe(BigInt(1)); + expect(result.items[1].metadata.$type).toBe("ArrayItem"); + expect(result.items[2].arr).toBeInstanceOf(Uint8Array); + expect(result.items[2].arr[0]).toBe(255); + }); + }); + + describe("Edge cases", () => { + test("should handle empty objects", () => { + const input = {}; + const json = jsonStringifyCompat(input); + const result = jsonParseCompat(json); + + expect(result).toEqual({}); + }); + + test("should handle null values", () => { + const input = { value: null }; + const json = jsonStringifyCompat(input); + const result = jsonParseCompat(json); + + expect(result.value).toBeNull(); + }); + + test("should handle undefined values", () => { + const input = { value: undefined, other: 123 }; + const json = jsonStringifyCompat(input); + const result = jsonParseCompat(json); + + // undefined gets dropped by JSON.stringify + expect(result.value).toBeUndefined(); + expect(result.other).toBe(123); + }); + + test("should escape and unescape user arrays with unknown $ types", () => { + const input = { arr: ["$CustomType", "custom value"] }; + const json = jsonStringifyCompat(input); + + // Should be escaped to $$CustomType + expect(json).toContain('"$$CustomType"'); + + const result = jsonParseCompat(json); + + // Should be unescaped back to original + expect(result.arr).toEqual(["$CustomType", "custom value"]); + expect(Array.isArray(result.arr)).toBe(true); + }); + + test("should throw error on unrecognized type starting with single $", () => { + // Manually construct invalid JSON (this shouldn't happen if encoding worked) + const invalidJson = '{"arr":["$UnknownType","value"]}'; + + expect(() => jsonParseCompat(invalidJson)).toThrow( + "Unknown JSON encoding type: $UnknownType", + ); + }); + }); +}); From d7975572bec95d3ca207c295b1a3d5d190de0686 Mon Sep 17 00:00:00 2001 From: NathanFlurry Date: Tue, 7 Oct 2025 23:39:01 +0000 Subject: [PATCH 2/2] chore(core): remove unused bufferutil (#1337) --- packages/rivetkit/package.json | 3 +-- pnpm-lock.yaml | 7 +++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/packages/rivetkit/package.json b/packages/rivetkit/package.json index 426cd0931..3a04cdbd4 100644 --- a/packages/rivetkit/package.json +++ b/packages/rivetkit/package.json @@ -188,8 +188,7 @@ "tsx": "^4.19.4", "typescript": "^5.7.3", "vitest": "^3.1.1", - "ws": "^8.18.1", - "bufferutil": "^4.0.9" + "ws": "^8.18.1" }, "peerDependencies": { "@hono/node-server": "^1.14.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4eb476cf7..06d5ff2a9 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1287,9 +1287,6 @@ importers: '@vitest/ui': specifier: 3.1.1 version: 3.1.1(vitest@3.2.4) - bufferutil: - specifier: ^4.0.9 - version: 4.0.9 bundle-require: specifier: ^5.1.0 version: 5.1.0(esbuild@0.25.5) @@ -6743,6 +6740,7 @@ snapshots: bufferutil@4.0.9: dependencies: node-gyp-build: 4.8.4 + optional: true bun-types@1.2.23(@types/react@19.1.8): dependencies: @@ -7626,7 +7624,8 @@ snapshots: detect-libc: 2.0.4 optional: true - node-gyp-build@4.8.4: {} + node-gyp-build@4.8.4: + optional: true node-releases@2.0.19: {}