[POC] Run query directly on MotherDuck

package.json

@@ -1,7 +1,9 @@
 {
 	"devDependencies": {
 		"@changesets/cli": "2.21.0",
-		"@duckdb/duckdb-wasm": "1.28.0",
+		"@duckdb/duckdb-wasm": "1.29.0",
+		"@motherduck/wasm-client": "0.6.5",
+		"apache-arrow": "17.0.0",
 		"@evidence-dev/bigquery": "link:packages/datasources/bigquery",
 		"@evidence-dev/component-utilities": "link:packages/lib/component-utilities",
 		"@evidence-dev/core-components": "link:packages/ui/core-components",
@@ -27,6 +29,7 @@
 		"@evidence-dev/trino": "link:packages/datasources/trino",
 		"@evidence-dev/universal-sql": "link:packages/lib/universal-sql",
 		"@parcel/packager-ts": "2.12.0",
+		"@parcel/core": "2.12.0",
 		"@parcel/transformer-typescript-types": "2.12.0",
 		"@playwright/test": "^1.45.3",
 		"@sveltejs/adapter-static": "3.0.1",
@@ -63,6 +66,7 @@
 		"svelte-icons": "2.1.0",
 		"svelte-preprocess": "5.1.3",
 		"svelte2tsx": "0.7.4",
+		"svgo": "3.3.2",
 		"typescript": "5.4.2",
 		"unified": "9.1.0",
 		"unist-util-visit": "4.1.2",

packages/lib/universal-sql/package.json

@@ -14,8 +14,9 @@
 	},
 	"type": "module",
 	"dependencies": {
-		"@duckdb/duckdb-wasm": "1.28.0",
-		"apache-arrow": "^13.0.0",
+		"@duckdb/duckdb-wasm": "1.29.0",
+		"@motherduck/wasm-client": "0.6.5",
+		"apache-arrow": "17.0.0",
 		"chalk": "^5.2.0",
 		"cli-progress": "^3.12.0",
 		"lodash.chunk": "^4.2.0",

packages/lib/universal-sql/src/client-duckdb/both.js

@@ -5,7 +5,7 @@ import { Type } from 'apache-arrow';
  *
  * @param {import("apache-arrow").Type} type
  */
-function apacheToEvidenceType(type) {
+export function apacheToEvidenceType(type) {
 	switch (
 		type.typeId // maybe just replace with `typeof`
 	) {

packages/lib/universal-sql/src/client-duckdb/browser.js

@@ -1,4 +1,4 @@
-import { arrowTableToJSON, getPromise, withTimeout } from './both.js';
+import { arrowTableToJSON, apacheToEvidenceType, getPromise, withTimeout } from './both.js';
 import {
 	AsyncDuckDB,
 	ConsoleLogger,
@@ -7,14 +7,21 @@ import {
 	VoidLogger
 } from '@duckdb/duckdb-wasm';
 
+import { Table } from 'apache-arrow';
+
 export { tableFromIPC } from 'apache-arrow';
 
+import { MDConnection } from '@motherduck/wasm-client';
+
 /** @type {import("@duckdb/duckdb-wasm").AsyncDuckDB} */
 let db;
 
 /** @type {import("@duckdb/duckdb-wasm").AsyncDuckDBConnection} */
 let connection;
 
+/** @type {import("@motherduck/wasm-client").MDConnection} */
+let md_connection;
+
 const { resolve: resolveInit, reject: rejectInit, promise: initPromise } = getPromise();
 const { resolve: resolveTables, reject: rejectTables, promise: tablesPromise } = getPromise();
 let initializing = false;
@@ -66,7 +73,15 @@ export async function initDB() {
 			}
 		});
 		connection = await db.connect();
+
 		resolveInit();
+
+		let token = prompt("Please enter a token:", "Token empty");
+		md_connection = MDConnection.create({
+			mdToken: token
+		});
+		await md_connection.isInitialized();
+
 	} catch (e) {
 		rejectInit(e);
 		throw e;
@@ -143,16 +158,89 @@ export async function setParquetURLs(urls, append = false) {
  * @param {string} sql
  * @returns {Promise<import("apache-arrow").Table | null>}
  */
-export async function query(sql) {
+export async function local_query(sql) {
 	// After this point, the database has been initialized
 	if (!db) await initDB();
 	// We need to wait for tables to be available
 	await withTimeout(tablesPromise);
 
 	// Now we can safely execute our query
-	const res = await connection.query(sql).then(arrowTableToJSON);
-
+	console.log(`"*** Running local query *** ${sql}:"`);
+	const res = await connection.query(sql).then((response) => {
+		console.log(`Received local response: ${response}`);
+		return arrowTableToJSON(response);
+	  });
+	console.log(`"Result for local query ${sql}:"`, res);
 	return res;
 }
 
-export { arrowTableToJSON };
+export function parseJSON(fields, json) {
+	const arr = JSON.parse(json);
+
+	Object.defineProperty(arr, '_evidenceColumnTypes', {
+		enumerable: false,
+		value: fields.map((field) => ({
+			name: field.name,
+			evidenceType: apacheToEvidenceType(field.type),
+			typeFidelity: 'precise'
+		}))
+	});
+
+	let rowIndex = 0;
+
+	arr.forEach(item => {
+		item.rowIndex = rowIndex;
+  		rowIndex++;
+	});
+
+	return arr;
+}
+
+
+/**
+ * Queries the database with the given SQL statement.
+ *
+ * @param {string} sql
+ * @returns {Promise<import("apache-arrow").Table | null>}
+ */
+export async function md_query(sql) {
+	console.log(`"*** Running MD query *** ${sql}:"`);
+	try {
+		const result = await md_connection.evaluateQuery(sql);
+		const fields = result.data.batches[0].recordBatch.schema.fields;
+		const rows = result.data.toRows();
+		const json = JSON.stringify(rows, (key, value) =>
+			typeof value === 'bigint'
+				? Number(value)
+				: value // return everything else unchanged
+		);
+		const table = result;
+		const res = parseJSON(fields, json);
+		console.log('*** Result for query ***: ' + sql, res);
+		return res;
+	} catch (err) {
+		console.log('*** Query failed ***: ' + sql, err);
+		return local_query(sql);
+	}
+}
+
+
+/**
+ * Queries the database with the given SQL statement.
+ *
+ * @param {string} sql
+ * @returns {Promise<import("apache-arrow").Table | null>}
+ */
+export async function query(sql) {
+	try {
+		const res = md_query(sql);
+		console.log(`*** MD Query Result ***: ${sql}`, res);
+		return res;
+		return res;
+	} catch (err) {
+		console.log(`*** MD Query failed ***: ${sql}`, err);
+		// return local_query(sql);
+	}
+}
+
+export { arrowTableToJSON, apacheToEvidenceType };

packages/ui/core-components/vite.config.js

@@ -5,5 +5,11 @@ export default defineConfig({
 	plugins: [sveltekit()],
 	test: {
 		include: ['src/**/*.{test,spec}.{js,ts}']
-	}
+	},
+	server: {
+		headers: {
+			"Cross-Origin-Embedder-Policy": "require-corp",
+			"Cross-Origin-Opener-Policy": "same-origin",
+		},
+	},
 });