Skip to content

Commit 8190f89

Browse files
committed
feat: query stores
1 parent 629837e commit 8190f89

38 files changed

+880
-35
lines changed

.npmrc

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
public-hoist-pattern[]=*apache-arrow*

package.json

+8-5
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,19 @@
11
{
22
"devDependencies": {
33
"@changesets/cli": "2.21.0",
4+
"@duckdb/duckdb-wasm": "1.27.0",
45
"@evidence-dev/component-utilities": "link:packages/component-utilities",
56
"@evidence-dev/core-components": "link:packages/core-components",
67
"@evidence-dev/db-orchestrator": "link:packages/db-orchestrator",
78
"@evidence-dev/evidence": "link:packages/evidence",
8-
"@evidence-dev/preprocess": "link:packages/preprocess",
9-
"@evidence-dev/universal-sql": "link:packages/universal-sql",
109
"@evidence-dev/plugin-connector": "link:packages/plugin-connector",
11-
"@evidence-dev/telemetry": "link:packages/telemetry",
10+
"@evidence-dev/preprocess": "link:packages/preprocess",
11+
"@evidence-dev/query-store": "link:packages/query-store",
1212
"@evidence-dev/tailwind": "link:packages/tailwind",
13+
"@evidence-dev/telemetry": "link:packages/telemetry",
14+
"@evidence-dev/universal-sql": "link:packages/universal-sql",
15+
"@parcel/packager-ts": "2.9.3",
16+
"@parcel/transformer-typescript-types": "2.9.3",
1317
"@sveltejs/adapter-static": "1.0.0",
1418
"@sveltejs/kit": "1.21.0",
1519
"@tidyjs/tidy": "2.4.4",
@@ -39,8 +43,7 @@
3943
"unified": "9.1.0",
4044
"unist-util-visit": "4.1.2",
4145
"uvu": "0.5.2",
42-
"vite": "4.3.9",
43-
"@duckdb/duckdb-wasm": "1.27.0"
46+
"vite": "4.3.9"
4447
},
4548
"scripts": {
4649
"build:dev-workspace": "run-s build:example-project",

packages/evidence/package.json

+2
Original file line numberDiff line numberDiff line change
@@ -43,12 +43,14 @@
4343
"@evidence-dev/universal-sql": "workspace:2.0.0-usql.9",
4444
"@evidence-dev/plugin-connector": "workspace:2.0.0-usql.16",
4545
"@evidence-dev/telemetry": "workspace:",
46+
"@evidence-dev/query-store": "workspace:",
4647
"@sveltejs/adapter-static": "1.0.0",
4748
"chokidar": "3.5.3",
4849
"fs-extra": "10.0.1",
4950
"sade": "^1.8.1"
5051
},
5152
"peerDependencies": {
53+
"@evidence-dev/query-store": "workspace:",
5254
"@evidence-dev/component-utilities": "workspace:2.0.0-usql.7",
5355
"@evidence-dev/core-components": "workspace:2.0.0-usql.11",
5456
"@evidence-dev/db-orchestrator": "workspace:3.0.0-usql.7",

packages/plugin-connector/package.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
"author": "",
1818
"license": "MIT",
1919
"dependencies": {
20-
"@evidence-dev/universal-sql": "workspace:2.0.0-usql.9",
20+
"@evidence-dev/universal-sql": "workspace:",
2121
"@types/estree": "^1.0.1",
2222
"chalk": "^5.2.0",
2323
"svelte": "3.55.0",

packages/preprocess/src/process-queries.cjs

+47-23
Original file line numberDiff line numberDiff line change
@@ -12,31 +12,55 @@ const createDefaultProps = function (filename, componentDevelopmentMode, duckdbQ
1212
const valid_ids = Object.keys(duckdbQueries).filter((queryId) =>
1313
queryId.match('^([a-zA-Z_$][a-zA-Z0-9d_$]*)$')
1414
);
15-
queryDeclarations += `
16-
import debounce from 'debounce';
17-
import { browser } from '$app/environment';
18-
import {profile} from '@evidence-dev/component-utilities/profile';
19-
20-
// partially bypasses weird reactivity stuff with \`select\` elements
21-
function data_update(data) {
22-
${valid_ids.map((id) => `${id} = data.${id} ?? [];`).join('\n')}
23-
}
24-
25-
$: data_update(data);
15+
// queryDeclarations += `
16+
// import debounce from 'debounce';
17+
// import { browser } from '$app/environment';
18+
// import {profile} from '@evidence-dev/component-utilities/profile';
19+
//
20+
// // partially bypasses weird reactivity stuff with \`select\` elements
21+
// function data_update(data) {
22+
// ${valid_ids.map((id) => `${id} = data.${id} ?? [];`).join('\n')}
23+
// }
24+
//
25+
// $: data_update(data);
26+
//
27+
//
28+
// ${valid_ids
29+
// .map(
30+
// (id) => `
31+
// let ${id} = data.${id} ?? [];
32+
// const _query_${id} = browser
33+
// ? debounce((query) => profile(__db.query, query).then((value) => ${id} = value), 200)
34+
// : (query) => (${id} = profile(__db.query, query, "${id}"));
35+
// $: _query_${id}(\`${duckdbQueries[id].replaceAll('`', '\\`')}\`);
36+
// `
37+
// )
38+
// .join('\n')}
39+
// `;
40+
41+
const queryStores = valid_ids.map(id => `
42+
const _${id} = new QueryStore(
43+
\`${duckdbQueries[id].replaceAll('`', '\\`')}\`,
44+
queryFunc,
45+
\`${id}\`,
46+
{ initialData: profile(__db.query, \`${duckdbQueries[id].replaceAll('`', '\\`')}\`) }
47+
);
48+
/** @type {QueryStore} */
49+
let ${id};
50+
$: ${id} = $_${id};
51+
`)
2652

2753

28-
${valid_ids
29-
.map(
30-
(id) => `
31-
let ${id} = data.${id} ?? [];
32-
const _query_${id} = browser
33-
? debounce((query) => profile(__db.query, query).then((value) => ${id} = value), 200)
34-
: (query) => (${id} = profile(__db.query, query, "${id}"));
35-
$: _query_${id}(\`${duckdbQueries[id].replaceAll('`', '\\`')}\`);
36-
`
37-
)
38-
.join('\n')}
39-
`;
54+
queryDeclarations += `
55+
import {browser} from "$app/environment";
56+
import {profile} from '@evidence-dev/component-utilities/profile';
57+
import debounce from 'debounce';
58+
import {QueryStore} from '@evidence-dev/query-store';
59+
60+
const queryFunc = q => profile(__db.query, q);
61+
62+
${queryStores.join("\n")}
63+
`
4064
}
4165

4266
let defaultProps = `

packages/query-store/.eslintrc.js

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
export default {
2+
root: false,
3+
// extends: ['eslint:recommended', 'prettier'],
4+
5+
}

packages/query-store/package.json

+37
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
{
2+
"name": "@evidence-dev/query-store",
3+
"version": "1.0.0",
4+
"description": "",
5+
"main": "dist/index.js",
6+
"source": "./src/index.ts",
7+
"types": "dist/index.d.ts",
8+
"type": "module",
9+
"scripts": {
10+
"build": "tsc -p ./tsconfig.json",
11+
"watch": "tsc -p ./tsconfig.json --watch",
12+
"test": "vitest --run",
13+
"test:watch": "vitest"
14+
},
15+
"keywords": [],
16+
"author": "The Evidence Engineering Team",
17+
"license": "MIT",
18+
"devDependencies": {
19+
"typescript": "^5.2.2",
20+
"vitest": "^0.34.1"
21+
},
22+
"dependencies": {
23+
"@evidence-dev/universal-sql": "workspace:",
24+
"@sqltools/formatter": "^1.2.5",
25+
"@types/lodash.debounce": "^4.0.7",
26+
"@types/nanoid": "^3.0.0",
27+
"@uwdata/mosaic-sql": "^0.3.2",
28+
"lodash.debounce": "^4.0.8",
29+
"nanoid": "^5.0.1",
30+
"sql-formatter": "^13.0.0"
31+
},
32+
"targets": {
33+
"main": {
34+
"context": "node"
35+
}
36+
}
37+
}
+137
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
import { beforeEach, describe, expect, it, vi } from 'vitest';
2+
import { QueryStore } from './QueryStore';
3+
import util from 'util';
4+
5+
const tick = () => new Promise((r) => setTimeout(r, 100));
6+
7+
describe('QueryStore', () => {
8+
const mockExec = vi.fn();
9+
const mockSubscription = vi.fn();
10+
11+
beforeEach(() => {
12+
vi.resetAllMocks();
13+
mockExec.mockImplementation((q: string) => {
14+
if (q.startsWith('--col-metadata'))
15+
return Promise.resolve([{ column_name: 'x', column_type: 'INTEGER' }]);
16+
if (q.startsWith('--len')) return Promise.resolve([{ length: 5 }]);
17+
return Promise.resolve([{ x: 1 }, { x: 2 }, { x: 3 }, { x: 4 }, { x: 5 }]);
18+
});
19+
});
20+
21+
it('should be defined', () => {
22+
expect(QueryStore).toBeDefined();
23+
});
24+
25+
it('should be subscribeable', () => {
26+
const store = new QueryStore('SELECT 1', mockExec);
27+
store.subscribe(mockSubscription);
28+
29+
expect(mockSubscription).toHaveBeenCalledOnce();
30+
});
31+
32+
it('should execute a query when accessing the .length property', async () => {
33+
const store = new QueryStore('SELECT 1', mockExec);
34+
store.subscribe(mockSubscription);
35+
36+
// Accessing the property should be enough
37+
store.proxy.length;
38+
39+
// let the event loop do its thing and the store hash things out
40+
await tick();
41+
42+
expect(store.proxy.length).toBe(5);
43+
44+
// 1. Initial Subscription
45+
// 2. Length loading started
46+
// 3. Length loading finished
47+
// 4. Column Descriptions become available
48+
expect(mockSubscription).toHaveBeenCalledTimes(4);
49+
// 1. Column Metadata
50+
// 2. Length
51+
expect(mockExec).toHaveBeenCalledTimes(2);
52+
53+
// Data was not touched
54+
// This query should not have fired
55+
expect(store.loaded).toBe(false);
56+
expect(store.loading).toBe(false);
57+
58+
// Length should have finished loading
59+
expect(store.lengthLoaded).toBe(true);
60+
expect(store.lengthLoading).toBe(false);
61+
});
62+
63+
it('should ensure that 2 queries with the same derivation are distinct', () => {
64+
const store1 = new QueryStore('SELECT 1;', mockExec, undefined, { disableCache: false });
65+
const store2 = new QueryStore('SELECT 2;', mockExec, undefined, { disableCache: false });
66+
67+
const limitedStore1 = store1.limit(0);
68+
const limitedStore2 = store2.limit(0);
69+
70+
expect(limitedStore1.text).not.toEqual(limitedStore2.text);
71+
});
72+
73+
it('should slice properly', async () => {
74+
const store = new QueryStore('SELECT 2;', mockExec, undefined, { disableCache: false }).proxy;
75+
76+
await store.fetch();
77+
78+
const sliced = store.slice();
79+
80+
console.error(sliced.length, store.length);
81+
expect(sliced.length).toEqual(store.length);
82+
83+
expect(sliced[0]).toEqual(store.proxy[0]);
84+
});
85+
86+
describe('Derived Stores', () => {
87+
describe.each<{ func: keyof QueryStore; args: unknown[] }>([
88+
{ func: 'where', args: [] },
89+
{ func: 'agg', args: [{}] },
90+
{ func: 'limit', args: [5] },
91+
{ func: 'orderBy', args: [{}] },
92+
{ func: 'offset', args: [] }
93+
])('$func', (opts) => {
94+
it(`should have the property ${opts.func}()`, () => {
95+
const store = new QueryStore('SELECT 1;', mockExec, undefined, { disableCache: true });
96+
expect(opts.func in store).toBe(true);
97+
expect(typeof store[opts.func]).toBe('function');
98+
});
99+
100+
it(`should return a new store when using .${opts.func}`, () => {
101+
const store = new QueryStore('SELECT 1;', mockExec, undefined, { disableCache: true });
102+
103+
const targetFunc = store[opts.func];
104+
expect(targetFunc).toBeTypeOf('function');
105+
106+
const childStore = (targetFunc as CallableFunction)(...opts.args);
107+
108+
// store !== childStore
109+
expect(store).not.toEqual(childStore);
110+
// childStore is a proxy
111+
expect(util.types.isProxy(childStore)).toBe(true);
112+
});
113+
it('should subscribe to derived stores', async () => {
114+
const store = new QueryStore('SELECT 1;', mockExec, undefined, { disableCache: true });
115+
const targetFunc = store[opts.func];
116+
expect(targetFunc).toBeTypeOf('function');
117+
118+
const childStore = (targetFunc as CallableFunction)(opts.args);
119+
store.subscribe(mockSubscription);
120+
121+
childStore.length;
122+
123+
await tick();
124+
125+
// 1. Parent Initial Load
126+
// 2. Parent Metadata
127+
// 3. Child Initial Load
128+
// 4. Child Metadata
129+
// 5. Child Length
130+
131+
expect(mockSubscription).toBeCalledTimes(5);
132+
133+
expect(childStore.length).toBe(5);
134+
});
135+
});
136+
});
137+
});

0 commit comments

Comments
 (0)