Skip to content

Commit 1e7513e

Browse files
gruuyagruuya
authored
Merge pull request #620 from splitgraph/snowflake-agg-pushdown-support-cu-23kfzuz
Snowflake aggregation pushdown support
2 parents 891966b + 73cdc71 commit 1e7513e

23 files changed

+3463
-1097
lines changed

‎engine/src/Multicorn

‎engine/src/postgres-elasticsearch-fdw

‎splitgraph/ingestion/snowflake/__init__.py

+10-3
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@ class SnowflakeDataSource(ForeignDataWrapperDataSource):
7575
},
7676
},
7777
],
78+
"description": "A password or a private key to be used for authentication",
7879
},
7980
"account": {
8081
"type": "string",
@@ -132,18 +133,24 @@ class SnowflakeDataSource(ForeignDataWrapperDataSource):
132133
$ sgr mount snowflake test_snowflake -o@- <<EOF
133134
{
134135
"username": "username",
135-
"password": "password",
136+
"secret": {
137+
"secret_type": "password",
138+
"password": "password"
139+
},
136140
"account": "acc-id.west-europe.azure",
137141
"database": "SNOWFLAKE_SAMPLE_DATA",
138-
"schema": "TPCH_SF100"
142+
"schema": "TPCH_SF100",
139143
"envvars": {"HTTPS_PROXY": "http://proxy.company.com"}
140144
}
141145
EOF
142146
\b
143147
$ sgr mount snowflake test_snowflake_subquery -o@- <<EOF
144148
{
145149
"username": "username",
146-
"private_key": "MIIEvQIBAD...",
150+
"secret": {
151+
"secret_type": "private_key",
152+
"private_key": "MIIEvQIBAD..."
153+
},
147154
"account": "acc-id.west-europe.azure",
148155
"database": "SNOWFLAKE_SAMPLE_DATA",
149156
"tables": {

‎test/architecture/data/pgorigin/accounts.csv

+1,001
Large diffs are not rendered by default.

‎test/architecture/data/pgorigin/load_account_data.sql

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
DROP TABLE IF EXISTS account;
2+
CREATE TABLE account (
3+
account_number integer,
4+
balance integer,
5+
firstname character varying(20),
6+
lastname character varying(20),
7+
age integer,
8+
gender character varying(1),
9+
address text,
10+
employer character varying(20),
11+
email text,
12+
city character varying(20),
13+
state character varying(5)
14+
);
15+
16+
COPY account from '/src/accounts.csv' DELIMITER ',' CSV HEADER;

‎test/architecture/data/pgorigin/setup.sql

-1
Original file line numberDiff line numberDiff line change
@@ -15,4 +15,3 @@ INSERT INTO fruits (name) VALUES ('orange');
1515

1616
INSERT INTO vegetables (name) VALUES ('potato');
1717
INSERT INTO vegetables (name) VALUES ('carrot');
18-

‎test/architecture/src/pgorigin/start.sh

+1
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ if [ ! -e '/done_setup' ]; then
1616

1717
if [ -e '/src/setup.sql' ]; then
1818
${SU} "psql ${ORIGIN_PG_DB} < /src/setup.sql"
19+
${SU} "psql ${ORIGIN_PG_DB} < /src/load_account_data.sql"
1920
fi
2021

2122
echo 1 > /done_setup

‎test/splitgraph/commands/snapshots/test_es_fdw_queries/test_aggregations_join_combinations/account_join_sub_aggs.yml ‎test/splitgraph/commands/snapshots/test_multicorn_fdws/test_aggregations_join_combinations/es/account_join_sub_aggs.yml

File renamed without changes.

‎test/splitgraph/commands/snapshots/test_multicorn_fdws/test_aggregations_join_combinations/pg/account_join_sub_aggs.yml

+84
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
- !!python/tuple
2+
- 20
3+
- 49568
4+
- 1650
5+
- !!python/tuple
6+
- 21
7+
- 49433
8+
- 2213
9+
- !!python/tuple
10+
- 22
11+
- 49404
12+
- 1133
13+
- !!python/tuple
14+
- 23
15+
- 49587
16+
- 1464
17+
- !!python/tuple
18+
- 24
19+
- 48745
20+
- 1011
21+
- !!python/tuple
22+
- 25
23+
- 49795
24+
- 1956
25+
- !!python/tuple
26+
- 26
27+
- 48466
28+
- 1447
29+
- !!python/tuple
30+
- 27
31+
- 46868
32+
- 1110
33+
- !!python/tuple
34+
- 28
35+
- 49222
36+
- 2889
37+
- !!python/tuple
38+
- 29
39+
- 49119
40+
- 3596
41+
- !!python/tuple
42+
- 30
43+
- 49334
44+
- 2726
45+
- !!python/tuple
46+
- 31
47+
- 48758
48+
- 2384
49+
- !!python/tuple
50+
- 32
51+
- 48294
52+
- 1031
53+
- !!python/tuple
54+
- 33
55+
- 48734
56+
- 1314
57+
- !!python/tuple
58+
- 34
59+
- 48997
60+
- 3001
61+
- !!python/tuple
62+
- 35
63+
- 49741
64+
- 1481
65+
- !!python/tuple
66+
- 36
67+
- 49989
68+
- 1249
69+
- !!python/tuple
70+
- 37
71+
- 47546
72+
- 1360
73+
- !!python/tuple
74+
- 38
75+
- 49339
76+
- 3022
77+
- !!python/tuple
78+
- 39
79+
- 47257
80+
- 3589
81+
- !!python/tuple
82+
- 40
83+
- 49671
84+
- 1932

‎test/splitgraph/commands/snapshots/test_es_fdw_queries/test_grouping_and_aggregations_bare/account_count_by_age.yml ‎test/splitgraph/commands/snapshots/test_multicorn_fdws/test_grouping_and_aggregations_bare/es/account_count_by_age.yml

File renamed without changes.

‎test/splitgraph/commands/snapshots/test_multicorn_fdws/test_grouping_and_aggregations_bare/pg/account_count_by_age.yml

+84
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
- !!python/tuple
2+
- 40
3+
- 45
4+
- 1932
5+
- !!python/tuple
6+
- 39
7+
- 60
8+
- 3589
9+
- !!python/tuple
10+
- 38
11+
- 39
12+
- 3022
13+
- !!python/tuple
14+
- 37
15+
- 42
16+
- 1360
17+
- !!python/tuple
18+
- 36
19+
- 52
20+
- 1249
21+
- !!python/tuple
22+
- 35
23+
- 52
24+
- 1481
25+
- !!python/tuple
26+
- 34
27+
- 49
28+
- 3001
29+
- !!python/tuple
30+
- 33
31+
- 50
32+
- 1314
33+
- !!python/tuple
34+
- 32
35+
- 52
36+
- 1031
37+
- !!python/tuple
38+
- 31
39+
- 61
40+
- 2384
41+
- !!python/tuple
42+
- 30
43+
- 47
44+
- 2726
45+
- !!python/tuple
46+
- 29
47+
- 35
48+
- 3596
49+
- !!python/tuple
50+
- 28
51+
- 51
52+
- 2889
53+
- !!python/tuple
54+
- 27
55+
- 39
56+
- 1110
57+
- !!python/tuple
58+
- 26
59+
- 59
60+
- 1447
61+
- !!python/tuple
62+
- 25
63+
- 42
64+
- 1956
65+
- !!python/tuple
66+
- 24
67+
- 42
68+
- 1011
69+
- !!python/tuple
70+
- 23
71+
- 42
72+
- 1464
73+
- !!python/tuple
74+
- 22
75+
- 51
76+
- 1133
77+
- !!python/tuple
78+
- 21
79+
- 46
80+
- 2213
81+
- !!python/tuple
82+
- 20
83+
- 44
84+
- 1650

0 commit comments

Comments
 (0)