diff --git a/Cargo.lock b/Cargo.lock index 67ddba84..f1858b10 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -245,19 +245,37 @@ version = "54.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5ec52ba94edeed950e4a41f75d35376df196e8cb04437f7280a5aa49f20f796" dependencies = [ - "arrow-arith", - "arrow-array", - "arrow-buffer", - "arrow-cast", + "arrow-arith 54.3.1", + "arrow-array 54.3.1", + "arrow-buffer 54.3.1", + "arrow-cast 54.3.1", + "arrow-data 54.3.1", + "arrow-ord 54.3.1", + "arrow-row 54.3.1", + "arrow-schema 54.3.1", + "arrow-select 54.3.1", + "arrow-string 54.3.1", +] + +[[package]] +name = "arrow" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3095aaf545942ff5abd46654534f15b03a90fba78299d661e045e5d587222f0d" +dependencies = [ + "arrow-arith 55.0.0", + "arrow-array 55.0.0", + "arrow-buffer 55.0.0", + "arrow-cast 55.0.0", "arrow-csv", - "arrow-data", + "arrow-data 55.0.0", "arrow-ipc", "arrow-json", - "arrow-ord", - "arrow-row", - "arrow-schema", - "arrow-select", - "arrow-string", + "arrow-ord 55.0.0", + "arrow-row 55.0.0", + "arrow-schema 55.0.0", + "arrow-select 55.0.0", + "arrow-string 55.0.0", "pyo3", ] @@ -267,10 +285,24 @@ version = "54.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8fc766fdacaf804cb10c7c70580254fcdb5d55cdfda2bc57b02baf5223a3af9e" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 54.3.1", + "arrow-buffer 54.3.1", + "arrow-data 54.3.1", + "arrow-schema 54.3.1", + "chrono", + "num", +] + +[[package]] +name = "arrow-arith" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00752064ff47cee746e816ddb8450520c3a52cbad1e256f6fa861a35f86c45e7" +dependencies = [ + "arrow-array 55.0.0", + "arrow-buffer 55.0.0", + "arrow-data 55.0.0", + "arrow-schema 55.0.0", "chrono", "num", ] @@ -282,9 +314,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a12fcdb3f1d03f69d3ec26ac67645a8fe3f878d77b5ebb0b15d64a116c212985" dependencies = [ "ahash 0.8.11", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-buffer 54.3.1", + "arrow-data 54.3.1", + "arrow-schema 54.3.1", + "chrono", + "half", + "hashbrown 0.15.2", + "num", +] + +[[package]] +name = "arrow-array" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cebfe926794fbc1f49ddd0cdaf898956ca9f6e79541efce62dabccfd81380472" +dependencies = [ + "ahash 0.8.11", + "arrow-buffer 55.0.0", + "arrow-data 55.0.0", + "arrow-schema 55.0.0", "chrono", "chrono-tz", "half", @@ -303,17 +351,49 @@ dependencies = [ "num", ] +[[package]] +name = "arrow-buffer" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0303c7ec4cf1a2c60310fc4d6bbc3350cd051a17bf9e9c0a8e47b4db79277824" +dependencies = [ + "bytes", + "half", + "num", +] + [[package]] name = "arrow-cast" version = "54.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ede6175fbc039dfc946a61c1b6d42fd682fcecf5ab5d148fbe7667705798cac9" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 54.3.1", + "arrow-buffer 54.3.1", + "arrow-data 54.3.1", + "arrow-schema 54.3.1", + "arrow-select 54.3.1", + "atoi", + "base64", + "chrono", + "comfy-table", + "half", + "lexical-core", + "num", + "ryu", +] + +[[package]] +name = "arrow-cast" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "335f769c5a218ea823d3760a743feba1ef7857cba114c01399a891c2fff34285" +dependencies = [ + "arrow-array 55.0.0", + "arrow-buffer 55.0.0", + "arrow-data 55.0.0", + "arrow-schema 55.0.0", + "arrow-select 55.0.0", "atoi", "base64", "chrono", @@ -326,13 +406,13 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "54.3.1" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1644877d8bc9a0ef022d9153dc29375c2bda244c39aec05a91d0e87ccf77995f" +checksum = "510db7dfbb4d5761826516cc611d97b3a68835d0ece95b034a052601109c0b1b" dependencies = [ - "arrow-array", - "arrow-cast", - "arrow-schema", + "arrow-array 55.0.0", + "arrow-cast 55.0.0", + "arrow-schema 55.0.0", "chrono", "csv", "csv-core", @@ -346,29 +426,41 @@ version = "54.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61cfdd7d99b4ff618f167e548b2411e5dd2c98c0ddebedd7df433d34c20a4429" dependencies = [ - "arrow-buffer", - "arrow-schema", + "arrow-buffer 54.3.1", + "arrow-schema 54.3.1", + "half", + "num", +] + +[[package]] +name = "arrow-data" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8affacf3351a24039ea24adab06f316ded523b6f8c3dbe28fbac5f18743451b" +dependencies = [ + "arrow-buffer 55.0.0", + "arrow-schema 55.0.0", "half", "num", ] [[package]] name = "arrow-flight" -version = "54.3.1" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a194f47959a4e111463cb6d02c8576fe084b3d7a3c092314baf3b9629b62595b" +checksum = "e2e0fad280f41a918d53ba48288a246ff04202d463b3b380fbc0edecdcb52cfd" dependencies = [ - "arrow-arith", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", + "arrow-arith 55.0.0", + "arrow-array 55.0.0", + "arrow-buffer 55.0.0", + "arrow-cast 55.0.0", + "arrow-data 55.0.0", "arrow-ipc", - "arrow-ord", - "arrow-row", - "arrow-schema", - "arrow-select", - "arrow-string", + "arrow-ord 55.0.0", + "arrow-row 55.0.0", + "arrow-schema 55.0.0", + "arrow-select 55.0.0", + "arrow-string 55.0.0", "base64", "bytes", "futures", @@ -381,29 +473,29 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "54.3.1" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62ff528658b521e33905334723b795ee56b393dbe9cf76c8b1f64b648c65a60c" +checksum = "69880a9e6934d9cba2b8630dd08a3463a91db8693b16b499d54026b6137af284" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 55.0.0", + "arrow-buffer 55.0.0", + "arrow-data 55.0.0", + "arrow-schema 55.0.0", "flatbuffers", "lz4_flex", ] [[package]] name = "arrow-json" -version = "54.3.1" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ee5b4ca98a7fb2efb9ab3309a5d1c88b5116997ff93f3147efdc1062a6158e9" +checksum = "d8dafd17a05449e31e0114d740530e0ada7379d7cb9c338fd65b09a8130960b0" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", - "arrow-schema", + "arrow-array 55.0.0", + "arrow-buffer 55.0.0", + "arrow-cast 55.0.0", + "arrow-data 55.0.0", + "arrow-schema 55.0.0", "chrono", "half", "indexmap 2.9.0", @@ -417,11 +509,11 @@ dependencies = [ [[package]] name = "arrow-odbc" -version = "16.0.0" +version = "16.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df5c9eaa58d98c0ba1d2a1a57c1fc2daa8ed25283c54a9b3b7daaf818f473914" +checksum = "212acf2aaf14506b805f0105a336466f77aee8d1dc4c0a4205fdb31fe33281a2" dependencies = [ - "arrow", + "arrow 55.0.0", "chrono", "log", "odbc-api", @@ -434,11 +526,24 @@ version = "54.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0a3334a743bd2a1479dbc635540617a3923b4b2f6870f37357339e6b5363c21" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 54.3.1", + "arrow-buffer 54.3.1", + "arrow-data 54.3.1", + "arrow-schema 54.3.1", + "arrow-select 54.3.1", +] + +[[package]] +name = "arrow-ord" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "895644523af4e17502d42c3cb6b27cb820f0cb77954c22d75c23a85247c849e1" +dependencies = [ + "arrow-array 55.0.0", + "arrow-buffer 55.0.0", + "arrow-data 55.0.0", + "arrow-schema 55.0.0", + "arrow-select 55.0.0", ] [[package]] @@ -447,10 +552,23 @@ version = "54.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d1d7a7291d2c5107e92140f75257a99343956871f3d3ab33a7b41532f79cb68" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 54.3.1", + "arrow-buffer 54.3.1", + "arrow-data 54.3.1", + "arrow-schema 54.3.1", + "half", +] + +[[package]] +name = "arrow-row" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9be8a2a4e5e7d9c822b2b8095ecd77010576d824f654d347817640acfc97d229" +dependencies = [ + "arrow-array 55.0.0", + "arrow-buffer 55.0.0", + "arrow-data 55.0.0", + "arrow-schema 55.0.0", "half", ] @@ -459,6 +577,15 @@ name = "arrow-schema" version = "54.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cfaf5e440be44db5413b75b72c2a87c1f8f0627117d110264048f2969b99e9" +dependencies = [ + "bitflags 2.9.0", +] + +[[package]] +name = "arrow-schema" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7450c76ab7c5a6805be3440dc2e2096010da58f7cab301fdc996a4ee3ee74e49" dependencies = [ "bitflags 2.9.0", "serde", @@ -471,10 +598,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69efcd706420e52cd44f5c4358d279801993846d1c2a8e52111853d61d55a619" dependencies = [ "ahash 0.8.11", - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 54.3.1", + "arrow-buffer 54.3.1", + "arrow-data 54.3.1", + "arrow-schema 54.3.1", + "num", +] + +[[package]] +name = "arrow-select" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa5f5a93c75f46ef48e4001535e7b6c922eeb0aa20b73cf58d09e13d057490d8" +dependencies = [ + "ahash 0.8.11", + "arrow-array 55.0.0", + "arrow-buffer 55.0.0", + "arrow-data 55.0.0", + "arrow-schema 55.0.0", "num", ] @@ -484,11 +625,28 @@ version = "54.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a21546b337ab304a32cfc0770f671db7411787586b45b78b4593ae78e64e2b03" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 54.3.1", + "arrow-buffer 54.3.1", + "arrow-data 54.3.1", + "arrow-schema 54.3.1", + "arrow-select 54.3.1", + "memchr", + "num", + "regex", + "regex-syntax 0.8.5", +] + +[[package]] +name = "arrow-string" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e7005d858d84b56428ba2a98a107fe88c0132c61793cf6b8232a1f9bfc0452b" +dependencies = [ + "arrow-array 55.0.0", + "arrow-buffer 55.0.0", + "arrow-data 55.0.0", + "arrow-schema 55.0.0", + "arrow-select 55.0.0", "memchr", "num", "regex", @@ -1333,13 +1491,13 @@ checksum = "575f75dfd25738df5b91b8e43e14d44bda14637a58fae779fd2b064f8bf3e010" [[package]] name = "datafusion" -version = "46.0.1" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "914e6f9525599579abbd90b0f7a55afcaaaa40350b9e9ed52563f126dfe45fd3" +checksum = "ffe060b978f74ab446be722adb8a274e052e005bf6dfd171caadc3abaad10080" dependencies = [ - "arrow", + "arrow 55.0.0", "arrow-ipc", - "arrow-schema", + "arrow-schema 55.0.0", "async-trait", "bytes", "bzip2", @@ -1349,6 +1507,9 @@ dependencies = [ "datafusion-common", "datafusion-common-runtime", "datafusion-datasource", + "datafusion-datasource-csv", + "datafusion-datasource-json", + "datafusion-datasource-parquet", "datafusion-execution", "datafusion-expr", "datafusion-expr-common", @@ -1363,10 +1524,11 @@ dependencies = [ "datafusion-physical-expr-common", "datafusion-physical-optimizer", "datafusion-physical-plan", + "datafusion-session", "datafusion-sql", "flate2", "futures", - "itertools 0.14.0", + "itertools", "log", "object_store", "parking_lot", @@ -1385,31 +1547,37 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "46.0.1" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "998a6549e6ee4ee3980e05590b2960446a56b343ea30199ef38acd0e0b9036e2" +checksum = "61fe34f401bd03724a1f96d12108144f8cd495a3cdda2bf5e091822fb80b7e66" dependencies = [ - "arrow", + "arrow 55.0.0", "async-trait", "dashmap", "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", "datafusion-execution", "datafusion-expr", + "datafusion-physical-expr", "datafusion-physical-plan", + "datafusion-session", "datafusion-sql", "futures", - "itertools 0.14.0", + "itertools", "log", + "object_store", "parking_lot", + "tokio", ] [[package]] name = "datafusion-catalog-listing" -version = "46.0.1" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5ac10096a5b3c0d8a227176c0e543606860842e943594ccddb45cf42a526e43" +checksum = "a4411b8e3bce5e0fc7521e44f201def2e2d5d1b5f176fb56e8cdc9942c890f00" dependencies = [ - "arrow", + "arrow 55.0.0", "async-trait", "datafusion-catalog", "datafusion-common", @@ -1419,6 +1587,7 @@ dependencies = [ "datafusion-physical-expr", "datafusion-physical-expr-common", "datafusion-physical-plan", + "datafusion-session", "futures", "log", "object_store", @@ -1427,12 +1596,12 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "46.0.1" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f53d7ec508e1b3f68bd301cee3f649834fad51eff9240d898a4b2614cfd0a7a" +checksum = "0734015d81c8375eb5d4869b7f7ecccc2ee8d6cb81948ef737cd0e7b743bd69c" dependencies = [ "ahash 0.8.11", - "arrow", + "arrow 55.0.0", "arrow-ipc", "base64", "half", @@ -1452,27 +1621,27 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "46.0.1" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0fcf41523b22e14cc349b01526e8b9f59206653037f2949a4adbfde5f8cb668" +checksum = "5167bb1d2ccbb87c6bc36c295274d7a0519b14afcfdaf401d53cbcaa4ef4968b" dependencies = [ + "futures", "log", "tokio", ] [[package]] name = "datafusion-datasource" -version = "46.0.1" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf7f37ad8b6e88b46c7eeab3236147d32ea64b823544f498455a8d9042839c92" +checksum = "04e602dcdf2f50c2abf297cc2203c73531e6f48b29516af7695d338cf2a778b1" dependencies = [ - "arrow", + "arrow 55.0.0", "async-compression", "async-trait", "bytes", "bzip2", "chrono", - "datafusion-catalog", "datafusion-common", "datafusion-common-runtime", "datafusion-execution", @@ -1480,13 +1649,16 @@ dependencies = [ "datafusion-physical-expr", "datafusion-physical-expr-common", "datafusion-physical-plan", + "datafusion-session", "flate2", "futures", "glob", - "itertools 0.14.0", + "itertools", "log", "object_store", + "parquet", "rand 0.8.5", + "tempfile", "tokio", "tokio-util", "url", @@ -1494,19 +1666,100 @@ dependencies = [ "zstd", ] +[[package]] +name = "datafusion-datasource-csv" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bb2253952dc32296ed5b84077cb2e0257fea4be6373e1c376426e17ead4ef6" +dependencies = [ + "arrow 55.0.0", + "async-trait", + "bytes", + "datafusion-catalog", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "object_store", + "regex", + "tokio", +] + +[[package]] +name = "datafusion-datasource-json" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b8c7f47a5d2fe03bfa521ec9bafdb8a5c82de8377f60967c3663f00c8790352" +dependencies = [ + "arrow 55.0.0", + "async-trait", + "bytes", + "datafusion-catalog", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "object_store", + "serde_json", + "tokio", +] + +[[package]] +name = "datafusion-datasource-parquet" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27d15868ea39ed2dc266728b554f6304acd473de2142281ecfa1294bb7415923" +dependencies = [ + "arrow 55.0.0", + "async-trait", + "bytes", + "datafusion-catalog", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions-aggregate", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-optimizer", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "itertools", + "log", + "object_store", + "parking_lot", + "parquet", + "rand 0.8.5", + "tokio", +] + [[package]] name = "datafusion-doc" -version = "46.0.1" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7db7a0239fd060f359dc56c6e7db726abaa92babaed2fb2e91c3a8b2fff8b256" +checksum = "a91f8c2c5788ef32f48ff56c68e5b545527b744822a284373ac79bba1ba47292" [[package]] name = "datafusion-execution" -version = "46.0.1" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0938f9e5b6bc5782be4111cdfb70c02b7b5451bf34fd57e4de062a7f7c4e31f1" +checksum = "06f004d100f49a3658c9da6fb0c3a9b760062d96cd4ad82ccc3b7b69a9fb2f84" dependencies = [ - "arrow", + "arrow 55.0.0", "dashmap", "datafusion-common", "datafusion-expr", @@ -1521,11 +1774,11 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "46.0.1" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b36c28b00b00019a8695ad7f1a53ee1673487b90322ecbd604e2cf32894eb14f" +checksum = "7a4e4ce3802609be38eeb607ee72f6fe86c3091460de9dbfae9e18db423b3964" dependencies = [ - "arrow", + "arrow 55.0.0", "chrono", "datafusion-common", "datafusion-doc", @@ -1542,22 +1795,22 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -version = "46.0.1" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18f0a851a436c5a2139189eb4617a54e6a9ccb9edc96c4b3c83b3bb7c58b950e" +checksum = "422ac9cf3b22bbbae8cdf8ceb33039107fde1b5492693168f13bd566b1bcc839" dependencies = [ - "arrow", + "arrow 55.0.0", "datafusion-common", "indexmap 2.9.0", - "itertools 0.14.0", + "itertools", "paste", ] [[package]] name = "datafusion-federation" -version = "0.3.7" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db96303adde1dff393be5f76377ce81f78d91aeacf8ed6836d66ace0ec8944a0" +checksum = "f82216f64b28516d822e2f3d317159270ee17c2923d7c4600d17bb08745a177e" dependencies = [ "arrow-json", "async-stream", @@ -1568,12 +1821,13 @@ dependencies = [ [[package]] name = "datafusion-ffi" -version = "46.0.1" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d740dd9f32a4f4ed1b907e6934201bb059efe6c877532512c661771d973c7b21" +checksum = "5cf3fe9ab492c56daeb7beed526690d33622d388b8870472e0b7b7f55490338c" dependencies = [ "abi_stable", - "arrow", + "arrow 55.0.0", + "arrow-schema 55.0.0", "async-ffi", "async-trait", "datafusion", @@ -1587,12 +1841,12 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "46.0.1" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3196e37d7b65469fb79fee4f05e5bb58a456831035f9a38aa5919aeb3298d40" +checksum = "2ddf0a0a2db5d2918349c978d42d80926c6aa2459cd8a3c533a84ec4bb63479e" dependencies = [ - "arrow", - "arrow-buffer", + "arrow 55.0.0", + "arrow-buffer 55.0.0", "base64", "blake2", "blake3", @@ -1604,7 +1858,7 @@ dependencies = [ "datafusion-expr-common", "datafusion-macros", "hex", - "itertools 0.14.0", + "itertools", "log", "md-5", "rand 0.8.5", @@ -1616,12 +1870,12 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "46.0.1" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adfc2d074d5ee4d9354fdcc9283d5b2b9037849237ddecb8942a29144b77ca05" +checksum = "408a05dafdc70d05a38a29005b8b15e21b0238734dab1e98483fcb58038c5aba" dependencies = [ "ahash 0.8.11", - "arrow", + "arrow 55.0.0", "datafusion-common", "datafusion-doc", "datafusion-execution", @@ -1637,12 +1891,12 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" -version = "46.0.1" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cbceba0f98d921309a9121b702bcd49289d383684cccabf9a92cda1602f3bbb" +checksum = "756d21da2dd6c9bef97af1504970ff56cbf35d03fbd4ffd62827f02f4d2279d4" dependencies = [ "ahash 0.8.11", - "arrow", + "arrow 55.0.0", "datafusion-common", "datafusion-expr-common", "datafusion-physical-expr-common", @@ -1650,12 +1904,12 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "46.0.1" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "170e27ce4baa27113ddf5f77f1a7ec484b0dbeda0c7abbd4bad3fc609c8ab71a" +checksum = "8d8d50f6334b378930d992d801a10ac5b3e93b846b39e4a05085742572844537" dependencies = [ - "arrow", - "arrow-ord", + "arrow 55.0.0", + "arrow-ord 55.0.0", "datafusion-common", "datafusion-doc", "datafusion-execution", @@ -1664,18 +1918,18 @@ dependencies = [ "datafusion-functions-aggregate", "datafusion-macros", "datafusion-physical-expr-common", - "itertools 0.14.0", + "itertools", "log", "paste", ] [[package]] name = "datafusion-functions-table" -version = "46.0.1" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d3a06a7f0817ded87b026a437e7e51de7f59d48173b0a4e803aa896a7bd6bb5" +checksum = "cc9a97220736c8fff1446e936be90d57216c06f28969f9ffd3b72ac93c958c8a" dependencies = [ - "arrow", + "arrow 55.0.0", "async-trait", "datafusion-catalog", "datafusion-common", @@ -1687,9 +1941,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "46.0.1" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6c608b66496a1e05e3d196131eb9bebea579eed1f59e88d962baf3dda853bc6" +checksum = "cefc2d77646e1aadd1d6a9c40088937aedec04e68c5f0465939912e1291f8193" dependencies = [ "datafusion-common", "datafusion-doc", @@ -1704,9 +1958,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "46.0.1" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da2f9d83348957b4ad0cd87b5cb9445f2651863a36592fe5484d43b49a5f8d82" +checksum = "dd4aff082c42fa6da99ce0698c85addd5252928c908eb087ca3cfa64ff16b313" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -1714,9 +1968,9 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "46.0.1" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4800e1ff7ecf8f310887e9b54c9c444b8e215ccbc7b21c2f244cfae373b1ece7" +checksum = "df6f88d7ee27daf8b108ba910f9015176b36fbc72902b1ca5c2a5f1d1717e1a1" dependencies = [ "datafusion-expr", "quote", @@ -1725,17 +1979,17 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "46.0.1" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "971c51c54cd309001376fae752fb15a6b41750b6d1552345c46afbfb6458801b" +checksum = "084d9f979c4b155346d3c34b18f4256e6904ded508e9554d90fed416415c3515" dependencies = [ - "arrow", + "arrow 55.0.0", "chrono", "datafusion-common", "datafusion-expr", "datafusion-physical-expr", "indexmap 2.9.0", - "itertools 0.14.0", + "itertools", "log", "recursive", "regex", @@ -1744,12 +1998,12 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "46.0.1" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1447c2c6bc8674a16be4786b4abf528c302803fafa186aa6275692570e64d85" +checksum = "64c536062b0076f4e30084065d805f389f9fe38af0ca75bcbac86bc5e9fbab65" dependencies = [ "ahash 0.8.11", - "arrow", + "arrow 55.0.0", "datafusion-common", "datafusion-expr", "datafusion-expr-common", @@ -1758,7 +2012,7 @@ dependencies = [ "half", "hashbrown 0.14.5", "indexmap 2.9.0", - "itertools 0.14.0", + "itertools", "log", "paste", "petgraph", @@ -1766,25 +2020,25 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "46.0.1" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69f8c25dcd069073a75b3d2840a79d0f81e64bdd2c05f2d3d18939afb36a7dcb" +checksum = "f8a92b53b3193fac1916a1c5b8e3f4347c526f6822e56b71faa5fb372327a863" dependencies = [ "ahash 0.8.11", - "arrow", + "arrow 55.0.0", "datafusion-common", "datafusion-expr-common", "hashbrown 0.14.5", - "itertools 0.14.0", + "itertools", ] [[package]] name = "datafusion-physical-optimizer" -version = "46.0.1" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68da5266b5b9847c11d1b3404ee96b1d423814e1973e1ad3789131e5ec912763" +checksum = "6fa0a5ac94c7cf3da97bedabd69d6bbca12aef84b9b37e6e9e8c25286511b5e2" dependencies = [ - "arrow", + "arrow 55.0.0", "datafusion-common", "datafusion-execution", "datafusion-expr", @@ -1792,21 +2046,21 @@ dependencies = [ "datafusion-physical-expr", "datafusion-physical-expr-common", "datafusion-physical-plan", - "itertools 0.14.0", + "itertools", "log", "recursive", ] [[package]] name = "datafusion-physical-plan" -version = "46.0.1" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88cc160df00e413e370b3b259c8ea7bfbebc134d32de16325950e9e923846b7f" +checksum = "690c615db468c2e5fe5085b232d8b1c088299a6c63d87fd960a354a71f7acb55" dependencies = [ "ahash 0.8.11", - "arrow", - "arrow-ord", - "arrow-schema", + "arrow 55.0.0", + "arrow-ord 55.0.0", + "arrow-schema 55.0.0", "async-trait", "chrono", "datafusion-common", @@ -1820,7 +2074,7 @@ dependencies = [ "half", "hashbrown 0.14.5", "indexmap 2.9.0", - "itertools 0.14.0", + "itertools", "log", "parking_lot", "pin-project-lite", @@ -1829,11 +2083,11 @@ dependencies = [ [[package]] name = "datafusion-proto" -version = "46.0.1" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f6ef4c6eb52370cb48639e25e2331a415aac0b2b0a0a472b36e26603bdf184f" +checksum = "a4a1afb2bdb05de7ff65be6883ebfd4ec027bd9f1f21c46aa3afd01927160a83" dependencies = [ - "arrow", + "arrow 55.0.0", "chrono", "datafusion", "datafusion-common", @@ -1845,22 +2099,46 @@ dependencies = [ [[package]] name = "datafusion-proto-common" -version = "46.0.1" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5faf4a9bbb0d0a305fea8a6db21ba863286b53e53a212e687d2774028dd6f03f" +checksum = "35b7a5876ebd6b564fb9a1fd2c3a2a9686b787071a256b47e4708f0916f9e46f" dependencies = [ - "arrow", + "arrow 55.0.0", "datafusion-common", "prost", ] +[[package]] +name = "datafusion-session" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad229a134c7406c057ece00c8743c0c34b97f4e72f78b475fe17b66c5e14fa4f" +dependencies = [ + "arrow 55.0.0", + "async-trait", + "dashmap", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-plan", + "datafusion-sql", + "futures", + "itertools", + "log", + "object_store", + "parking_lot", + "tokio", +] + [[package]] name = "datafusion-sql" -version = "46.0.1" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "325a212b67b677c0eb91447bf9a11b630f9fc4f62d8e5d145bf859f5a6b29e64" +checksum = "64f6ab28b72b664c21a27b22a2ff815fd390ed224c26e89a93b5a8154a4e8607" dependencies = [ - "arrow", + "arrow 55.0.0", "bigdecimal", "datafusion-common", "datafusion-expr", @@ -1876,12 +2154,12 @@ name = "datafusion-table-providers" version = "0.4.0" dependencies = [ "anyhow", - "arrow", - "arrow-array", + "arrow 55.0.0", + "arrow-array 55.0.0", "arrow-flight", "arrow-json", "arrow-odbc", - "arrow-schema", + "arrow-schema 55.0.0", "async-stream", "async-trait", "base64", @@ -1907,7 +2185,7 @@ dependencies = [ "geo-types", "geozero", "insta", - "itertools 0.14.0", + "itertools", "libduckdb-sys", "mysql_async", "native-tls", @@ -1947,7 +2225,7 @@ dependencies = [ name = "datafusion-table-providers-python" version = "0.4.0" dependencies = [ - "arrow", + "arrow 55.0.0", "arrow-flight", "datafusion", "datafusion-ffi", @@ -2130,11 +2408,11 @@ checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" [[package]] name = "flatbuffers" -version = "24.12.23" +version = "25.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f1baf0dbf96932ec9a3038d57900329c015b0bfb7b63d904f3bc27e2b02a096" +checksum = "1045398c1bfd89168b5fd3f1fc11f6e70b34f6f66300c87d44d3de849463abf1" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.9.0", "rustc_version", ] @@ -2145,6 +2423,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ced92e76e966ca2fd84c8f7aa01a4aea65b0eb6648d72f7c8f3e2764a67fece" dependencies = [ "crc32fast", + "libz-rs-sys", "libz-sys", "miniz_oxide", ] @@ -2964,15 +3243,6 @@ version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" -[[package]] -name = "itertools" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" -dependencies = [ - "either", -] - [[package]] name = "itertools" version = "0.14.0" @@ -3179,6 +3449,15 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "libz-rs-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6489ca9bd760fe9642d7644e827b0c9add07df89857b0416ee15c1cc1a3b8c5a" +dependencies = [ + "zlib-rs", +] + [[package]] name = "libz-sys" version = "1.1.22" @@ -3790,19 +4069,20 @@ dependencies = [ [[package]] name = "object_store" -version = "0.11.2" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cfccb68961a56facde1163f9319e0d15743352344e7808a11795fb99698dcaf" +checksum = "e9ce831b09395f933addbc56d894d889e4b226eba304d4e7adbab591e26daf1e" dependencies = [ "async-trait", "bytes", "chrono", "futures", + "http", "humantime", - "itertools 0.13.0", + "itertools", "parking_lot", "percent-encoding", - "snafu", + "thiserror 2.0.12", "tokio", "tracing", "url", @@ -3928,18 +4208,18 @@ dependencies = [ [[package]] name = "parquet" -version = "54.3.1" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfb15796ac6f56b429fd99e33ba133783ad75b27c36b4b5ce06f1f82cc97754e" +checksum = "cd31a8290ac5b19f09ad77ee7a1e6a541f1be7674ad410547d5f1eef6eef4a9c" dependencies = [ "ahash 0.8.11", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", + "arrow-array 55.0.0", + "arrow-buffer 55.0.0", + "arrow-cast 55.0.0", + "arrow-data 55.0.0", "arrow-ipc", - "arrow-schema", - "arrow-select", + "arrow-schema 55.0.0", + "arrow-select 55.0.0", "base64", "brotli", "bytes", @@ -3958,7 +4238,7 @@ dependencies = [ "snap", "thrift", "tokio", - "twox-hash 1.6.3", + "twox-hash 2.1.0", "zstd", ] @@ -4218,7 +4498,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools", "proc-macro2", "quote", "syn 2.0.100", @@ -4264,9 +4544,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.23.5" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7778bffd85cf38175ac1f545509665d0b9b92a198ca7941f131f85f7a4f9a872" +checksum = "e5203598f366b11a02b13aa20cab591229ff0a89fd121a308a5df751d5fc9219" dependencies = [ "cfg-if", "indoc", @@ -4282,9 +4562,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.23.5" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94f6cbe86ef3bf18998d9df6e0f3fc1050a8c5efa409bf712e661a4366e010fb" +checksum = "99636d423fa2ca130fa5acde3059308006d46f98caac629418e53f7ebb1e9999" dependencies = [ "once_cell", "target-lexicon", @@ -4292,9 +4572,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.23.5" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9f1b4c431c0bb1c8fb0a338709859eed0d030ff6daa34368d3b152a63dfdd8d" +checksum = "78f9cf92ba9c409279bc3305b5409d90db2d2c22392d443a87df3a1adad59e33" dependencies = [ "libc", "pyo3-build-config", @@ -4302,9 +4582,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.23.5" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbc2201328f63c4710f68abdf653c89d8dbc2858b88c5d88b0ff38a75288a9da" +checksum = "0b999cb1a6ce21f9a6b147dcf1be9ffedf02e0043aec74dc390f3007047cecd9" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -4314,9 +4594,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.23.5" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fca6726ad0f3da9c9de093d6f116a93c1a38e417ed73bf138472cf4064f72028" +checksum = "822ece1c7e1012745607d5cf0bcb2874769f0f7cb34c4cde03b9358eb9ef911a" dependencies = [ "heck 0.5.0", "proc-macro2", @@ -5121,7 +5401,7 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9293086270aff32097c27af628107e8f33358de8993ebd9769fd3f72ac2250d" dependencies = [ - "arrow", + "arrow 54.3.1", "cast", "fallible-iterator 0.3.0", "fallible-streaming-iterator", @@ -5138,9 +5418,9 @@ dependencies = [ [[package]] name = "sqlparser" -version = "0.54.0" +version = "0.55.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c66e3b7374ad4a6af849b08b3e7a6eda0edbd82f0fd59b57e22671bf16979899" +checksum = "c4521174166bac1ff04fe16ef4524c70144cd29682a45978978ca3d7f4e0be11" dependencies = [ "log", "recursive", @@ -5320,9 +5600,9 @@ dependencies = [ [[package]] name = "target-lexicon" -version = "0.12.16" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" +checksum = "e502f78cdbb8ba4718f566c418c52bc729126ffd16baee5baa718cf25dd5a69a" [[package]] name = "tempfile" @@ -6769,6 +7049,12 @@ dependencies = [ "syn 2.0.100", ] +[[package]] +name = "zlib-rs" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "868b928d7949e09af2f6086dfc1e01936064cc7a819253bce650d4e2a2d63ba8" + [[package]] name = "zstd" version = "0.13.3" diff --git a/Cargo.toml b/Cargo.toml index 3f5330d5..dc9a2d8d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,21 +31,21 @@ license = "Apache-2.0" description = "Extend the capabilities of DataFusion to support additional data sources via implementations of the `TableProvider` trait." [workspace.dependencies] -arrow = "54.2.1" -arrow-array = { version = "54.2.1" } -arrow-flight = { version = "54.2.1", features = [ +arrow = "55.0.0" +arrow-array = { version = "55.0.0" } +arrow-flight = { version = "55.0.0", features = [ "flight-sql-experimental", "tls", ] } -arrow-schema = { version = "54.2.1", features = ["serde"] } -arrow-json = "54.2.1" -arrow-odbc = { version = "16" } -datafusion = { version = "46", default-features = false } -datafusion-expr = { version = "46" } -datafusion-federation = { version = "=0.3.7" } -datafusion-ffi = { version = "46" } -datafusion-proto = { version = "46" } -datafusion-physical-expr = { version = "46" } -datafusion-physical-plan = { version = "46" } +arrow-schema = { version = "55.0.0", features = ["serde"] } +arrow-json = "55.0.0" +arrow-odbc = { version = "16.0.1" } +datafusion = { version = "47", default-features = false } +datafusion-expr = { version = "47" } +datafusion-federation = { version = "=0.4.2" } +datafusion-ffi = { version = "47" } +datafusion-proto = { version = "47" } +datafusion-physical-expr = { version = "47" } +datafusion-physical-plan = { version = "47" } datafusion-table-providers = { path = "core" } duckdb = { version = "=1.2.1", package = "spiceai_duckdb_fork" } # Forked to add support for duckdb_scan_arrow, pending: https://github.com/duckdb/duckdb-rs/pull/488 diff --git a/core/src/duckdb/creator.rs b/core/src/duckdb/creator.rs index d028a029..37007fb8 100644 --- a/core/src/duckdb/creator.rs +++ b/core/src/duckdb/creator.rs @@ -731,13 +731,13 @@ pub(crate) mod tests { dbconnection::duckdbconn::DuckDbConnection, duckdbpool::DuckDbConnectionPool, }, }; - use datafusion::arrow::array::RecordBatch; + use datafusion::{arrow::array::RecordBatch, datasource::sink::DataSink}; use datafusion::{ common::SchemaExt, execution::{SendableRecordBatchStream, TaskContext}, logical_expr::dml::InsertOp, parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder, - physical_plan::{insert::DataSink, memory::MemoryStream}, + physical_plan::memory::MemoryStream, }; use tracing::subscriber::DefaultGuard; use tracing_subscriber::EnvFilter; diff --git a/core/src/duckdb/federation.rs b/core/src/duckdb/federation.rs index ce86092a..8e31298a 100644 --- a/core/src/duckdb/federation.rs +++ b/core/src/duckdb/federation.rs @@ -2,7 +2,9 @@ use crate::sql::db_connection_pool::dbconnection::{get_schema, Error as DbError} use crate::sql::sql_provider_datafusion::{get_stream, to_execution_error}; use datafusion::arrow::datatypes::SchemaRef; use datafusion::sql::unparser::dialect::Dialect; -use datafusion_federation::sql::{SQLExecutor, SQLFederationProvider, SQLTableSource}; +use datafusion_federation::sql::{ + RemoteTableRef, SQLExecutor, SQLFederationProvider, SQLTableSource, +}; use datafusion_federation::{FederatedTableProviderAdaptor, FederatedTableSource}; use futures::TryStreamExt; use snafu::ResultExt; @@ -23,14 +25,14 @@ impl DuckDBTable { fn create_federated_table_source( self: Arc, ) -> DataFusionResult> { - let table_name = self.base_table.table_reference.to_quoted_string(); + let table_reference = self.base_table.table_reference.clone(); let schema = Arc::clone(&Arc::clone(&self).base_table.schema()); let fed_provider = Arc::new(SQLFederationProvider::new(self)); Ok(Arc::new(SQLTableSource::new_with_schema( fed_provider, - table_name, + RemoteTableRef::from(table_reference), schema, - )?)) + ))) } pub fn create_federated_table_provider( diff --git a/core/src/duckdb/write.rs b/core/src/duckdb/write.rs index cd71b52e..4ff572f9 100644 --- a/core/src/duckdb/write.rs +++ b/core/src/duckdb/write.rs @@ -15,17 +15,14 @@ use arrow_schema::ArrowError; use async_trait::async_trait; use datafusion::catalog::Session; use datafusion::common::{Constraints, SchemaExt}; +use datafusion::datasource::sink::{DataSink, DataSinkExec}; use datafusion::logical_expr::dml::InsertOp; use datafusion::{ datasource::{TableProvider, TableType}, error::DataFusionError, execution::{SendableRecordBatchStream, TaskContext}, logical_expr::Expr, - physical_plan::{ - insert::{DataSink, DataSinkExec}, - metrics::MetricsSet, - DisplayAs, DisplayFormatType, ExecutionPlan, - }, + physical_plan::{metrics::MetricsSet, DisplayAs, DisplayFormatType, ExecutionPlan}, }; use duckdb::Transaction; use futures::StreamExt; diff --git a/core/src/flight/exec.rs b/core/src/flight/exec.rs index dddd6c76..e1a60d27 100644 --- a/core/src/flight/exec.rs +++ b/core/src/flight/exec.rs @@ -268,7 +268,7 @@ fn find_matching_column( impl DisplayAs for FlightExec { fn fmt_as(&self, t: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result { match t { - DisplayFormatType::Default => write!( + DisplayFormatType::Default | DisplayFormatType::TreeRender => write!( f, "FlightExec: origin={}, streams={}", self.config.origin, diff --git a/core/src/mysql/federation.rs b/core/src/mysql/federation.rs index 5db723c1..83840786 100644 --- a/core/src/mysql/federation.rs +++ b/core/src/mysql/federation.rs @@ -4,7 +4,9 @@ use arrow::datatypes::SchemaRef; use async_trait::async_trait; use datafusion::sql::sqlparser::ast::{self, VisitMut}; use datafusion::sql::unparser::dialect::Dialect; -use datafusion_federation::sql::{AstAnalyzer, SQLExecutor, SQLFederationProvider, SQLTableSource}; +use datafusion_federation::sql::{ + AstAnalyzer, RemoteTableRef, SQLExecutor, SQLFederationProvider, SQLTableSource, +}; use datafusion_federation::{FederatedTableProviderAdaptor, FederatedTableSource}; use futures::TryStreamExt; use snafu::ResultExt; @@ -24,14 +26,14 @@ impl MySQLTable { fn create_federated_table_source( self: Arc, ) -> DataFusionResult> { - let table_name = self.base_table.table_reference.to_quoted_string(); + let table_reference = self.base_table.table_reference.clone(); let schema = Arc::clone(&Arc::clone(&self).base_table.schema()); let fed_provider = Arc::new(SQLFederationProvider::new(self)); Ok(Arc::new(SQLTableSource::new_with_schema( fed_provider, - table_name, + RemoteTableRef::from(table_reference), schema, - )?)) + ))) } pub fn create_federated_table_provider( diff --git a/core/src/mysql/mysql_window.rs b/core/src/mysql/mysql_window.rs index 0af37c15..7d5c698e 100644 --- a/core/src/mysql/mysql_window.rs +++ b/core/src/mysql/mysql_window.rs @@ -1,4 +1,6 @@ -use datafusion::sql::sqlparser::ast::{Expr, Function, Ident, VisitorMut, WindowType}; +use datafusion::sql::sqlparser::ast::{ + Expr, Function, Ident, ObjectNamePart, VisitorMut, WindowType, +}; use std::ops::ControlFlow; #[derive(PartialEq, Eq)] @@ -41,7 +43,7 @@ impl VisitorMut for MySQLWindowVisitor { fn pre_visit_expr(&mut self, expr: &mut Expr) -> ControlFlow { if let Expr::Function(func) = expr { - if let Some(Ident { value, .. }) = func.name.0.first() { + if let Some(ObjectNamePart::Identifier(Ident { value, .. })) = func.name.0.first() { // match for some scalars that support window functions // all of them need to remove nulls first/last, but only rank removes the frame clause if let Some(func_type) = FunctionType::from_str(value) { @@ -58,7 +60,7 @@ impl MySQLWindowVisitor { pub fn remove_nulls_first_last(func: &mut Function) { if let Some(WindowType::WindowSpec(spec)) = func.over.as_mut() { for order_by in &mut spec.order_by { - order_by.nulls_first = None; // nulls first/last are not supported in MySQL + order_by.options.nulls_first = None; // nulls first/last are not supported in MySQL } } } @@ -83,11 +85,11 @@ mod test { #[test] fn test_remove_frame_clause() { let mut func = Function { - name: ObjectName(vec![Ident { + name: ObjectName(vec![ObjectNamePart::Identifier(Ident { value: "RANK".to_string(), quote_style: None, span: Span::empty(), - }]), + })]), args: ast::FunctionArguments::None, over: Some(WindowType::WindowSpec(ast::WindowSpec { window_name: None, @@ -96,8 +98,10 @@ mod test { expr: sqlparser::ast::Expr::Wildcard(AttachedToken(TokenWithSpan::wrap( Token::Char('*'), ))), - asc: None, - nulls_first: Some(true), + options: sqlparser::ast::OrderByOptions { + asc: None, + nulls_first: Some(true), + }, with_fill: None, }], window_frame: Some(WindowFrame { @@ -120,8 +124,10 @@ mod test { expr: sqlparser::ast::Expr::Wildcard(AttachedToken(TokenWithSpan::wrap( Token::Char('*'), ))), - asc: None, - nulls_first: Some(true), + options: sqlparser::ast::OrderByOptions { + asc: None, + nulls_first: Some(true), + }, with_fill: None, }], window_frame: None, @@ -135,11 +141,11 @@ mod test { #[test] fn test_remove_nulls_first_last() { let mut func = Function { - name: ObjectName(vec![Ident { + name: ObjectName(vec![ObjectNamePart::Identifier(Ident { value: "RANK".to_string(), quote_style: None, span: Span::empty(), - }]), + })]), args: sqlparser::ast::FunctionArguments::None, over: Some(WindowType::WindowSpec(sqlparser::ast::WindowSpec { window_name: None, @@ -148,8 +154,10 @@ mod test { expr: sqlparser::ast::Expr::Wildcard(AttachedToken(TokenWithSpan::wrap( Token::Char('*'), ))), - asc: None, - nulls_first: Some(true), + options: sqlparser::ast::OrderByOptions { + asc: None, + nulls_first: Some(true), + }, with_fill: None, }], window_frame: Some(WindowFrame { @@ -172,8 +180,10 @@ mod test { expr: sqlparser::ast::Expr::Wildcard(AttachedToken(TokenWithSpan::wrap( Token::Char('*'), ))), - asc: None, - nulls_first: None, + options: sqlparser::ast::OrderByOptions { + asc: None, + nulls_first: None, + }, with_fill: None, }], window_frame: Some(WindowFrame { diff --git a/core/src/mysql/write.rs b/core/src/mysql/write.rs index 484b4abc..602a7f9b 100644 --- a/core/src/mysql/write.rs +++ b/core/src/mysql/write.rs @@ -4,16 +4,13 @@ use crate::util::retriable_error::check_and_mark_retriable_error; use crate::util::{constraints, to_datafusion_error}; use async_trait::async_trait; use datafusion::arrow::datatypes::SchemaRef; +use datafusion::datasource::sink::{DataSink, DataSinkExec}; use datafusion::{ catalog::Session, datasource::{TableProvider, TableType}, execution::{SendableRecordBatchStream, TaskContext}, logical_expr::{dml::InsertOp, Expr}, - physical_plan::{ - insert::{DataSink, DataSinkExec}, - metrics::MetricsSet, - DisplayAs, DisplayFormatType, ExecutionPlan, - }, + physical_plan::{metrics::MetricsSet, DisplayAs, DisplayFormatType, ExecutionPlan}, }; use futures::StreamExt; use mysql_async::TxOpts; diff --git a/core/src/postgres/write.rs b/core/src/postgres/write.rs index 9e3b5517..91c237f4 100644 --- a/core/src/postgres/write.rs +++ b/core/src/postgres/write.rs @@ -6,14 +6,13 @@ use async_trait::async_trait; use datafusion::{ catalog::Session, common::{Constraints, SchemaExt}, - datasource::{TableProvider, TableType}, + datasource::{ + sink::{DataSink, DataSinkExec}, + TableProvider, TableType, + }, execution::{SendableRecordBatchStream, TaskContext}, logical_expr::{dml::InsertOp, Expr}, - physical_plan::{ - insert::{DataSink, DataSinkExec}, - metrics::MetricsSet, - DisplayAs, DisplayFormatType, ExecutionPlan, - }, + physical_plan::{metrics::MetricsSet, DisplayAs, DisplayFormatType, ExecutionPlan}, }; use futures::StreamExt; use snafu::prelude::*; diff --git a/core/src/sql/arrow_sql_gen/postgres.rs b/core/src/sql/arrow_sql_gen/postgres.rs index 1ed47529..66d7f14a 100644 --- a/core/src/sql/arrow_sql_gen/postgres.rs +++ b/core/src/sql/arrow_sql_gen/postgres.rs @@ -356,8 +356,7 @@ pub fn rows_to_arrow(rows: &[Row], projected_schema: &Option) -> Resu let Some(builder) = builder else { return NoBuilderForIndexSnafu { index: i }.fail(); }; - let Some(builder) = builder.as_any_mut().downcast_mut::() - else { + let Some(builder) = builder.as_any_mut().downcast_mut::() else { return FailedToDowncastBuilderSnafu { postgres_type: format!("{postgres_type}"), } diff --git a/core/src/sql/db_connection_pool/dbconnection/duckdbconn.rs b/core/src/sql/db_connection_pool/dbconnection/duckdbconn.rs index 7dd5dcc9..0b9d8ff5 100644 --- a/core/src/sql/db_connection_pool/dbconnection/duckdbconn.rs +++ b/core/src/sql/db_connection_pool/dbconnection/duckdbconn.rs @@ -1,6 +1,6 @@ use std::any::Any; -use std::sync::Arc; use std::collections::HashSet; +use std::sync::Arc; use arrow::array::RecordBatch; use arrow_schema::{DataType, Field}; diff --git a/core/src/sql/sql_provider_datafusion/federation.rs b/core/src/sql/sql_provider_datafusion/federation.rs index c8ebc0fc..484865c7 100644 --- a/core/src/sql/sql_provider_datafusion/federation.rs +++ b/core/src/sql/sql_provider_datafusion/federation.rs @@ -1,6 +1,8 @@ use crate::sql::db_connection_pool::{dbconnection::get_schema, JoinPushDown}; use async_trait::async_trait; -use datafusion_federation::sql::{SQLExecutor, SQLFederationProvider, SQLTableSource}; +use datafusion_federation::sql::{ + RemoteTableRef, SQLExecutor, SQLFederationProvider, SQLTableSource, +}; use datafusion_federation::{FederatedTableProviderAdaptor, FederatedTableSource}; use futures::TryStreamExt; use snafu::prelude::*; @@ -20,14 +22,14 @@ impl SqlTable { fn create_federated_table_source( self: Arc, ) -> DataFusionResult> { - let table_name = self.table_reference.to_quoted_string(); + let table_reference = self.table_reference.clone(); let schema = Arc::clone(&self.schema); let fed_provider = Arc::new(SQLFederationProvider::new(self)); Ok(Arc::new(SQLTableSource::new_with_schema( fed_provider, - table_name, + RemoteTableRef::from(table_reference), schema, - )?)) + ))) } pub fn create_federated_table_provider( diff --git a/core/src/sqlite/federation.rs b/core/src/sqlite/federation.rs index 1897a68b..725b00aa 100644 --- a/core/src/sqlite/federation.rs +++ b/core/src/sqlite/federation.rs @@ -4,7 +4,9 @@ use async_trait::async_trait; use datafusion::arrow::datatypes::SchemaRef; use datafusion::sql::sqlparser::ast::{self, VisitMut}; use datafusion::sql::unparser::dialect::Dialect; -use datafusion_federation::sql::{AstAnalyzer, SQLExecutor, SQLFederationProvider, SQLTableSource}; +use datafusion_federation::sql::{ + AstAnalyzer, RemoteTableRef, SQLExecutor, SQLFederationProvider, SQLTableSource, +}; use datafusion_federation::{FederatedTableProviderAdaptor, FederatedTableSource}; use futures::TryStreamExt; use snafu::ResultExt; @@ -24,14 +26,14 @@ impl SQLiteTable { fn create_federated_table_source( self: Arc, ) -> DataFusionResult> { - let table_name = self.base_table.table_reference.to_quoted_string(); + let table_reference = self.base_table.table_reference.clone(); let schema = Arc::clone(&Arc::clone(&self).base_table.schema()); let fed_provider = Arc::new(SQLFederationProvider::new(self)); Ok(Arc::new(SQLTableSource::new_with_schema( fed_provider, - table_name, + RemoteTableRef::from(table_reference), schema, - )?)) + ))) } pub fn create_federated_table_provider( diff --git a/core/src/sqlite/sqlite_interval.rs b/core/src/sqlite/sqlite_interval.rs index 193e1501..98dddcf9 100644 --- a/core/src/sqlite/sqlite_interval.rs +++ b/core/src/sqlite/sqlite_interval.rs @@ -1,7 +1,7 @@ use datafusion::error::DataFusionError; use datafusion::sql::sqlparser::ast::{ self, BinaryOperator, Expr, FunctionArg, FunctionArgExpr, FunctionArgumentList, Ident, - VisitorMut, + ObjectNamePart, VisitorMut, }; use std::fmt::Display; use std::ops::ControlFlow; @@ -145,7 +145,10 @@ impl SQLiteIntervalVisitor { fn parse_interval(interval: &Expr) -> Result { if let Expr::Interval(interval_expr) = interval { - if let Expr::Value(ast::Value::SingleQuotedString(value)) = interval_expr.value.as_ref() + if let Expr::Value(ast::ValueWithSpan { + value: ast::Value::SingleQuotedString(value), + span: _, + }) = interval_expr.value.as_ref() { return SQLiteIntervalVisitor::parse_interval_string(value); } @@ -229,7 +232,9 @@ impl SQLiteIntervalVisitor { .collect(); let datetime_function = Expr::Function(ast::Function { - name: ast::ObjectName(vec![Ident::new(interval_date_type.to_string())]), + name: ast::ObjectName(vec![ObjectNamePart::Identifier(Ident::new( + interval_date_type.to_string(), + ))]), args: ast::FunctionArguments::List(FunctionArgumentList { duplicate_treatment: None, args: function_args, @@ -255,7 +260,7 @@ impl SQLiteIntervalVisitor { if value == 0 { None } else { - Some(FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::Value( + Some(FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::value( ast::Value::SingleQuotedString(format!("{value:+} {unit}")), )))) } @@ -275,7 +280,7 @@ impl SQLiteIntervalVisitor { String::new() }; - Some(FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::Value( + Some(FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::value( ast::Value::SingleQuotedString(format!("{value:+}{fraction_str} {unit}")), )))) } @@ -372,7 +377,7 @@ mod test { #[test] fn test_create_date_function() { - let target = Expr::Value(ast::Value::SingleQuotedString("1995-01-01".to_string())); + let target = Expr::value(ast::Value::SingleQuotedString("1995-01-01".to_string())); let interval = IntervalParts::new() .with_years(1) .with_months(2) @@ -386,20 +391,20 @@ mod test { let expected = Expr::Cast { expr: Box::new(Expr::Function(ast::Function { - name: ast::ObjectName(vec![Ident::new("date")]), + name: ast::ObjectName(vec![ObjectNamePart::Identifier(Ident::new("date"))]), args: ast::FunctionArguments::List(FunctionArgumentList { duplicate_treatment: None, args: vec![ - FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::Value( + FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::value( ast::Value::SingleQuotedString("1995-01-01".to_string()), ))), - FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::Value( + FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::value( ast::Value::SingleQuotedString("+1 years".to_string()), ))), - FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::Value( + FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::value( ast::Value::SingleQuotedString("+2 months".to_string()), ))), - FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::Value( + FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::value( ast::Value::SingleQuotedString("+3 days".to_string()), ))), ], @@ -422,7 +427,7 @@ mod test { #[test] fn test_create_datetime_function() { - let target = Expr::Value(ast::Value::SingleQuotedString("1995-01-01".to_string())); + let target = Expr::value(ast::Value::SingleQuotedString("1995-01-01".to_string())); let interval = IntervalParts::new() .with_years(0) .with_months(0) @@ -436,20 +441,20 @@ mod test { let expected = Expr::Cast { expr: Box::new(Expr::Function(ast::Function { - name: ast::ObjectName(vec![Ident::new("datetime")]), + name: ast::ObjectName(vec![ObjectNamePart::Identifier(Ident::new("datetime"))]), args: ast::FunctionArguments::List(FunctionArgumentList { duplicate_treatment: None, args: vec![ - FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::Value( + FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::value( ast::Value::SingleQuotedString("1995-01-01".to_string()), ))), - FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::Value( + FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::value( ast::Value::SingleQuotedString("+1 hours".to_string()), ))), - FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::Value( + FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::value( ast::Value::SingleQuotedString("+2 minutes".to_string()), ))), - FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::Value( + FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::value( ast::Value::SingleQuotedString("+3 seconds".to_string()), ))), ], diff --git a/core/src/sqlite/write.rs b/core/src/sqlite/write.rs index 8fda4fc8..cf4f9f99 100644 --- a/core/src/sqlite/write.rs +++ b/core/src/sqlite/write.rs @@ -2,6 +2,7 @@ use std::{any::Any, fmt, sync::Arc}; use async_trait::async_trait; use datafusion::arrow::{array::RecordBatch, datatypes::SchemaRef}; +use datafusion::datasource::sink::{DataSink, DataSinkExec}; use datafusion::{ catalog::Session, common::Constraints, @@ -9,11 +10,7 @@ use datafusion::{ error::DataFusionError, execution::{SendableRecordBatchStream, TaskContext}, logical_expr::{dml::InsertOp, Expr}, - physical_plan::{ - insert::{DataSink, DataSinkExec}, - metrics::MetricsSet, - DisplayAs, DisplayFormatType, ExecutionPlan, - }, + physical_plan::{metrics::MetricsSet, DisplayAs, DisplayFormatType, ExecutionPlan}, }; use futures::StreamExt; use snafu::prelude::*; diff --git a/core/src/util/test.rs b/core/src/util/test.rs index d4a2af29..970d5363 100644 --- a/core/src/util/test.rs +++ b/core/src/util/test.rs @@ -69,7 +69,9 @@ impl MockExec { impl DisplayAs for MockExec { fn fmt_as(&self, t: DisplayFormatType, f: &mut std::fmt::Formatter) -> std::fmt::Result { match t { - DisplayFormatType::Default | DisplayFormatType::Verbose => { + DisplayFormatType::Default + | DisplayFormatType::Verbose + | DisplayFormatType::TreeRender => { write!(f, "MockExec") } } diff --git a/core/tests/postgres/mod.rs b/core/tests/postgres/mod.rs index 03c87aa4..7ba1a2f5 100644 --- a/core/tests/postgres/mod.rs +++ b/core/tests/postgres/mod.rs @@ -289,11 +289,7 @@ async fn test_postgres_jsonb_type(port: usize) { ('{"nested": {"key": "value"}}'); "#; - let schema = Arc::new(Schema::new(vec![Field::new( - "data", - DataType::Utf8, - true, - )])); + let schema = Arc::new(Schema::new(vec![Field::new("data", DataType::Utf8, true)])); // Parse and re-serialize the JSON to ensure consistent ordering let expected_values = vec![ @@ -314,9 +310,7 @@ async fn test_postgres_jsonb_type(port: usize) { let expected_record = RecordBatch::try_new( Arc::clone(&schema), - vec![Arc::new(arrow::array::StringArray::from( - expected_values, - ))], + vec![Arc::new(arrow::array::StringArray::from(expected_values))], ) .expect("Failed to create arrow record batch"); diff --git a/python/Cargo.toml b/python/Cargo.toml index 06786c98..70c005a3 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -19,6 +19,6 @@ arrow-flight = {workspace = true} datafusion = { workspace = true, features = ["pyarrow"] } datafusion-ffi = { workspace = true } datafusion-table-providers = { workspace = true, features = ["sqlite", "duckdb", "odbc", "mysql", "postgres", "flight"] } -pyo3 = { version = "0.23" } +pyo3 = { version = "0.24.2" } tokio = { version = "1.44", features = ["macros", "rt", "rt-multi-thread", "sync"] } duckdb = { workspace = true }