diff --git a/rust/Cargo.lock b/rust/Cargo.lock index dfb8a2731..06244aafc 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -17,6 +17,31 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", +] + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "const-random", + "getrandom 0.3.1", + "once_cell", + "version_check", + "zerocopy 0.8.27", +] + [[package]] name = "aho-corasick" version = "1.1.3" @@ -26,6 +51,21 @@ dependencies = [ "memchr", ] +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" +dependencies = [ + "alloc-no-stdlib", +] + [[package]] name = "android-tzdata" version = "0.1.1" @@ -47,17 +87,163 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" +[[package]] +name = "anstream" +version = "0.6.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + [[package]] name = "anstyle" version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd" +[[package]] +name = "anstyle-parse" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e231f6134f61b71076a3eab506c379d4f36122f2af15a9ff04415ea4c3339e2" +dependencies = [ + "windows-sys 0.60.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e0633414522a32ffaac8ac6cc8f748e090c5717661fddeea04219e2344f5f2a" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.60.2", +] + [[package]] name = "anyhow" -version = "1.0.98" +version = "1.0.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" + +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] + +[[package]] +name = "arrow-array" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8548ca7c070d8db9ce7aa43f37393e4bfcf3f2d3681df278490772fd1673d08d" +dependencies = [ + "ahash", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "half", + "hashbrown 0.16.0", + "num", +] + +[[package]] +name = "arrow-buffer" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e003216336f70446457e280807a73899dd822feaf02087d31febca1363e2fccc" +dependencies = [ + "bytes", + "half", + "num", +] + +[[package]] +name = "arrow-cast" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "919418a0681298d3a77d1a315f625916cb5678ad0d74b9c60108eb15fd083023" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "atoi", + "base64 0.22.1", + "chrono", + "half", + "lexical-core", + "num", + "ryu", +] + +[[package]] +name = "arrow-data" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5c64fff1d142f833d78897a772f2e5b55b36cb3e6320376f0961ab0db7bd6d0" +dependencies = [ + "arrow-buffer", + "arrow-schema", + "half", + "num", +] + +[[package]] +name = "arrow-ipc" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d3594dcddccc7f20fd069bc8e9828ce37220372680ff638c5e00dea427d88f5" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "flatbuffers", +] + +[[package]] +name = "arrow-schema" +version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" +checksum = "b3aa9e59c611ebc291c28582077ef25c97f1975383f1479b12f3b9ffee2ffabe" + +[[package]] +name = "arrow-select" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c41dbbd1e97bfcaee4fcb30e29105fb2c75e4d82ae4de70b792a5d3f66b2e7a" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "num", +] [[package]] name = "async-stream" @@ -92,6 +278,15 @@ dependencies = [ "syn", ] +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -163,7 +358,7 @@ dependencies = [ "miniz_oxide", "object", "rustc-demangle", - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -184,6 +379,36 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "brotli" +version = "8.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bd8b9603c7aa97359dbd97ecf258968c95f3adddd6db2f7e7a5bef101c84560" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "874bb8112abecc98cbd6d81ea4fa7e94fb9449648c93cc89aa40c81c24d7de03" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + [[package]] name = "bumpalo" version = "3.17.0" @@ -208,6 +433,15 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3c8f83209414aacf0eeae3cf730b18d6981697fba62f200fcfb92b9f082acba" +[[package]] +name = "bzip2" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bea8dcd42434048e4f7a304411d9273a411f647446c1234a65ce0554923f4cff" +dependencies = [ + "libbz2-rs-sys", +] + [[package]] name = "cast" version = "0.3.0" @@ -220,6 +454,8 @@ version = "1.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be714c154be609ec7f5dad223a33bf1482fff90472de28f7362806e6d4832b8c" dependencies = [ + "jobserver", + "libc", "shlex", ] @@ -240,7 +476,7 @@ dependencies = [ "js-sys", "num-traits", "wasm-bindgen", - "windows-link", + "windows-link 0.1.0", ] [[package]] @@ -270,6 +506,16 @@ dependencies = [ "half", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + [[package]] name = "clap" version = "4.5.48" @@ -277,6 +523,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2134bb3ea021b78629caa971416385309e0131b351b25e01dc16fb54e1b5fae" dependencies = [ "clap_builder", + "clap_derive", ] [[package]] @@ -285,8 +532,32 @@ version = "4.5.48" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2ba64afa3c0a6df7fa517765e31314e983f51dda798ffba27b988194fb65dc9" dependencies = [ + "anstream", "anstyle", "clap_lex", + "strsim", + "terminal_size", +] + +[[package]] +name = "clap_complete" +version = "4.5.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75bf0b32ad2e152de789bb635ea4d3078f6b838ad7974143e99b99f45a04af4a" +dependencies = [ + "clap", +] + +[[package]] +name = "clap_derive" +version = "4.5.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbfd7eae0b0f1a6e63d4b13c9c478de77c2eb546fba158ad50b4203dc24b9f9c" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -295,6 +566,70 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" +[[package]] +name = "colorchoice" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" + +[[package]] +name = "console" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b430743a6eb14e9764d4260d4c0d8123087d504eeb9c48f2b2a5e810dd369df4" +dependencies = [ + "encode_unicode", + "libc", + "once_cell", + "unicode-width", + "windows-sys 0.61.2", +] + +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.15", + "once_cell", + "tiny-keccak", +] + +[[package]] +name = "constant_time_eq" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" + +[[package]] +name = "convert_case" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb402b8d4c85569410425650ce3eddc7d698ed96d39a73f941b08fb63082f1e7" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation" version = "0.10.0" @@ -311,6 +646,30 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crc" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9710d3b3739c2e349eb44fe848ad0b7c8cb1e42bd87ee49371df2f7acaf3e675" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + [[package]] name = "crc32fast" version = "1.4.2" @@ -381,6 +740,33 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +[[package]] +name = "crossterm" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8b9f2e4c67f833b660cdb0a3523065869fb35570177239812ed4c905aeff87b" +dependencies = [ + "bitflags", + "crossterm_winapi", + "derive_more", + "document-features", + "mio", + "parking_lot", + "rustix", + "signal-hook", + "signal-hook-mio", + "winapi", +] + +[[package]] +name = "crossterm_winapi" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" +dependencies = [ + "winapi", +] + [[package]] name = "crunchy" version = "0.2.4" @@ -388,59 +774,204 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" [[package]] -name = "dirs" -version = "6.0.0" +name = "crypto-common" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ - "dirs-sys", + "generic-array", + "typenum", ] [[package]] -name = "dirs-sys" -version = "0.5.0" +name = "csv" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" +checksum = "acdc4883a9c96732e4733212c01447ebd805833b7275a73ca3ee080fd77afdaf" dependencies = [ - "libc", - "option-ext", - "redox_users", - "windows-sys 0.59.0", + "csv-core", + "itoa", + "ryu", + "serde", ] [[package]] -name = "either" -version = "1.15.0" +name = "csv-core" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +checksum = "7d02f3b0da4c6504f86e9cd789d8dbafab48c2321be74e9987593de5a894d93d" +dependencies = [ + "memchr", +] [[package]] -name = "equivalent" -version = "1.0.2" +name = "deflate64" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +checksum = "26bf8fc351c5ed29b5c2f0cbbac1b209b74f60ecd62e675a998df72c49af5204" [[package]] -name = "errno" -version = "0.3.10" +name = "deranged" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" +checksum = "a41953f86f8a05768a6cda24def994fd2f424b04ec5c719cf89989779f199071" dependencies = [ - "libc", - "windows-sys 0.59.0", + "powerfmt", ] [[package]] -name = "fastrand" -version = "2.3.0" +name = "derive_arbitrary" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] [[package]] -name = "fixedbitset" -version = "0.4.2" +name = "derive_more" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" +checksum = "093242cf7570c207c83073cf82f79706fe7b8317e98620a47d5be7c3d8497678" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bda628edc44c4bb645fbe0f758797143e4e07926f7ebf4e9bdfbd3d2ce621df3" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", +] + +[[package]] +name = "dirs" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.61.2", +] + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "document-features" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95249b50c6c185bee49034bcb378a49dc2b5dff0be90ff6616d31d64febab05d" +dependencies = [ + "litrs", +] + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" +dependencies = [ + "libc", + "windows-sys 0.59.0", +] + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + +[[package]] +name = "flatbuffers" +version = "25.9.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09b6620799e7340ebd9968d2e0708eb82cf1971e9a16821e2091b6d6e475eed5" +dependencies = [ + "bitflags", + "rustc_version", +] + +[[package]] +name = "flate2" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc5a4e564e38c699f2880d3fda590bedc2e69f3f84cd48b457bd892ce61d0aa9" +dependencies = [ + "crc32fast", + "libz-rs-sys", + "miniz_oxide", +] [[package]] name = "fnv" @@ -448,6 +979,30 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + [[package]] name = "fuchsia-cprng" version = "0.1.1" @@ -543,6 +1098,16 @@ dependencies = [ "slab", ] +[[package]] +name = "generic-array" +version = "0.14.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dc8f7d2ded5f9209535e4b3fd4d39c002f30902ff5ce9f64e2c33d549576500" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.2.15" @@ -563,7 +1128,7 @@ dependencies = [ "cfg-if", "libc", "wasi 0.13.3+wasi-0.2.2", - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -599,6 +1164,7 @@ checksum = "459196ed295495a68f7d7fe1d84f6c4b7ff0e21fe3017b2f283c6fac3ad803c9" dependencies = [ "cfg-if", "crunchy", + "num-traits", ] [[package]] @@ -613,6 +1179,12 @@ version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" +[[package]] +name = "hashbrown" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d" + [[package]] name = "heck" version = "0.5.0" @@ -625,6 +1197,15 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + [[package]] name = "http" version = "1.2.0" @@ -694,6 +1275,22 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-rustls" +version = "0.27.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service", +] + [[package]] name = "hyper-timeout" version = "0.5.2" @@ -707,23 +1304,46 @@ dependencies = [ "tower-service", ] +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + [[package]] name = "hyper-util" -version = "0.1.10" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4" +checksum = "3c6995591a8f1380fcb4ba966a252a4b29188d51d2b89e3a252f5305be65aea8" dependencies = [ + "base64 0.22.1", "bytes", "futures-channel", + "futures-core", "futures-util", "http", "http-body", "hyper", + "ipnet", + "libc", + "percent-encoding", "pin-project-lite", - "socket2", + "socket2 0.6.0", + "system-configuration", "tokio", "tower-service", "tracing", + "windows-registry", ] [[package]] @@ -749,6 +1369,113 @@ dependencies = [ "cc", ] +[[package]] +name = "icu_collections" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "200072f5d0e3614556f94a9930d5dc3e0662a652823904c3a75dc3b0af7fee47" +dependencies = [ + "displaydoc", + "potential_utf", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cde2700ccaed3872079a65fb1a78f6c0a36c91570f28755dda67bc8f7d9f00a" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "436880e8e18df4d7bbc06d58432329d6458cc84531f7ac5f024e93deadb37979" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00210d6893afc98edb752b664b8890f0ef174c8adbb8d0be9710fa66fbbf72d3" + +[[package]] +name = "icu_properties" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "016c619c1eeb94efb86809b015c58f479963de65bdb6253345c1a1276f22e32b" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "potential_utf", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "298459143998310acd25ffe6810ed544932242d3f07083eee1084d83a71bd632" + +[[package]] +name = "icu_provider" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03c80da27b5f4187909049ee2d72f276f0d9f99a42c306bd0131ecfe04d8e5af" +dependencies = [ + "displaydoc", + "icu_locale_core", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + [[package]] name = "indexmap" version = "1.9.3" @@ -769,12 +1496,40 @@ dependencies = [ "hashbrown 0.15.2", ] +[[package]] +name = "indicatif" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70a646d946d06bedbbc4cac4c218acf4bbf2d87757a784857025f4d447e4e1cd" +dependencies = [ + "console", + "portable-atomic", + "unicode-width", + "unit-prefix", + "web-time", +] + [[package]] name = "indoc" version = "2.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f4c7245a08504955605670dbf141fceab975f15ca21570696aebe9d2e71576bd" +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", +] + +[[package]] +name = "integer-encoding" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" + [[package]] name = "inventory" version = "0.3.20" @@ -784,6 +1539,33 @@ dependencies = [ "rustversion", ] +[[package]] +name = "io-uring" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "046fa2d4d00aea763528b4950358d0ead425372445dc8ff86312b3c69ff7727b" +dependencies = [ + "bitflags", + "cfg-if", + "libc", +] + +[[package]] +name = "ipnet" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" + +[[package]] +name = "iri-string" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc5ebe9c3a1a7a5127f920a418f7585e9e758e911d0466ed004f393b0e380b2" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "is-terminal" version = "0.4.16" @@ -795,6 +1577,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + [[package]] name = "itertools" version = "0.10.5" @@ -819,6 +1607,15 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" +[[package]] +name = "jobserver" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" +dependencies = [ + "libc", +] + [[package]] name = "js-sys" version = "0.3.77" @@ -835,11 +1632,80 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "lexical-core" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d8d125a277f807e55a77304455eb7b1cb52f2b18c143b60e766c120bd64a594" +dependencies = [ + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", +] + +[[package]] +name = "lexical-parse-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56" +dependencies = [ + "lexical-parse-integer", + "lexical-util", +] + +[[package]] +name = "lexical-parse-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34" +dependencies = [ + "lexical-util", +] + +[[package]] +name = "lexical-util" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17" + +[[package]] +name = "lexical-write-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50c438c87c013188d415fbabbb1dceb44249ab81664efbd31b14ae55dabb6361" +dependencies = [ + "lexical-util", + "lexical-write-integer", +] + +[[package]] +name = "lexical-write-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "409851a618475d2d5796377cad353802345cba92c867d9fbcde9cf4eac4e14df" +dependencies = [ + "lexical-util", +] + +[[package]] +name = "libbz2-rs-sys" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c4a545a15244c7d945065b5d392b2d2d7f21526fba56ce51467b06ed445e8f7" + [[package]] name = "libc" -version = "0.2.170" +version = "0.2.176" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58f929b4d672ea937a23a1ab494143d968337a5f47e56d0815df1e0890ddf174" + +[[package]] +name = "libm" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "875b3680cb2f8f71bdcf9a30f38d48282f5d3c95cbf9b3fa57269bb5d5c06828" +checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" [[package]] name = "libredox" @@ -851,12 +1717,33 @@ dependencies = [ "libc", ] +[[package]] +name = "libz-rs-sys" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "840db8cf39d9ec4dd794376f38acc40d0fc65eec2a8f484f7fd375b84602becd" +dependencies = [ + "zlib-rs", +] + [[package]] name = "linux-raw-sys" version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fe7db12097d22ec582439daf8618b8fdd1a7bef6270e9af3b1ebcd30893cf413" +[[package]] +name = "litemap" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" + +[[package]] +name = "litrs" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5e54036fe321fd421e10d732f155734c4e4afd610dd556d9a82833ab3ee0bed" + [[package]] name = "lock_api" version = "0.4.12" @@ -873,6 +1760,25 @@ version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" +[[package]] +name = "lz4_flex" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08ab2867e3eeeca90e844d1940eab391c9dc5228783db2ed999acbc0a9ed375a" +dependencies = [ + "twox-hash", +] + +[[package]] +name = "lzma-rust2" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c60a23ffb90d527e23192f1246b14746e2f7f071cb84476dd879071696c18a4a" +dependencies = [ + "crc", + "sha2", +] + [[package]] name = "maplit" version = "1.0.2" @@ -932,6 +1838,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e3e04debbb59698c15bacbb6d93584a8c0ca9cc3213cb423d31f760d8843ce5" dependencies = [ "adler2", + "simd-adler32", ] [[package]] @@ -941,6 +1848,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" dependencies = [ "libc", + "log", "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys 0.52.0", ] @@ -951,6 +1859,23 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "defc4c55412d89136f966bbb339008b474350e5e6e78d2714439c386b3137a03" +[[package]] +name = "native-tls" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework 2.11.1", + "security-framework-sys", + "tempfile", +] + [[package]] name = "ndarray" version = "0.16.1" @@ -976,6 +1901,30 @@ dependencies = [ "winapi", ] +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-complex" version = "0.4.6" @@ -985,6 +1934,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "num-integer" version = "0.1.46" @@ -994,6 +1949,28 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -1001,6 +1978,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", + "libm", ] [[package]] @@ -1034,24 +2012,77 @@ version = "1.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "945462a4b81e43c4e3ba96bd7b49d834c6f61198356aa858733bc4acf3cbe62e" +[[package]] +name = "once_cell_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad" + [[package]] name = "oorandom" version = "11.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" +[[package]] +name = "openssl" +version = "0.10.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8505734d46c8ab1e19a1dce3aef597ad87dcb4c37e7188231769bd6bd51cebf8" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "openssl-probe" version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" +[[package]] +name = "openssl-sys" +version = "0.9.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90096e2e47630d78b7d1c20952dc621f957103f8bc2c8359ec81290d75238571" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "option-ext" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "ordered-float" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" +dependencies = [ + "num-traits", +] + [[package]] name = "overload" version = "0.1.1" @@ -1078,9 +2109,48 @@ dependencies = [ "libc", "redox_syscall", "smallvec", - "windows-targets", + "windows-targets 0.52.6", +] + +[[package]] +name = "parquet" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0dbd48ad52d7dccf8ea1b90a3ddbfaea4f69878dd7683e51c507d4bc52b5b27" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-ipc", + "arrow-schema", + "arrow-select", + "base64 0.22.1", + "brotli", + "bytes", + "chrono", + "flate2", + "half", + "hashbrown 0.16.0", + "lz4_flex", + "num", + "num-bigint", + "paste", + "seq-macro", + "simdutf8", + "snap", + "thrift", + "twox-hash", + "zstd", ] +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + [[package]] name = "pbjson" version = "0.7.0" @@ -1118,6 +2188,16 @@ dependencies = [ "serde", ] +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest", + "hmac", +] + [[package]] name = "percent-encoding" version = "2.3.1" @@ -1166,6 +2246,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + [[package]] name = "plotters" version = "0.3.7" @@ -1209,13 +2295,34 @@ dependencies = [ "portable-atomic", ] +[[package]] +name = "potential_utf" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84df19adbe5b5a0782edcab45899906947ab039ccf4573713735ee7de1e6b08a" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppmd-rust" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c834641d8ad1b348c9ee86dec3b9840d805acd5f24daa5f90c788951a52ff59b" + [[package]] name = "ppv-lite86" version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" dependencies = [ - "zerocopy", + "zerocopy 0.7.35", ] [[package]] @@ -1573,6 +2680,46 @@ dependencies = [ "winapi", ] +[[package]] +name = "reqwest" +version = "0.12.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d429f34c8092b2d42c7c93cec323bb4adeb7c67698f70839adec842ec10c7ceb" +dependencies = [ + "base64 0.22.1", + "bytes", + "encoding_rs", + "futures-core", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-tls", + "hyper-util", + "js-sys", + "log", + "mime", + "native-tls", + "percent-encoding", + "pin-project-lite", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-native-tls", + "tower 0.5.2", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "ring" version = "0.17.11" @@ -1599,6 +2746,15 @@ version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + [[package]] name = "rustix" version = "1.0.3" @@ -1636,7 +2792,7 @@ dependencies = [ "openssl-probe", "rustls-pki-types", "schannel", - "security-framework", + "security-framework 3.2.0", ] [[package]] @@ -1701,6 +2857,19 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags", + "core-foundation 0.9.4", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + [[package]] name = "security-framework" version = "3.2.0" @@ -1708,7 +2877,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316" dependencies = [ "bitflags", - "core-foundation", + "core-foundation 0.10.0", "core-foundation-sys", "libc", "security-framework-sys", @@ -1724,6 +2893,18 @@ dependencies = [ "libc", ] +[[package]] +name = "semver" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" + +[[package]] +name = "seq-macro" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" + [[package]] name = "serde" version = "1.0.226" @@ -1776,6 +2957,40 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -1804,6 +3019,29 @@ dependencies = [ "sift_stream", ] +[[package]] +name = "sift_cli" +version = "0.6.0" +dependencies = [ + "anyhow", + "chrono", + "clap", + "clap_complete", + "crossterm", + "csv", + "dirs", + "flate2", + "indicatif", + "indoc", + "parquet", + "pbjson-types", + "reqwest", + "sift_rs", + "tokio", + "toml", + "zip", +] + [[package]] name = "sift_connect" version = "0.6.0" @@ -1873,6 +3111,27 @@ dependencies = [ "uuid", ] +[[package]] +name = "signal-hook" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d881a16cf4426aa584979d30bd82cb33429027e42122b169753d6ef1085ed6e2" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-mio" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34db1a06d485c9142248b7a054f034b349b212551f3dfd19c94d45a754a217cd" +dependencies = [ + "libc", + "mio", + "signal-hook", +] + [[package]] name = "signal-hook-registry" version = "1.4.2" @@ -1882,6 +3141,18 @@ dependencies = [ "libc", ] +[[package]] +name = "simd-adler32" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + [[package]] name = "slab" version = "0.4.9" @@ -1897,6 +3168,12 @@ version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fcf8323ef1faaee30a44a340193b1ac6814fd9b7b4e88e9d4519a3e4abe1cfd" +[[package]] +name = "snap" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" + [[package]] name = "socket2" version = "0.5.8" @@ -1907,6 +3184,28 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "socket2" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "233504af464074f9d066d7b5416c5f9b894a5862a6506e306f7b816cdd6f1807" +dependencies = [ + "libc", + "windows-sys 0.59.0", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "subtle" version = "2.6.1" @@ -1929,6 +3228,41 @@ name = "sync_wrapper" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "system-configuration" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" +dependencies = [ + "bitflags", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] [[package]] name = "target-lexicon" @@ -1959,6 +3293,16 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "terminal_size" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60b8cb979cb11c32ce1603f8137b22262a9d131aaa5c37b5678025f22b8becd0" +dependencies = [ + "rustix", + "windows-sys 0.60.2", +] + [[package]] name = "thiserror" version = "2.0.12" @@ -1989,6 +3333,55 @@ dependencies = [ "once_cell", ] +[[package]] +name = "thrift" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09" +dependencies = [ + "byteorder", + "integer-encoding", + "ordered-float", +] + +[[package]] +name = "time" +version = "0.3.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e7d9e3bb61134e77bde20dd4825b97c010155709965fedf0f49bb138e52a9d" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde", + "time-core", +] + +[[package]] +name = "time-core" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40868e7c1d2f0b8d73e4a8c7f0ff63af4f6d19be117e90bd73eb1d62cf831c6b" + +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + +[[package]] +name = "tinystr" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d4f6d1145dcb577acf783d4e601bc1d76a13337bb54e6233add580b07344c8b" +dependencies = [ + "displaydoc", + "zerovec", +] + [[package]] name = "tinytemplate" version = "1.2.1" @@ -2001,20 +3394,22 @@ dependencies = [ [[package]] name = "tokio" -version = "1.44.1" +version = "1.47.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f382da615b842244d4b8738c82ed1275e6c5dd90c459a30941cd07080b06c91a" +checksum = "89e49afdadebb872d3145a5638b59eb0691ea23e46ca484037cfab3b76b95038" dependencies = [ "backtrace", "bytes", + "io-uring", "libc", "mio", "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2", + "slab", + "socket2 0.6.0", "tokio-macros", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -2028,6 +3423,16 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.26.2" @@ -2126,7 +3531,7 @@ dependencies = [ "prost", "rustls-native-certs", "rustls-pemfile", - "socket2", + "socket2 0.5.8", "tokio", "tokio-rustls", "tokio-stream", @@ -2151,22 +3556,41 @@ dependencies = [ "rand 0.8.5", "slab", "tokio", - "tokio-util", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", "tower-layer", "tower-service", - "tracing", ] [[package]] -name = "tower" -version = "0.5.2" +name = "tower-http" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" +checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2" dependencies = [ - "futures-core", + "bitflags", + "bytes", "futures-util", + "http", + "http-body", + "iri-string", "pin-project-lite", - "sync_wrapper", + "tower 0.5.2", "tower-layer", "tower-service", ] @@ -2272,24 +3696,77 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "twox-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" + +[[package]] +name = "typenum" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" + [[package]] name = "unicode-ident" version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "00e2473a93778eb0bad35909dff6a10d28e63f792f16ed15e404fca9d5eeedbe" +[[package]] +name = "unicode-segmentation" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + [[package]] name = "unindent" version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" +[[package]] +name = "unit-prefix" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "323402cff2dd658f39ca17c789b502021b3f18707c91cdf22e3838e1b4023817" + [[package]] name = "untrusted" version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "url" +version = "2.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "uuid" version = "1.16.0" @@ -2305,6 +3782,18 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + [[package]] name = "walkdir" version = "2.5.0" @@ -2365,6 +3854,19 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61" +dependencies = [ + "cfg-if", + "js-sys", + "once_cell", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.100" @@ -2407,6 +3909,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "webpki-roots" version = "0.26.8" @@ -2438,7 +3950,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -2453,7 +3965,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -2462,13 +3974,48 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dccfd733ce2b1753b03b6d3c65edf020262ea35e20ccdf3e288043e6dd620e3" +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-registry" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c44a98275e31bfd112bb06ba96c8ab13c03383a3753fdddd715406a1824c7e0" +dependencies = [ + "windows-link 0.1.0", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-result" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06374efe858fab7e4f881500e6e86ec8bc28f9462c47e5a9941a0142ad86b189" +dependencies = [ + "windows-link 0.1.0", +] + +[[package]] +name = "windows-strings" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87fa48cc5d406560701792be122a10132491cff9d0aeb23583cc2dcafc847319" +dependencies = [ + "windows-link 0.1.0", +] + [[package]] name = "windows-sys" version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -2477,7 +4024,25 @@ version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link 0.2.1", ] [[package]] @@ -2486,14 +4051,31 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link 0.2.1", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", ] [[package]] @@ -2502,48 +4084,96 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + [[package]] name = "windows_i686_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + [[package]] name = "winnow" version = "0.7.11" @@ -2562,6 +4192,36 @@ dependencies = [ "bitflags", ] +[[package]] +name = "writeable" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" + +[[package]] +name = "yoke" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f41bb01b8226ef4bfd589436a297c53d118f65921786300e427be8d487695cc" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + [[package]] name = "zerocopy" version = "0.7.35" @@ -2569,7 +4229,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" dependencies = [ "byteorder", - "zerocopy-derive", + "zerocopy-derive 0.7.35", +] + +[[package]] +name = "zerocopy" +version = "0.8.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0894878a5fa3edfd6da3f88c4805f4c8558e2b996227a3d864f47fe11e38282c" +dependencies = [ + "zerocopy-derive 0.8.27", ] [[package]] @@ -2583,8 +4252,160 @@ dependencies = [ "syn", ] +[[package]] +name = "zerocopy-derive" +version = "0.8.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerofrom" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + [[package]] name = "zeroize" version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerotrie" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36f0bbd478583f79edad978b407914f61b2972f5af6fa089686016be8f9af595" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7aa2bd55086f1ab526693ecbe444205da57e25f4489879da80635a46d90e73b" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zip" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb2a05c7c36fde6c09b08576c9f7fb4cda705990f73b58fe011abf7dfb24168b" +dependencies = [ + "aes", + "arbitrary", + "bzip2", + "constant_time_eq", + "crc32fast", + "deflate64", + "flate2", + "getrandom 0.3.1", + "hmac", + "indexmap 2.7.1", + "lzma-rust2", + "memchr", + "pbkdf2", + "ppmd-rust", + "sha1", + "time", + "zeroize", + "zopfli", + "zstd", +] + +[[package]] +name = "zlib-rs" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f06ae92f42f5e5c42443fd094f245eb656abf56dd7cce9b8b263236565e00f2" + +[[package]] +name = "zopfli" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edfc5ee405f504cd4984ecc6f14d02d55cfda60fa4b689434ef4102aae150cd7" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/rust/Cargo.toml b/rust/Cargo.toml index ea361c649..339e2dfe8 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -7,6 +7,7 @@ members = [ "crates/sift_error", "crates/sift_connect", "crates/sift_stream_bindings", + "crates/sift_cli", ] [workspace.package] diff --git a/rust/crates/sift_cli/Cargo.toml b/rust/crates/sift_cli/Cargo.toml new file mode 100644 index 000000000..218f906ef --- /dev/null +++ b/rust/crates/sift_cli/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "sift_cli" +authors.workspace = true +version.workspace = true +edition.workspace = true +categories.workspace = true +homepage.workspace = true +repository.workspace = true +keywords.workspace = true +readme.workspace = true +license.workspace = true +description = "CLI to streamline programmatic workflows with Sift's API" + +[dependencies] +anyhow = "1.0.100" +chrono.workspace = true +clap = { version = "4.5.48", features = ["cargo", "derive", "wrap_help"] } +clap_complete = "4.5.58" +crossterm = "0.29.0" +csv = "1.3.1" +dirs = "6.0.0" +flate2 = "1.1.2" +indicatif = "0.18.0" +parquet = "56.2.0" +pbjson-types = { workspace = true } +reqwest = "0.12.23" +sift_rs = { workspace = true } +tokio = { version = "1.47.1", features = ["full", "net", "time"] } +toml = "0.8.23" +zip = "6.0.0" + +[dev-dependencies] +indoc = "2.0.6" diff --git a/rust/crates/sift_cli/src/cli/channel.rs b/rust/crates/sift_cli/src/cli/channel.rs new file mode 100644 index 000000000..01a898d03 --- /dev/null +++ b/rust/crates/sift_cli/src/cli/channel.rs @@ -0,0 +1,39 @@ +use clap::ValueEnum; +use sift_rs::common::r#type::v1::ChannelDataType; + +#[derive(Debug, Clone, ValueEnum)] +pub enum DataType { + /// Asks the program to infer the type so user can just focus on setting things like unit, + /// description, etc. + Infer, + Double, + String, + Enum, + BitField, + Bool, + Float, + Int32, + Uint32, + Int64, + Uint64, + Bytes, +} + +impl From for ChannelDataType { + fn from(dt: DataType) -> Self { + match dt { + DataType::Double => Self::Double, + DataType::String => Self::String, + DataType::Enum => Self::Enum, + DataType::BitField => Self::BitField, + DataType::Bool => Self::Bool, + DataType::Float => Self::Float, + DataType::Int32 => Self::Int32, + DataType::Uint32 => Self::Uint32, + DataType::Int64 => Self::Int64, + DataType::Uint64 => Self::Uint64, + DataType::Bytes => Self::Bytes, + DataType::Infer => Self::Unspecified, + } + } +} diff --git a/rust/crates/sift_cli/src/cli/export.rs b/rust/crates/sift_cli/src/cli/export.rs new file mode 100644 index 000000000..f82cd495a --- /dev/null +++ b/rust/crates/sift_cli/src/cli/export.rs @@ -0,0 +1,17 @@ +use clap::ValueEnum; +use sift_rs::exports::v1::ExportOutputFormat; + +#[derive(Debug, Copy, Clone, ValueEnum)] +pub enum Format { + Csv, + Sun, +} + +impl From for ExportOutputFormat { + fn from(val: Format) -> Self { + match val { + Format::Csv => ExportOutputFormat::Csv, + Format::Sun => ExportOutputFormat::Sun, + } + } +} diff --git a/rust/crates/sift_cli/src/cli/mod.rs b/rust/crates/sift_cli/src/cli/mod.rs new file mode 100644 index 000000000..29366e47b --- /dev/null +++ b/rust/crates/sift_cli/src/cli/mod.rs @@ -0,0 +1,317 @@ +use clap::{Parser, Subcommand, crate_description, crate_version}; +use clap_complete::Shell; +use parquet::ComplexTypesMode; +use std::path::PathBuf; + +pub mod channel; +use channel::DataType; + +pub mod export; + +pub mod parquet; + +pub mod time; +use time::TimeFormat; + +#[derive(Parser)] +#[command( + version = crate_version!(), + about = crate_description!(), +)] +pub struct Args { + #[command(subcommand)] + pub cmd: Cmd, + + /// The profile to use + #[arg(long, global = true)] + pub profile: Option, + + /// Disable TLS for non-cloud Sift environments + #[arg(long, global = true)] + pub disable_tls: bool, +} + +#[derive(Subcommand)] +pub enum Cmd { + /// Manage Sift CLI configuration + #[command(subcommand)] + Config(ConfigCmd), + + /// Manage shell autocompletions + #[command(subcommand)] + Completions(CompletionsCmd), + + /// Export asset/run data from Sift + #[command(subcommand)] + Export(ExportCmd), + + /// Import time series files into Sift + #[command(subcommand)] + Import(ImportCmd), +} + +#[derive(Subcommand)] +pub enum ExportCmd { + /// Export data for a run + Run(ExportRunArgs), + + /// Export data for an asset + Asset(ExportAssetArgs), +} + +#[derive(clap::Args)] +pub struct ExportRunArgs { + /// The name of the run + #[arg(short, long, group = "run_identifier")] + pub name: Option, + + /// The ID of the run + #[arg(short, long, group = "run_identifier")] + pub run_id: Option, + + /// The client key of the run + #[arg(short = 'k', long, group = "run_identifier")] + pub client_key: Option, + + #[command(flatten)] + pub common: ExportArgs, +} + +#[derive(clap::Args)] +pub struct ExportAssetArgs { + /// The name of the asset + pub asset: String, + + #[command(flatten)] + pub common: ExportArgs, +} + +#[derive(clap::Args)] +pub struct ExportArgs { + /// The file to generate + #[arg(short, long)] + pub output: PathBuf, + + /// File format for the output file + #[arg(short, long)] + pub format: export::Format, + + /// Regular expression used to filter channels to include in the export + #[arg(short = 'x', long)] + pub channel_regex: Option, + + /// Name of channel to include in the export; can be specified multiple times + #[arg(short, long)] + pub channel: Vec, + + /// ID of channel to include in the export; can be specified multiple times + #[arg(long)] + pub channel_id: Vec, + + /// Start time in RFC 3339 format (required for asset exports) + #[arg(long)] + pub start: Option, + + /// Stop time in RFC 3339 format (required for asset exports) + #[arg(long)] + pub stop: Option, +} + +#[derive(Subcommand)] +pub enum CompletionsCmd { + /// Print completions for your shell + Print(CompletionsPrintArgs), + + /// Attempts to automatically update this CLI's completions file for the current shell + Update, +} + +#[derive(clap::Args)] +pub struct CompletionsPrintArgs { + /// The shell to print completions for. If empty the program will try to infer the user shell + /// by reading the "$SHELL" environment variable. + #[arg(short, long)] + pub shell: Option, +} + +#[derive(Subcommand)] +pub enum ImportCmd { + /// Import a CSV file into Sift. Unless manually specified all columns are inferred to type + /// string or double. + Csv(ImportCsvArgs), + + /// Import a Parquet file into Sift. + #[command(subcommand)] + Parquet(ImportParquetCmd), +} + +#[derive(Subcommand)] +pub enum ConfigCmd { + /// Display the contents of the current config file + Show, + + /// Show the path to the current config file + Where, + + /// Create a new config file (fails if one already exists) + Create, + + /// Update fields in the existing config file + Update(ConfigUpdateArgs), +} + +#[derive(clap::Args)] +pub struct ConfigUpdateArgs { + /// Edit or create a profile interactively (ignores other flags) + #[arg(short, long)] + pub interactive: bool, + + /// Base gRPC endpoint for Sift + #[arg(short, long)] + pub grpc_uri: Option, + + /// Base REST endpoint for Sift + #[arg(short, long)] + pub rest_uri: Option, + + /// API key used for authentication + #[arg(short = 'k', long)] + pub api_key: Option, +} + +#[derive(clap::Args)] +pub struct ImportCsvArgs { + /// Path to the CSV file to import + pub path: PathBuf, + + /// Name of the asset this data belongs to + #[arg(short, long)] + pub asset: String, + + /// Optional run name to associate with this import + #[arg(short, long)] + pub run: Option, + + /// Row number containing column headers (1-based) + #[arg(long, default_value_t = 1)] + pub header_row: usize, + + /// Row number where data starts (1-based) + #[arg(long, default_value_t = 2)] + pub first_data_row: usize, + + /// 1-based column indices to override; can appear multiple times + #[arg(short, long)] + pub channel_column: Vec, + + /// Data type for each channel in `--channel-column`. Use `"infer"` to have the program infer + /// the data type which is useful when wanting to just specify `--unit` and/or `--description` + #[arg(short, long)] + pub data_type: Vec, + + /// Unit for each channel in `--channel-column` (can be empty) + #[arg(short, long)] + pub unit: Vec, + + /// Description for each channel in `--channel-column` (can be empty) + #[arg(short = 'n', long)] + pub description: Vec, + + /// Enum configuration pairs `` (e.g. `"0,start|1,stop"`) for enum-type channels + #[arg(short, long)] + pub enum_config: Vec, + + /// Bit-field configuration triplets `` (e.g. `"12v,0,4|led,4,4"`) + #[arg(short, long)] + pub bit_field_config: Vec, + + /// 1-based index of the time column + #[arg(short, long, default_value_t = 1)] + pub time_column: usize, + + /// Time format used in the file + #[arg(short = 'f', long, default_value_t = TimeFormat::default())] + pub time_format: TimeFormat, + + /// Start time (RFC3339) to use if time format is relative + #[arg(short = 's')] + pub relative_start_time: Option, + + /// Wait until the import finishes processing + #[arg(short, long)] + pub wait: bool, + + /// Preview the parsed schema without uploading + #[arg(short, long)] + pub preview: bool, +} + +#[derive(Subcommand)] +pub enum ImportParquetCmd { + /// A parquet file where every column is exclusive to a single channel except for the time + /// column + FlatDataset(FlatDatasetArgs), +} + +#[derive(clap::Args)] +pub struct FlatDatasetArgs { + /// Path to the Parquet file to import + pub path: PathBuf, + + /// Name of the asset this data belongs to + #[arg(short, long)] + pub asset: String, + + /// Optional run name to associate with this import + #[arg(short, long)] + pub run: Option, + + /// Paths of data columns to import; can be specified multiple times + #[arg(short, long)] + pub channel_path: Vec, + + /// Data type for each channel in `--channel-path`. Use `"infer"` to have the program infer + /// the data type which is useful when wanting to just specify `--unit` and/or `--description` + #[arg(short, long)] + pub data_type: Vec, + + /// Unit for each channel in `--channel-path` (can be empty) + #[arg(short, long)] + pub unit: Vec, + + /// Description for each channel in `--channel-path` (can be empty) + #[arg(short = 'n', long)] + pub description: Vec, + + /// Enum configuration pairs `` for enum-type channels + #[arg(short, long)] + pub enum_config: Vec, + + /// Bit-field configuration triplets `` for bit-field channels + #[arg(short, long)] + pub bit_field_config: Vec, + + /// Path to the time column + #[arg(short, long, default_value_t = String::from("timestamp"))] + pub time_path: String, + + /// Time format used in the file + #[arg(short = 'f', long, default_value_t = TimeFormat::default())] + pub time_format: TimeFormat, + + /// Start time (RFC3339) to use if time format is relative + #[arg(short = 's')] + pub relative_start_time: Option, + + /// Strategy for handling complex types (maps, lists, structs) + #[arg(short = 'm', long, default_value_t = ComplexTypesMode::default())] + pub complex_types_mode: ComplexTypesMode, + + /// Wait until the import finishes processing + #[arg(short, long)] + pub wait: bool, + + /// Preview the parsed schema without uploading + #[arg(short, long)] + pub preview: bool, +} diff --git a/rust/crates/sift_cli/src/cli/parquet.rs b/rust/crates/sift_cli/src/cli/parquet.rs new file mode 100644 index 000000000..a9d267edf --- /dev/null +++ b/rust/crates/sift_cli/src/cli/parquet.rs @@ -0,0 +1,40 @@ +use std::fmt::{self, Display}; + +use clap::ValueEnum; +use sift_rs::data_imports::v2::ParquetComplexTypesImportMode; + +/// Specifies how to handle columns that are complex types i.e. maps, lists and structs. +#[derive(Debug, Clone, ValueEnum, Default)] +pub enum ComplexTypesMode { + /// Ignore columns containing complex types + #[default] + Ignore, + /// Import complex types as both Arrow bytes and JSON strings. + Both, + /// Import complex types as JSON strings + String, + /// Import complex types as Arrow bytes. + Bytes, +} + +impl From for ParquetComplexTypesImportMode { + fn from(mode: ComplexTypesMode) -> Self { + match mode { + ComplexTypesMode::Both => Self::Both, + ComplexTypesMode::Bytes => Self::Bytes, + ComplexTypesMode::Ignore => Self::Ignore, + ComplexTypesMode::String => Self::String, + } + } +} + +impl Display for ComplexTypesMode { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Both => write!(f, "both"), + Self::Bytes => write!(f, "bytes"), + Self::Ignore => write!(f, "ignore"), + Self::String => write!(f, "string"), + } + } +} diff --git a/rust/crates/sift_cli/src/cli/time.rs b/rust/crates/sift_cli/src/cli/time.rs new file mode 100644 index 000000000..a70c1dac0 --- /dev/null +++ b/rust/crates/sift_cli/src/cli/time.rs @@ -0,0 +1,59 @@ +use std::fmt::{self, Display}; + +use clap::ValueEnum; +use sift_rs::data_imports::v2::TimeFormat as ProtoTimeFormat; + +#[derive(Debug, Copy, Clone, ValueEnum, Default)] +pub enum TimeFormat { + #[default] + AbsoluteRfc3339, + AbsoluteDatetime, + AbsoluteUnixSeconds, + AbsoluteUnixMilliseconds, + AbsoluteUnixMicroseconds, + AbsoluteUnixNanoseconds, + RelativeNanoseconds, + RelativeMicroseconds, + RelativeMilliseconds, + RelativeSeconds, + RelativeMinutes, + RelativeHours, +} + +impl From for ProtoTimeFormat { + fn from(tf: TimeFormat) -> Self { + match tf { + TimeFormat::RelativeNanoseconds => Self::RelativeNanoseconds, + TimeFormat::RelativeMicroseconds => Self::RelativeMicroseconds, + TimeFormat::RelativeMilliseconds => Self::RelativeMilliseconds, + TimeFormat::RelativeSeconds => Self::RelativeSeconds, + TimeFormat::RelativeMinutes => Self::RelativeMinutes, + TimeFormat::RelativeHours => Self::RelativeHours, + TimeFormat::AbsoluteRfc3339 => Self::AbsoluteRfc3339, + TimeFormat::AbsoluteDatetime => Self::AbsoluteDatetime, + TimeFormat::AbsoluteUnixSeconds => Self::AbsoluteUnixSeconds, + TimeFormat::AbsoluteUnixMilliseconds => Self::AbsoluteUnixMilliseconds, + TimeFormat::AbsoluteUnixMicroseconds => Self::AbsoluteUnixMicroseconds, + TimeFormat::AbsoluteUnixNanoseconds => Self::AbsoluteUnixNanoseconds, + } + } +} + +impl Display for TimeFormat { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::AbsoluteRfc3339 => write!(f, "absolute-rfc3339"), + Self::AbsoluteDatetime => write!(f, "absolute-datetime"), + Self::AbsoluteUnixSeconds => write!(f, "absolute-unix-seconds"), + Self::AbsoluteUnixMilliseconds => write!(f, "absolute-unix-milliseconds"), + Self::AbsoluteUnixMicroseconds => write!(f, "absolute-unix-microseconds"), + Self::AbsoluteUnixNanoseconds => write!(f, "absolute-unix-nanoseconds"), + Self::RelativeNanoseconds => write!(f, "relative-nanoseconds"), + Self::RelativeMicroseconds => write!(f, "relative-microseconds"), + Self::RelativeMilliseconds => write!(f, "relative-milliseconds"), + Self::RelativeSeconds => write!(f, "relative-seconds"), + Self::RelativeMinutes => write!(f, "relative-minutes"), + Self::RelativeHours => write!(f, "relative-hours"), + } + } +} diff --git a/rust/crates/sift_cli/src/cmd/completions.rs b/rust/crates/sift_cli/src/cmd/completions.rs new file mode 100644 index 000000000..be0b13dd3 --- /dev/null +++ b/rust/crates/sift_cli/src/cmd/completions.rs @@ -0,0 +1,155 @@ +use std::{ + env, + fs::{self, File}, + io, + path::{Path, PathBuf}, + process::ExitCode, + str::FromStr, +}; + +use anyhow::{Context, Result, anyhow}; +use clap::{CommandFactory, crate_name}; +use clap_complete::{Shell, generate}; +use crossterm::style::Stylize; + +use crate::{ + cli::{self, CompletionsPrintArgs}, + util::tty::Output, +}; + +pub fn print(args: CompletionsPrintArgs) -> Result { + let shell = match args.shell { + Some(sh) => sh, + None => try_get_shell()?, + }; + + let mut cmd = cli::Args::command(); + let mut stdout = io::stdout(); + + generate(shell, &mut cmd, crate_name!(), &mut stdout); + + Ok(ExitCode::SUCCESS) +} + +pub fn update() -> Result { + let shell = try_get_shell()?; + let root = get_config_root()?; + + let (dir_path, filename) = match try_get_shell()? { + Shell::Zsh => (root.join(".zsh-complete"), format!("_{}", crate_name!())), + Shell::Bash => (root.join(".bash_completion.d"), String::from(crate_name!())), + Shell::Fish => ( + get_fish_completions_dir(&root), + format!("{}.fish", crate_name!()), + ), + _ => { + Output::new() + .line(format!( + "failed to automatically update completions due to unsupported shell {shell}" + )) + .tip(format!( + "try manually generating completions with `{}`", + "completions print".cyan() + )) + .eprint(); + return Ok(ExitCode::FAILURE); + } + }; + + fs::create_dir_all(&dir_path)?; + let completions_file_path = dir_path.join(filename); + + let mut completions_file = File::options() + .create(true) + .write(true) + .truncate(true) + .open(&completions_file_path) + .with_context(|| format!("failed to create/open {}", completions_file_path.display()))?; + + let mut cmd = cli::Args::command(); + generate(shell, &mut cmd, crate_name!(), &mut completions_file); + + let mut out = Output::new(); + out.line(format!( + "{} {}", + "Updated".green(), + completions_file_path.display() + )); + + match shell { + Shell::Zsh => { + out.tip(format!( + "Ensure \"{}\" is set in your {} and restart your shell (sourcing doesn't always work)", + "fpath=($HOME/.zsh-complete $fpath)".cyan(), + "$HOME/.zshrc".cyan(), + )); + } + Shell::Bash => { + out.tip("Don't forget to restart your shell (sourcing doesn't always work)"); + } + Shell::Fish => { + out.tip(format!( + "Fish will automatically load completions from {}. \ + If you don’t see them right away, restart your shell with `{}`.", + "~/.config/fish/completions".cyan(), + "exec fish".yellow(), + )); + } + _ => (), + } + out.print(); + + Ok(ExitCode::SUCCESS) +} + +fn try_get_shell() -> Result { + env::var_os("SHELL") + .map(PathBuf::from) + .and_then(|path| { + path.as_path() + .file_name() + .and_then(|n| n.to_str()) + .and_then(|n| Shell::from_str(n).ok()) + }) + .ok_or(anyhow!( + "failed to infer user shell from \"$SHELL\" environment variable" + )) +} + +#[cfg(target_os = "macos")] +pub fn get_fish_completions_dir(root: &Path) -> PathBuf { + root.join(".config").join("fish").join("completions") +} + +#[cfg(target_os = "linux")] +pub fn get_fish_completions_dir(root: &Path) -> PathBuf { + root.join("fish").join("completions") +} + +#[cfg(not(any(target_os = "macos", target_os = "linux")))] +pub fn get_fish_completions_dir(root: &Path) -> PathBuf { + unreachable!("automatic updating of completions is only supported on macos and linux") +} + +/// Home directory for macos. +#[cfg(target_os = "macos")] +pub fn get_config_root() -> Result { + env::home_dir().ok_or(anyhow!("unable to determine home directory")) +} + +/// `XDG_CONFIG_HOME` or `HOME` for linux. +#[cfg(target_os = "linux")] +pub fn get_config_root() -> Result { + dirs::config_local_dir() + .or_else(env::home_dir) + .ok_or(anyhow!( + "unable to determine config directory from \"$XDG_CONFIG_HOME\" or \"$HOME\"" + )) +} + +#[cfg(not(any(target_os = "macos", target_os = "linux")))] +pub fn get_config_root() -> Result { + Err(anyhow!( + "automatic updating of completions is only supported on macos and linux" + )) +} diff --git a/rust/crates/sift_cli/src/cmd/config.rs b/rust/crates/sift_cli/src/cmd/config.rs new file mode 100644 index 000000000..10432f16f --- /dev/null +++ b/rust/crates/sift_cli/src/cmd/config.rs @@ -0,0 +1,191 @@ +use anyhow::{Context, Result, anyhow}; +use crossterm::style::Stylize; +use std::{ + fs::{File, OpenOptions, metadata, read_to_string}, + io::Write, + path::PathBuf, + process::ExitCode, +}; +use toml::{Table, Value}; + +use crate::{ + cli::ConfigUpdateArgs, + util::tty::{Output, PromptUser}, +}; + +pub const CONFIG_FILE_NAME: &str = "sift.toml"; + +pub fn show() -> Result { + let p = get_config_file_path()?; + let contents = read_to_string(p).context("failed to read config file")?; + Output::new().line(contents).print(); + Ok(ExitCode::SUCCESS) +} + +pub fn create() -> Result { + let (_, path) = create_config_file()?; + let p = path.display().to_string(); + + Output::new() + .line(format!( + "An empty config file has been created at '{}'.", + p.yellow() + )) + .tip(format!( + "Use '{}' to configure it.", + "sift_cli config update".green() + )) + .print(); + + Ok(ExitCode::SUCCESS) +} + +pub fn update(profile: Option, args: ConfigUpdateArgs) -> Result { + let prof = profile.clone(); + let mut target = prof.unwrap_or_else(|| String::from("default")); + + let updated_config = { + if !args.interactive { + if is_update_empty(&args) { + Output::new().line("Nothing to update.").print(); + return Ok(ExitCode::SUCCESS); + } + get_updated_config(profile, args.grpc_uri, args.rest_uri, args.api_key)? + } else { + let [prof, grpc, rest, key]: [Option; 4] = PromptUser::new() + .header("Any blank values will be ignored preserving the original.") + .prompt(" Specify the profile to configure (leave blank for default profile): ") + .prompt(" Specify the gRPC API base URL: ") + .prompt(" Specify the REST API base URL: ") + .prompt(" Provide your Sift API key: ") + .run()? + .try_into() + .unwrap(); + + if let Some(p) = prof.as_ref() { + target = p.clone(); + } + let updated = get_updated_config(prof, grpc, rest, key)?; + let divider = "-".repeat(40); + + let [confirmation]: [Option; 1] = PromptUser::new() + .prompt(format!( + "\n{divider}\n{updated}\n{divider}\nDoes this look correct? [y/n]: " + )) + .run()? + .try_into() + .unwrap(); + + if confirmation.is_none_or(|c| c != "y") { + Output::new().line("Operation aborted.").print(); + return Ok(ExitCode::SUCCESS); + } + updated + } + }; + + update_config_file(updated_config)?; + + Output::new() + .line(format!( + "Successfully configured the '{}' profile.", + target.yellow() + )) + .print(); + + Ok(ExitCode::SUCCESS) +} + +pub fn config_where() -> Result { + let expected_path = get_config_file_path()?; + let p = expected_path.display().to_string(); + + if metadata(&expected_path).is_err() { + Output::new() + .line(format!("'{}' not found.", p.yellow())) + .tip(format!( + "try running '{}' first.", + "sift_cli config create".green() + )) + .eprint(); + return Ok(ExitCode::FAILURE); + } + Output::new().line(p.to_string()).print(); + Ok(ExitCode::SUCCESS) +} + +pub(super) fn get_config_file_path() -> Result { + dirs::config_dir() + .map(|p| p.join(CONFIG_FILE_NAME)) + .ok_or(anyhow!("user config directory not found")) +} + +fn create_config_file() -> Result<(File, PathBuf)> { + let path = get_config_file_path()?; + + let config_file = File::create_new(&path).context("failed to create config file")?; + + Ok((config_file, path)) +} + +fn get_updated_config( + profile: Option, + grpc_uri: Option, + rest_uri: Option, + api_key: Option, +) -> Result { + let path = get_config_file_path()?; + + let contents = read_to_string(path).context("failed to read config file")?; + + let mut config_toml = contents + .parse::() + .context("config file is invalid TOML")?; + + let target = match profile { + Some(prof) => match config_toml.get_mut(&prof) { + Some(Value::Table(profile_config)) => profile_config, + _ => { + config_toml.insert(prof.clone(), Value::Table(Table::new())); + config_toml[&prof].as_table_mut().unwrap() + } + }, + None => &mut config_toml, + }; + + if let Some(uri) = grpc_uri { + target.insert(String::from("grpc_uri"), Value::String(uri)); + } + if let Some(uri) = rest_uri { + target.insert(String::from("rest_uri"), Value::String(uri)); + } + if let Some(token) = api_key { + target.insert(String::from("apikey"), Value::String(token)); + } + + Ok(config_toml.to_string()) +} + +fn update_config_file(updated: String) -> Result<()> { + let path = get_config_file_path()?; + + let mut config = OpenOptions::new() + .write(true) + .truncate(true) + .open(path) + .context("failed to open config file")?; + + write!(config, "{updated}").context("failed to update config file") +} + +fn is_update_empty(args: &ConfigUpdateArgs) -> bool { + let ConfigUpdateArgs { + grpc_uri, + rest_uri, + api_key, + .. + } = args; + grpc_uri.as_ref().is_none_or(|s| s.is_empty()) + || rest_uri.as_ref().is_none_or(|s| s.is_empty()) + || api_key.as_ref().is_none_or(|s| s.is_empty()) +} diff --git a/rust/crates/sift_cli/src/cmd/export.rs b/rust/crates/sift_cli/src/cmd/export.rs new file mode 100644 index 000000000..0b69e3339 --- /dev/null +++ b/rust/crates/sift_cli/src/cmd/export.rs @@ -0,0 +1,388 @@ +use std::{ + env::temp_dir, + fs::{File, remove_file}, + io::{self, Write}, + path::PathBuf, + process::ExitCode, + time::Duration, +}; + +use anyhow::{Context as AnyhowContext, Result, anyhow}; +use chrono::DateTime; +use crossterm::style::Stylize; +use parquet::data_type::AsBytes; +use pbjson_types::Timestamp; +use reqwest::Client; +use sift_rs::{ + SiftChannel, + assets::v1::{ListAssetsRequest, ListAssetsResponse, asset_service_client::AssetServiceClient}, + exports::v1::{ + AssetsAndTimeRange, ExportDataRequest, ExportDataResponse, ExportOutputFormat, + GetDownloadUrlRequest, GetDownloadUrlResponse, RunsAndTimeRange, + export_data_request::TimeSelection, export_service_client::ExportServiceClient, + }, + jobs::v1::JobStatus, + runs::v2::{ListRunsRequest, ListRunsResponse, run_service_client::RunServiceClient}, +}; +use tokio::time::sleep; +use zip::ZipArchive; + +use crate::{ + cli::{ExportAssetArgs, ExportRunArgs}, + util::{ + api::create_grpc_channel, channel::filter_channels, job::JobServiceWrapper, + progress::Spinner, tty::Output, + }, +}; + +use super::Context; + +pub async fn run(ctx: Context, args: ExportRunArgs) -> Result { + let grpc_channel = create_grpc_channel(&ctx)?; + let mut run_service = RunServiceClient::new(grpc_channel.clone()); + + let filter = { + if let Some(run_name) = args.name.as_ref() { + format!("name == \"{run_name}\"") + } else if let Some(run_id) = args.run_id.as_ref() { + format!("run_id == \"{run_id}\"") + } else if let Some(client_key) = args.client_key.as_ref() { + format!("client_key == \"{client_key}\"") + } else { + return Err(anyhow!( + "at least one of the following arguments is required: `{}`, `{}`, `{}`", + "--run".cyan(), + "--id".cyan(), + "--client-key".cyan(), + )); + } + }; + + let ListRunsResponse { runs, .. } = run_service + .list_runs(ListRunsRequest { + filter, + ..Default::default() + }) + .await + .context("failed to query run")? + .into_inner(); + + if runs.is_empty() { + return Err(anyhow!("no run found")); + } else if runs.len() > 1 { + return Err(anyhow!( + "multiple runs found. Try providing a unique identifier with `{}` or `{}`", + "--id".cyan(), + "--client-key".cyan(), + )); + } + + let run = runs.first().unwrap(); + + if run.asset_ids.is_empty() { + return Err(anyhow!( + "run '{}' isn't associated with any assets", + run.name.clone().yellow() + )); + } + let asset_ids_cel = run + .asset_ids + .iter() + .map(|a| format!("'{a}'")) + .collect::>() + .join(","); + + let mut channel_ids = args.common.channel_id; + + if !args.common.channel.is_empty() { + let channel_names_cel = args + .common + .channel + .iter() + .map(|c| format!("'{c}'")) + .collect::>() + .join(","); + + let filter = format!("asset_id in [{asset_ids_cel}] && name in [{channel_names_cel}]"); + let query_res = filter_channels(grpc_channel.clone(), &filter).await?; + + for channel in query_res { + channel_ids.push(channel.channel_id); + } + } + + if let Some(re) = args.common.channel_regex { + let filter = format!("asset_id in [{asset_ids_cel}] && name.matches(\"{re}\")"); + let query_res = filter_channels(grpc_channel.clone(), &filter).await?; + + for channel in query_res { + channel_ids.push(channel.channel_id); + } + } + + let start_time = args + .common + .start + .as_deref() + .and_then(|t| { + DateTime::parse_from_rfc3339(t) + .map(|d| Timestamp::from(d.to_utc())) + .ok() + }) + .or(run.start_time); + + let stop_time = args + .common + .stop + .as_deref() + .and_then(|t| { + DateTime::parse_from_rfc3339(t) + .map(|d| Timestamp::from(d.to_utc())) + .ok() + }) + .or(run.stop_time); + + let export_req = ExportDataRequest { + channel_ids, + output_format: ExportOutputFormat::from(args.common.format).into(), + time_selection: Some(TimeSelection::RunsAndTimeRange(RunsAndTimeRange { + start_time, + stop_time, + run_ids: vec![run.run_id.clone()], + })), + ..Default::default() + }; + + export(grpc_channel, export_req, args.common.output).await +} + +pub async fn asset(ctx: Context, args: ExportAssetArgs) -> Result { + let start_time = args + .common + .start + .as_deref() + .and_then(|t| { + DateTime::parse_from_rfc3339(t) + .map(|d| Timestamp::from(d.to_utc())) + .ok() + }) + .ok_or_else(|| anyhow!("missing required argument `{}`", "--start".yellow()))?; + + let stop_time = args + .common + .stop + .as_deref() + .and_then(|t| { + DateTime::parse_from_rfc3339(t) + .map(|d| Timestamp::from(d.to_utc())) + .ok() + }) + .ok_or_else(|| anyhow!("missing required argument `{}`", "--stop".yellow()))?; + + let grpc_channel = create_grpc_channel(&ctx)?; + let mut asset_service = AssetServiceClient::new(grpc_channel.clone()); + + let filter = format!("name == '{}'", args.asset); + + let ListAssetsResponse { assets, .. } = asset_service + .list_assets(ListAssetsRequest { + filter, + ..Default::default() + }) + .await + .context("failed to query asset")? + .into_inner(); + + if assets.is_empty() { + return Err(anyhow!("no run found")); + } + let asset = assets.first().unwrap(); + let asset_id = &asset.asset_id; + + let mut channel_ids = args.common.channel_id; + + if !args.common.channel.is_empty() { + let channel_names_cel = args + .common + .channel + .iter() + .map(|c| format!("'{c}'")) + .collect::>() + .join(","); + + let filter = format!("asset_id == '{asset_id}' && name in [{channel_names_cel}]"); + let query_res = filter_channels(grpc_channel.clone(), &filter).await?; + + for channel in query_res { + channel_ids.push(channel.channel_id); + } + } + + if let Some(re) = args.common.channel_regex { + let filter = format!("asset_id == '{asset_id}' && name.matches(\"{re}\")"); + let query_res = filter_channels(grpc_channel.clone(), &filter).await?; + + for channel in query_res { + channel_ids.push(channel.channel_id); + } + } + + let export_req = ExportDataRequest { + channel_ids, + output_format: ExportOutputFormat::from(args.common.format).into(), + time_selection: Some(TimeSelection::AssetsAndTimeRange(AssetsAndTimeRange { + asset_ids: vec![asset_id.to_string()], + start_time: Some(start_time), + stop_time: Some(stop_time), + })), + ..Default::default() + }; + + export(grpc_channel, export_req, args.common.output).await +} + +async fn export( + grpc_channel: SiftChannel, + req: ExportDataRequest, + output: PathBuf, +) -> Result { + let mut export_service = ExportServiceClient::new(grpc_channel.clone()); + let ExportDataResponse { job_id, .. } = export_service + .export_data(req) + .await + .context("failed to initiate export")? + .into_inner(); + + let mut job_service = JobServiceWrapper::new(grpc_channel); + let mut job = job_service + .get_job(&job_id) + .await? + .ok_or_else(|| anyhow!("failed to find job {job_id}"))?; + + let spinner = Spinner::new(); + spinner.set_message(format!("{} export", "Processing".green())); + + loop { + sleep(Duration::from_secs(3)).await; + + match job.job_status() { + JobStatus::Created => (), + JobStatus::Running => { + spinner.set_message(format!("{} export", "Processing".green())); + } + JobStatus::CancelRequested => { + spinner.set_message(format!( + "{} was requested but the job may still finish", + "Cancellation".green() + )); + } + JobStatus::Cancelled => { + spinner.finish_and_clear(); + Output::new() + .line(format!("{} data export job", "Cancelled".green())) + .print(); + break; + } + JobStatus::Failed => { + spinner.finish_and_clear(); + Output::new() + .line("Processing failed") + .tip("Please check the Sift jobs manage page for further details") + .eprint(); + return Ok(ExitCode::FAILURE); + } + JobStatus::Finished => { + spinner.finish_and_clear(); + Output::new() + .line(format!("{} export", "Downloading".green())) + .print(); + break; + } + _ => (), + } + + job = job_service + .get_job(&job_id) + .await? + .ok_or_else(|| anyhow!("failed to find job {job_id}"))?; + } + + let GetDownloadUrlResponse { presigned_url } = export_service + .get_download_url(GetDownloadUrlRequest { job_id }) + .await + .context("failed to get download URL for export")? + .into_inner(); + + let http_client = Client::new(); + + let mut output_file = match File::create_new(&output) { + Ok(fd) => fd, + Err(err) => { + spinner.finish_and_clear(); + Output::new() + .line(format!("Failed to create output file: {err}")) + .tip("the export is available to download in the jobs manage page") + .eprint(); + return Ok(ExitCode::FAILURE); + } + }; + + let zip_file_path = temp_dir().join(format!("{}.zip", output.display())); + let mut zip_file = match File::create_new(&zip_file_path) { + Ok(fd) => fd, + Err(err) => { + spinner.finish_and_clear(); + Output::new() + .line(format!("Failed to create output zip file: {err}")) + .tip("the export is available to download in the jobs manage page") + .eprint(); + return Ok(ExitCode::FAILURE); + } + }; + + let mut resp = match http_client.get(presigned_url).send().await { + Ok(res) => res, + Err(err) => { + spinner.finish_and_clear(); + Output::new() + .line(format!( + "Something went wrong while downloading export: {err}" + )) + .tip("the export is available to download in the jobs manage page") + .eprint(); + return Ok(ExitCode::FAILURE); + } + }; + + while let Some(chunk) = resp + .chunk() + .await + .context("error while streaming response body")? + { + zip_file + .write_all(chunk.as_bytes()) + .context("failed to write to output file")?; + } + zip_file.sync_all()?; + + let mut zip_reader = ZipArchive::new(zip_file).context("failed to read zip")?; + let mut zip_item = zip_reader + .by_index(0) + .context("unexpected empty zip file")?; + io::copy(&mut zip_item, &mut output_file)?; + output_file.sync_all()?; + + let _ = remove_file(zip_file_path); + + spinner.finish_and_clear(); + + Output::new() + .line(format!("{} download", "Completed".green())) + .tip(format!( + "download can be located at '{}'", + output.display().to_string().cyan() + )) + .print(); + + Ok(ExitCode::SUCCESS) +} diff --git a/rust/crates/sift_cli/src/cmd/import/csv.rs b/rust/crates/sift_cli/src/cmd/import/csv.rs new file mode 100644 index 000000000..627556efd --- /dev/null +++ b/rust/crates/sift_cli/src/cmd/import/csv.rs @@ -0,0 +1,620 @@ +use std::{ + collections::{HashMap, HashSet}, + fs::File, + io::{self, Seek}, + process::ExitCode, +}; + +use anyhow::{Context as AnyhowContext, Result, anyhow}; +use chrono::DateTime; +use crossterm::style::Stylize; +use pbjson_types::Timestamp; +use reqwest::header::{CONTENT_ENCODING, CONTENT_TYPE}; +use sift_rs::{ + common::r#type::v1::{ChannelConfig, ChannelDataType}, + data_imports::v2::{ + CreateDataImportFromUploadRequest, CreateDataImportFromUploadResponse, CsvConfig, + CsvTimeColumn, TimeFormat as PbTimeFormat, + data_import_service_client::DataImportServiceClient, + }, +}; + +use crate::{ + cli::ImportCsvArgs, + cmd::{ + Context, + import::utils::{try_parse_bit_field_config, try_parse_enum_config}, + }, + util::{ + api::{create_grpc_channel, create_rest_client}, + tty::Output, + }, +}; + +use super::{ + preview_import_config, + utils::{gzip_file, validate_time_format}, + wait_for_job_completion, +}; + +pub async fn run(ctx: Context, args: ImportCsvArgs) -> Result { + let mut csv_file = File::open(&args.path)?; + let csv_reader = csv::ReaderBuilder::new() + .has_headers(false) + .from_reader(&csv_file); + + let grpc_channel = create_grpc_channel(&ctx)?; + let create_data_import_req = create_data_import_request(csv_reader, &args)?; + let mut data_imports_client = DataImportServiceClient::new(grpc_channel.clone()); + + if args.preview { + let csv_conf = create_data_import_req.csv_config.unwrap(); + + let channel_confs = csv_conf + .data_columns + .values() + .collect::>(); + + preview_import_config( + &csv_conf.asset_name, + if csv_conf.run_id.is_empty() { + csv_conf.run_name.as_str() + } else { + csv_conf.run_id.as_str() + }, + &channel_confs, + ); + + return Ok(ExitCode::SUCCESS); + } + + let CreateDataImportFromUploadResponse { upload_url, .. } = data_imports_client + .create_data_import_from_upload(create_data_import_req) + .await + .context("error creating data import")? + .into_inner(); + + csv_file.rewind()?; + let compressed_data = gzip_file(csv_file)?; + + let rest_client = create_rest_client(&ctx)?; + let res = rest_client + .post(upload_url) + .header(CONTENT_ENCODING, "gzip") + .header(CONTENT_TYPE, "text/csv") + .body(compressed_data) + .send() + .await + .context("failed to upload CSV file")?; + + if !res.status().is_success() { + let status = res.status(); + let text = res + .text() + .await + .unwrap_or_else(|_| "".into()); + return Err(anyhow!( + "failed to upload CSV with http status {status}: {text}" + )); + } + + let location = args.run.as_ref().map_or_else( + || format!("asset '{}'", args.asset.cyan()), + |r| format!("run '{}'", r.clone().cyan()), + ); + + if !args.wait { + Output::new() + .line(format!("{} file for processing", "Uploaded".green())) + .tip(format!( + "Once processing is complete the data will be available on the {location}." + )) + .print(); + + return Ok(ExitCode::SUCCESS); + } + wait_for_job_completion(grpc_channel, location).await +} + +fn create_data_import_request( + csv_reader: csv::Reader, + args: &ImportCsvArgs, +) -> Result { + let num_overrides = args.channel_column.len(); + + if ![ + args.data_type.len(), + args.unit.len(), + args.description.len(), + ] + .iter() + .all(|n| *n == num_overrides) + { + return Err(anyhow!( + "occurrences of --data-type, --units, and --descriptions must equal --channel-column" + )) + .context("keep in mind that --units and --descriptions can be empty strings"); + } + + validate_time_format(args.time_format, &args.relative_start_time)?; + + let relative_start_time = match &args.relative_start_time { + Some(start) => { + let rs = DateTime::parse_from_rfc3339(start) + .context("--relative-start-time is not valid RFC3339")?; + let utc = rs.to_utc(); + Some(Timestamp::from(utc)) + } + None => None, + }; + + if args.header_row == 0 { + return Err(anyhow!("--header-row cannot be 0 due to 1-based indexing")); + } + if args.first_data_row == 0 { + return Err(anyhow!( + "--first-data-row cannot be 0 due to 1-based indexing" + )); + } + if args.header_row >= args.first_data_row { + return Err(anyhow!("--header-row must come before --first-data-row")); + } + + let data_types = args + .data_type + .iter() + .map(|dt| dt.clone().into()) + .collect::>(); + + let mut enum_configs_iter = { + let mut parsed_enum_configs = Vec::with_capacity(args.enum_config.len()); + + for config in &args.enum_config { + let parsed = try_parse_enum_config(config)?; + parsed_enum_configs.push(parsed); + } + parsed_enum_configs.into_iter() + }; + + let mut bit_field_configs_iter = { + let mut parsed_bit_field_configs = Vec::with_capacity(args.bit_field_config.len()); + + for config in &args.bit_field_config { + let parsed = try_parse_bit_field_config(config)?; + parsed_bit_field_configs.push(parsed); + } + parsed_bit_field_configs.into_iter() + }; + let mut records_iter = csv_reader.into_records().enumerate(); + let mut current_row = 1; + + // Find the header row + let headers = { + let mut values = Vec::new(); + + while current_row < args.header_row { + current_row += 1; + records_iter.next(); + } + + let Some((idx, header_row)) = records_iter.next() else { + return Err(anyhow!( + "CSV prematurely reached EOF while looking for header row" + )) + .context("double check --header-row"); + }; + current_row += 1; + let row_num = idx + 1; + + let parsed_record = header_row.context(anyhow!("failed to parse row {row_num}"))?; + + for col in &parsed_record { + values.push(col.to_string()); + } + values + }; + if headers.is_empty() { + return Err(anyhow!("no headers were found given the --header-row")); + } + if headers.len() < 2 { + return Err(anyhow!( + "expected at least two columns: a timestamp column and a channel column" + )); + } + let num_columns = headers.len(); + + let mut channel_columns_set = HashSet::new(); + + let data_columns = { + let mut values = HashMap::::new(); + + while current_row < args.first_data_row { + if records_iter.next().is_none() { + return Err(anyhow!( + "CSV reached EOF with the provided --first-data-row" + )); + } + current_row += 1; + } + + // Create a config for every single column + for (i, record) in records_iter { + // All data columns have been accounted for + if values.len() == num_columns - 1 { + break; + } + let row_num = i + 1; + + let parsed_record = record.context(anyhow!("failed to parse row {row_num}"))?; + + for (j, col_val) in parsed_record.iter().enumerate() { + let col_num = j + 1; + + if col_num == args.time_column { + continue; + } + let name = headers.get(j).unwrap().to_string(); + + if values.contains_key(&(col_num as u32)) { + continue; + } + + // Is there an override specified for a particular column? + if let Some((idx, col)) = args + .channel_column + .iter() + .enumerate() + .find(|(_, col)| **col == col_num) + { + if !channel_columns_set.insert(col) { + return Err(anyhow!( + "cannot have redundant values '{col}' for --channel-column" + )); + } + + // Safe to unwrap all these because of top-level validation ensuring all + // vectors are of equal length with channel_columns; enum and bit filed configs + // follow other validation rules. + let data_type: i32 = { + let raw_data_type = data_types.get(idx).unwrap(); + + if matches!(raw_data_type, ChannelDataType::Unspecified) { + // Maybe a value will be present in a future iteration + if col_val.is_empty() { + continue; + } else if col_val.parse::().is_ok() { + ChannelDataType::Double.into() + } else if col_val.parse::().is_ok() { + ChannelDataType::String.into() + } else { + return Err(anyhow!("failed to infer type of column {col_num}")); + } + } else { + (*raw_data_type).into() + } + }; + + let unit = args.unit.get(idx).unwrap().clone(); + let description = args.description.get(idx).unwrap().clone(); + + let mut enum_configs = Vec::new(); + let mut bit_field_configs = Vec::new(); + + if data_type == ChannelDataType::Enum.into() { + let Some(configs) = enum_configs_iter.next() else { + return Err(anyhow!( + "'{name}' was declared as type enum but --enum-config was not specified" + )); + }; + enum_configs = configs; + } else if data_type == ChannelDataType::BitField.into() { + let Some(configs) = bit_field_configs_iter.next() else { + return Err(anyhow!( + "'{name}' was declared as type bit-field but --bit-field-config was not specified" + )); + }; + bit_field_configs = configs; + } + values.insert( + *col as u32, + ChannelConfig { + name, + description, + data_type, + units: unit, + bit_field_elements: bit_field_configs, + enum_types: enum_configs, + ..Default::default() + }, + ); + } else if col_val.is_empty() { + // Maybe a value will be present in a future iteration + continue; + } else if col_val.parse::().is_ok() { + values.insert( + col_num as u32, + ChannelConfig { + name, + data_type: ChannelDataType::Double.into(), + ..Default::default() + }, + ); + } else { + values.insert( + col_num as u32, + ChannelConfig { + name, + data_type: ChannelDataType::String.into(), + ..Default::default() + }, + ); + } + } + } + values + }; + + for col_num in &args.channel_column { + if !channel_columns_set.contains(col_num) { + return Err(anyhow!( + "an override was specified for column {col_num} but it doesn't refer to a channel" + )); + } + } + + Ok(CreateDataImportFromUploadRequest { + csv_config: Some(CsvConfig { + asset_name: args.asset.clone(), + data_columns, + first_data_row: args.first_data_row as u32, + run_name: args.run.clone().unwrap_or_default(), + time_column: Some(CsvTimeColumn { + relative_start_time, + column_number: args.time_column as u32, + format: PbTimeFormat::from(args.time_format).into(), + }), + ..Default::default() + }), + ..Default::default() + }) +} + +#[cfg(test)] +mod test_create_data_import_request { + use std::path::PathBuf; + + use crate::cli::{ImportCsvArgs, channel::DataType, time::TimeFormat}; + use indoc::indoc; + use sift_rs::{ + common::r#type::v1::ChannelDataType, data_imports::v2::TimeFormat as PbTimeFormat, + }; + + use super::create_data_import_request; + + #[test] + fn simple_case() { + let test_csv = indoc! {" + time,channel + 2025-10-04T21:58:13Z,1.0 + "}; + let csv_reader = csv::ReaderBuilder::new() + .has_headers(false) + .from_reader(test_csv.as_bytes()); + + let req = create_data_import_request( + csv_reader, + &ImportCsvArgs { + path: PathBuf::default(), + asset: "test_asset".into(), + run: None, + header_row: 1, + first_data_row: 2, + channel_column: Vec::default(), + data_type: Vec::default(), + unit: Vec::default(), + description: Vec::default(), + enum_config: Vec::default(), + bit_field_config: Vec::default(), + time_column: 1, + time_format: TimeFormat::default(), + relative_start_time: None, + wait: false, + preview: false, + }, + ) + .expect("expected Result::Ok"); + + let csv_config = req.csv_config.expect("expected Option::Some"); + assert_eq!(String::from("test_asset"), csv_config.asset_name); + assert!(csv_config.run_id.is_empty()); + assert!(csv_config.run_name.is_empty()); + assert_eq!(2, csv_config.first_data_row); + + let time_config = csv_config.time_column.unwrap(); + assert_eq!(1, time_config.column_number); + assert_eq!(PbTimeFormat::AbsoluteRfc3339 as i32, time_config.format); + assert!(time_config.relative_start_time.is_none()); + + assert_eq!(1, csv_config.data_columns.len()); + let config = csv_config.data_columns.get(&2).unwrap(); + assert_eq!(ChannelDataType::Double, config.data_type()); + assert_eq!(String::from("channel"), config.name); + } + + #[test] + fn simple_type_override() { + let test_csv = indoc! {" + time,channel + 2025-10-04T21:58:13Z,1.0 + "}; + let csv_reader = csv::ReaderBuilder::new() + .has_headers(false) + .from_reader(test_csv.as_bytes()); + + let req = create_data_import_request( + csv_reader, + &ImportCsvArgs { + path: PathBuf::default(), + asset: "test_asset".into(), + run: Some("test_run".into()), + header_row: 1, + first_data_row: 2, + channel_column: vec![2], + data_type: vec![DataType::Float], + unit: vec!["km/hr".into()], + description: vec!["some_description".into()], + enum_config: Vec::default(), + bit_field_config: Vec::default(), + time_column: 1, + time_format: TimeFormat::default(), + relative_start_time: None, + wait: false, + preview: false, + }, + ) + .expect("expected Result::Ok"); + + let csv_config = req.csv_config.expect("expected Option::Some"); + assert_eq!(1, csv_config.data_columns.len()); + let config = csv_config.data_columns.get(&2).unwrap(); + assert_eq!(String::from("test_run"), csv_config.run_name); + assert_eq!(ChannelDataType::Float, config.data_type()); + assert_eq!(String::from("channel"), config.name); + assert_eq!(String::from("km/hr"), config.units); + assert_eq!(String::from("some_description"), config.description); + } + + #[test] + fn enum_type_override() { + let test_csv = indoc! {" + time,channel + 2025-10-04T21:58:13Z,0 + 2025-10-04T21:58:13Z,1 + "}; + let csv_reader = csv::ReaderBuilder::new() + .has_headers(false) + .from_reader(test_csv.as_bytes()); + + let req = create_data_import_request( + csv_reader, + &ImportCsvArgs { + path: PathBuf::default(), + asset: "test_asset".into(), + run: Some("test_run".into()), + header_row: 1, + first_data_row: 2, + channel_column: vec![2], + data_type: vec![DataType::Enum], + unit: vec![String::new()], + description: vec![String::new()], + enum_config: vec!["0,stop|1,go".into()], + bit_field_config: Vec::default(), + time_column: 1, + time_format: TimeFormat::default(), + relative_start_time: None, + wait: false, + preview: false, + }, + ) + .expect("expected Result::Ok"); + + let csv_config = req.csv_config.expect("expected Option::Some"); + assert_eq!(1, csv_config.data_columns.len()); + let config = csv_config.data_columns.get(&2).unwrap(); + assert_eq!(ChannelDataType::Enum, config.data_type()); + assert_eq!(String::from("channel"), config.name); + assert!(config.units.is_empty()); + assert!(config.description.is_empty()); + assert_eq!(2, config.enum_types.len()); + assert!( + config + .enum_types + .iter() + .find(|c| c.name == "stop" && c.key == 0) + .is_some() + ); + assert!( + config + .enum_types + .iter() + .find(|c| c.name == "go" && c.key == 1) + .is_some() + ); + } + + #[test] + fn multi_channel_with_overrides_and_empty_cells() { + // string_channel will have no override and will be inferred + let test_csv = indoc! {" + time,float_channel,enum_channel,string_channel + 2025-10-04T21:58:13Z,1.0,, + 2025-10-04T21:58:14Z,1.2,0, + 2025-10-04T21:58:14Z,,1,cthulhu + "}; + let csv_reader = csv::ReaderBuilder::new() + .has_headers(false) + .from_reader(test_csv.as_bytes()); + + let req = create_data_import_request( + csv_reader, + &ImportCsvArgs { + path: PathBuf::default(), + asset: "test_asset".into(), + run: Some("test_run".into()), + header_row: 1, + first_data_row: 2, + channel_column: vec![2, 3], + data_type: vec![DataType::Float, DataType::Enum], + unit: vec!["km/hr".into(), String::new()], + description: vec!["float channel".into(), "enum channel".into()], + enum_config: vec!["0,stop|1,go".into()], + bit_field_config: Vec::default(), + time_column: 1, + time_format: TimeFormat::default(), + relative_start_time: None, + wait: false, + preview: false, + }, + ) + .expect("expected Result::Ok"); + + let csv_config = req.csv_config.expect("expected Option::Some"); + assert_eq!(3, csv_config.data_columns.len()); + + // enum channel + let config = csv_config.data_columns.get(&3).unwrap(); + assert_eq!(ChannelDataType::Enum, config.data_type()); + assert_eq!(String::from("enum_channel"), config.name); + assert!(config.units.is_empty()); + assert_eq!("enum channel".to_string(), config.description); + assert_eq!(2, config.enum_types.len()); + assert!( + config + .enum_types + .iter() + .find(|c| c.name == "stop" && c.key == 0) + .is_some() + ); + assert!( + config + .enum_types + .iter() + .find(|c| c.name == "go" && c.key == 1) + .is_some() + ); + + // float channel + let config = csv_config.data_columns.get(&2).unwrap(); + assert_eq!(ChannelDataType::Float, config.data_type()); + assert_eq!(String::from("float_channel"), config.name); + assert_eq!("km/hr".to_string(), config.units); + assert_eq!("float channel".to_string(), config.description); + + // string channel + let config = csv_config.data_columns.get(&4).unwrap(); + assert_eq!(ChannelDataType::String, config.data_type()); + assert_eq!(String::from("string_channel"), config.name); + assert!(config.units.is_empty()); + assert!(config.description.is_empty()); + } +} diff --git a/rust/crates/sift_cli/src/cmd/import/mod.rs b/rust/crates/sift_cli/src/cmd/import/mod.rs new file mode 100644 index 000000000..b979c1808 --- /dev/null +++ b/rust/crates/sift_cli/src/cmd/import/mod.rs @@ -0,0 +1,178 @@ +use anyhow::Result; +use crossterm::style::Stylize; +use std::{process::ExitCode, time::Duration}; +use tokio::time::sleep; + +use sift_rs::{ + SiftChannel, + common::r#type::v1::ChannelConfig, + jobs::v1::{JobStatus, JobType}, +}; + +use crate::util::{job::JobServiceWrapper, progress::Spinner, tty::Output, user::get_user_id}; + +pub mod csv; +pub mod parquet; +mod utils; + +const INDENT_1: &str = " "; +const INDENT_2: &str = " "; +const INDENT_3: &str = " "; +const INDENT_4: &str = " "; + +pub async fn wait_for_job_completion( + grpc_channel: SiftChannel, + import_output_location: String, +) -> Result { + let spinner = Spinner::new(); + spinner.set_message(format!("{} file for processing", "Uploaded".green())); + + let user_id = get_user_id(grpc_channel.clone()).await?; + let mut job_service = JobServiceWrapper::new(grpc_channel.clone()); + + let Some(mut job) = job_service + .get_latest_job_for_user(&user_id, JobType::DataImport) + .await? + else { + spinner.finish_and_clear(); + + Output::new() + .line("The file was successfully uploaded but the job was unexpectedly not found") + .tip("Please notify Sift about this bug") + .eprint(); + return Ok(ExitCode::FAILURE); + }; + + loop { + sleep(Duration::from_secs(3)).await; + + let Some(updated_job) = job_service.get_job(&job.job_id).await? else { + spinner.finish_and_clear(); + Output::new() + .line("The file was successfully uploaded but the job was unexpectedly not found") + .tip("Please notify Sift about this bug") + .eprint(); + return Ok(ExitCode::FAILURE); + }; + job = updated_job; + + match job.job_status() { + JobStatus::Created => (), + JobStatus::Running => { + spinner.set_message(format!("{} imported file", "Processing".green())); + } + JobStatus::CancelRequested => { + spinner.set_message(format!( + "{} was requested but the job may still finish", + "Cancellation".green() + )); + } + JobStatus::Cancelled => { + spinner.finish_and_clear(); + Output::new() + .line(format!("{} data import job", "Cancelled".green())) + .print(); + break; + } + JobStatus::Failed => { + spinner.finish_and_clear(); + Output::new() + .line("Processing failed") + .tip("Please check the Sift jobs manage page for further details") + .eprint(); + return Ok(ExitCode::FAILURE); + } + JobStatus::Finished => { + spinner.finish_and_clear(); + Output::new() + .line(format!("{} data import job", "Completed".green())) + .tip(format!( + "The data should be available on the {import_output_location}" + )) + .print(); + break; + } + _ => (), + } + } + Ok(ExitCode::SUCCESS) +} + +fn preview_import_config(asset: &str, run: &str, channel_configs: &[&ChannelConfig]) { + let mut asset_run = Output::new(); + asset_run.line(format!("{}: {asset}", "Asset".green())); + + if !run.is_empty() { + asset_run.line(format!("{}: {run}", "Run".green())); + } + asset_run.print(); + + let mut configs = Output::new(); + + if channel_configs.is_empty() { + configs.line(format!("{}: {{}}", "Channels".green())); + configs.print(); + return; + } + + configs.line(format!("{}: {{", "Channels".green())); + + for conf in channel_configs { + configs.line(format!( + "{INDENT_1}{:<27} {}", + conf.data_type().as_str_name(), + conf.name.clone().cyan() + )); + + if !conf.units.is_empty() { + configs.line(format!("{INDENT_2}{} {}", "units".yellow(), conf.units)); + } + + if !conf.description.is_empty() { + configs.line(format!( + "{INDENT_2}{} {}", + "description".yellow(), + conf.description + )); + } + + if !conf.enum_types.is_empty() { + configs.line(format!("{INDENT_2}{} {{", "enum-config".yellow())); + + let mut enum_confs = Vec::new(); + + for enum_conf in &conf.enum_types { + enum_confs.push((enum_conf.key, &enum_conf.name)); + } + enum_confs.sort_by(|(k_a, _), (k_b, _)| k_a.cmp(k_b)); + + for (k, v) in enum_confs { + configs.line(format!("{INDENT_3}{k} => {v}")); + } + configs.line(format!("{INDENT_2}}}")); + } + + if !conf.bit_field_elements.is_empty() { + configs.line(format!("{INDENT_2}{} {{", "bit-field-elements".yellow())); + + let mut bit_field_elements = Vec::new(); + + for el in &conf.bit_field_elements { + bit_field_elements.push((el.index, el.bit_count, &el.name)); + } + bit_field_elements.sort_by(|(k_a, _, _), (k_b, _, _)| k_a.cmp(k_b)); + + for (idx, length, name) in bit_field_elements { + configs + .line(format!("{INDENT_3}{{")) + .line(format!("{INDENT_4}element: {name}")) + .line(format!("{INDENT_4}index: {idx}")) + .line(format!("{INDENT_4}length: {length}")) + .line(format!("{INDENT_3}}}")); + } + configs.line(format!("{INDENT_2}}}")); + } + } + configs.line("}"); + configs.print(); +} diff --git a/rust/crates/sift_cli/src/cmd/import/parquet/flat_dataset.rs b/rust/crates/sift_cli/src/cmd/import/parquet/flat_dataset.rs new file mode 100644 index 000000000..c3de1d8d6 --- /dev/null +++ b/rust/crates/sift_cli/src/cmd/import/parquet/flat_dataset.rs @@ -0,0 +1,291 @@ +use std::{collections::HashMap, fs::File, io::Seek, process::ExitCode}; + +use anyhow::{Context as AnyhowContext, Result, anyhow}; +use chrono::DateTime; +use crossterm::style::Stylize; +use pbjson_types::Timestamp; +use reqwest::header::{CONTENT_ENCODING, CONTENT_TYPE}; +use sift_rs::{ + common::r#type::v1::{ChannelConfig, ChannelDataType}, + data_imports::v2::{ + CreateDataImportFromUploadRequest, CreateDataImportFromUploadResponse, DataTypeKey, + DetectConfigRequest, ParquetComplexTypesImportMode, ParquetConfig, ParquetTimeColumn, + TimeFormat, data_import_service_client::DataImportServiceClient, parquet_config::Config, + }, +}; + +use crate::{ + cli::{FlatDatasetArgs, channel::DataType}, + cmd::{ + Context, + import::{ + parquet::{FooterMetadata, get_footer}, + preview_import_config, + utils::{ + gzip_file, try_parse_bit_field_config, try_parse_enum_config, validate_time_format, + }, + wait_for_job_completion, + }, + }, + util::{ + api::{create_grpc_channel, create_rest_client}, + tty::Output, + }, +}; + +pub async fn run(ctx: Context, args: FlatDatasetArgs) -> Result { + let grpc_channel = create_grpc_channel(&ctx)?; + let mut data_imports_client = DataImportServiceClient::new(grpc_channel.clone()); + let mut file = File::open(&args.path).context("failed to open Parquet file")?; + let footer_md = FooterMetadata::try_from(&mut file)?; + + let mut config = { + let footer = get_footer(&mut file, footer_md)?; + let resp = data_imports_client + .detect_config(DetectConfigRequest { + data: footer, + r#type: DataTypeKey::ParquetFlatdataset.into(), + }) + .await + .context("failed to parse Parquet schema")? + .into_inner(); + + resp.parquet_config + .ok_or(anyhow!("unexpected empty Parquet config"))? + }; + + update_config_with_overrides(&mut config, &args)?; + let create_data_import_req = create_data_import_request(&args, config, footer_md)?; + + if args.preview { + let parquet_conf = create_data_import_req.parquet_config.unwrap(); + let Config::FlatDataset(flatset_conf) = parquet_conf.config.unwrap(); + + let channel_confs = flatset_conf + .data_columns + .iter() + .filter_map(|col| col.channel_config.as_ref()) + .collect::>(); + + preview_import_config( + &parquet_conf.asset_name, + if parquet_conf.run_id.is_empty() { + parquet_conf.run_name.as_str() + } else { + parquet_conf.run_id.as_str() + }, + &channel_confs, + ); + return Ok(ExitCode::SUCCESS); + } + + let CreateDataImportFromUploadResponse { upload_url, .. } = data_imports_client + .create_data_import_from_upload(create_data_import_req) + .await + .context("error creating data import")? + .into_inner(); + + file.rewind()?; + let compressed_data = gzip_file(file)?; + + let rest_client = create_rest_client(&ctx)?; + let res = rest_client + .post(upload_url) + .header(CONTENT_ENCODING, "gzip") + .header(CONTENT_TYPE, "application/vnd.apache.parquet") + .body(compressed_data) + .send() + .await + .context("failed to upload Parquet file")?; + + if !res.status().is_success() { + let status = res.status(); + let text = res + .text() + .await + .unwrap_or_else(|_| "".into()); + return Err(anyhow!( + "failed to upload Parquet with http status {status}: {text}" + )); + } + + let location = args.run.as_ref().map_or_else( + || format!("asset '{}'", args.asset.cyan()), + |r| format!("run '{}'", r.clone().cyan()), + ); + + if !args.wait { + Output::new() + .line(format!("{} file for processing", "Uploaded".green())) + .tip(format!( + "Once processing is complete the data will be available on the {location}." + )) + .print(); + + return Ok(ExitCode::SUCCESS); + } + wait_for_job_completion(grpc_channel, location).await +} + +fn update_config_with_overrides( + parquet_config: &mut ParquetConfig, + args: &FlatDatasetArgs, +) -> Result<()> { + let Some(Config::FlatDataset(flat_dataset_conf)) = parquet_config.config.as_mut() else { + return Err(anyhow!("unexpected missing Parquet file config")); + }; + if flat_dataset_conf.data_columns.is_empty() { + return Err(anyhow!( + "failed to find any channel data columns in the provided Parquet file" + )); + } + validate_time_format(args.time_format, &args.relative_start_time)?; + + let relative_start_time = match &args.relative_start_time { + Some(start) => { + let rs = DateTime::parse_from_rfc3339(start) + .context("--relative-start-time is not valid RFC3339")?; + let utc = rs.to_utc(); + Some(Timestamp::from(utc)) + } + None => None, + }; + + flat_dataset_conf.time_column = Some(ParquetTimeColumn { + relative_start_time, + path: args.time_path.clone(), + format: TimeFormat::from(args.time_format).into(), + }); + + let num_overrides = args.channel_path.len(); + + if ![ + args.data_type.len(), + args.unit.len(), + args.description.len(), + ] + .iter() + .all(|n| *n == num_overrides) + { + return Err(anyhow!( + "occurrences of --data-type, --units, and --descriptions must equal --channel-paths" + )) + .context("keep in mind that --units and --descriptions can be empty strings"); + } + + if num_overrides == 0 { + return Ok(()); + } + let path_index_lookup = { + let mut lookup = HashMap::new(); + for (i, config) in flat_dataset_conf.data_columns.iter().enumerate() { + lookup.insert(config.path.clone(), i); + } + lookup + }; + + let mut enum_configs_iter = { + let mut parsed_enum_configs = Vec::with_capacity(args.enum_config.len()); + + for config in &args.enum_config { + let parsed = try_parse_enum_config(config)?; + parsed_enum_configs.push(parsed); + } + parsed_enum_configs.into_iter() + }; + + let mut bit_field_configs_iter = { + let mut parsed_bit_field_configs = Vec::with_capacity(args.bit_field_config.len()); + + for config in &args.bit_field_config { + let parsed = try_parse_bit_field_config(config)?; + parsed_bit_field_configs.push(parsed); + } + parsed_bit_field_configs.into_iter() + }; + + for (i, channel) in args.channel_path.iter().enumerate() { + let Some(idx) = path_index_lookup.get(channel) else { + return Err(anyhow!( + "override for {channel} was specified but it wasn't found in the Parquet file" + )); + }; + + let dt = args.data_type.get(i).unwrap(); + let units = args.unit.get(i).unwrap(); + let description = args.description.get(i).unwrap(); + + let mut updated_config = ChannelConfig { + name: channel.clone(), + units: units.clone(), + description: description.clone(), + ..Default::default() + }; + + match dt { + DataType::Infer => { + updated_config.data_type = flat_dataset_conf + .data_columns + .get(*idx) + .unwrap() + .channel_config + .as_ref() + .unwrap() + .data_type; + } + DataType::BitField => { + let Some(bf_conf) = bit_field_configs_iter.next() else { + return Err(anyhow!( + "'{channel}' was declared as type bit-field but --bit-field-config was not specified" + )); + }; + updated_config.data_type = ChannelDataType::BitField.into(); + updated_config.bit_field_elements = bf_conf; + } + DataType::Enum => { + let Some(enum_conf) = enum_configs_iter.next() else { + return Err(anyhow!( + "'{channel}' was declared as type enum but --enum-config was not specified" + )); + }; + updated_config.data_type = ChannelDataType::Enum.into(); + updated_config.enum_types = enum_conf; + } + _ => updated_config.data_type = ChannelDataType::from(dt.clone()).into(), + } + + let target = flat_dataset_conf + .data_columns + .get_mut(*idx) + .unwrap() + .channel_config + .as_mut() + .unwrap(); + *target = updated_config; + } + Ok(()) +} + +fn create_data_import_request( + args: &FlatDatasetArgs, + config: ParquetConfig, + footer_md: FooterMetadata, +) -> Result { + let req = CreateDataImportFromUploadRequest { + parquet_config: Some(ParquetConfig { + asset_name: args.asset.clone(), + run_name: args.run.clone().unwrap_or_default(), + footer_offset: footer_md.offset, + footer_length: u32::try_from(footer_md.length) + .context("parquet footer length too large")?, + complex_types_import_mode: ParquetComplexTypesImportMode::from( + args.complex_types_mode.clone(), + ) + .into(), + config: config.config, + ..Default::default() + }), + ..Default::default() + }; + Ok(req) +} diff --git a/rust/crates/sift_cli/src/cmd/import/parquet/mod.rs b/rust/crates/sift_cli/src/cmd/import/parquet/mod.rs new file mode 100644 index 000000000..ae18fb6c8 --- /dev/null +++ b/rust/crates/sift_cli/src/cmd/import/parquet/mod.rs @@ -0,0 +1,57 @@ +use anyhow::{Result, anyhow}; +use std::{ + fs::File, + io::{Read, Seek, SeekFrom}, + os::unix::fs::FileExt, +}; + +pub mod flat_dataset; + +/// Metadata about the Parquet's footer +#[derive(Copy, Clone)] +struct FooterMetadata { + /// Offset to the Parquet file's footer + offset: u64, + /// Length of the Parquet file's footer + length: u64, +} + +fn get_footer(file: &mut File, footer_metadata: FooterMetadata) -> Result> { + let FooterMetadata { length, offset } = footer_metadata; + let mut buf = vec![0u8; length as usize]; + file.read_exact_at(&mut buf, offset)?; + Ok(buf) +} + +/// Note that this will advance the cursor. +impl TryFrom<&mut File> for FooterMetadata { + type Error = anyhow::Error; + + fn try_from(file: &mut File) -> Result { + file.seek(SeekFrom::End(-8))?; + + let footer_len = { + let mut buf = [0u8; 4]; + file.read_exact(&mut buf)?; + u32::from_le_bytes(buf) + }; + + let mut magic = [0u8; 4]; + file.read_exact(&mut magic)?; + if &magic != b"PAR1" { + return Err(anyhow!("invalid Parquet magic bytes")); + } + + let file_len = file.metadata()?.len(); + if u64::from(footer_len) + 8 > file_len { + return Err(anyhow!( + "footer length ({footer_len}) exceeds file size ({file_len})", + )); + } + + Ok(Self { + offset: file_len - u64::from(footer_len) - 8, + length: u64::from(footer_len), + }) + } +} diff --git a/rust/crates/sift_cli/src/cmd/import/utils.rs b/rust/crates/sift_cli/src/cmd/import/utils.rs new file mode 100644 index 000000000..a1e5bff98 --- /dev/null +++ b/rust/crates/sift_cli/src/cmd/import/utils.rs @@ -0,0 +1,102 @@ +use std::{ + fs::File, + io::{self, BufReader, Read}, +}; + +use anyhow::{Context, Result, anyhow}; +use flate2::{Compression, write::GzEncoder}; +use sift_rs::common::r#type::v1::{ChannelBitFieldElement, ChannelEnumType}; + +use crate::cli::time::TimeFormat; + +/// Be sure that the file's cursor is rewinded to the start before hand. +pub fn gzip_file(file: File) -> Result> { + let mut reader = BufReader::new(file); + let mut buffer = Vec::new(); + reader.read_to_end(&mut buffer)?; + + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + io::copy(&mut buffer.as_slice(), &mut encoder)?; + let compressed_data = encoder.finish()?; + Ok(compressed_data) +} + +pub fn validate_time_format( + time_format: TimeFormat, + relative_start_time: &Option, +) -> Result<()> { + match time_format { + TimeFormat::RelativeNanoseconds + | TimeFormat::RelativeMicroseconds + | TimeFormat::RelativeMilliseconds + | TimeFormat::RelativeSeconds + | TimeFormat::RelativeMinutes + | TimeFormat::RelativeHours => { + if relative_start_time.is_none() { + return Err(anyhow!( + "--relative-start-time is required if time format is relative" + )); + } + Ok(()) + } + _ => Ok(()), + } +} + +pub fn try_parse_enum_config(val: &str) -> Result> { + let values = val.split("|").collect::>(); + + if values.is_empty() { + return Err(anyhow!("blank --enum-config argument not allowed")); + } + + let mut result = Vec::new(); + for key_value in values { + let parts = key_value.split(",").collect::>(); + if parts.len() != 2 { + return Err(anyhow!( + "expected --enum-config argument to contain pairs delimited by \"|\"" + )) + .context(format!("bad argument: {val}")); + } + let key = parts[0].parse::() + .with_context(|| format!("expected first value in comma-separated list for enum config to be a number for '{val}'"))?; + let name = parts[1].to_string(); + + result.push(ChannelEnumType { + key, + name, + ..Default::default() + }) + } + Ok(result) +} + +pub fn try_parse_bit_field_config(val: &str) -> Result> { + let values = val.split("|").collect::>(); + + if values.is_empty() { + return Err(anyhow!("blank --bit-field-config argument not allowed")); + } + + let mut result = Vec::new(); + for element in values { + let parts = element.split(",").collect::>(); + if parts.len() != 3 { + return Err(anyhow!("expected --bit-field-config argument to contain triplets delimited by \"|\"")) + .context(format!("bad argument: {val}")); + } + let name = parts[0].to_string(); + let index = parts[1].parse::() + .with_context(|| format!("expected first value in comma-separated list for bit-field config to be a number for '{val}'"))?; + let bit_count = parts[2].parse::() + .with_context(|| format!("expected third value in comma-separated list for bit-field config to be a number for '{val}'"))?; + + result.push(ChannelBitFieldElement { + name, + index, + bit_count, + }); + } + Ok(result) +} diff --git a/rust/crates/sift_cli/src/cmd/mod.rs b/rust/crates/sift_cli/src/cmd/mod.rs new file mode 100644 index 000000000..c508a4d6e --- /dev/null +++ b/rust/crates/sift_cli/src/cmd/mod.rs @@ -0,0 +1,102 @@ +use std::{fs::read_to_string, io::ErrorKind}; + +use anyhow::{Context as AnyhowContext, Result, anyhow}; +use crossterm::style::Stylize; +use toml::{Table, Value}; + +pub mod completions; +pub mod config; +pub mod export; +pub mod import; + +pub struct Context { + pub grpc_uri: String, + pub api_key: String, + pub disable_tls: bool, + + #[allow(dead_code)] + pub rest_uri: String, +} + +impl Context { + pub fn new(profile: Option, disable_tls: bool) -> Result { + let config_path = config::get_config_file_path()?; + let p = config_path.display().to_string(); + + let config_txt = match read_to_string(config_path) { + Ok(txt) => txt, + Err(err) => match err.kind() { + ErrorKind::NotFound => { + return Err(anyhow!("expected to find '{}'.", p.yellow())).context(format!( + "Create a config using '{}'.", + "sift_cli config create".green() + )); + } + _ => return Err(anyhow!("failed to read config file")), + }, + }; + + let config_toml = config_txt + .parse::
() + .context("failed to parse config file")?; + + let target_profile = match profile { + Some(prof) => { + let Some(Value::Table(target)) = config_toml.get(&prof) else { + return Err(anyhow!( + "Profile '{}' not found or not a TOML table.", + prof.yellow() + )); + }; + target + } + None => &config_toml, + }; + + let Some(Value::String(grpc_uri)) = target_profile.get("grpc_uri").cloned() else { + return Err(anyhow!( + "Expected value of '{}' to be a string", + "grpc_uri".yellow() + )); + }; + if grpc_uri.is_empty() { + return Err(anyhow!( + "Expected value of '{}' to be present", + "grpc_uri".yellow() + )); + } + + let Some(Value::String(rest_uri)) = target_profile.get("rest_uri").cloned() else { + return Err(anyhow!( + "Expected value of '{}' to be a string", + "rest_uri".yellow() + )); + }; + if rest_uri.is_empty() { + return Err(anyhow!( + "Expected value of '{}' to be present", + "rest_uri".yellow() + )); + } + + let Some(Value::String(api_key)) = target_profile.get("apikey").cloned() else { + return Err(anyhow!( + "Expected value of '{}' to be a string", + "apikey".yellow() + )); + }; + if api_key.is_empty() { + return Err(anyhow!( + "Expected value of '{}' to be present", + "apikey".yellow() + )); + } + + Ok(Self { + grpc_uri, + rest_uri, + api_key, + disable_tls, + }) + } +} diff --git a/rust/crates/sift_cli/src/main.rs b/rust/crates/sift_cli/src/main.rs new file mode 100644 index 000000000..210d7a81b --- /dev/null +++ b/rust/crates/sift_cli/src/main.rs @@ -0,0 +1,83 @@ +use anyhow::{Context as AnyhowContext, Result}; +use crossterm::style::Stylize; +use std::process::ExitCode; +use tokio::runtime; + +mod cli; +use cli::{Cmd, ConfigCmd}; + +mod cmd; +use cmd::Context; + +mod util; +use util::tty::Output; + +use clap::Parser; + +fn main() -> ExitCode { + let args = cli::Args::parse(); + + match run(args) { + Err(err) => { + Output::new().line(format!("{err:?}")).eprint(); + ExitCode::FAILURE + } + Ok(exit_code) => exit_code, + } +} + +fn run_future(fut: F) -> Result +where + F: Future> + 'static, +{ + let runtime = runtime::Builder::new_current_thread() + .enable_all() + .build() + .context("failed to initialize Tokio runtime")?; + + runtime.block_on(fut) +} + +fn run(clargs: cli::Args) -> Result { + // These commands don't require `Context` + match clargs.cmd { + Cmd::Config(cmd) => match cmd { + ConfigCmd::Show => return cmd::config::show(), + ConfigCmd::Create => return cmd::config::create(), + ConfigCmd::Where => return cmd::config::config_where(), + ConfigCmd::Update(args) => return cmd::config::update(clargs.profile, args), + }, + Cmd::Completions(cmd) => match cmd { + cli::CompletionsCmd::Print(args) => return cmd::completions::print(args), + cli::CompletionsCmd::Update => return cmd::completions::update(), + }, + _ => (), + } + + let profile = clargs + .profile + .as_ref() + .map_or_else(|| "default".to_string().cyan(), |s| s.clone().cyan()); + let ctx = Context::new(clargs.profile, clargs.disable_tls)?; + + Output::new() + .line(format!("{} profile '{profile}'", "Using".green())) + .print(); + + // These commands require `Context` + match clargs.cmd { + Cmd::Import(cmd) => match cmd { + cli::ImportCmd::Csv(args) => run_future(cmd::import::csv::run(ctx, args)), + cli::ImportCmd::Parquet(cmd) => match cmd { + cli::ImportParquetCmd::FlatDataset(args) => { + run_future(cmd::import::parquet::flat_dataset::run(ctx, args)) + } + }, + }, + Cmd::Export(cmd) => match cmd { + cli::ExportCmd::Run(args) => run_future(cmd::export::run(ctx, args)), + cli::ExportCmd::Asset(args) => run_future(cmd::export::asset(ctx, args)), + }, + _ => Ok(ExitCode::SUCCESS), + } +} diff --git a/rust/crates/sift_cli/src/util/api.rs b/rust/crates/sift_cli/src/util/api.rs new file mode 100644 index 000000000..89ae605a5 --- /dev/null +++ b/rust/crates/sift_cli/src/util/api.rs @@ -0,0 +1,36 @@ +use std::collections::HashMap; + +use anyhow::Result; +use reqwest::{ClientBuilder, header::AUTHORIZATION}; +use sift_rs::{Credentials, SiftChannel, SiftChannelBuilder}; + +use crate::cmd::Context; + +pub fn create_grpc_channel( + Context { + grpc_uri, + api_key, + disable_tls, + .. + }: &Context, +) -> Result { + let mut builder = SiftChannelBuilder::new(Credentials::Config { + uri: grpc_uri.into(), + apikey: api_key.into(), + }); + if *disable_tls { + builder = builder.use_tls(false) + } + Ok(builder.build()?) +} + +pub fn create_rest_client(Context { api_key, .. }: &Context) -> Result { + let mut http_headers = HashMap::::new(); + http_headers.insert(AUTHORIZATION.to_string(), format!("Bearer {api_key}")); + + let rest_client = ClientBuilder::new() + .default_headers((&http_headers).try_into()?) + .build()?; + + Ok(rest_client) +} diff --git a/rust/crates/sift_cli/src/util/channel.rs b/rust/crates/sift_cli/src/util/channel.rs new file mode 100644 index 000000000..a5e75c8c9 --- /dev/null +++ b/rust/crates/sift_cli/src/util/channel.rs @@ -0,0 +1,39 @@ +use anyhow::{Context, Result}; +use sift_rs::{ + SiftChannel, + channels::v3::{ + Channel, ListChannelsRequest, ListChannelsResponse, + channel_service_client::ChannelServiceClient, + }, +}; + +pub async fn filter_channels(grpc_channel: SiftChannel, filter: &str) -> Result> { + let mut channel_service = ChannelServiceClient::new(grpc_channel); + let mut page_token = String::new(); + let mut query_result = Vec::new(); + + loop { + let ListChannelsResponse { + channels, + next_page_token, + .. + } = channel_service + .list_channels(ListChannelsRequest { + page_token, + filter: filter.to_string(), + page_size: 1000, + ..Default::default() + }) + .await + .context("failed to query channels")? + .into_inner(); + + query_result.extend(channels.into_iter()); + + if next_page_token.is_empty() { + break; + } + page_token = next_page_token; + } + Ok(query_result) +} diff --git a/rust/crates/sift_cli/src/util/job.rs b/rust/crates/sift_cli/src/util/job.rs new file mode 100644 index 000000000..e24fccc63 --- /dev/null +++ b/rust/crates/sift_cli/src/util/job.rs @@ -0,0 +1,65 @@ +use std::ops::{Deref, DerefMut}; + +use anyhow::{Context, Result}; +use sift_rs::{ + SiftChannel, + jobs::v1::{Job, JobType, ListJobsRequest, job_service_client::JobServiceClient}, +}; + +pub struct JobServiceWrapper(JobServiceClient); + +impl Deref for JobServiceWrapper { + type Target = JobServiceClient; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for JobServiceWrapper { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl JobServiceWrapper { + pub fn new(grpc_channel: SiftChannel) -> Self { + let job_service = JobServiceClient::new(grpc_channel); + JobServiceWrapper(job_service) + } + + pub async fn get_latest_job_for_user( + &mut self, + user_id: &str, + job_type: JobType, + ) -> Result> { + let jt = job_type.as_str_name(); + + let res = self + .list_jobs(ListJobsRequest { + page_size: 1, + filter: format!("job_type == '{jt}' && created_by_user_id == '{user_id}'"), + order_by: "created_date desc".into(), + ..Default::default() + }) + .await + .context("failed to retrieve latest user job")? + .into_inner(); + + Ok(res.jobs.first().cloned()) + } + + pub async fn get_job(&mut self, job_id: &str) -> Result> { + let res = self + .list_jobs(ListJobsRequest { + page_size: 1, + filter: format!("job_id == '{job_id}'"), + ..Default::default() + }) + .await + .context("failed to retrieve job by ID")? + .into_inner(); + + Ok(res.jobs.first().cloned()) + } +} diff --git a/rust/crates/sift_cli/src/util/mod.rs b/rust/crates/sift_cli/src/util/mod.rs new file mode 100644 index 000000000..3165a95f8 --- /dev/null +++ b/rust/crates/sift_cli/src/util/mod.rs @@ -0,0 +1,6 @@ +pub mod api; +pub mod channel; +pub mod job; +pub mod progress; +pub mod tty; +pub mod user; diff --git a/rust/crates/sift_cli/src/util/progress.rs b/rust/crates/sift_cli/src/util/progress.rs new file mode 100644 index 000000000..c0a56b500 --- /dev/null +++ b/rust/crates/sift_cli/src/util/progress.rs @@ -0,0 +1,32 @@ +use std::{ops::Deref, time::Duration}; + +use indicatif::{ProgressBar, ProgressStyle}; + +pub struct Spinner(ProgressBar); + +impl Spinner { + pub fn new() -> Self { + let spinner = ProgressBar::new_spinner().with_style( + ProgressStyle::default_spinner() + .tick_chars("⣾⣷⣯⣟⡿⢿⣻⣽⣾") + .template("{spinner:.green} {msg}") + .unwrap(), + ); + spinner.enable_steady_tick(Duration::from_millis(100)); + Self(spinner) + } +} + +impl Deref for Spinner { + type Target = ProgressBar; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl Drop for Spinner { + fn drop(&mut self) { + self.finish_and_clear(); + } +} diff --git a/rust/crates/sift_cli/src/util/tty.rs b/rust/crates/sift_cli/src/util/tty.rs new file mode 100644 index 000000000..0a3064023 --- /dev/null +++ b/rust/crates/sift_cli/src/util/tty.rs @@ -0,0 +1,103 @@ +use std::io::{self, Write}; + +use anyhow::Result; +use crossterm::style::Stylize; + +#[derive(Default)] +pub struct PromptUser { + header: Option, + prompts: Vec, +} + +impl PromptUser { + pub fn new() -> Self { + Self::default() + } + + pub fn header>(&mut self, header: S) -> &mut Self { + self.header.replace(header.into()); + self + } + + pub fn prompt>(&mut self, prompt: S) -> &mut Self { + self.prompts.push(prompt.into()); + self + } + + pub fn run(&self) -> Result>> { + if self.prompts.is_empty() { + return Ok(Vec::new()); + } + let mut user_inputs = Vec::with_capacity(self.prompts.len()); + + let stdin = io::stdin(); + let mut stdout = io::stdout(); + + if let Some(header) = self.header.as_ref() { + writeln!(stdout, "{header}")?; + } + + for prompt in &self.prompts { + write!(stdout, "{prompt}")?; + stdout.flush()?; + + let user_input = { + let mut buf = String::new(); + stdin.read_line(&mut buf)?; + buf.trim().to_string() + }; + if user_input.is_empty() { + user_inputs.push(None); + } else { + user_inputs.push(Some(user_input)); + } + } + Ok(user_inputs) + } +} + +#[derive(Default)] +pub struct Output { + lines: Vec, + tip: Option, +} + +impl Output { + pub fn new() -> Self { + Self::default() + } + + pub fn line>(&mut self, txt: S) -> &mut Self { + self.lines.push(txt.into()); + self + } + + pub fn tip>(&mut self, txt: S) -> &mut Self { + self.tip.replace(txt.into()); + self + } + + pub fn print(&self) { + let out = self.lines.join("\n"); + + if let Some(help) = self.tip.as_ref() { + println!("{out}\n\n{}: {help}", "Tip".bold().underlined()); + return; + } + println!("{out}") + } + + pub fn eprint(&self) { + let out = self.lines.join("\n"); + + if let Some(help) = self.tip.as_ref() { + eprintln!( + "{}: {out}\n\n{}: {help}", + "error".red(), + "Tip".bold().underlined() + ); + return; + } + eprintln!("{}: {out}", "error".red()) + } +} diff --git a/rust/crates/sift_cli/src/util/user.rs b/rust/crates/sift_cli/src/util/user.rs new file mode 100644 index 000000000..6331cb7d8 --- /dev/null +++ b/rust/crates/sift_cli/src/util/user.rs @@ -0,0 +1,15 @@ +use anyhow::{Context, Result}; +use sift_rs::{ + SiftChannel, + me::v2::{GetMeRequest, me_service_client::MeServiceClient}, +}; + +pub async fn get_user_id(grpc_channel: SiftChannel) -> Result { + let mut me_service = MeServiceClient::new(grpc_channel); + let res = me_service + .get_me(GetMeRequest::default()) + .await + .context("failed to retrieve user info from provided --profile")? + .into_inner(); + Ok(res.user_id) +}