diff --git a/.cargo/config.toml b/.cargo/config.toml deleted file mode 100644 index 1b42b4e..0000000 --- a/.cargo/config.toml +++ /dev/null @@ -1,2 +0,0 @@ -[env] -HDW_DIR = { value = "target/hdw/", relative = true } \ No newline at end of file diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml new file mode 100644 index 0000000..7e4e2b8 --- /dev/null +++ b/.github/workflows/CI.yml @@ -0,0 +1,180 @@ +# This file is autogenerated by maturin v1.7.0 +# To update, run +# +# maturin generate-ci github --zig +# +name: CI + +on: + push: + branches: + - main + tags: + - '*' + pull_request: + branches: + - main + workflow_dispatch: + +permissions: + contents: read + +jobs: + linux: + runs-on: ${{ matrix.platform.runner }} + strategy: + matrix: + platform: + - runner: ubuntu-latest + target: x86_64 + - runner: ubuntu-latest + target: x86 + - runner: ubuntu-latest + target: aarch64 + - runner: ubuntu-latest + target: armv7 + - runner: ubuntu-latest + target: s390x + # - runner: ubuntu-latest + # target: ppc64le + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: 3.x + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.platform.target }} + args: --release --out dist --zig + sccache: 'false' + manylinux: auto + before-script-linux: | + sudo apt update -y && sudo apt-get install -y libssl-dev openssl pkg-config + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: wheels-linux-${{ matrix.platform.target }} + path: dist + + musllinux: + runs-on: ${{ matrix.platform.runner }} +# if: "startsWith(github.ref, 'refs/tags/')" + strategy: + matrix: + platform: + - runner: ubuntu-latest + target: x86_64 + - runner: ubuntu-latest + target: x86 + - runner: ubuntu-latest + target: aarch64 + - runner: ubuntu-latest + target: armv7 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: 3.x + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.platform.target }} + args: --release --out dist + sccache: 'false' + manylinux: musllinux_1_2 + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: wheels-musllinux-${{ matrix.platform.target }} + path: dist + + windows: + runs-on: ${{ matrix.platform.runner }} +# if: "startsWith(github.ref, 'refs/tags/')" + strategy: + matrix: + platform: + - runner: windows-latest + target: x64 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: 3.x + architecture: ${{ matrix.platform.target }} + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.platform.target }} + args: --release --out dist + sccache: 'false' + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: wheels-windows-${{ matrix.platform.target }} + path: dist + + macos: + runs-on: ${{ matrix.platform.runner }} +# if: "startsWith(github.ref, 'refs/tags/')" + strategy: + matrix: + platform: + - runner: macos-26-intel + target: x86_64 + - runner: macos-14 + target: aarch64 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: 3.x + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.platform.target }} + args: --release --out dist + sccache: 'false' + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: wheels-macos-${{ matrix.platform.target }} + path: dist + + sdist: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: 3.x + - name: Build sdist + uses: PyO3/maturin-action@v1 + with: + command: sdist + args: --out dist + - name: Upload sdist + uses: actions/upload-artifact@v4 + with: + name: wheels-sdist + path: dist + + release: + name: Release + runs-on: ubuntu-latest + if: "startsWith(github.ref, 'refs/tags/')" + needs: [linux, musllinux, windows, macos, sdist] + permissions: + id-token: write + environment: release + steps: + - uses: actions/download-artifact@v4 + - uses: actions/setup-python@v5 + with: + python-version: 3.x + - name: Publish to PyPI + uses: PyO3/maturin-action@v1 + with: + command: upload + args: --non-interactive --skip-existing wheels-*/* diff --git a/Cargo.lock b/Cargo.lock index 6ad0924..8c9efe1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,25 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "aacgmv2-rs" +version = "0.1.0" +source = "git+https://github.com/SuperDARNCanada/aacgmv2-rs.git?branch=riir#a9fb764733c8fe06992ef1410ea07c077671c317" +dependencies = [ + "chrono", + "lazy_static", + "thiserror 2.0.18", +] + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + [[package]] name = "android_system_properties" version = "0.1.5" @@ -17,53 +36,80 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" +[[package]] +name = "ansi_term" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +dependencies = [ + "winapi", +] + [[package]] name = "anstream" -version = "0.3.2" +version = "0.6.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163" +checksum = "64e15c1ab1f89faffbf04a634d5e1962e9074f2741eef6d97f3c4e322426d526" dependencies = [ "anstyle", "anstyle-parse", "anstyle-query", "anstyle-wincon", "colorchoice", - "is-terminal", + "is_terminal_polyfill", "utf8parse", ] [[package]] name = "anstyle" -version = "1.0.0" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41ed9a86bf92ae6580e0a31281f65a1b1d867c0cc68d5346e2ae128dddfa6a7d" +checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1" [[package]] name = "anstyle-parse" -version = "0.2.0" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e765fd216e48e067936442276d1d57399e37bce53c264d6fefbe298080cb57ee" +checksum = "eb47de1e80c2b463c735db5b217a0ddc39d612e7ac9e2e96a5aed1f57616c1cb" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.0.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" +checksum = "6d36fc52c7f6c869915e99412912f22093507da8d9e942ceaf66fe4b7c14422a" dependencies = [ - "windows-sys", + "windows-sys 0.52.0", ] [[package]] name = "anstyle-wincon" -version = "1.0.1" +version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "180abfa45703aebe0093f79badacc01b8fd4ea2e35118747e5811127f926e188" +checksum = "5bf74e1b6e971609db8ca7a9ce79fd5768ab6ae46441c572e46cf596f59e57f8" dependencies = [ "anstyle", - "windows-sys", + "windows-sys 0.52.0", +] + +[[package]] +name = "approx" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab112f0a86d568ea0e627cc1d6be74a1e9cd55214684db5561995f6dad897c6" +dependencies = [ + "num-traits", +] + +[[package]] +name = "assert_unordered" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c74323b7881323eb351134e08ee5331594826789557afef8e309baf481b2264" +dependencies = [ + "ansi_term", ] [[package]] @@ -72,32 +118,16 @@ version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" dependencies = [ - "hermit-abi 0.1.19", + "hermit-abi", "libc", "winapi", ] [[package]] name = "autocfg" -version = "1.1.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" - -[[package]] -name = "backscatter-rs" -version = "0.1.0" -dependencies = [ - "bytemuck", - "chrono", - "clap 4.2.7", - "criterion", - "dmap", - "git2", - "is_close", - "itertools", - "rayon", - "rust-embed", -] +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "bitflags" @@ -105,6 +135,12 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitflags" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" + [[package]] name = "block-buffer" version = "0.10.4" @@ -116,15 +152,36 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.12.1" +version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b1ce199063694f33ffb7dd4e0ee620741495c32833cde5aa08f02a0bf96f0c8" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] -name = "bytemuck" -version = "1.13.1" +name = "byteorder" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17febce684fd15d89027105661fec94afb475cb995fbc59d2865198446ba2eea" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bzip2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" +dependencies = [ + "bzip2-sys", + "libc", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.11+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" +dependencies = [ + "cc", + "libc", + "pkg-config", +] [[package]] name = "cast" @@ -134,11 +191,13 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.0.79" +version = "1.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" +checksum = "b16803a61b81d9eabb7eae2588776c4c1e584b738ede45fdbb4c972cec1e9945" dependencies = [ "jobserver", + "libc", + "shlex", ] [[package]] @@ -149,24 +208,22 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.24" +version = "0.4.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e3c5919066adf22df73762e50cffcde3a758f2a848b113b586d1f86728b673b" +checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" dependencies = [ "iana-time-zone", "js-sys", - "num-integer", "num-traits", - "time", "wasm-bindgen", - "winapi", + "windows-link", ] [[package]] name = "ciborium" -version = "0.2.0" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0c137568cc60b904a7724001b35ce2630fd00d5d84805fbb608ab89509d788f" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" dependencies = [ "ciborium-io", "ciborium-ll", @@ -175,15 +232,15 @@ dependencies = [ [[package]] name = "ciborium-io" -version = "0.2.0" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "346de753af073cc87b52b2083a506b38ac176a44cfb05497b622e27be899b369" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" [[package]] name = "ciborium-ll" -version = "0.2.0" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "213030a2b5a4e0c0892b6652260cf6ccac84827b83a85a534e178e3906c4cf1b" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" dependencies = [ "ciborium-io", "half", @@ -195,46 +252,44 @@ version = "3.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" dependencies = [ - "bitflags", + "bitflags 1.3.2", "clap_lex 0.2.4", - "indexmap", + "indexmap 1.9.3", "textwrap", ] [[package]] name = "clap" -version = "4.2.7" +version = "4.5.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34d21f9bf1b425d2968943631ec91202fe5e837264063503708b83013f8fc938" +checksum = "b97f376d85a664d5837dbae44bf546e6477a679ff6610010f17276f686d867e8" dependencies = [ "clap_builder", "clap_derive", - "once_cell", ] [[package]] name = "clap_builder" -version = "4.2.7" +version = "4.5.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "914c8c79fb560f238ef6429439a30023c862f7a28e688c58f7203f12b29970bd" +checksum = "19bc80abd44e4bed93ca373a0704ccbd1b710dc5749406201bb018272808dc54" dependencies = [ "anstream", "anstyle", - "bitflags", - "clap_lex 0.4.1", + "clap_lex 0.7.2", "strsim", ] [[package]] name = "clap_derive" -version = "4.2.0" +version = "4.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9644cd56d6b87dbe899ef8b053e331c0637664e9e21a33dfcdc36093f5c5c4" +checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.15", + "syn", ] [[package]] @@ -248,37 +303,27 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.4.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a2dd5a6fe8c6e3502f568a6353e5273bbb15193ad9a89e457b9970798efbea1" - -[[package]] -name = "codespan-reporting" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" -dependencies = [ - "termcolor", - "unicode-width", -] +checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" [[package]] name = "colorchoice" -version = "1.0.0" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" +checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0" [[package]] name = "core-foundation-sys" -version = "0.8.4" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.7" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e4c1eaa2012c47becbbad2ab175484c2a84d1185b566fb2cc5b8707343dfe58" +checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0" dependencies = [ "libc", ] @@ -295,7 +340,7 @@ dependencies = [ "ciborium", "clap 3.2.25", "criterion-plot", - "itertools", + "itertools 0.10.5", "lazy_static", "num-traits", "oorandom", @@ -316,51 +361,39 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" dependencies = [ "cast", - "itertools", -] - -[[package]] -name = "crossbeam-channel" -version = "0.5.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" -dependencies = [ - "cfg-if", - "crossbeam-utils", + "itertools 0.10.5", ] [[package]] name = "crossbeam-deque" -version = "0.8.3" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" dependencies = [ - "cfg-if", "crossbeam-epoch", "crossbeam-utils", ] [[package]] name = "crossbeam-epoch" -version = "0.9.14" +version = "0.9.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46bd5f3f85273295a9d14aedfb86f6aadbff6d8f5295c4a9edb08e819dcf5695" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" dependencies = [ - "autocfg", - "cfg-if", "crossbeam-utils", - "memoffset", - "scopeguard", ] [[package]] name = "crossbeam-utils" -version = "0.8.15" +version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b" -dependencies = [ - "cfg-if", -] +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" + +[[package]] +name = "crunchy" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" [[package]] name = "crypto-common" @@ -373,100 +406,70 @@ dependencies = [ ] [[package]] -name = "cxx" -version = "1.0.94" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f61f1b6389c3fe1c316bf8a4dccc90a38208354b330925bce1f74a6c4756eb93" -dependencies = [ - "cc", - "cxxbridge-flags", - "cxxbridge-macro", - "link-cplusplus", -] - -[[package]] -name = "cxx-build" -version = "1.0.94" +name = "darn-dmap" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12cee708e8962df2aeb38f594aae5d827c022b6460ac71a7a3e2c3c2aae5a07b" +checksum = "e3154dcd983eb7731d8d81d429da48d9b0664ce91936fc0a8edf93a93056308f" dependencies = [ - "cc", - "codespan-reporting", - "once_cell", - "proc-macro2", - "quote", - "scratch", - "syn 2.0.15", + "bzip2", + "indexmap 2.6.0", + "itertools 0.13.0", + "lazy_static", + "numpy", + "paste", + "pyo3", + "rayon", + "thiserror 1.0.64", + "zerocopy", ] [[package]] -name = "cxxbridge-flags" -version = "1.0.94" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7944172ae7e4068c533afbb984114a56c46e9ccddda550499caa222902c7f7bb" - -[[package]] -name = "cxxbridge-macro" -version = "1.0.94" +name = "deranged" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2345488264226bf682893e25de0769f3360aac9957980ec49361b083ddaa5bc5" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.15", + "powerfmt", ] [[package]] name = "digest" -version = "0.10.6" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", ] [[package]] -name = "dmap" -version = "0.1.0" -source = "git+https://github.com/SuperDARNCanada/dmap.git?branch=develop#7ee466297a313324710e9805ccb0d71ac259c97c" +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ - "bytemuck", - "itertools", + "proc-macro2", + "quote", + "syn", ] [[package]] name = "either" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" - -[[package]] -name = "errno" -version = "0.3.1" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" -dependencies = [ - "errno-dragonfly", - "libc", - "windows-sys", -] +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" [[package]] -name = "errno-dragonfly" -version = "0.1.2" +name = "equivalent" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" -dependencies = [ - "cc", - "libc", -] +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "form_urlencoded" -version = "1.1.0" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" dependencies = [ "percent-encoding", ] @@ -483,11 +486,11 @@ dependencies = [ [[package]] name = "git2" -version = "0.17.1" +version = "0.20.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b7905cdfe33d31a88bb2e8419ddd054451f5432d1da9eaf2ac7804ee1ea12d5" +checksum = "7b88256088d75a56f8ecfa070513a775dd9107f6530ef14919dac831af9cfe2b" dependencies = [ - "bitflags", + "bitflags 2.11.0", "libc", "libgit2-sys", "log", @@ -496,11 +499,21 @@ dependencies = [ "url", ] +[[package]] +name = "glob" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" + [[package]] name = "half" -version = "1.8.2" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" +checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" +dependencies = [ + "cfg-if", + "crunchy", +] [[package]] name = "hashbrown" @@ -508,11 +521,17 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +[[package]] +name = "hashbrown" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" + [[package]] name = "heck" -version = "0.4.1" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "hermit-abi" @@ -524,52 +543,138 @@ dependencies = [ ] [[package]] -name = "hermit-abi" -version = "0.2.6" +name = "iana-time-zone" +version = "0.1.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" +checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220" dependencies = [ - "libc", + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", ] [[package]] -name = "hermit-abi" -version = "0.3.1" +name = "iana-time-zone-haiku" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] [[package]] -name = "iana-time-zone" -version = "0.1.56" +name = "icu_collections" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0722cd7114b7de04316e7ea5456a0bbb20e4adb46fd27a3697adb812cff0f37c" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" dependencies = [ - "android_system_properties", - "core-foundation-sys", - "iana-time-zone-haiku", - "js-sys", - "wasm-bindgen", - "windows", + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", ] [[package]] -name = "iana-time-zone-haiku" -version = "0.1.1" +name = "icu_locale_core" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" dependencies = [ - "cxx", - "cxx-build", + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", ] [[package]] name = "idna" -version = "0.3.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" dependencies = [ - "unicode-bidi", - "unicode-normalization", + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "igrf" +version = "0.2.1" +source = "git+https://github.com/SuperDARNCanada/igrf.git?branch=main#333373d5d2a07537b5cff8a996d38de88150a497" +dependencies = [ + "thiserror 1.0.64", + "time", ] [[package]] @@ -579,31 +684,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ "autocfg", - "hashbrown", + "hashbrown 0.12.3", ] [[package]] -name = "io-lifetimes" -version = "1.0.10" +name = "indexmap" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c66c74d2ae7e79a5a8f7ac924adbe38ee42a859c6539ad869eb51f0b52dc220" +checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" dependencies = [ - "hermit-abi 0.3.1", - "libc", - "windows-sys", + "equivalent", + "hashbrown 0.15.0", ] [[package]] -name = "is-terminal" -version = "0.4.7" +name = "indoc" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adcf93614601c8129ddf72e2d5633df827ba6551541c6d8c59520a371475be1f" -dependencies = [ - "hermit-abi 0.3.1", - "io-lifetimes", - "rustix", - "windows-sys", -] +checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" [[package]] name = "is_close" @@ -614,6 +712,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + [[package]] name = "itertools" version = "0.10.5" @@ -623,47 +727,56 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itoa" -version = "1.0.6" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] name = "jobserver" -version = "0.1.26" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2" +checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" dependencies = [ "libc", ] [[package]] name = "js-sys" -version = "0.3.61" +version = "0.3.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730" +checksum = "6a88f1bda2bd75b0452a14784937d796722fdebfe50df998aeb3f0b7603019a9" dependencies = [ "wasm-bindgen", ] [[package]] name = "lazy_static" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.142" +version = "0.2.170" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a987beff54b60ffa6d51982e1aa1146bc42f19bd26be28b0586f252fccf5317" +checksum = "875b3680cb2f8f71bdcf9a30f38d48282f5d3c95cbf9b3fa57269bb5d5c06828" [[package]] name = "libgit2-sys" -version = "0.15.1+1.6.4" +version = "0.18.3+1.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb4577bde8cdfc7d6a2a4bcb7b049598597de33ffd337276e9c7db6cd4a2cee7" +checksum = "c9b3acc4b91781bb0b3386669d325163746af5f6e4f73e6d2d630e09a35f3487" dependencies = [ "cc", "libc", @@ -689,9 +802,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.9" +version = "1.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56ee889ecc9568871456d42f603d6a0ce59ff328d291063a45cbdf0036baf6db" +checksum = "d2d16453e800a8cf6dd2fc3eb4bc99b786a9b90c663b8559a5b1a041bf89e472" dependencies = [ "cc", "libc", @@ -700,78 +813,118 @@ dependencies = [ ] [[package]] -name = "link-cplusplus" -version = "1.0.8" +name = "litemap" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecd207c9c713c34f95a097a5b029ac2ce6010530c7b49d7fea24d977dede04f5" -dependencies = [ - "cc", -] +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" [[package]] -name = "linux-raw-sys" -version = "0.3.7" +name = "log" +version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ece97ea872ece730aed82664c424eb4c8291e1ff2480247ccf7409044bc6479f" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" [[package]] -name = "log" -version = "0.4.17" +name = "matrixmultiply" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" +checksum = "9380b911e3e96d10c1f415da0876389aaf1b56759054eeb0de7df940c456ba1a" dependencies = [ - "cfg-if", + "autocfg", + "rawpointer", ] +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + [[package]] name = "memoffset" -version = "0.8.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" dependencies = [ "autocfg", ] +[[package]] +name = "ndarray" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "882ed72dce9365842bf196bdeedf5055305f11fc8c03dee7bb0194a6cad34841" +dependencies = [ + "approx", + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "portable-atomic", + "portable-atomic-util", + "rawpointer", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-conv" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967" + [[package]] name = "num-integer" -version = "0.1.45" +version = "0.1.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" dependencies = [ - "autocfg", "num-traits", ] [[package]] name = "num-traits" -version = "0.2.15" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", ] [[package]] -name = "num_cpus" -version = "1.15.0" +name = "numpy" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" +checksum = "9b2dba356160b54f5371b550575b78130a54718b4c6e46b3f33a6da74a27e78b" dependencies = [ - "hermit-abi 0.2.6", "libc", + "ndarray", + "num-complex", + "num-integer", + "num-traits", + "pyo3", + "pyo3-build-config", + "rustc-hash", ] [[package]] name = "once_cell" -version = "1.17.1" +version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oorandom" -version = "11.1.3" +version = "11.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" +checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9" [[package]] name = "openssl-probe" @@ -779,41 +932,57 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +[[package]] +name = "openssl-src" +version = "300.4.2+3.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "168ce4e058f975fe43e89d9ccf78ca668601887ae736090aacc23ae353c298e2" +dependencies = [ + "cc", +] + [[package]] name = "openssl-sys" -version = "0.9.87" +version = "0.9.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e17f59264b2809d77ae94f0e1ebabc434773f370d6ca667bd223ea10e06cc7e" +checksum = "8bb61ea9811cc39e3c2069f40b8b8e2e70d8569b361f879786cc7ed48b777cdd" dependencies = [ "cc", "libc", + "openssl-src", "pkg-config", "vcpkg", ] [[package]] name = "os_str_bytes" -version = "6.5.0" +version = "6.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" + +[[package]] +name = "paste" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ceedf44fb00f2d1984b0bc98102627ce622e083e49a5bacdb3e514fa4238e267" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" [[package]] name = "percent-encoding" -version = "2.2.0" +version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pkg-config" -version = "0.3.27" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" +checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" [[package]] name = "plotters" -version = "0.3.4" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2538b639e642295546c50fcd545198c9d64ee2a38620a628724a3b266d5fbf97" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" dependencies = [ "num-traits", "plotters-backend", @@ -824,42 +993,166 @@ dependencies = [ [[package]] name = "plotters-backend" -version = "0.3.4" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "193228616381fecdc1224c62e96946dfbc73ff4384fba576e052ff8c1bea8142" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" [[package]] name = "plotters-svg" -version = "0.3.3" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9a81d2759aae1dae668f783c308bc5c8ebd191ff4184aaa1b37f65a6ae5a56f" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" dependencies = [ "plotters-backend", ] +[[package]] +name = "portable-atomic" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc9c68a3f6da06753e9335d63e27f6b9754dd1920d941135b7ea8224f141adb2" + +[[package]] +name = "portable-atomic-util" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcdd8420072e66d54a407b3316991fe946ce3ab1083a7f575b2463866624704d" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "proc-macro2" -version = "1.0.56" +version = "1.0.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435" +checksum = "7c3a7fc5db1e57d5a779a352c8cdb57b29aa4c40cc69c3a68a7fedc815fbf2f9" dependencies = [ "unicode-ident", ] +[[package]] +name = "procdarn" +version = "0.1.0" +dependencies = [ + "aacgmv2-rs", + "approx", + "assert_unordered", + "chrono", + "clap 4.5.20", + "criterion", + "darn-dmap", + "git2", + "glob", + "igrf", + "indexmap 2.6.0", + "is_close", + "itertools 0.10.5", + "ndarray", + "numpy", + "pyo3", + "rayon", + "rust-embed", + "thiserror 1.0.64", + "time", +] + +[[package]] +name = "pyo3" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ba0117f4212101ee6544044dae45abe1083d30ce7b29c4b5cbdfa2354e07383" +dependencies = [ + "indexmap 2.6.0", + "indoc", + "libc", + "memoffset", + "once_cell", + "portable-atomic", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", + "unindent", +] + +[[package]] +name = "pyo3-build-config" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fc6ddaf24947d12a9aa31ac65431fb1b851b8f4365426e182901eabfb87df5f" +dependencies = [ + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "025474d3928738efb38ac36d4744a74a400c901c7596199e20e45d98eb194105" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e64eb489f22fe1c95911b77c44cc41e7c19f3082fc81cce90f657cdc42ffded" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "100246c0ecf400b475341b8455a9213344569af29a3c841d29270e53102e0fcf" +dependencies = [ + "heck", + "proc-macro2", + "pyo3-build-config", + "quote", + "syn", +] + [[package]] name = "quote" -version = "1.0.26" +version = "1.0.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" dependencies = [ "proc-macro2", ] +[[package]] +name = "rawpointer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" + [[package]] name = "rayon" -version = "1.7.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" dependencies = [ "either", "rayon-core", @@ -867,36 +1160,48 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.11.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" dependencies = [ - "crossbeam-channel", "crossbeam-deque", "crossbeam-utils", - "num_cpus", ] [[package]] name = "regex" -version = "1.8.1" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38200e5ee88914975b69f657f0801b6f6dccafd44fd9326302a4aaeecfacb1d8" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af83e617f331cc6ae2da5443c602dfa5af81e517212d9d611a5b3ba1777b5370" +checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3" dependencies = [ + "aho-corasick", + "memchr", "regex-syntax", ] [[package]] name = "regex-syntax" -version = "0.7.1" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5996294f19bd3aae0453a862ad728f60e6600695733dd5df01da90c54363a3c" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "rust-embed" -version = "6.6.1" +version = "6.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b68543d5527e158213414a92832d2aab11a84d2571a5eb021ebe22c43aab066" +checksum = "a36224c3276f8c4ebc8c20f158eca7ca4359c8db89991c4925132aaaf6702661" dependencies = [ "rust-embed-impl", "rust-embed-utils", @@ -905,46 +1210,38 @@ dependencies = [ [[package]] name = "rust-embed-impl" -version = "6.5.0" +version = "6.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d4e0f0ced47ded9a68374ac145edd65a6c1fa13a96447b873660b2a568a0fd7" +checksum = "49b94b81e5b2c284684141a2fb9e2a31be90638caf040bf9afbc5a0416afe1ac" dependencies = [ "proc-macro2", "quote", "rust-embed-utils", - "syn 1.0.109", + "syn", "walkdir", ] [[package]] name = "rust-embed-utils" -version = "7.5.0" +version = "7.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "512b0ab6853f7e14e3c8754acb43d6f748bb9ced66aa5915a6553ac8213f7731" +checksum = "9d38ff6bf570dc3bb7100fce9f7b60c33fa71d80e88da3f2580df4ff2bdded74" dependencies = [ "sha2", "walkdir", ] [[package]] -name = "rustix" -version = "0.37.19" +name = "rustc-hash" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acf8729d8542766f1b2cf77eb034d52f40d375bb8b615d0b147089946e16613d" -dependencies = [ - "bitflags", - "errno", - "io-lifetimes", - "libc", - "linux-raw-sys", - "windows-sys", -] +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" [[package]] name = "ryu" -version = "1.0.13" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" [[package]] name = "same-file" @@ -956,70 +1253,87 @@ dependencies = [ ] [[package]] -name = "scopeguard" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" - -[[package]] -name = "scratch" -version = "1.0.5" +name = "serde" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1792db035ce95be60c3f8853017b3999209281c24e2ba5bc8e59bf97a0c590c1" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] [[package]] -name = "serde" -version = "1.0.162" +name = "serde_core" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71b2f6e1ab5c2b98c05f0f35b236b22e8df7ead6ffbf51d7808da7f8817e7ab6" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.162" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2a0814352fd64b58489904a44ea8d90cb1a91dcb6b4f5ebabc32c8318e93cb6" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.15", + "syn", ] [[package]] name = "serde_json" -version = "1.0.96" +version = "1.0.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" +checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" dependencies = [ "itoa", + "memchr", "ryu", "serde", ] [[package]] name = "sha2" -version = "0.10.6" +version = "0.10.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" dependencies = [ "cfg-if", "cpufeatures", "digest", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + [[package]] name = "strsim" -version = "0.10.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "syn" -version = "1.0.109" +version = "2.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" dependencies = [ "proc-macro2", "quote", @@ -1027,116 +1341,148 @@ dependencies = [ ] [[package]] -name = "syn" -version = "2.0.15" +name = "synstructure" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a34fcf3e8b60f57e6a14301a2e916d323af98b0ea63c599441eec8558660c822" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "unicode-ident", + "syn", ] [[package]] -name = "termcolor" -version = "1.2.0" +name = "target-lexicon" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" -dependencies = [ - "winapi-util", -] +checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" [[package]] name = "textwrap" -version = "0.16.0" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" +checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9" [[package]] -name = "time" -version = "0.1.45" +name = "thiserror" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" +checksum = "d50af8abc119fb8bb6dbabcfa89656f46f84aa0ac7688088608076ad2b459a84" dependencies = [ - "libc", - "wasi", - "winapi", + "thiserror-impl 1.0.64", ] [[package]] -name = "tinytemplate" -version = "1.2.1" +name = "thiserror" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" dependencies = [ - "serde", - "serde_json", + "thiserror-impl 2.0.18", ] [[package]] -name = "tinyvec" -version = "1.6.0" +name = "thiserror-impl" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3" dependencies = [ - "tinyvec_macros", + "proc-macro2", + "quote", + "syn", ] [[package]] -name = "tinyvec_macros" -version = "0.1.1" +name = "thiserror-impl" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] [[package]] -name = "typenum" -version = "1.16.0" +name = "time" +version = "0.3.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", +] [[package]] -name = "unicode-bidi" -version = "0.3.13" +name = "time-core" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" [[package]] -name = "unicode-ident" -version = "1.0.8" +name = "tinystr" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] [[package]] -name = "unicode-normalization" -version = "0.1.22" +name = "tinytemplate" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" dependencies = [ - "tinyvec", + "serde", + "serde_json", ] [[package]] -name = "unicode-width" -version = "0.1.10" +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + +[[package]] +name = "unicode-ident" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" + +[[package]] +name = "unindent" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" +checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce" [[package]] name = "url" -version = "2.3.1" +version = "2.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" dependencies = [ "form_urlencoded", "idna", "percent-encoding", + "serde", ] +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "utf8parse" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "vcpkg" @@ -1146,56 +1492,51 @@ checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" [[package]] name = "version_check" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] name = "walkdir" -version = "2.3.3" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36df944cda56c7d8d8b7496af378e6b16de9284591917d307c9b4d313c44e698" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" dependencies = [ "same-file", "winapi-util", ] -[[package]] -name = "wasi" -version = "0.10.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" - [[package]] name = "wasm-bindgen" -version = "0.2.84" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" +checksum = "128d1e363af62632b8eb57219c8fd7877144af57558fb2ef0368d0087bddeb2e" dependencies = [ "cfg-if", + "once_cell", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.84" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9" +checksum = "cb6dd4d3ca0ddffd1dd1c9c04f94b868c37ff5fac97c30b97cff2d74fce3a358" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 1.0.109", + "syn", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.84" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" +checksum = "e79384be7f8f5a9dd5d7167216f022090cf1f9ec128e6e6a482a2cb5c5422c56" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -1203,28 +1544,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.84" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" +checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.84" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" +checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d" [[package]] name = "web-sys" -version = "0.3.61" +version = "0.3.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97" +checksum = "f6488b90108c040df0fe62fa815cbdee25124641df01814dd7282749234c6112" dependencies = [ "js-sys", "wasm-bindgen", @@ -1248,11 +1589,11 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.5" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "winapi", + "windows-sys 0.59.0", ] [[package]] @@ -1262,32 +1603,48 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] -name = "windows" -version = "0.48.0" +name = "windows-core" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ "windows-targets", ] +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + [[package]] name = "windows-sys" -version = "0.48.0" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" dependencies = [ "windows-targets", ] [[package]] name = "windows-targets" -version = "0.48.0" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ "windows_aarch64_gnullvm", "windows_aarch64_msvc", "windows_i686_gnu", + "windows_i686_gnullvm", "windows_i686_msvc", "windows_x86_64_gnu", "windows_x86_64_gnullvm", @@ -1296,42 +1653,152 @@ dependencies = [ [[package]] name = "windows_aarch64_gnullvm" -version = "0.48.0" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" [[package]] name = "windows_aarch64_msvc" -version = "0.48.0" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" [[package]] name = "windows_i686_gnu" -version = "0.48.0" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" [[package]] name = "windows_i686_msvc" -version = "0.48.0" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" [[package]] name = "windows_x86_64_gnu" -version = "0.48.0" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" [[package]] name = "windows_x86_64_gnullvm" -version = "0.48.0" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] name = "windows_x86_64_msvc" -version = "0.48.0" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "yoke" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +dependencies = [ + "byteorder", + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerofrom" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69faa1f2a1ea75661980b013019ed6687ed0e83d069bc1114e2cc74c6c04c4df" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/Cargo.toml b/Cargo.toml index 613a706..35be1d4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,28 +1,39 @@ [package] -name = "backscatter-rs" +name = "procdarn" version = "0.1.0" edition = "2021" +rust-version = "1.74.1" [dependencies] -bytemuck = "1.13.1" -chrono = "0.4.24" +aacgmv2-rs = { git = "https://github.com/SuperDARNCanada/aacgmv2-rs.git", branch = "riir" } +chrono = "0.4.44" clap = { version = "4.2.7", features = ["derive"] } +darn-dmap = "0.7.0" +igrf = { git = "https://github.com/SuperDARNCanada/igrf.git", branch = "main" } +indexmap = "2.3.0" is_close = "0.1.3" itertools = "0.10.5" -dmap = { git = "https://github.com/SuperDARNCanada/dmap.git", branch = "develop" } +numpy = "0.26.0" +pyo3 = { version = "0.26.0", features = ["extension-module", "indexmap", "abi3-py38"] } rust-embed = "6.6.1" rayon = "1.7.0" +thiserror = "1.0.64" +time = "0.3.41" [build-dependencies] -git2 = "0.17.1" +git2 = { version = "0.20.4", features = ["vendored-openssl"] } [dev-dependencies] +approx = "0.5.1" +assert_unordered = "0.3.5" criterion = { version = "0.4", features = ["html_reports"] } +glob = "0.3.2" +ndarray = { version = "0.16.1", features = ["approx"] } [[bench]] -name = "backscatter_benchmark" +name = "fitacf3" harness = false [lib] -name = "backscatter_rs" -path = "src/lib.rs" \ No newline at end of file +name = "procdarn" +path = "src/lib.rs" diff --git a/README.md b/README.md new file mode 100644 index 0000000..ed1f675 --- /dev/null +++ b/README.md @@ -0,0 +1,73 @@ +Core SuperDARN Processing Tools +=============================== + +[![github]](https://github.com/SuperDARNCanada/procdarn) + +[github]: https://img.shields.io/badge/github-8da0cb?style=for-the-badge&labelColor=555555&logo=github + +## Installation + +### Rust +Add the crate to your dependencies in your `Cargo.toml` file, specifying as a git dependency. + +### Python +`pip install procdarn` + +### From source +If you want to build from source, you first need to have Rust installed on your machine. Then: +1. Clone the repository: `git clone https://github.com/SuperDARNCanada/procdarn` +2. Run `cargo build` in the repository directory +3. If wanting to install the Python API, create a virtual environment and source it, then install `maturin` +4. In the project directory, run `maturin develop` to build and install the Python bindings. This will make a wheel file based on your operating system and architecture that you can install directly on any compatible machine. + +## Python API + +### In-memory fitting +Use FITACF3 algorithm to fit a list of RAWACF records +```python +fitacf3(recs: list[dict]) -> list[dict] +``` + +#### Example +```python +import procdarn +import dmap +infile = "path/to/rawacf" +rawacf_records = dmap.read_rawacf(infile) +fitacf_records = procdarn.fitacf3(rawacf_records) +dmap.write_fitacf(fitacf_records, "path/to/fitacf") +``` + +### File-to-file fitting + +Fit a RAWACF file into a FITACF file +```python +fitacf3_file(rawacf_file: str, fitacf_file: str) +``` + +#### Example +```python +import procdarn +procdarn.fitacf3_file("path/to/rawacf", "path/to/fitacf") +``` + +## Rust API + +* `procdarn::fitacf3_file(raw_file: PathBuf, fit_file: PathBuf) -> Result<(), Fitacf3Error>`: Fits a RAWACF file into a FITACF file +* `procdarn::fitacf3(Vec) -> Result, Fitacf3Error>`: parallelized FITACFv3 on collection of `RawacfRecord`s (from `dmap` crate) +* `procdarn::fitacf3_single_threaded(Vec) -> Result, Fitacf3Error>`: single-threaded FITACFv3 implementation + +## Binary +`raw2fit`: Command-line tool for fitting a RAWACF file using the FITACF3 algorithm. + +``` +Usage: raw2fit + +Arguments: + Rawacf file to fit + Output fitacf file path + +Options: + -h, --help Print help + -V, --version Print version +``` \ No newline at end of file diff --git a/benches/backscatter_benchmark.rs b/benches/backscatter_benchmark.rs deleted file mode 100644 index b53ec74..0000000 --- a/benches/backscatter_benchmark.rs +++ /dev/null @@ -1,68 +0,0 @@ -use backscatter_rs::fitting::fitacf3::fitacf_v3::{fit_rawacf_record, Fitacf3Error}; -use backscatter_rs::utils::hdw::HdwInfo; -use chrono::NaiveDateTime; -use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion}; -use dmap::formats::{DmapRecord, FitacfRecord, RawacfRecord}; - -use dmap; -use rayon::prelude::*; -use std::fs::File; - -fn criterion_benchmark(c: &mut Criterion) { - c.bench_function("Fitacf3", |b| b.iter(|| fitacf3())); - c.bench_function("Parallel Fitacf3", |b| b.iter(|| rayon_fitacf3())); -} - -fn fitacf3() { - let file = - File::open("tests/test_files/20210607.1801.00.cly.a.rawacf").expect("Test file not found"); - let rawacf = RawacfRecord::read_records(file).expect("Could not read records"); - let mut fitacf_records = vec![]; - - let rec = &rawacf[0]; - let file_datetime = NaiveDateTime::parse_from_str( - format!( - "{:4}{:0>2}{:0>2} {:0>2}:{:0>2}:{:0>2}", - rec.year, rec.month, rec.day, rec.hour, rec.minute, rec.second - ) - .as_str(), - "%Y%m%d %H:%M:%S", - ) - .expect("Unable to interpret record timestamp"); - let hdw = HdwInfo::new(rec.station_id, file_datetime).expect("Unable to read hdw file"); - // fitacf_records.push(fit_rawacf_record(&rawacf[0]).expect("Could not fit rawacf record")); - for rec in rawacf { - fitacf_records.push(fit_rawacf_record(&rec, &hdw).expect("Could not fit record")); - } - dmap::formats::to_file("tests/test_files/temp.fitacf", &fitacf_records) - .expect("Unable to write to file"); -} - -fn rayon_fitacf3() { - let file = - File::open("tests/test_files/20210607.1801.00.cly.a.rawacf").expect("Test file not found"); - let rawacf = RawacfRecord::read_records(file).expect("Could not read records"); - let fitacf_records: Vec; - - let rec = &rawacf[0]; - let file_datetime = NaiveDateTime::parse_from_str( - format!( - "{:4}{:0>2}{:0>2} {:0>2}:{:0>2}:{:0>2}", - rec.year, rec.month, rec.day, rec.hour, rec.minute, rec.second - ) - .as_str(), - "%Y%m%d %H:%M:%S", - ) - .expect("Unable to interpret record timestamp"); - let hdw = HdwInfo::new(rec.station_id, file_datetime).expect("Unable to read utils file"); - - fitacf_records = rawacf - .par_iter() - .map(|rec| fit_rawacf_record(&rec, &hdw).expect("Could not fit record")) - .collect(); - dmap::formats::to_file("tests/test_files/temp.fitacf", &fitacf_records) - .expect("Unable to write to file"); -} - -criterion_group!(benches, criterion_benchmark); -criterion_main!(benches); diff --git a/benches/fitacf3.rs b/benches/fitacf3.rs new file mode 100644 index 0000000..623d1c2 --- /dev/null +++ b/benches/fitacf3.rs @@ -0,0 +1,32 @@ +use criterion::{criterion_group, criterion_main, BatchSize, Criterion}; +use dmap::{RawacfRecord, Record}; +use procdarn::fitting::fitacf3::fitacf_v3::{fitacf3, par_fitacf3}; + +const TEST_FILE: &str = "tests/test_files/large.rawacf"; +// const TEST_FILE: &str = "/data/dmap_files/20221107.2200.00.rkn.a.rawacf"; // widebeam file + +fn criterion_benchmark(c: &mut Criterion) { + let rawacf = RawacfRecord::read_file(TEST_FILE.to_string()).expect("Could not read records"); + + c.bench_function("Fitacf3", |b| { + b.iter_batched( + || rawacf.clone(), + |rawacf| fitacf3(rawacf), + BatchSize::SmallInput, + ) + }); + c.bench_function("Parallel Fitacf3", |b| { + b.iter_batched( + || rawacf.clone(), + |rawacf| par_fitacf3(rawacf), + BatchSize::SmallInput, + ) + }); +} + +criterion_group! { + name = benches; + config = Criterion::default(); + targets = criterion_benchmark +} +criterion_main!(benches); diff --git a/build.rs b/build.rs index 60053e9..6f326e6 100644 --- a/build.rs +++ b/build.rs @@ -1,15 +1,17 @@ use git2::Repository; -use std::env; use std::path::Path; fn main() { // clone the utils repo - let out_dir = env::var("HDW_DIR").expect("HDW_DIR not set"); + let out_dir = "target/hdw/"; let url = "https://github.com/SuperDARN/hdw"; - if !Path::new(&out_dir).is_dir() { + println!("Installing {url} to {out_dir}"); + if Path::new(&out_dir).is_dir() { + println!("{out_dir} already exists"); + } else { match Repository::clone(url, out_dir) { Ok(r) => r, - Err(err) => panic!("failed to clone: {}", err), + Err(err) => panic!("failed to clone: {err}"), }; } } diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..f51d9fd --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,28 @@ +[build-system] +requires = ["maturin>=1,<2", "numpy<3"] +build-backend = "maturin" + +[project] +name = "procdarn" +version = "0.1.0" +requires-python = ">=3.8" +authors = [ + { name = "Remington Rohel" } +] +classifiers = [ + "Programming Language :: Python", + "Programming Language :: Rust" +] +dependencies = ["numpy<3"] + +[tool.maturin] +bindings = "pyo3" +features = ["pyo3/extension-module"] +profile = "release" +compatibility = "manylinux2014" +auditwheel = "repair" +strip = true + +[project.scripts] +raw2fit = "procdarn:raw2fit" +fit2grid = "procdarn:fit2grid" diff --git a/src/bin/fit_fitacf3.rs b/src/bin/fit_fitacf3.rs deleted file mode 100644 index d693ca4..0000000 --- a/src/bin/fit_fitacf3.rs +++ /dev/null @@ -1,62 +0,0 @@ -use backscatter_rs::fitting::fitacf3::fitacf_v3::{fit_rawacf_record, Fitacf3Error}; -use backscatter_rs::utils::hdw::HdwInfo; -use chrono::NaiveDateTime; -use clap::Parser; -use dmap::formats::{to_file, DmapRecord, FitacfRecord, RawacfRecord}; -use rayon::prelude::*; -use std::fs::File; -use std::path::PathBuf; - -pub type BinResult> = Result; - -fn main() { - if let Err(e) = bin_main() { - eprintln!("error: {e}"); - if let Some(e) = e.source() { - eprintln!("error: {e}") - } - std::process::exit(1); - } -} - -#[derive(Parser, Debug)] -#[command(author, version, about, long_about = None)] -struct Args { - /// Rawacf file to fit - #[arg(short, long)] - infile: PathBuf, - - /// Output fitacf file path - #[arg(short, long)] - outfile: PathBuf, -} - -fn bin_main() -> BinResult<()> { - let args = Args::parse(); - - let rawacf = File::open(args.infile)?; - let rawacf_records = RawacfRecord::read_records(rawacf)?; - - let rec = &rawacf_records[0]; - let file_datetime = NaiveDateTime::parse_from_str( - format!( - "{:4}{:0>2}{:0>2} {:0>2}:{:0>2}:{:0>2}", - rec.year, rec.month, rec.day, rec.hour, rec.minute, rec.second - ) - .as_str(), - "%Y%m%d %H:%M:%S", - ) - .map_err(|_| Fitacf3Error::Message("Unable to interpret record timestamp".to_string()))?; - let hdw = HdwInfo::new(rec.station_id, file_datetime) - .map_err(|e| Fitacf3Error::Message(e.details))?; - - // Fit the records! - let fitacf_records: Vec = rawacf_records - .par_iter() - .map(|rec| fit_rawacf_record(rec, &hdw).expect("Unable to fit record")) - .collect(); - - // Write to file - to_file(args.outfile, &fitacf_records)?; - Ok(()) -} diff --git a/src/error.rs b/src/error.rs index ad4899c..2dac3ff 100644 --- a/src/error.rs +++ b/src/error.rs @@ -1,21 +1,40 @@ -use std::fmt; -use std::fmt::Formatter; +//! Error type for `procdarn`. +use crate::utils::hdw::HdwError; +use dmap::error::DmapError; +use thiserror::Error; -#[derive(Debug)] -pub struct BackscatterError { - pub details: String, -} +/// Top-level error object for all processing functions. +#[derive(Error, Debug)] +pub enum ProcdarnError { + /// Represents a bad DMAP record + #[error("{0}")] + Dmap(#[from] DmapError), -impl fmt::Display for BackscatterError { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.details) - } -} + /// Unable to get hdw file information + #[error("{0}")] + Hdw(#[from] HdwError), + + /// Error in igrf crate usage + #[error("{0}")] + Igrf(#[from] igrf::Error), + + /// Invalid timestamp in a record + #[error("invalid timestamp `{0}`")] + Timestamp(String), + + /// Field missing from a record + #[error("missing `{0}`")] + MissingField(&'static str), + + /// Field from a record has the wrong type + #[error("wrong type for `{0}`")] + WrongType(&'static str), + + /// Zero records available + #[error("zero records {0}")] + ZeroRecords(&'static str), -impl BackscatterError { - pub fn new(details: &str) -> BackscatterError { - BackscatterError { - details: details.to_string(), - } - } + /// Invalid channel specifier + #[error("invalid channel `{0}`")] + Channel(String), } diff --git a/src/fitting/fitacf3/determinations.rs b/src/fitting/fitacf3/determinations.rs index b5b9623..de907f7 100644 --- a/src/fitting/fitacf3/determinations.rs +++ b/src/fitting/fitacf3/determinations.rs @@ -1,26 +1,35 @@ use crate::fitting::fitacf3::fitacf_v3::Fitacf3Error; use crate::fitting::fitacf3::fitstruct::RangeNode; use crate::utils::hdw::HdwInfo; -use dmap::formats::{FitacfRecord, RawacfRecord}; -use dmap::{DmapVec, InDmap}; +use crate::utils::rawacf::Rawacf; +use chrono::Utc; +use dmap::formats::fitacf::FitacfRecord; +use dmap::record::Record; +use dmap::types::DmapField; +use indexmap::IndexMap; +use numpy::ndarray::{Array, Array1}; use std::f32::consts::PI as PI_f32; +use std::f64::consts::PI as PI_f64; use std::iter::zip; pub const FITACF_REVISION_MAJOR: i32 = 3; pub const FITACF_REVISION_MINOR: i32 = 0; +const LIGHTSPEED: f32 = 299_792_458.0; +const KHZ_TO_HZ: f32 = 1000.0; +const US_TO_S: f32 = 1e-6; +pub const ORIGIN_CODE: i8 = 1; pub const V_MAX: f32 = 30.0; pub const W_MAX: f32 = 90.0; -pub fn determinations( - rec: &RawacfRecord, - ranges: Vec, +pub(crate) fn determinations( + rec: &Rawacf, + ranges: &[RangeNode], noise_power: f32, hdw: &HdwInfo, ) -> Result { let range_list: Vec = ranges.iter().map(|r| r.range_num as i16).collect(); - let lag_0_power_db: Vec = rec - .lag_zero_power - .data + let lag_0_power_db: Array1 = rec + .pwr0 .iter() .map(|p| { if p - noise_power > 0.0 { @@ -30,105 +39,94 @@ pub fn determinations( } }) .collect(); - if range_list.is_empty() { - Ok(FitacfRecord { - radar_revision_major: rec.radar_revision_major, - radar_revision_minor: rec.radar_revision_minor, - origin_code: rec.origin_code, - origin_time: "".to_string(), // TODO: Get current time - origin_command: "".to_string(), // TODO: Get this - control_program: rec.control_program, - station_id: rec.station_id, - year: rec.year, - month: rec.month, - day: rec.day, - hour: rec.hour, - minute: rec.minute, - second: rec.second, - microsecond: rec.microsecond, - tx_power: rec.tx_power, - num_averages: rec.num_averages, - attenuation: rec.attenuation, - lag_to_first_range: rec.lag_to_first_range, - sample_separation: rec.sample_separation, - error_code: rec.error_code, - agc_status: rec.agc_status, - low_power_status: rec.low_power_status, - search_noise: rec.search_noise, - mean_noise: rec.mean_noise, - channel: rec.channel, - beam_num: rec.beam_num, - beam_azimuth: rec.beam_azimuth, - scan_flag: rec.scan_flag, - offset: rec.offset, - rx_rise_time: rec.rx_rise_time, - intt_second: rec.intt_second, - intt_microsecond: rec.intt_microsecond, - tx_pulse_length: rec.tx_pulse_length, - multi_pulse_increment: rec.multi_pulse_increment, - num_pulses: rec.num_pulses, - num_lags: rec.num_lags, - num_lags_extras: rec.num_lags_extras, - if_mode: rec.if_mode, - num_ranges: rec.num_ranges, - first_range: rec.first_range, - range_sep: rec.range_sep, - xcf_flag: rec.xcf_flag, - tx_freq: rec.tx_freq, - max_power: rec.max_power, - max_noise_level: rec.max_noise_level, - comment: rec.comment.clone(), - algorithm: None, - fitacf_revision_major: FITACF_REVISION_MAJOR, - fitacf_revision_minor: FITACF_REVISION_MINOR, - sky_noise: noise_power, - lag_zero_noise: 0.0, - velocity_noise: 0.0, - tdiff: None, - pulse_table: rec.pulse_table.clone(), - lag_table: rec.lag_table.clone(), - lag_zero_power: convert_to_dmapvec(lag_0_power_db), - range_list: convert_to_dmapvec(vec![]), - fitted_points: convert_to_dmapvec(vec![]), - quality_flag: convert_to_dmapvec(vec![]), - ground_flag: convert_to_dmapvec(vec![]), - lambda_power: convert_to_dmapvec(vec![]), - lambda_power_error: convert_to_dmapvec(vec![]), - sigma_power: convert_to_dmapvec(vec![]), - sigma_power_error: convert_to_dmapvec(vec![]), - velocity: convert_to_dmapvec(vec![]), - velocity_error: convert_to_dmapvec(vec![]), - lambda_spectral_width: convert_to_dmapvec(vec![]), - lambda_spectral_width_error: convert_to_dmapvec(vec![]), - sigma_spectral_width: convert_to_dmapvec(vec![]), - sigma_spectral_width_error: convert_to_dmapvec(vec![]), - lambda_std_dev: convert_to_dmapvec(vec![]), - sigma_std_dev: convert_to_dmapvec(vec![]), - phi_std_dev: convert_to_dmapvec(vec![]), - xcf_quality_flag: None, - xcf_ground_flag: None, - lambda_xcf_power: None, - lambda_xcf_power_error: None, - sigma_xcf_power: None, - sigma_xcf_power_error: None, - xcf_velocity: None, - xcf_velocity_error: None, - lambda_xcf_spectral_width: None, - lambda_xcf_spectral_width_error: None, - sigma_xcf_spectral_width: None, - sigma_xcf_spectral_width_error: None, - lag_zero_phi: None, - lag_zero_phi_error: None, - elevation: None, - elevation_fitted: None, - elevation_error: None, - elevation_low: None, - elevation_high: None, - lambda_xcf_std_dev: None, - sigma_xcf_std_dev: None, - phi_xcf_std_dev: None, - }) + + let mut fit_rec: IndexMap = IndexMap::new(); + + fit_rec.insert( + "radar.revision.major".to_string(), + rec.radar_revision_major.into(), + ); + fit_rec.insert( + "radar.revision.minor".to_string(), + rec.radar_revision_minor.into(), + ); + fit_rec.insert("origin.code".to_string(), ORIGIN_CODE.into()); + let now: chrono::DateTime = std::time::SystemTime::now().into(); + fit_rec.insert( + "origin.time".to_string(), + format!("{}", now.format("%a %b %e %T %Y")).into(), + ); + fit_rec.insert( + "origin.command".to_string(), + rec.origin_command.clone().into(), + ); // todo: Get the invocation + fit_rec.insert("cp".to_string(), rec.cp.into()); + fit_rec.insert("stid".to_string(), rec.stid.into()); + fit_rec.insert("time.yr".to_string(), rec.time_yr.into()); + fit_rec.insert("time.mo".to_string(), rec.time_mo.into()); + fit_rec.insert("time.dy".to_string(), rec.time_dy.into()); + fit_rec.insert("time.hr".to_string(), rec.time_hr.into()); + fit_rec.insert("time.mt".to_string(), rec.time_mt.into()); + fit_rec.insert("time.sc".to_string(), rec.time_sc.into()); + fit_rec.insert("time.us".to_string(), rec.time_us.into()); + fit_rec.insert("txpow".to_string(), rec.txpow.into()); + fit_rec.insert("nave".to_string(), rec.nave.into()); + fit_rec.insert("atten".to_string(), rec.atten.into()); + fit_rec.insert("lagfr".to_string(), rec.lagfr.into()); + fit_rec.insert("smsep".to_string(), rec.smsep.into()); + fit_rec.insert("ercod".to_string(), rec.ercod.into()); + fit_rec.insert("stat.agc".to_string(), rec.stat_agc.into()); + fit_rec.insert("stat.lopwr".to_string(), rec.stat_lopwr.into()); + fit_rec.insert("noise.search".to_string(), rec.noise_search.into()); + fit_rec.insert("noise.mean".to_string(), rec.noise_mean.into()); + fit_rec.insert("channel".to_string(), rec.channel.into()); + fit_rec.insert("bmnum".to_string(), rec.bmnum.into()); + fit_rec.insert("bmazm".to_string(), rec.bmazm.into()); + fit_rec.insert("scan".to_string(), rec.scan.into()); + fit_rec.insert("offset".to_string(), rec.offset.into()); + fit_rec.insert("rxrise".to_string(), rec.rxrise.into()); + fit_rec.insert("intt.sc".to_string(), rec.intt_sc.into()); + fit_rec.insert("intt.us".to_string(), rec.intt_us.into()); + fit_rec.insert("txpl".to_string(), rec.txpl.into()); + fit_rec.insert("mpinc".to_string(), rec.mpinc.into()); + fit_rec.insert("mppul".to_string(), rec.mppul.into()); + fit_rec.insert("mplgs".to_string(), rec.mplgs.into()); + fit_rec.insert("nrang".to_string(), rec.nrang.into()); + fit_rec.insert("frang".to_string(), rec.frang.into()); + fit_rec.insert("rsep".to_string(), rec.rsep.into()); + fit_rec.insert("xcf".to_string(), rec.xcf.into()); + fit_rec.insert("tfreq".to_string(), (rec.tfreq.round() as i16).into()); + fit_rec.insert("mxpwr".to_string(), rec.mxpwr.into()); + fit_rec.insert("lvmax".to_string(), rec.lvmax.into()); + fit_rec.insert("combf".to_string(), rec.combf.clone().into()); + fit_rec.insert("ptab".to_string(), rec.ptab.clone().into_dyn().into()); + fit_rec.insert("ltab".to_string(), rec.ltab.clone().into_dyn().into()); + fit_rec.insert("algorithm".to_string(), "fitacf3".to_string().into()); + fit_rec.insert("tdiff".to_string(), hdw.tdiff_a.into()); + fit_rec.insert( + "fitacf.revision.major".to_string(), + FITACF_REVISION_MAJOR.into(), + ); + fit_rec.insert( + "fitacf.revision.minor".to_string(), + FITACF_REVISION_MINOR.into(), + ); + fit_rec.insert("noise.lag0".to_string(), 0.0_f32.into()); + fit_rec.insert("noise.vel".to_string(), 0.0_f32.into()); + if let Some(x) = rec.ifmode { + fit_rec.insert("ifmode".to_string(), x.into()); } else { + fit_rec.insert("ifmode".to_string(), >::from(0)); + } + if let Some(x) = rec.mplgexs { + fit_rec.insert("mplgexs".to_string(), x.into()); + } else { + fit_rec.insert("mplgexs".to_string(), 0_i16.into()); + } + fit_rec.insert("noise.sky".to_string(), noise_power.into()); + fit_rec.insert("pwr0".to_string(), lag_0_power_db.into_dyn().into()); + + if !range_list.is_empty() { let num_lags: Vec = ranges .iter() .map(|r| r.powers.ln_power.len() as i16) @@ -143,7 +141,7 @@ pub fn determinations( .as_ref() .expect("Unable to make fitacf without linear fitted power") .intercept as f32 - / (10.0_f32).ln() + / 10.0_f32.ln() - noise_db }) .collect(); @@ -156,7 +154,7 @@ pub fn determinations( .expect("Unable to make fitacf without linear fitted power error") .variance_intercept as f32) .sqrt() - / (10.0_f32).ln() + / 10.0_f32.ln() }) .collect(); let power_quadratic: Vec = ranges @@ -167,7 +165,7 @@ pub fn determinations( .as_ref() .expect("Unable to make fitacf without quadratic fitted power") .intercept as f32) - / (10.0_f32).ln() + / 10.0_f32.ln() - noise_db }) .collect(); @@ -180,11 +178,11 @@ pub fn determinations( .expect("Unable to make fitacf without quadratic fitted power error") .variance_intercept as f32) .sqrt() - / (10.0_f32).ln() + / 10.0_f32.ln() }) .collect(); let velocity_conversion: f32 = - 299792458.0 * hdw.velocity_sign / (4.0 * PI_f32 * rec.tx_freq as f32 * 1000.0); + LIGHTSPEED * hdw.velocity_sign / (4.0 * PI_f32 * rec.tfreq * KHZ_TO_HZ); let velocity: Vec = ranges .iter() .map(|r| { @@ -206,8 +204,7 @@ pub fn determinations( * velocity_conversion }) .collect(); - let width_conversion: f32 = - 299792458.0 * 2.0 / (4.0 * PI_f32 * rec.tx_freq as f32 * 1000.0); + let width_conversion: f32 = LIGHTSPEED * 2.0 / (4.0 * PI_f32 * rec.tfreq * KHZ_TO_HZ); let spectral_width_linear: Vec = ranges .iter() .map(|r| { @@ -231,7 +228,7 @@ pub fn determinations( }) .collect(); let quadratic_width_conversion: f32 = - 299792458.0 * (2.0_f32).ln().sqrt() / (PI_f32 * rec.tx_freq as f32 * 1000.0); + LIGHTSPEED * 2.0_f32.ln().sqrt() / (PI_f32 * rec.tfreq * KHZ_TO_HZ); let spectral_width_quadratic: Vec = ranges .iter() .map(|r| { @@ -281,20 +278,12 @@ pub fn determinations( }) .collect(); let groundscatter_flag: Vec = zip(velocity.iter(), spectral_width_linear.iter()) - .map(|(v, w)| (v.abs() - (V_MAX - w * (V_MAX / W_MAX)) < 1.0) as i8) + .map(|(v, w)| i8::from(v.abs() - (V_MAX - w * (V_MAX / W_MAX)) < 0.0)) .collect(); - let xcfs = &rec - .xcfs - .as_ref() - .expect("Unable to make fitacf xcf_phi0") - .data; + let xcfs = &rec.xcfd.as_ref().expect("Unable to make fitacf xcf_phi0"); let xcf_phi0: Vec = ranges .iter() - .map(|r| { - xcfs[r.range_idx * rec.num_lags as usize * 2 + 1] - .atan2(xcfs[r.range_idx * rec.num_lags as usize * 2]) - * hdw.phase_sign - }) + .map(|r| xcfs[[r.range_idx, 0, 1]].atan2(xcfs[[r.range_idx, 0, 0]]) * hdw.phase_sign) .collect(); let xcf_phi0_err: Vec = ranges .iter() @@ -315,232 +304,316 @@ pub fn determinations( .chi_squared as f32 }) .collect(); - let (elevation_low, elevation_normal, elevation_high) = - calculate_elevation(&ranges, rec, &xcf_phi0, hdw); - - let float_zeros = DmapVec { - data: quality_flag.iter().map(|_| 0.0_f32).collect(), - dimensions: vec![quality_flag.len() as i32], - }; - let i8_zeros = DmapVec { - data: quality_flag.iter().map(|_| 0_i8).collect(), - dimensions: vec![quality_flag.len() as i32], - }; + let (elevation_phi0, elevation_intercept) = + calculate_elevation_v2(ranges, rec, &xcf_phi0, hdw); + let (_, elevation_intercept_error, _) = calculate_elevation(ranges, rec, &xcf_phi0, hdw); - Ok(FitacfRecord { - radar_revision_major: rec.radar_revision_major, - radar_revision_minor: rec.radar_revision_minor, - origin_code: rec.origin_code, - origin_time: "".to_string(), // TODO: Get current time - origin_command: "".to_string(), // TODO: Get this - control_program: rec.control_program, - station_id: rec.station_id, - year: rec.year, - month: rec.month, - day: rec.day, - hour: rec.hour, - minute: rec.minute, - second: rec.second, - microsecond: rec.microsecond, - tx_power: rec.tx_power, - num_averages: rec.num_averages, - attenuation: rec.attenuation, - lag_to_first_range: rec.lag_to_first_range, - sample_separation: rec.sample_separation, - error_code: rec.error_code, - agc_status: rec.agc_status, - low_power_status: rec.low_power_status, - search_noise: rec.search_noise, - mean_noise: rec.mean_noise, - channel: rec.channel, - beam_num: rec.beam_num, - beam_azimuth: rec.beam_azimuth, - scan_flag: rec.scan_flag, - offset: rec.offset, - rx_rise_time: rec.rx_rise_time, - intt_second: rec.intt_second, - intt_microsecond: rec.intt_microsecond, - tx_pulse_length: rec.tx_pulse_length, - multi_pulse_increment: rec.multi_pulse_increment, - num_pulses: rec.num_pulses, - num_lags: rec.num_lags, - num_lags_extras: rec.num_lags_extras, - if_mode: rec.if_mode, - num_ranges: rec.num_ranges, - first_range: rec.first_range, - range_sep: rec.range_sep, - xcf_flag: rec.xcf_flag, - tx_freq: rec.tx_freq, - max_power: rec.max_power, - max_noise_level: rec.max_noise_level, - comment: rec.comment.clone(), - algorithm: None, - fitacf_revision_major: FITACF_REVISION_MAJOR, - fitacf_revision_minor: FITACF_REVISION_MINOR, - sky_noise: noise_power, - lag_zero_noise: 0.0, - velocity_noise: 0.0, - tdiff: None, - pulse_table: rec.pulse_table.clone(), - lag_table: rec.lag_table.clone(), - lag_zero_power: convert_to_dmapvec(lag_0_power_db), - range_list: convert_to_dmapvec(range_list), - fitted_points: convert_to_dmapvec(num_lags), - quality_flag: convert_to_dmapvec(quality_flag), - ground_flag: convert_to_dmapvec(groundscatter_flag), - lambda_power: convert_to_dmapvec(power_linear), - lambda_power_error: convert_to_dmapvec(power_linear_error), - sigma_power: convert_to_dmapvec(power_quadratic), - sigma_power_error: convert_to_dmapvec(power_quadratic_error), - velocity: convert_to_dmapvec(velocity), - velocity_error: convert_to_dmapvec(velocity_error), - lambda_spectral_width: convert_to_dmapvec(spectral_width_linear), - lambda_spectral_width_error: convert_to_dmapvec(spectral_width_linear_error), - sigma_spectral_width: convert_to_dmapvec(spectral_width_quadratic), - sigma_spectral_width_error: convert_to_dmapvec(spectral_width_quadratic_error), - lambda_std_dev: convert_to_dmapvec(std_dev_linear), - sigma_std_dev: convert_to_dmapvec(std_dev_quadratic), - phi_std_dev: convert_to_dmapvec(std_dev_phi), - xcf_quality_flag: Some(i8_zeros.clone()), - xcf_ground_flag: Some(i8_zeros), - lambda_xcf_power: Some(float_zeros.clone()), - lambda_xcf_power_error: Some(float_zeros.clone()), - sigma_xcf_power: Some(float_zeros.clone()), - sigma_xcf_power_error: Some(float_zeros.clone()), - xcf_velocity: Some(float_zeros.clone()), - xcf_velocity_error: Some(float_zeros.clone()), - lambda_xcf_spectral_width: Some(float_zeros.clone()), - lambda_xcf_spectral_width_error: Some(float_zeros.clone()), - sigma_xcf_spectral_width: Some(float_zeros.clone()), - sigma_xcf_spectral_width_error: Some(float_zeros.clone()), - lag_zero_phi: Some(convert_to_dmapvec(xcf_phi0)), - lag_zero_phi_error: Some(convert_to_dmapvec(xcf_phi0_err)), - elevation: Some(convert_to_dmapvec(elevation_normal)), - elevation_fitted: None, - elevation_error: None, - elevation_low: Some(convert_to_dmapvec(elevation_low)), - elevation_high: Some(convert_to_dmapvec(elevation_high)), - lambda_xcf_std_dev: Some(float_zeros.clone()), - sigma_xcf_std_dev: Some(float_zeros), - phi_xcf_std_dev: Some(convert_to_dmapvec(xcf_phi_std_dev)), - }) - } -} - -fn convert_to_dmapvec(vals: Vec) -> DmapVec { - DmapVec { - dimensions: vec![vals.len() as i32], - data: vals, + fit_rec.insert( + "slist".to_string(), + Array::from_vec(range_list).into_dyn().into(), + ); + fit_rec.insert( + "nlag".to_string(), + Array::from_vec(num_lags).into_dyn().into(), + ); + fit_rec.insert( + "qflg".to_string(), + Array::from_vec(quality_flag).into_dyn().into(), + ); + fit_rec.insert( + "gflg".to_string(), + Array::from_vec(groundscatter_flag).into_dyn().into(), + ); + fit_rec.insert( + "p_l".to_string(), + Array::from_vec(power_linear).into_dyn().into(), + ); + fit_rec.insert( + "p_l_e".to_string(), + Array::from_vec(power_linear_error).into_dyn().into(), + ); + fit_rec.insert( + "p_s".to_string(), + Array::from_vec(power_quadratic).into_dyn().into(), + ); + fit_rec.insert( + "p_s_e".to_string(), + Array::from_vec(power_quadratic_error).into_dyn().into(), + ); + fit_rec.insert("v".to_string(), Array::from_vec(velocity).into_dyn().into()); + fit_rec.insert( + "v_e".to_string(), + Array::from_vec(velocity_error).into_dyn().into(), + ); + fit_rec.insert( + "w_l".to_string(), + Array::from_vec(spectral_width_linear).into_dyn().into(), + ); + fit_rec.insert( + "w_l_e".to_string(), + Array::from_vec(spectral_width_linear_error) + .into_dyn() + .into(), + ); + fit_rec.insert( + "w_s".to_string(), + Array::from_vec(spectral_width_quadratic).into_dyn().into(), + ); + fit_rec.insert( + "w_s_e".to_string(), + Array::from_vec(spectral_width_quadratic_error) + .into_dyn() + .into(), + ); + fit_rec.insert( + "sd_l".to_string(), + Array::from_vec(std_dev_linear).into_dyn().into(), + ); + fit_rec.insert( + "sd_s".to_string(), + Array::from_vec(std_dev_quadratic).into_dyn().into(), + ); + fit_rec.insert( + "sd_phi".to_string(), + Array::from_vec(std_dev_phi).into_dyn().into(), + ); + fit_rec.insert( + "phi0".to_string(), + Array::from_vec(xcf_phi0).into_dyn().into(), + ); + fit_rec.insert( + "phi0_e".to_string(), + Array::from_vec(xcf_phi0_err).into_dyn().into(), + ); + fit_rec.insert( + "elv".to_string(), + Array::from_vec(elevation_phi0).into_dyn().into(), + ); + fit_rec.insert( + "elv_error".to_string(), + Array::from_vec(elevation_intercept_error).into_dyn().into(), + ); + fit_rec.insert( + "elv_fitted".to_string(), + Array::from_vec(elevation_intercept).into_dyn().into(), + ); + fit_rec.insert( + "x_sd_phi".to_string(), + Array::from_vec(xcf_phi_std_dev).into_dyn().into(), + ); } + let new_rec = FitacfRecord::new(&mut fit_rec).map_err(|e| { + Fitacf3Error::BadFit(format!( + "Could not create valid Fitacf record from results: {e}" + )) + })?; + Ok(new_rec) } fn calculate_elevation( ranges: &[RangeNode], - rec: &RawacfRecord, + rec: &Rawacf, xcf_phi0: &[f32], hdw: &HdwInfo, ) -> (Vec, Vec, Vec) { - let x = hdw.intf_offset_x; - let y = hdw.intf_offset_y; - let z = hdw.intf_offset_z; + let x = hdw.intf_offset_x as f64; + let y = hdw.intf_offset_y as f64; + let z = hdw.intf_offset_z as f64; - let array_separation: f32 = (x * x + y * y + z * z).sqrt(); + let array_separation: f64 = (x * x + y * y + z * z).sqrt(); let mut elevation_corr = (z / array_separation).asin(); - let phi_sign: f32; + let phi_sign: f64; if y > 0.0 { phi_sign = 1.0; } else { phi_sign = -1.0; elevation_corr *= -1.0; } - let azimuth_offset = hdw.max_num_beams as f32 / 2.0 - 0.5; - let phi_0 = - (hdw.beam_separation * (rec.beam_num as f32 - azimuth_offset) * PI_f32 / 180.0).cos(); - let wave_num = 2.0 * PI_f32 * rec.tx_freq as f32 * 1000.0 / 299792458.0; - let cable_offset = -2.0 * PI_f32 * rec.tx_freq as f32 * 1000.0 * hdw.tdiff_a * 1.0e-6; - let phase_diff_max = phi_sign * wave_num * array_separation * phi_0 + cable_offset; - let mut psi: Vec = ranges + let azimuth_offset = f32::from(hdw.max_num_beams) / 2.0 - 0.5; + let cos_phi_0 = (hdw.boresight_shift + + hdw.beam_separation * (f32::from(rec.bmnum) - azimuth_offset)) + .to_radians() + .cos() as f64; + let wave_num = 2.0 * PI_f64 * (rec.tfreq * KHZ_TO_HZ) as f64 / LIGHTSPEED as f64; + let cable_offset = + -2.0 * PI_f64 * (rec.tfreq * KHZ_TO_HZ) as f64 * (hdw.tdiff_a * US_TO_S) as f64; + let phase_diff_max = phi_sign * wave_num * array_separation * cos_phi_0 + cable_offset; + + let mut psi: Vec = ranges .iter() .map(|r| { - let x = r + let a = r .elev_fit .as_ref() .expect("Unable to find elevation without fitted elevation") - .intercept as f32; - let mut y = - x + 2.0 * PI_f32 * ((phase_diff_max - x) / (2.0 * PI_f32)).floor() - cable_offset; + .intercept; + let mut psi_raw = a + 2.0 * PI_f64 * ((phase_diff_max - a) / (2.0 * PI_f64)).floor(); if phi_sign < 0.0 { - y += 2.0 * PI_f32; + psi_raw += 2.0 * PI_f64; } - y + psi_raw - cable_offset }) .collect(); - let mut psi_kd: Vec = psi + let mut psi_kd: Vec = psi .iter() .map(|p| p / (wave_num * array_separation)) .collect(); - let mut theta: Vec = psi_kd.iter().map(|p| phi_0 * phi_0 - p * p).collect(); - let elevation: Vec = theta + let mut theta: Vec = psi_kd + .iter() + .map(|p| cos_phi_0 * cos_phi_0 - p * p) + .collect(); + let elevation_intercept: Vec = theta .iter() .map(|&t| { if t < 0.0 || t.abs() > 1.0 { - -elevation_corr + -elevation_corr.to_degrees() as f32 } else { - t.sqrt().asin() + t.sqrt().asin().to_degrees() as f32 } }) .collect(); - let elevation_high: Vec = elevation - .iter() - .map(|e| (e + elevation_corr) * 180.0 / PI_f32) - .collect(); - let psi_k2d2: Vec = psi + let psi_k2d2: Vec = psi .iter() .map(|p| p / (wave_num * wave_num * array_separation * array_separation)) .collect(); - let df_by_dy: Vec = zip(psi_k2d2.iter(), theta.iter()) - .map(|(p, t)| p / (t * (1.0 - t)).sqrt()) + let df_by_dy: Vec = zip(psi_k2d2.iter(), theta.iter()) + .map(|(p, t)| p / (t - t * t).sqrt()) .collect(); - let errors: Vec = ranges + let errors: Vec = ranges .iter() .map(|r| { r.elev_fit .as_ref() .expect("Unable to calculate elevation errors") - .variance_intercept as f32 + .variance_intercept }) .collect(); - let elevations_low: Vec = zip(errors.iter(), df_by_dy.iter()) - .map(|(e, d)| e.sqrt() * d.abs() * 180.0 / PI_f32) + let elevation_intercept_error: Vec = zip(errors.iter(), df_by_dy.iter()) + .map(|(e, d)| (e.sqrt() * d.abs()).to_degrees() as f32) .collect(); // This time, use the xcf lag0 phase psi = xcf_phi0 .iter() - .map(|&x| { - let mut y = x as f32 - + 2.0 * PI_f32 * ((phase_diff_max - x as f32) / (2.0 * PI_f32)).floor() + .map(|&p| { + let mut ps = p as f64 + + 2.0 * PI_f64 * ((phase_diff_max - p as f64) / (2.0 * PI_f64)).floor() - cable_offset; if phi_sign < 0.0 { - y += 2.0 * PI_f32; + ps += 2.0 * PI_f64; } - y + ps }) .collect(); psi_kd = psi .iter() .map(|p| p / (wave_num * array_separation)) .collect(); - theta = psi_kd.iter().map(|p| phi_0 * phi_0 - p * p).collect(); - let elevation_normal: Vec = theta + theta = psi_kd + .iter() + .map(|p| cos_phi_0 * cos_phi_0 - p * p) + .collect(); + let elevation_phi0: Vec = theta .iter() .map(|&t| { if t < 0.0 || t.abs() > 1.0 { - -180.0 / PI_f32 * elevation_corr + -elevation_corr.to_degrees() as f32 } else { - (t + elevation_corr).sqrt().asin() * 180.0 / PI_f32 + (t + elevation_corr).sqrt().asin().to_degrees() as f32 } }) .collect(); - (elevations_low, elevation_normal, elevation_high) + ( + elevation_phi0, + elevation_intercept_error, + elevation_intercept, + ) +} + +fn calculate_elevation_v2( + ranges: &[RangeNode], + rec: &Rawacf, + xcf_phi0: &[f32], + hdw: &HdwInfo, +) -> (Vec, Vec) { + let x = hdw.intf_offset_x; + let y = hdw.intf_offset_y; + let z = hdw.intf_offset_z; + + let psi_sign: f32 = if y > 0.0 { 1.0 } else { -1.0 }; + + let azimuth_offset = f32::from(hdw.max_num_beams) / 2.0 - 0.5; + let phi_0 = (hdw.boresight_shift + + hdw.beam_separation * (f32::from(rec.bmnum) - azimuth_offset)) + .to_radians(); + let cos_phi_0 = phi_0.cos(); // cp0 + let sin_phi_0 = phi_0.sin(); // sp0 + + let wave_num = 2.0 * PI_f32 * rec.tfreq * KHZ_TO_HZ / LIGHTSPEED; + let cable_offset_rad = -2.0 * PI_f32 * rec.tfreq * KHZ_TO_HZ * hdw.tdiff_a * US_TO_S; // psi_ele + + let mut elv_of_max_psi = (psi_sign * z * cos_phi_0 / (y * y + z * z).sqrt()).asin(); // a0 + if elv_of_max_psi < 0. { + elv_of_max_psi = 0.0; + } + + let cos_elv_of_max_psi = elv_of_max_psi.cos(); // ca0 + let sin_elv_of_max_psi = elv_of_max_psi.sin(); // sa0 + + let psi_max = cable_offset_rad + + wave_num + * (x * sin_phi_0 + + y * (cos_elv_of_max_psi * cos_elv_of_max_psi - sin_phi_0 * sin_phi_0).sqrt() + + z * sin_elv_of_max_psi); + + let num_phase_jump_func = { + if y > 0.0 { + f64::floor + } else { + f64::ceil + } + }; + let psi_calc = |p: &f32| -> f64 { + let delta_psi = (psi_max - p) as f64; + (p + 0.) as f64 + 2.0 * PI_f64 * num_phase_jump_func(delta_psi / (2.0 * PI_f64)) + }; + let e_calc = |p: &f64| -> f64 { + (p / (2.0 * PI_f64 * (rec.tfreq * KHZ_TO_HZ) as f64) + (hdw.tdiff_a * US_TO_S) as f64) + * LIGHTSPEED as f64 + - (x * sin_phi_0) as f64 + }; + let elv_calc = |e: &f64| -> f32 { + (e * z as f64 + + (e * e * (z * z) as f64 + - ((y * y) as f64 + (z * z) as f64) + * (e * e - (y * y) as f64 * (cos_phi_0 * cos_phi_0) as f64)) + .sqrt() + / ((y * y) as f64 + (z * z) as f64)) + .asin() + .to_degrees() as f32 + }; + + let psi_normal: Vec = xcf_phi0.iter().map(psi_calc).collect(); + let e_normal: Vec = psi_normal.iter().map(e_calc).collect(); + let elv_normal = e_normal // called alpha in RST + .iter() + .map(elv_calc) + .collect(); + + let psi_fitted: Vec = ranges + .iter() + .map(|r| { + let p = r + .elev_fit + .as_ref() + .expect("Unable to find elevation without fitted elevation") + .intercept as f32 + * hdw.phase_sign; + psi_calc(&p) + }) + .collect(); + let e_fitted: Vec = psi_fitted.iter().map(e_calc).collect(); + let elv_fitted = e_fitted.iter().map(elv_calc).collect(); + + (elv_normal, elv_fitted) } diff --git a/src/fitting/fitacf3/filtering.rs b/src/fitting/fitacf3/filtering.rs index 7187d70..77431ee 100644 --- a/src/fitting/fitacf3/filtering.rs +++ b/src/fitting/fitacf3/filtering.rs @@ -2,69 +2,68 @@ use crate::fitting::fitacf3::fitacf_v3::{ Fitacf3Error, ALPHA_CUTOFF, FLUCTUATION_CUTOFF_COEFFICIENT, MIN_LAGS, }; use crate::fitting::fitacf3::fitstruct::{LagNode, RangeNode}; -use dmap::formats::RawacfRecord; +use crate::utils::rawacf::Rawacf; use is_close::is_close; -/// passing -pub fn mark_bad_samples(rec: &RawacfRecord) -> Vec { +/// Finds all samples that were collected during transmission of a pulse. +pub(crate) fn mark_bad_samples(rec: &Rawacf) -> Vec { let mut pulses_in_us: Vec = rec - .pulse_table - .data + .ptab .iter() - .map(|&p| p as i32 * rec.multi_pulse_increment as i32) + .map(|&p| i32::from(p) * i32::from(rec.mpinc)) .collect(); if rec.offset != 0 { if rec.channel == 1 { let pulses_stereo: Vec = pulses_in_us .iter() - .map(|&p| p - rec.offset as i32) + .map(|&p| p - i32::from(rec.offset)) .collect(); pulses_in_us.extend(pulses_stereo); } else if rec.channel == 2 { let pulses_stereo: Vec = pulses_in_us .iter() - .map(|&p| p + rec.offset as i32) + .map(|&p| p + i32::from(rec.offset)) .collect(); pulses_in_us.extend(pulses_stereo); } } pulses_in_us.sort(); - let mut ts = rec.lag_to_first_range as i32; + let mut ts = i32::from(rec.lagfr); let mut t1; let mut t2; let mut sample = 0; let mut bad_samples = vec![]; for pulse_us in pulses_in_us { - t1 = pulse_us - rec.tx_pulse_length as i32 / 2; - t2 = t1 + 3 * rec.tx_pulse_length as i32 / 2 + 100; + t1 = pulse_us - i32::from(rec.txpl) / 2; + t2 = t1 + 3 * i32::from(rec.txpl) / 2 + 100; // Start incrementing the sample until we find a sample that lies within a pulse while ts < t1 { sample += 1; - ts += rec.sample_separation as i32; + ts += i32::from(rec.smsep); } // Blank all samples within the pulse duration while (ts >= t1) && (ts <= t2) { bad_samples.push(sample); sample += 1; - ts += rec.sample_separation as i32; + ts += i32::from(rec.smsep); } } bad_samples } -/// passing -pub fn filter_tx_overlapped_lags( - rec: &RawacfRecord, - lags: Vec, - ranges: &mut Vec, +/// Removes all lags that contain samples collected during transmission of a pulse. +pub(crate) fn filter_tx_overlapped_lags( + rec: &Rawacf, + lags: &[LagNode], + ranges: &mut [RangeNode], ) { let bad_samples = mark_bad_samples(rec); - for range_node in ranges { + for range_node in ranges.iter_mut() { let mut bad_indices = vec![]; for (idx, lag) in lags.iter().enumerate() { let sample_1 = lag.sample_base_1 + range_node.range_num as i32; @@ -83,8 +82,8 @@ pub fn filter_tx_overlapped_lags( } } -/// passing -pub fn filter_infinite_lags(ranges: &mut Vec) { +/// Removes all lags that have infinite power values. +pub(crate) fn filter_infinite_lags(ranges: &mut Vec) { for range in ranges { let mut infinite_indices = vec![]; for i in 0..range.powers.ln_power.len() { @@ -99,9 +98,9 @@ pub fn filter_infinite_lags(ranges: &mut Vec) { } } -/// passing -pub fn filter_low_power_lags(rec: &RawacfRecord, ranges: &mut Vec) { - if rec.num_averages <= 0 { +/// Removes all lags after a lag with low power +pub(crate) fn filter_low_power_lags(rec: &Rawacf, ranges: &mut Vec) { + if rec.nave <= 0 { return; } for range in ranges { @@ -109,15 +108,14 @@ pub fn filter_low_power_lags(rec: &RawacfRecord, ranges: &mut Vec) { if range.powers.ln_power.is_empty() { continue; } - let log_sigma_fluc = (FLUCTUATION_CUTOFF_COEFFICIENT as f32 - * rec.lag_zero_power.data[range_num] - / ((2 * rec.num_averages) as f32).sqrt()) + let log_sigma_fluc = (FLUCTUATION_CUTOFF_COEFFICIENT * rec.pwr0[range_num as usize] + / f32::from(2 * rec.nave).sqrt()) .ln(); let mut bad_indices = vec![]; - let mut cutoff_lag = rec.num_lags as usize + 1; + let mut cutoff_lag = rec.mplgs as usize + 1; for idx in 0..range.powers.ln_power.len() { - if idx > cutoff_lag as usize { + if idx > cutoff_lag { bad_indices.push(idx); } else { let log_power = range.powers.ln_power[idx]; @@ -140,16 +138,15 @@ pub fn filter_low_power_lags(rec: &RawacfRecord, ranges: &mut Vec) { } } -/// passing -pub fn filter_bad_acfs(rec: &RawacfRecord, ranges: &mut Vec, noise_power: f32) { - if rec.num_averages <= 0 { +/// Removes range gates that either contain too weak of a fit, or used too few lags when fitting. +pub(crate) fn filter_bad_acfs(rec: &Rawacf, ranges: &mut Vec, noise_power: f32) { + if rec.nave <= 0 { return; } let cutoff_power = noise_power * 2.0; let mut bad_indices = vec![]; for (idx, range) in ranges.iter().enumerate() { - let range_num = range.range_num as usize; - let power = rec.lag_zero_power.data[range_num]; + let power = rec.pwr0[range.range_num as usize]; let num_powers = range.powers.ln_power.len(); if (power <= cutoff_power) || (num_powers < MIN_LAGS as usize) { bad_indices.push(idx); @@ -171,15 +168,15 @@ pub fn filter_bad_acfs(rec: &RawacfRecord, ranges: &mut Vec, noise_po } } -/// presumed passing -pub fn filter_bad_fits(ranges: &mut Vec) -> Result<(), Fitacf3Error> { +/// Removes all ranges that have not had phase, lambda power, or quadratic power fitted +pub(crate) fn filter_bad_fits(ranges: &mut Vec) -> Result<(), Fitacf3Error> { let mut bad_indices = vec![]; for (idx, range) in ranges.iter().enumerate() { if (range .phase_fit .as_ref() .ok_or_else(|| { - Fitacf3Error::Message("Cannot filter fits since phase not fit".to_string()) + Fitacf3Error::BadFit("Cannot filter fits since phase not fit".to_string()) })? .slope == 0.0) @@ -187,7 +184,7 @@ pub fn filter_bad_fits(ranges: &mut Vec) -> Result<(), Fitacf3Error> .lin_pwr_fit .as_ref() .ok_or_else(|| { - Fitacf3Error::Message( + Fitacf3Error::BadFit( "Cannot filter fits since power not linearly fit".to_string(), ) })? @@ -197,7 +194,7 @@ pub fn filter_bad_fits(ranges: &mut Vec) -> Result<(), Fitacf3Error> .quad_pwr_fit .as_ref() .ok_or_else(|| { - Fitacf3Error::Message( + Fitacf3Error::BadFit( "Cannot filter fits since power not quadratically fit".to_string(), ) })? diff --git a/src/fitting/fitacf3/fitacf_v3.rs b/src/fitting/fitacf3/fitacf_v3.rs index eb149e5..0600e8b 100644 --- a/src/fitting/fitacf3/fitacf_v3.rs +++ b/src/fitting/fitacf3/fitacf_v3.rs @@ -1,99 +1,159 @@ -use crate::fitting::fitacf3::fitstruct::{LagNode, RangeNode}; - +//! Top-level objects and functions for the FITACF3 fitting method. +use crate::error::ProcdarnError; use crate::fitting::fitacf3::determinations::determinations; use crate::fitting::fitacf3::filtering; +use crate::fitting::fitacf3::fitstruct::{LagNode, RangeNode}; use crate::fitting::fitacf3::fitting; use crate::utils::hdw::HdwInfo; -use dmap::formats::{FitacfRecord, RawacfRecord}; -use std::error::Error; +use crate::utils::rawacf::{get_hdw, Rawacf}; +use dmap::error::DmapError; +use dmap::formats::{fitacf::FitacfRecord, rawacf::RawacfRecord}; +use pyo3::exceptions::PyValueError; +use pyo3::PyErr; +use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; use std::f64::consts::PI; -use std::fmt; -use std::fmt::Display; +use thiserror::Error; type Result = std::result::Result; -pub const FLUCTUATION_CUTOFF_COEFFICIENT: f32 = 2.0; -pub const ALPHA_CUTOFF: f32 = 2.0; -pub const ACF_SNR_CUTOFF: f64 = 1.0; -pub const MIN_LAGS: i16 = 3; +pub(crate) const FLUCTUATION_CUTOFF_COEFFICIENT: f32 = 2.0; +pub(crate) const ALPHA_CUTOFF: f32 = 2.0; +pub(crate) const ACF_SNR_CUTOFF: f64 = 1.0; +pub(crate) const MIN_LAGS: i16 = 3; -#[derive(Debug, Clone)] +/// Enum of the possible error variants that may be encountered. +#[derive(Error, Debug)] pub enum Fitacf3Error { - Message(String), - Lookup(String), - Mismatch { msg: String }, -} + /// Represents an error in the Rawacf record that is attempting to be fitted + #[error("{0}")] + InvalidRawacf(String), -impl Error for Fitacf3Error {} + /// Represents a bad fit of the record, for any reason + #[error("{0}")] + BadFit(String), -impl Display for Fitacf3Error { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match self { - Fitacf3Error::Message(msg) => write!(f, "{}", msg), - Fitacf3Error::Lookup(msg) => write!(f, "{}", msg), - Fitacf3Error::Mismatch { msg } => write!(f, "{}", msg), - } + /// Unable to get hardware file information + #[error("{0}")] + Hdw(#[from] ProcdarnError), + + /// Invalid DMAP file + #[error("{0}")] + Dmap(#[from] DmapError), +} + +impl From for PyErr { + fn from(value: Fitacf3Error) -> Self { + let msg = value.to_string(); + PyValueError::new_err(msg) } } -pub fn fit_rawacf_record(record: &RawacfRecord, hdw: &HdwInfo) -> Result { - let lags = create_lag_list(record); +/// Fits a single `RawacfRecord` into a `FitacfRecord` +/// +/// # Errors +/// Will return `Err` if the `RawacfRecord` does not have all required fields for fitting, +/// or if the data within the `RawacfRecord` is unsuitable for fitting for any reason. +fn fit_rawacf_record(record: &RawacfRecord, hdw: &HdwInfo) -> Result { + let raw: Rawacf = Rawacf::try_from(record).map_err(|e| { + Fitacf3Error::InvalidRawacf(format!( + "Could not extract all required fields from rawacf record: {e}" + )) + })?; + let lags = create_lag_list(&raw); - let noise_power = if record.num_averages <= 0 { + let noise_power = if raw.nave <= 0 { 1.0 } else { - acf_cutoff_power(record) + acf_cutoff_power(&raw) }; let mut range_list = vec![]; - for i in 0..record.range_list.data.len() { - let range_num = record.range_list.data[i]; - if record.lag_zero_power.data[range_num as usize] != 0.0 { - range_list.push(RangeNode::new(i, range_num as usize, record, &lags)?) + for i in 0..raw.slist.len() { + let range_num = raw.slist[i]; + if raw.pwr0[range_num as usize] != 0.0 { + range_list.push(RangeNode::new(i, range_num as usize, &raw, &lags)?); } } - filtering::filter_tx_overlapped_lags(record, lags, &mut range_list); + filtering::filter_tx_overlapped_lags(&raw, &lags, &mut range_list); filtering::filter_infinite_lags(&mut range_list); - filtering::filter_low_power_lags(record, &mut range_list); - filtering::filter_bad_acfs(record, &mut range_list, noise_power); + filtering::filter_low_power_lags(&raw, &mut range_list); + filtering::filter_bad_acfs(&raw, &mut range_list, noise_power); fitting::acf_power_fitting(&mut range_list)?; - fitting::calculate_phase_and_elev_sigmas(&mut range_list, record)?; + fitting::calculate_phase_and_elev_sigmas(&mut range_list, &raw)?; fitting::acf_phase_unwrap(&mut range_list); fitting::acf_phase_fitting(&mut range_list)?; filtering::filter_bad_fits(&mut range_list)?; fitting::xcf_phase_unwrap(&mut range_list)?; fitting::xcf_phase_fitting(&mut range_list)?; - determinations(record, range_list, noise_power, hdw) + determinations(&raw, &range_list, noise_power, hdw) +} + +/// Fits a collection of [`RawacfRecord`]s into [`FitacfRecord`]s using the FITACF3 algorithm without parallelization. +/// +/// # Errors +/// Will return `Err` if the [`RawacfRecord`]s do not have all required fields for fitting, +/// or if the data within the [`RawacfRecord`]s are unsuitable for fitting for any reason. +pub fn fitacf3_single_threaded(raw_recs: Vec) -> Result> { + let hdw = get_hdw(&raw_recs[0])?; + + let mut fitacf_records = vec![]; + for rec in raw_recs { + fitacf_records.push(fit_rawacf_record(&rec, &hdw)?); + } + Ok(fitacf_records) +} + +/// Fits a collection of [`RawacfRecord`]s into [`FitacfRecord`]s using the FITACF3 algorithm. +/// +/// # Errors +/// Will return `Err` if the [`RawacfRecord`]s do not have all required fields for fitting, +/// or if the data within the [`RawacfRecord`]s are unsuitable for fitting for any reason. +pub fn fitacf3(raw_recs: Vec) -> Result> { + let hdw = get_hdw(&raw_recs[0])?; + + // Fit the records! + let fitacf_results: Vec> = raw_recs + .par_iter() + .map(|rec| fit_rawacf_record(rec, &hdw)) + .collect(); + + let mut fitacf_records = vec![]; + for res in fitacf_results { + match res { + Ok(x) => fitacf_records.push(x), + Err(e) => Err(e)?, + } + } + Ok(fitacf_records) } /// Creates the lag table based on the data. -fn create_lag_list(record: &RawacfRecord) -> Vec { - let lag_table = &record.lag_table; - let pulse_table = &record.pulse_table; - let multi_pulse_increment = record.multi_pulse_increment; - let sample_separation = record.sample_separation; +fn create_lag_list(record: &Rawacf) -> Vec { + let lag_table = &record.ltab; + let pulse_table = &record.ptab; + let multi_pulse_increment = record.mpinc; + let sample_separation = record.smsep; let mut lags = vec![]; - for i in 0..record.num_lags as usize { + for i in 0..record.mplgs as usize { let mut pulse_1_idx = 0; let mut pulse_2_idx = 0; - let number = lag_table.data[2 * i + 1] - lag_table.data[2 * i]; // flattened, we want row i, cols 1 and 0 - for j in 0..record.num_pulses as usize { - if lag_table.data[2 * i] == pulse_table.data[j] { + let number = lag_table[[i, 1]] - lag_table[[i, 0]]; + for j in 0..record.mppul as usize { + if lag_table[[i, 0]] == pulse_table[j] { pulse_1_idx = j; } - if lag_table.data[2 * i + 1] == pulse_table.data[j] { + if lag_table[[i, 1]] == pulse_table[j] { pulse_2_idx = j; } } let sample_base_1 = - (lag_table.data[2 * i] * (multi_pulse_increment / sample_separation)) as i32; + i32::from(lag_table[[i, 0]] * (multi_pulse_increment / sample_separation)); let sample_base_2 = - (lag_table.data[2 * i + 1] * (multi_pulse_increment / sample_separation)) as i32; + i32::from(lag_table[[i, 1]] * (multi_pulse_increment / sample_separation)); lags.push(LagNode { - lag_num: number as i32, + lag_num: i32::from(number), pulses: [pulse_1_idx, pulse_2_idx], - lag_idx: 0, sample_base_1, sample_base_2, }); @@ -101,14 +161,14 @@ fn create_lag_list(record: &RawacfRecord) -> Vec { lags } -/// Calculates the minimum power value for ACFs in the record (passing) -fn acf_cutoff_power(rec: &RawacfRecord) -> f32 { - let mut sorted_power_levels = rec.lag_zero_power.data.clone(); - sorted_power_levels.sort_by(|a, b| a.total_cmp(b)); // sort floats +/// Calculates the minimum power value for ACFs in the record +fn acf_cutoff_power(rec: &Rawacf) -> f32 { + let mut sorted_power_levels = rec.pwr0.clone().to_vec(); + sorted_power_levels.sort_by(f32::total_cmp); // sort floats let mut i: usize = 0; let mut j: f64 = 0.0; let mut min_power: f64 = 0.0; - while j < 10.0 && i < rec.num_ranges as usize / 3 { + while j < 10.0 && i < rec.nrang as usize / 3 { if sorted_power_levels[i] > 0.0 { j += 1.0; } @@ -119,25 +179,26 @@ fn acf_cutoff_power(rec: &RawacfRecord) -> f32 { j = 1.0; } min_power *= cutoff_power_correction(rec) / j; - if min_power < ACF_SNR_CUTOFF && rec.search_noise > 0.0 { - min_power = rec.search_noise as f64; + let search_noise = rec.noise_search; + if min_power < ACF_SNR_CUTOFF && search_noise != 0.0 { + min_power = search_noise as f64; } min_power as f32 } -/// Passing -fn cutoff_power_correction(rec: &RawacfRecord) -> f64 { - let std_dev = 1.0 / (rec.num_averages as f64).sqrt(); +/// Applies a correction to the noise power estimate to account for selecting least-powerful ranges +fn cutoff_power_correction(rec: &Rawacf) -> f64 { + let std_dev = 1.0 / (rec.nave as f64).sqrt(); - let mut i = 0.0; - let mut cumulative_pdf = 0.0; - let mut cumulative_pdf_x_norm_power = 0.0; - let mut normalized_power; - while cumulative_pdf < (10.0 / rec.num_ranges as f64) { + let mut i = 0.0_f64; + let mut cumulative_pdf = 0.0_f64; + let mut cumulative_pdf_x_norm_power = 0.0_f64; + let mut normalized_power: f64; + while cumulative_pdf < (10.0 / rec.nrang as f64) { // Normalized power for calculating model PDF (Gaussian) normalized_power = i / 1000.0; let x = -(normalized_power - 1.0) * (normalized_power - 1.0) / (2.0 * std_dev * std_dev); - let pdf = x.exp() / std_dev / (2.0 * PI).sqrt() / 1000.0; + let pdf = x.exp() / (std_dev * (2.0 * PI).sqrt() * 1000.0); cumulative_pdf += pdf; // Cumulative value of PDF * x -> needed for calculating the mean diff --git a/src/fitting/fitacf3/fitstruct.rs b/src/fitting/fitacf3/fitstruct.rs index 421ebfe..12fffea 100644 --- a/src/fitting/fitacf3/fitstruct.rs +++ b/src/fitting/fitacf3/fitstruct.rs @@ -1,13 +1,14 @@ use crate::fitting::fitacf3::fitacf_v3::Fitacf3Error; -use dmap::formats::RawacfRecord; +use crate::utils::rawacf::Rawacf; +use numpy::ndarray::prelude::*; use std::iter::zip; #[derive(Debug)] -pub struct RangeNode { - pub range_num: usize, +pub(crate) struct RangeNode { + pub range_num: u16, pub range_idx: usize, - pub cross_range_interference: Vec, - pub refractive_idx: f32, + // pub cross_range_interference: Vec, + // pub refractive_idx: f32, pub power_alpha_2: Vec, pub phase_alpha_2: Vec, pub phases: PhaseNode, @@ -21,24 +22,24 @@ pub struct RangeNode { pub elev_fit: Option, } impl RangeNode { - pub fn new( + pub(crate) fn new( index: usize, range_num: usize, - record: &RawacfRecord, + record: &Rawacf, lags: &[LagNode], ) -> Result { let cross_range_interference = RangeNode::calculate_cross_range_interference(range_num, record); let alpha_2 = RangeNode::calculate_alphas(range_num, &cross_range_interference, record, lags); - let phases = PhaseNode::new(record, "acfd", lags, index)?; - let elevations = PhaseNode::new(record, "xcfd", lags, index)?; + let phases = PhaseNode::new(record, &PhaseFitType::Acf, lags, index)?; + let elevations = PhaseNode::new(record, &PhaseFitType::Xcf, lags, index)?; let powers = PowerNode::new(record, lags, index, range_num, &alpha_2); Ok(RangeNode { range_idx: index, - range_num, - cross_range_interference, - refractive_idx: 1.0, + range_num: range_num as u16, + // cross_range_interference, + // refractive_idx: 1.0, power_alpha_2: alpha_2.clone(), phase_alpha_2: alpha_2, phases, @@ -52,22 +53,22 @@ impl RangeNode { elev_fit: None, }) } - fn calculate_cross_range_interference(range_num: usize, rec: &RawacfRecord) -> Vec { - let tau: i16 = if rec.sample_separation != 0 { - rec.multi_pulse_increment / rec.sample_separation + fn calculate_cross_range_interference(range_num: usize, rec: &Rawacf) -> Vec { + let tau: i16 = if rec.smsep != 0 { + rec.mpinc / rec.smsep } else { // TODO: Log warning? - rec.multi_pulse_increment / rec.tx_pulse_length + rec.mpinc / rec.txpl }; let mut interference_for_pulses: Vec = vec![]; - for pulse_to_check in 0..rec.num_pulses as usize { + for pulse_to_check in 0..rec.mppul as usize { let mut total_interference: f64 = 0.0; - for pulse in 0..rec.num_pulses as usize { - let pulse_diff = rec.pulse_table.data[pulse_to_check] - rec.pulse_table.data[pulse]; + for pulse in 0..rec.mppul as usize { + let pulse_diff = rec.ptab[pulse_to_check] - rec.ptab[pulse]; let range_to_check = (pulse_diff * tau + range_num as i16) as usize; - if (pulse != pulse_to_check) && (range_to_check < rec.num_ranges as usize) { - total_interference += rec.lag_zero_power.data[range_to_check] as f64; + if (pulse != pulse_to_check) && (range_to_check < rec.nrang as usize) { + total_interference += rec.pwr0[range_to_check] as f64; } } interference_for_pulses.push(total_interference); @@ -77,14 +78,14 @@ impl RangeNode { fn calculate_alphas( range_num: usize, cross_range_interference: &[f64], - rec: &RawacfRecord, + rec: &Rawacf, lags: &[LagNode], ) -> Vec { let mut alpha_2: Vec = vec![]; - for lag in lags.iter() { - let pulse_1_interference = cross_range_interference[lag.pulses[0] as usize]; - let pulse_2_interference = cross_range_interference[lag.pulses[1] as usize]; - let lag_zero_power = rec.lag_zero_power.data[range_num] as f64; + for lag in lags { + let pulse_1_interference = cross_range_interference[lag.pulses[0]]; + let pulse_2_interference = cross_range_interference[lag.pulses[1]]; + let lag_zero_power = rec.pwr0[range_num] as f64; alpha_2.push( lag_zero_power * lag_zero_power / ((lag_zero_power + pulse_1_interference) @@ -96,42 +97,42 @@ impl RangeNode { } #[derive(Debug)] -pub struct PhaseNode { +pub(crate) struct PhaseNode { pub phases: Vec, pub t: Vec, pub std_dev: Vec, } impl PhaseNode { - pub fn new( - rec: &RawacfRecord, - phase_type: &str, + pub(crate) fn new( + rec: &Rawacf, + phase_type: &PhaseFitType, lags: &[LagNode], range_idx: usize, ) -> Result { let acfd = match phase_type { - "acfd" => &rec.acfs.data, - "xcfd" => match &rec.xcfs { - Some(x) => &x.data, - None => Err(Fitacf3Error::Message( + PhaseFitType::Acf => &rec.acfd, + PhaseFitType::Xcf => match &rec.xcfd { + Some(ref x) => x, + None => Err(Fitacf3Error::InvalidRawacf( "Cannot find xcfs in data".to_string(), ))?, }, - _ => Err(Fitacf3Error::Message(format!( - "Unknown type for PhaseNode: {}", - phase_type - )))?, }; - let start_idx = range_idx * 2 * rec.num_lags as usize; - let end_idx = start_idx + 2 * rec.num_lags as usize; - let phases = acfd[start_idx..end_idx] - .chunks_exact(2) - .map(|x| (x[1] as f64).atan2(x[0] as f64)) - .collect(); + let phases = zip( + acfd.slice(s![range_idx, .., 0]), + acfd.slice(s![range_idx, .., 1]), + ) + .map(|(&x, &y)| { + let real = x as f64; + let imag = y as f64; + imag.atan2(real) + }) + .collect(); let t = lags .iter() - .map(|x| (x.lag_num * rec.multi_pulse_increment as i32) as f64 * 1.0e-6) + .map(|x| (x.lag_num * rec.mpinc as i32) as f64 * 1.0e-6) .collect(); - let std_dev = (0..rec.num_lags).map(|_| 0.0).collect(); + let std_dev = (0..rec.mplgs).map(|_| 0.0).collect(); Ok(PhaseNode { phases, t, std_dev }) } pub fn remove(&mut self, idx: usize) { @@ -142,41 +143,41 @@ impl PhaseNode { } #[derive(Debug)] -pub struct PowerNode { +pub(crate) struct PowerNode { pub ln_power: Vec, pub t: Vec, pub std_dev: Vec, } impl PowerNode { - pub fn new( - rec: &RawacfRecord, + pub(crate) fn new( + rec: &Rawacf, lags: &[LagNode], range_idx: usize, range_num: usize, alpha_2: &[f64], ) -> PowerNode { - let pwr_0 = rec.lag_zero_power.data[range_num] as f64; + let pwr_0 = rec.pwr0[range_num] as f64; // acfs stores as [num_ranges, num_lags, 2] in memory, with 2 corresponding to real, imag - let start_idx = range_idx * 2 * rec.num_lags as usize; - let end_idx = start_idx + 2 * rec.num_lags as usize; - let powers: Vec = rec.acfs.data[start_idx..end_idx] - .chunks_exact(2) - .map(|x| { - let real = x[0] as f64; - let imag = x[1] as f64; - (real * real + imag * imag).sqrt() - }) - .collect(); + let powers: Vec = zip( + rec.acfd.slice(s![range_idx, .., 0]), + rec.acfd.slice(s![range_idx, .., 1]), + ) + .map(|(&x, &y)| { + let real = x as f64; + let imag = y as f64; + (real * real + imag * imag).sqrt() + }) + .collect(); let normalized_power: Vec = powers.iter().map(|x| x * x / (pwr_0 * pwr_0)).collect(); let sigmas: Vec = zip(normalized_power.iter(), alpha_2.iter()) .map(|(pwr_norm, alpha)| { - pwr_0 * ((pwr_norm + 1.0 / alpha) / (2.0 * rec.num_averages as f64)).sqrt() + pwr_0 * ((pwr_norm + 1.0 / alpha) / (2.0 * rec.nave as f64)).sqrt() }) .collect(); let t = lags .iter() - .map(|x| (x.lag_num * rec.multi_pulse_increment as i32) as f64 * 1.0e-6) + .map(|x| (x.lag_num * rec.mpinc as i32) as f64 * 1.0e-6) .collect(); PowerNode { ln_power: powers.iter().map(|x| x.ln()).collect(), @@ -184,7 +185,7 @@ impl PowerNode { std_dev: sigmas, } } - pub fn remove(&mut self, idx: usize) { + pub(crate) fn remove(&mut self, idx: usize) { self.ln_power.remove(idx); self.t.remove(idx); self.std_dev.remove(idx); @@ -192,16 +193,15 @@ impl PowerNode { } #[derive(Debug)] -pub struct LagNode { +pub(crate) struct LagNode { pub lag_num: i32, pub pulses: [usize; 2], - pub lag_idx: i32, pub sample_base_1: i32, pub sample_base_2: i32, } #[derive(Default, Debug)] -pub struct FittedData { +pub(crate) struct FittedData { pub delta: f64, pub intercept: f64, pub slope: f64, @@ -211,12 +211,11 @@ pub struct FittedData { pub delta_slope: f64, pub covariance_intercept_slope: f64, pub residual_intercept_slope: f64, - pub quality: f64, pub chi_squared: f64, } #[derive(Default, Debug)] -pub struct Sums { +pub(crate) struct Sums { pub sum: f64, pub sum_x: f64, pub sum_y: f64, @@ -224,7 +223,14 @@ pub struct Sums { pub sum_xy: f64, } -pub enum FitType { +#[derive(Copy, Clone)] +pub(crate) enum PowerFitType { Linear, Quadratic, } + +#[derive(Copy, Clone)] +pub(crate) enum PhaseFitType { + Acf, + Xcf, +} diff --git a/src/fitting/fitacf3/fitting.rs b/src/fitting/fitacf3/fitting.rs index f1d0748..f01a93b 100644 --- a/src/fitting/fitacf3/fitting.rs +++ b/src/fitting/fitacf3/fitting.rs @@ -1,113 +1,116 @@ use crate::fitting::fitacf3::fitacf_v3::Fitacf3Error; -use crate::fitting::fitacf3::fitstruct::{FitType, RangeNode}; +use crate::fitting::fitacf3::fitstruct::{PowerFitType, RangeNode}; use crate::fitting::fitacf3::least_squares::LeastSquares; -use dmap::formats::RawacfRecord; +use crate::utils::rawacf::Rawacf; use std::f64::consts::PI; use std::iter::zip; type Result = std::result::Result; -/// passing -pub fn acf_power_fitting(ranges: &mut Vec) -> Result<()> { +/// Fits the power of ACF data. +pub(crate) fn acf_power_fitting(ranges: &mut Vec) -> Result<()> { let lsq = LeastSquares::new(1, 1); - for mut range in ranges { + for range in ranges { let log_powers = &range.powers.ln_power; let sigmas = &range.powers.std_dev; let t = &range.powers.t; let num_points = range.powers.ln_power.len(); if t.len() != num_points || sigmas.len() != num_points { - Err(Fitacf3Error::Message( + Err(Fitacf3Error::BadFit( "Cannot perform acf power fitting - dimension mismatch".to_string(), - ))? + ))?; } range.lin_pwr_fit = - Some(lsq.two_parameter_line_fit(t, log_powers, sigmas, FitType::Linear)); + Some(lsq.two_parameter_line_fit(t, log_powers, sigmas, &PowerFitType::Linear)); range.quad_pwr_fit = - Some(lsq.two_parameter_line_fit(t, log_powers, sigmas, FitType::Quadratic)); + Some(lsq.two_parameter_line_fit(t, log_powers, sigmas, &PowerFitType::Quadratic)); let log_corrected_sigmas: Vec = zip(sigmas.iter(), log_powers.iter()) .map(|(s, l)| s / l.exp()) .collect(); - range.lin_pwr_fit_err = - Some(lsq.two_parameter_line_fit(t, log_powers, &log_corrected_sigmas, FitType::Linear)); + range.lin_pwr_fit_err = Some(lsq.two_parameter_line_fit( + t, + log_powers, + &log_corrected_sigmas, + &PowerFitType::Linear, + )); range.quad_pwr_fit_err = Some(lsq.two_parameter_line_fit( t, log_powers, &log_corrected_sigmas, - FitType::Quadratic, + &PowerFitType::Quadratic, )); } Ok(()) } -/// passing -pub fn acf_phase_fitting(ranges: &mut Vec) -> Result<()> { +/// Fits the phase of ACF data. +pub(crate) fn acf_phase_fitting(ranges: &mut Vec) -> Result<()> { let lsq = LeastSquares::new(1, 1); - for mut range in ranges { + for range in ranges { let phases = &range.phases.phases; let sigmas = &range.phases.std_dev; let t = &range.phases.t; let num_points = t.len(); if phases.len() != num_points || sigmas.len() != num_points { - Err(Fitacf3Error::Message( + Err(Fitacf3Error::BadFit( "Cannot perform acf phase fitting - dimension mismatch".to_string(), - ))? + ))?; } range.phase_fit = Some(lsq.one_parameter_line_fit(t, phases, sigmas)); } Ok(()) } -/// passing -pub fn xcf_phase_fitting(ranges: &mut Vec) -> Result<()> { +/// Fits the phase of XCF data. +pub(crate) fn xcf_phase_fitting(ranges: &mut Vec) -> Result<()> { let lsq = LeastSquares::new(1, 1); - for mut range in ranges { + for range in ranges { let phases = &range.elev.phases; let sigmas = &range.elev.std_dev; let t = &range.elev.t; let num_points = t.len(); if phases.len() != num_points || sigmas.len() != num_points { - Err(Fitacf3Error::Message( + Err(Fitacf3Error::BadFit( "Cannot perform xcf phase fitting - dimension mismatch".to_string(), - ))? + ))?; } - range.elev_fit = Some(lsq.two_parameter_line_fit(t, phases, sigmas, FitType::Linear)); + range.elev_fit = Some(lsq.two_parameter_line_fit(t, phases, sigmas, &PowerFitType::Linear)); } Ok(()) } -/// passing -pub fn calculate_phase_and_elev_sigmas( +/// Calculates standard deviations for phase and elevation fits. +pub(crate) fn calculate_phase_and_elev_sigmas( ranges: &mut Vec, - rec: &RawacfRecord, + rec: &Rawacf, ) -> Result<()> { - for mut range in ranges { + for range in ranges { let inverse_alpha_2: Vec = range.phase_alpha_2.iter().map(|x| 1.0 / x).collect(); - // let elevs_inverse_alpha_2: Vec = range.alpha_2.iter().map(|x| 1.0 / x).collect(); let pwr_values: Vec = range .phases .t .iter() - .map(|t| (-1.0 * range.lin_pwr_fit.as_ref().unwrap().slope.abs() * t).exp()) + .map(|t| (-range.lin_pwr_fit.as_ref().unwrap().slope.abs() * t).exp()) .collect(); let inverse_pwr_squared: Vec = pwr_values.iter().map(|x| 1.0 / (x * x)).collect(); let phase_numerator: Vec = zip(inverse_alpha_2.iter(), inverse_pwr_squared.iter()) .map(|(x, y)| x * y - 1.0) .collect(); - let denominator = 2.0 * rec.num_averages as f64; + let denominator = 2.0 * rec.nave as f64; let mut phase_sigmas: Vec = phase_numerator .iter() .map(|x| (x / denominator).sqrt()) .collect(); if phase_sigmas.iter().filter(|&x| !x.is_finite()).count() > 0 { - Err(Fitacf3Error::Message(format!( - "Phase sigmas bad at range {}", + Err(Fitacf3Error::BadFit(format!( + "Phase sigmas infinite at range {}", range.range_idx - )))? + )))?; } range.phases.std_dev = phase_sigmas.clone(); // Since lag 0 phase is included for elevation fit, set lag 0 sigma the same as lag 1 sigma @@ -117,9 +120,9 @@ pub fn calculate_phase_and_elev_sigmas( Ok(()) } -/// passing -pub fn acf_phase_unwrap(ranges: &mut Vec) { - for mut range in ranges { +/// Applies 2Ï€ phase unwrapping to ACF phases. +pub(crate) fn acf_phase_unwrap(ranges: &mut Vec) { + for range in ranges { let (mut slope_numerator, mut slope_denominator) = (0.0, 0.0); let phases = &range.phases.phases; @@ -133,19 +136,19 @@ pub fn acf_phase_unwrap(ranges: &mut Vec) { let mut first_time = true; for (p, (s, t)) in zip(phases.iter(), zip(sigmas.iter(), t.iter())) { - if !first_time { + if first_time { + first_time = false; + } else { let phase_diff = p - phase_prev; let sigma_bar = (s + sigma_prev) / 2.0; let t_diff = t - t_prev; if phase_diff.abs() < PI { slope_numerator += phase_diff / (sigma_bar * sigma_bar * t_diff); - slope_denominator += 1.0 / (sigma_bar * sigma_bar) + slope_denominator += 1.0 / (sigma_bar * sigma_bar); } phase_prev = *p; sigma_prev = *s; t_prev = *t; - } else { - first_time = false; } } @@ -176,7 +179,7 @@ pub fn acf_phase_unwrap(ranges: &mut Vec) { } let orig_slope_estimate = sum_xy / sum_xx; let mut orig_slope_error = 0.0; - for (p, (s, t)) in zip(new_phases.iter(), zip(sigmas.iter(), t.iter())) { + for (p, (s, t)) in zip(phases.iter(), zip(sigmas.iter(), t.iter())) { if *s > 0.0 { let temp = orig_slope_estimate * t - p; orig_slope_error += temp * temp / (s * s); @@ -189,9 +192,9 @@ pub fn acf_phase_unwrap(ranges: &mut Vec) { } } -/// passing -pub fn xcf_phase_unwrap(ranges: &mut Vec) -> Result<()> { - for mut range in ranges { +/// Applies 2Ï€ phase unwrapping to XCF phases +pub(crate) fn xcf_phase_unwrap(ranges: &mut Vec) -> Result<()> { + for range in ranges { let (mut sum_xy, mut sum_xx) = (0.0, 0.0); let phases = &range.elev.phases; @@ -199,7 +202,7 @@ pub fn xcf_phase_unwrap(ranges: &mut Vec) -> Result<()> { let t = &range.elev.t; match range.phase_fit.as_ref() { - None => Err(Fitacf3Error::Message( + None => Err(Fitacf3Error::BadFit( "Phase fit must be defined to unwrap XCF phase".to_string(), ))?, Some(fit) => { @@ -219,17 +222,23 @@ pub fn xcf_phase_unwrap(ranges: &mut Vec) -> Result<()> { Ok(()) } -/// passing +/// Determines which points need a 2Ï€ phase correction applied fn phase_correction(slope_estimate: f64, phases: &[f64], times: &[f64]) -> (Vec, i32) { let phase_predicted: Vec = times.iter().map(|t| t * slope_estimate).collect(); - // Round to 4 decimals places, so that 0.4999 rounds to 0.5, then up to 1.0 + // Round to 5 decimals places, so that 0.49999 rounds to 0.5, then up to 1.0 let phase_diff: Vec = zip(phases.iter(), phase_predicted.iter()) - .map(|(p, pred)| ((((pred - p) / (2.0 * PI)) * 10000.0).round() / 10000.0).round() as i32) + .map(|(p, pred)| { + ((((pred - p) / (2.0 * PI)) * 100_000.0).round() / 100_000.0).round() as i32 + }) .collect(); let corrected_phase: Vec = zip(phases.iter(), phase_diff.iter()) .map(|(p, &corr)| p + corr as f64 * 2.0 * PI) .collect(); - let total_corrections: i32 = phase_diff.iter().map(|x| x.abs()).sum(); + let total_corrections: i32 = phase_diff + .iter() + .map(|x| x.abs()) + .max() + .unwrap_or(0); (corrected_phase, total_corrections) } diff --git a/src/fitting/fitacf3/least_squares.rs b/src/fitting/fitacf3/least_squares.rs index 6f63387..c48e43f 100644 --- a/src/fitting/fitacf3/least_squares.rs +++ b/src/fitting/fitacf3/least_squares.rs @@ -1,13 +1,13 @@ -use crate::fitting::fitacf3::fitstruct::{FitType, FittedData, Sums}; +use crate::fitting::fitacf3::fitstruct::{FittedData, PowerFitType, Sums}; #[derive(Debug)] -pub struct LeastSquares { +pub(crate) struct LeastSquares { pub delta_chi_2: [[f64; 2]; 6], pub confidence: usize, pub degrees_of_freedom: usize, } impl LeastSquares { - pub fn new(confidence: usize, degrees_of_freedom: usize) -> LeastSquares { + pub(crate) fn new(confidence: usize, degrees_of_freedom: usize) -> LeastSquares { let delta_chi_2 = [ [1.00, 2.30], [2.71, 4.61], @@ -22,37 +22,37 @@ impl LeastSquares { degrees_of_freedom: degrees_of_freedom - 1, } } - pub fn two_parameter_line_fit( + pub(crate) fn two_parameter_line_fit( &self, x_vals: &[f64], y_vals: &[f64], sigmas: &[f64], - fit_type: FitType, + fit_type: &PowerFitType, ) -> FittedData { - let mut fitted: FittedData = Default::default(); - let sums = Self::find_sums(x_vals, y_vals, sigmas, &fit_type); + let mut fitted: FittedData = FittedData::default(); + let sums = Self::find_sums(x_vals, y_vals, sigmas, fit_type); fitted.delta = sums.sum * sums.sum_xx - sums.sum_x * sums.sum_x; fitted.intercept = (sums.sum_xx * sums.sum_y - sums.sum_x * sums.sum_xy) / fitted.delta; fitted.slope = (sums.sum * sums.sum_xy - sums.sum_x * sums.sum_y) / fitted.delta; fitted.variance_intercept = sums.sum_xx / fitted.delta; fitted.variance_slope = sums.sum / fitted.delta; - fitted.covariance_intercept_slope = (-1.0 * sums.sum_x) / fitted.delta; - fitted.residual_intercept_slope = (-1.0 * sums.sum_x) / (sums.sum * sums.sum_xx).sqrt(); + fitted.covariance_intercept_slope = -sums.sum_x / fitted.delta; + fitted.residual_intercept_slope = -sums.sum_x / (sums.sum * sums.sum_xx).sqrt(); let delta_chi_2 = self.delta_chi_2[self.confidence][self.degrees_of_freedom]; fitted.delta_intercept = delta_chi_2.sqrt() * fitted.variance_intercept.sqrt(); fitted.delta_slope = delta_chi_2.sqrt() * fitted.variance_slope.sqrt(); - fitted.chi_squared = Self::calculate_chi_2(&fitted, x_vals, y_vals, sigmas, &fit_type); + fitted.chi_squared = Self::calculate_chi_2(&fitted, x_vals, y_vals, sigmas, fit_type); fitted } - pub fn one_parameter_line_fit( + pub(crate) fn one_parameter_line_fit( &self, x_vals: &[f64], y_vals: &[f64], sigmas: &[f64], ) -> FittedData { - let mut fitted: FittedData = Default::default(); - let sums = Self::find_sums(x_vals, y_vals, sigmas, &FitType::Linear); + let mut fitted: FittedData = FittedData::default(); + let sums = Self::find_sums(x_vals, y_vals, sigmas, &PowerFitType::Linear); fitted.slope = sums.sum_xy / sums.sum_xx; fitted.variance_slope = 1.0 / sums.sum_xx; @@ -61,15 +61,15 @@ impl LeastSquares { fitted.delta_slope = delta_chi_2.sqrt() * fitted.variance_slope.sqrt(); fitted.delta_intercept = delta_chi_2.sqrt() * fitted.variance_intercept.sqrt(); fitted.chi_squared = - Self::calculate_chi_2(&fitted, x_vals, y_vals, sigmas, &FitType::Linear); + Self::calculate_chi_2(&fitted, x_vals, y_vals, sigmas, &PowerFitType::Linear); fitted } /// passing - fn find_sums(x_vals: &[f64], y_vals: &[f64], sigmas: &[f64], fit_type: &FitType) -> Sums { + fn find_sums(x_vals: &[f64], y_vals: &[f64], sigmas: &[f64], fit_type: &PowerFitType) -> Sums { let nonzero_sigma: Vec = sigmas .iter() .enumerate() - .filter_map(|(i, &x)| if x != 0.0 { Some(i) } else { None }) + .filter_map(|(i, &x)| if x == 0.0 { None } else { Some(i) }) .collect(); let sigma_squared: Vec = nonzero_sigma .iter() @@ -83,7 +83,7 @@ impl LeastSquares { let mut sum_xy: f64 = 0.0; match fit_type { - FitType::Linear => { + PowerFitType::Linear => { for (new, &orig) in nonzero_sigma.iter().enumerate() { sum_x += x_vals[orig] / sigma_squared[new]; sum_y += y_vals[orig] / sigma_squared[new]; @@ -91,7 +91,7 @@ impl LeastSquares { sum_xy += x_vals[orig] * y_vals[orig] / sigma_squared[new]; } } - FitType::Quadratic => { + PowerFitType::Quadratic => { for (new, &orig) in nonzero_sigma.iter().enumerate() { sum_x += x_vals[orig] * x_vals[orig] / sigma_squared[new]; sum_y += y_vals[orig] / sigma_squared[new]; @@ -114,25 +114,25 @@ impl LeastSquares { x_vals: &[f64], y_vals: &[f64], sigmas: &[f64], - fit_type: &FitType, + fit_type: &PowerFitType, ) -> f64 { let nonzero_sigma: Vec = sigmas .iter() .enumerate() - .filter_map(|(i, &x)| if x != 0.0 { Some(i) } else { None }) + .filter_map(|(i, &x)| if x == 0.0 { None } else { Some(i) }) .collect(); let mut chi: Vec = vec![]; match fit_type { - FitType::Linear => { - for &i in nonzero_sigma.iter() { + PowerFitType::Linear => { + for &i in &nonzero_sigma { chi.push( ((y_vals[i] - fitted.intercept) - (fitted.slope * x_vals[i])) / sigmas[i], ); } chi.iter().map(|x| x * x).sum() } - FitType::Quadratic => { - for &i in nonzero_sigma.iter() { + PowerFitType::Quadratic => { + for &i in &nonzero_sigma { chi.push( ((y_vals[i] - fitted.intercept) - (fitted.slope * x_vals[i] * x_vals[i])) / sigmas[i], diff --git a/src/fitting/mod.rs b/src/fitting/mod.rs index ce63c24..3a40d06 100644 --- a/src/fitting/mod.rs +++ b/src/fitting/mod.rs @@ -1 +1,4 @@ +//! Fitting of RAWACF data for ionospheric parameter determination. +//! +//! The only fitting algorithm currently implemented is FITACF3, due to its ubiquity. pub mod fitacf3; diff --git a/src/gridding/filter.rs b/src/gridding/filter.rs new file mode 100644 index 0000000..5aa6534 --- /dev/null +++ b/src/gridding/filter.rs @@ -0,0 +1,492 @@ +//! Filtering kernel functionality for gridding. +use crate::error::ProcdarnError; +use crate::utils::scan::{RadarBeam, RadarCell, RadarScan}; +use chrono::{DateTime, TimeDelta, Utc}; + +#[allow(dead_code)] +pub const MAX_BEAM: i32 = 256; +pub const FILTER_HEIGHT: usize = 3; +pub const FILTER_WIDTH: usize = 3; +pub const FILTER_DEPTH: usize = 3; + +/// Calculates the mean and standard deviation of a parameter from the vector `v`. +/// `f` is used to extract the parameter from an entry of `v`. +fn calculate_mean_sigma(v: &Vec<&RadarCell>, f: fn(&RadarCell) -> f32) -> (f32, f32) { + let mut mean = 0.0; + let mut variance = 0.0; + + // Calculate the mean value + for &cell in v.iter() { + mean += f(cell); + } + mean /= v.len() as f32; + + // Calculate the variance of the velocity values + for &cell in v.iter() { + variance += (f(cell) - mean) * (f(cell) - mean); + } + variance /= v.len() as f32; + let sigma = variance.sqrt(); + + (mean, sigma) +} + +/// Calculates the median value of RadarCells in kernel, and the standard deviation +/// of those cells which are within two standard deviations of the mean of cells in kernel. +/// +/// The parameter `f` is a function which extracts the parameter from an entry of kernel. +/// The parameter `g` is used to extract the parameter for sorting (which may be different than the +/// parameter having its median value calculated) +/// +/// Returns the median and standard deviation. +fn calculate_median_sigma( + kernel: &mut Vec<&RadarCell>, + f: fn(&RadarCell) -> f32, + g: fn(&RadarCell) -> f32, +) -> (f32, f32) { + // Calculate mean and std deviation of kernel with respect to lambda power + let (mean, sigma) = calculate_mean_sigma(kernel, f); + + // Only keep values which fall within 2 std deviations of mean + let mut valid_cells: Vec<&RadarCell> = vec![]; + for &cell in kernel.iter() { + // If the cell deviates by more than 2 standard deviations from the mean, skip it + if (f(cell) - mean).abs() > 2.0 * sigma { + continue; + } + // Add the cell to the median structure + valid_cells.push(cell); + } + // Sort cells in median by their value according to `g` + valid_cells.sort_by(|&a, &b| g(a).partial_cmp(&g(b)).unwrap()); + + // Set the beam/range cell in out_scan to the median value + let median = f(valid_cells[valid_cells.len() / 2]); + + // Recalculate the standard deviation, this time including only valid cells + let (_, sigma) = calculate_mean_sigma(&valid_cells, f); + + (median, sigma) +} + +/// Performs median filtering on a collection of [`RadarScan`]. +/// +/// The filter operates on each range/beam cell, with a 3x3x3 kernel of range/beam/time. +/// If the weighted sum of valid cells in the kernel exceeds a threshold, the median value of each +/// parameter (velocity, power, and spectral width) is determined from the kernel. Otherwise, the +/// output cell is considered empty. The associated parameter errors are calculated from the +/// standard deviations of the input parameters. +/// +/// Called `FilterRadarScan` in `filter.c` of RST. +pub fn median_filter( + mode: i32, + depth: u32, + index: i32, + param: i32, + isort: bool, + scans: &[RadarScan], +) -> Result { + let mut out_scan = RadarScan { + ..Default::default() + }; + let mut max_beam: i32 = -1; + let mut max_range: usize = 1000; + let threshold = &[12, 24]; + let filter_depth = (depth as usize).min(FILTER_DEPTH); + + // Find the largest beam number and range number in all the scans + for scan in scans.iter().take(filter_depth) { + for beam in scan.beams.iter() { + if beam.beam >= max_beam { + max_beam = beam.beam + 1; // Add one since beam number is indexed from 0 + } + if max_range > beam.num_ranges as usize { + max_range = beam.num_ranges as usize; + } + } + } + + // Calculate weight of each cell in the kernel. + // <---> beam + // 1 1 1 2 2 2 1 1 1 ^ + // 1 2 1 2 4 2 1 2 1 | range + // 1 1 1 2 2 2 1 1 1 ⌄ + // <---------time--------> (previous scan, current scan, next scan) + let mut weights: [[[i32; FILTER_DEPTH]; FILTER_HEIGHT]; FILTER_WIDTH] = [ + [[0, 0, 0], [0, 0, 0], [0, 0, 0]], + [[0, 0, 0], [0, 0, 0], [0, 0, 0]], + [[0, 0, 0], [0, 0, 0], [0, 0, 0]], + ]; + let mut f: i32; + let mut w: i32; + for z in 0..FILTER_DEPTH { + if z == 1 { + f = 2; + } else { + f = 1; + } + for y in 0..FILTER_HEIGHT { + for x in 0..FILTER_WIDTH { + if x == 1 && y == 1 { + w = 2; + } else { + w = 1; + } + weights[x][y][z] = w * f; + } + } + } + + // [max_beams, depth, num_points] to store all observations grouped by beam number + let mut beam_pointers: Vec>> = Vec::with_capacity(max_beam as usize); + + // The largest amount of observations for a given beam, for any scan + let mut max_observations_for_a_beam: i32 = 0; + + // Add enough beams and ranges to the output RadarScan + for beam in 0..max_beam as usize { + out_scan.add_beam(max_range as i32); + out_scan.beams[beam].beam = -1; + beam_pointers.push(vec![vec![]; depth as usize]); + } + // beam_pointers should now be [max_beams, depth, 0], where the last dimension is an empty Vec + + for z in 0..depth as usize { + // Figure out if this scan is the current, previous, or next scan + let i: usize = { + if (index - (depth as i32 - 1) + z as i32) < 0 { + (index + 1 + z as i32) as usize + } else { + (index - (depth as i32 - 1) + z as i32) as usize + } + }; + + // Loop through the beams in this scan + for beam in scans[i].beams.iter() { + let beam_num = beam.beam as usize; + beam_pointers[beam_num][z].push(beam.clone()); + + // Update the largest amount of observations seen + if beam_pointers[beam_num][z].len() as i32 > max_observations_for_a_beam { + max_observations_for_a_beam = beam_pointers[beam_num][z].len() as i32; + } + } + } + + // Get index of center scan in temporal dimension + let i: usize = { + if index - 1 < 0 { + (depth as i32 + index - 1) as usize + } else { + index as usize - 1 + } + }; + + // Copy over parameters from center scan + out_scan.station_id = scans[i].station_id; + out_scan.version_major = scans[i].version_major; + out_scan.version_minor = scans[i].version_minor; + out_scan.start_time = scans[i].start_time; + out_scan.end_time = scans[i].end_time; + + // bitmap of (mode & 0x04) is nonzero + if (mode / 4) % 2 == 1 { + for beam_num in 0..max_beam as usize { + // If center scan doesn't have beams then skip this beam + if beam_pointers[beam_num][depth as usize / 2].is_empty() { + continue; + } + let beam = &beam_pointers[beam_num][depth as usize / 2][0]; // First beam + let b = &mut out_scan.beams[beam_num]; + + // Copy radar operating parameters from first beam into corresponding beam of out_scan + b.beam = beam_num as i32; + b.program_id = beam.program_id; + b.time = beam.time; + b.integration_time_s = beam.integration_time_s; + b.integration_time_us = beam.integration_time_us; + b.num_averages = beam.num_averages; + b.first_range = beam.first_range; + b.range_sep = beam.range_sep; + b.rx_rise = beam.rx_rise; + b.freq = beam.freq; + b.noise = beam.noise; + b.attenuation = beam.attenuation; + b.channel = beam.channel; + b.num_ranges = beam.num_ranges; + } + } else { + for beam_num in 0..max_beam as usize { + let b = &mut out_scan.beams[beam_num]; + + // Initialize radar operating parameters + b.program_id = -1; + b.time = DateTime::::default(); + b.integration_time_s = 0; + b.integration_time_us = 0; + b.first_range = 0; + b.range_sep = 0; + b.rx_rise = 0; + b.freq = 0; + b.noise = 0; + b.attenuation = 0; + b.channel = -1; + b.num_ranges = -1; + } + + for z in 0..depth as usize { + for beam_num in 0..max_beam as usize { + // If no beams previously found, continue + if beam_pointers[beam_num][z].is_empty() { + continue; + } + + // Corresponding beam in out_scan + let out_beam = &mut out_scan.beams[beam_num]; + + // Setting beam number in out_scan for this beam + out_beam.beam = beam_num as i32; + + // Go through all beams for this beam/time combo + for in_beam in beam_pointers[beam_num][z].iter() { + // If this is the first beam then use it to set program_id for out_scan beam + if out_beam.program_id == -1 { + out_beam.program_id = in_beam.program_id + } + + // Sum all the operating parameters, which will be averaged later once all beams + // have been added + out_beam.time = out_beam + .time + .checked_add_signed( + TimeDelta::seconds(in_beam.time.timestamp()) + + TimeDelta::nanoseconds(in_beam.time.timestamp_micros() % 1000), + ) + .ok_or_else(|| { + ProcdarnError::Timestamp( + "Could not add two grid times together without overflow" + .to_string(), + ) + })?; + out_beam.integration_time_s += in_beam.integration_time_s; + out_beam.integration_time_us += in_beam.integration_time_us; + if out_beam.integration_time_us > 1_000_000 { + out_beam.integration_time_us += 1; + out_beam.integration_time_us -= 1_000_000; + } + out_beam.num_averages += in_beam.num_averages; + out_beam.first_range += in_beam.first_range; + out_beam.range_sep += in_beam.range_sep; + out_beam.rx_rise += in_beam.rx_rise; + out_beam.freq += in_beam.freq; + out_beam.noise += in_beam.noise; + out_beam.attenuation += in_beam.attenuation; + if out_beam.channel == 0 { + out_beam.channel = in_beam.channel; + } + + // If this is the first beam in the time/beam combo then use max_range + // to set the number of range gates for the beam + if out_beam.num_ranges == -1 { + out_beam.num_ranges = max_range as i32; + } + } + } + } + + for beam_num in 0..max_beam as usize { + let mut count: i32 = 0; + + // Count all the observations for this beam, summing over all scans being averaged + for z in 0..depth as usize { + count += beam_pointers[beam_num][z].len() as i32; + } + // If this beam wasn't sampled, continue to the next one + if count == 0 { + continue; + } + + // Corresponding beam in out_scan + let out_beam = &mut out_scan.beams[beam_num]; + + out_beam.time = DateTime::::from_timestamp_micros( + out_beam.time.timestamp_micros() / count as i64, + ) + .ok_or_else(|| { + ProcdarnError::Timestamp("Could not average beam timestamps".to_string()) + })?; + out_beam.num_averages /= count; + out_beam.first_range /= count; + out_beam.range_sep /= count; + out_beam.rx_rise /= count; + out_beam.freq /= count; + out_beam.noise /= count; + out_beam.attenuation /= count; + out_beam.integration_time_us /= count; + let mut microseconds = (out_beam.integration_time_s * 1_000_000) / count; + out_beam.integration_time_s /= count; + microseconds -= out_beam.integration_time_s * 1_000_000; + out_beam.integration_time_us += microseconds; + } + } + + // 3 x 3 x 3 kernel for storing all values of data for median filtering + let mut kernel = vec![]; + + for beam_num in 0..max_beam as usize { + for range in 0..max_range { + kernel.clear(); + + // Set up the spatial 3x3 (beam by range) filtering boundaries + // saturating_sub will stop underflow for these usize types, limiting the result to 0 + let bbox = (beam_num as i32) - (FILTER_WIDTH as i32 / 2); + let bmin = beam_num.saturating_sub(FILTER_WIDTH / 2); + let mut bmax = beam_num + FILTER_WIDTH / 2; + let rbox = (range as i32) - (FILTER_HEIGHT as i32 / 2); + let rmin = range.saturating_sub(FILTER_WIDTH / 2); + let mut rmax = range + FILTER_HEIGHT / 2; + + // Set upper beam boundary to the highest beam when at other edge of FOV + if bmax >= max_beam as usize { + bmax = max_beam as usize - 1; + } + // Set upper range boundary to the farthest range gate when at other edge of FOV + if rmax >= max_range { + rmax = max_range - 1; + } + + // Initialize center cell weight to zero + let mut weight = 0; + + // Loop over beams + for x in bmin..bmax + 1 { + // Loop over ranges + for y in rmin..rmax + 1 { + // Loop over time + for z in 0..depth as usize { + // Loop over beams in time/beam combo + for beam in beam_pointers[x][z].iter() { + // Skip if this range gate is not in the beam + if y >= beam.scatter.len() { + continue; + } + + let bm_idx = (x as i32 - bbox) as usize; + let rg_idx = (y as i32 - rbox) as usize; + + // Check that there is scatter present in the beam/range/time cell + if beam.scatter[y] != 0 { + // Increment weight + weight += weights[bm_idx][rg_idx][z]; + // Add this observation to the kernel + kernel.push(&beam.cells[y]); + } + } + } + } + } + // If no cells with scatter found, continue + if kernel.is_empty() { + continue; + } + + // If the current beam is at the edge of the FOV then increase its weight by 50% + // TODO: What about near/far range edges? + // TODO: weight is an integer, this is kinda hacky + if beam_num == 0 || beam_num == usize::try_from(max_beam - 1).unwrap() { + weight += weight / 2; + } + + // If the sum of weights of cells with scatter in the kernel is less than the threshold + // then continue + if weight <= threshold[mode as usize % 2] { + continue; + } + + // Threshold was exceeded, so the output scan should have scatter in this beam/range cell + let out_beam = &mut out_scan.beams[beam_num]; + out_beam.scatter[range] = 1; + + // Initialize observation parameters to zero + let out_cell = &mut out_beam.cells[range]; + out_cell.groundscatter = 0; + out_cell.power_lin = 0.0; + out_cell.spectral_width_lin = 0.0; + out_cell.velocity = 0.0; + + // Perform velocity median filtering if specified + let mut compare_fn: fn(&RadarCell) -> f32 = |x| x.velocity; + if param % 2 == 1 { + // i.e. bitmap of (param & 0x01) is nonzero + (out_cell.velocity, out_cell.velocity_error) = + calculate_median_sigma(&mut kernel, |x| x.velocity, compare_fn); + } + + // Perform lambda power median filtering if specified + if (param / 2) % 2 == 1 { + // i.e. bitmap of (param & 0x02) is nonzero + if isort { + compare_fn = |x| x.power_lin; + } + (out_cell.power_lin, out_cell.power_lin_error) = + calculate_median_sigma(&mut kernel, |x| x.power_lin, compare_fn); + } + + // Perform spectral width median filtering if specified + if (param / 4) % 2 == 1 { + // i.e. bitmap of (param & 0x04) is nonzero + if isort { + compare_fn = |x| x.spectral_width_lin; + } + ( + out_cell.spectral_width_lin, + out_cell.spectral_width_lin_error, + ) = calculate_median_sigma(&mut kernel, |x| x.spectral_width_lin, compare_fn); + } + + // Perform lag0 power median filtering if specified + if (param / 8) % 2 == 1 { + // i.e. bitmap of (param & 0x08) is nonzero + if isort { + compare_fn = |x| x.power_lag_zero; + } + (out_cell.power_lag_zero, out_cell.power_error_lag_zero) = + calculate_median_sigma(&mut kernel, |x| x.power_lag_zero, compare_fn); + } + } + } + + Ok(out_scan) +} + +/// Checks to make sure the radar operating parameters do not change significantly between scans. +/// If the frequency, distance to first range, or range separation change between scans, then the +/// scattering location for a range gate will also change, so median filtering the data is +/// nonsensical. +/// Called FilterCheckOps in checkops.c of RST. +pub fn check_operational_params(scans: &[RadarScan], max_frequency_var: i32) -> bool { + // Choose the middle scan of scans being median filtered + let ref_scan = scans[scans.len() / 2].clone(); + + // Loop through other scans that are being median filtered + for scan in scans.iter().filter(|&s| *s != ref_scan) { + // Loop through beams of the reference scan + for ref_beam in ref_scan.beams.iter() { + // Loop through beams of the scan under consideration + for check_beam in scan.beams.iter().filter(|&b| b.beam == ref_beam.beam) { + // Check if the relevant operating parameters are equal or close to equal + if ref_beam.first_range != check_beam.first_range { + return false; + } + if ref_beam.range_sep != check_beam.range_sep { + return false; + } + if (ref_beam.freq - check_beam.freq).abs() > max_frequency_var { + return false; + } + } + } + } + // If relevant operating parameters match or are close enough, then return true + true +} diff --git a/src/gridding/grid.rs b/src/gridding/grid.rs new file mode 100644 index 0000000..c5f0543 --- /dev/null +++ b/src/gridding/grid.rs @@ -0,0 +1,669 @@ +//! Objects and functions for [`fit2grid`] functionality. +use aacgmv2_rs::aacgmv2::Aacgmv2; +use crate::error::ProcdarnError; +use crate::gridding::filter::{check_operational_params, median_filter}; +use crate::gridding::grid_table::GridTable; +use crate::utils::channel::{set_fix_channel, set_stereo_channel}; +use crate::utils::hdw::{HdwError, HdwInfo}; +use crate::utils::scan::RadarScan; +use crate::utils::search::fit_seek; +use chrono::{DateTime, Datelike, NaiveDate, NaiveDateTime, NaiveTime, TimeDelta, Utc}; +use clap::Parser; +use dmap::error::DmapError; +use dmap::formats::{fitacf::FitacfRecord, grid::GridRecord}; +use dmap::Record; +use pyo3::exceptions::PyValueError; +use pyo3::{FromPyObject, PyErr}; +use std::path::PathBuf; +use thiserror::Error; + +/// Enum of the possible error variants that may be encountered +#[derive(Error, Debug)] +pub enum GridError { + /// Represents an error in the Fitacf record that is attempting to be gridded + #[error("{0}")] + InvalidFitacf(String), + + /// Represents a field having the wrong type + #[error("{0}")] + WrongType(&'static str), + + /// Represents an error in processing of the record, for any reason + #[error("{0}")] + ProcessingError(#[from] ProcdarnError), + + /// Represents an error parsing a date, time, or datetime + #[error("{0}")] + Datetime(#[from] chrono::ParseError), + + /// Unable to get hardware file information + #[error("{0}")] + Hdw(#[from] HdwError), + + /// Invalid DMAP file + #[error("{0}")] + Dmap(#[from] DmapError), + + /// Error in `igrf` crate + #[error("{0}")] + Igrf(#[from] igrf::Error), + + /// Error in argument specification + #[error("{0}")] + BadArgs(String), + + #[error("{0}")] + Aacgmv2(#[from] aacgmv2_rs::AACGMv2Error), +} + +impl From for PyErr { + fn from(value: GridError) -> Self { + let msg = value.to_string(); + PyValueError::new_err(msg) + } +} + +/// Arguments for [`fit2grid`] customization. +#[derive(Parser, Debug, FromPyObject)] +#[command(author, version, about, long_about = None)] +pub struct GridArgs { + /// Start time in HH:MM format + #[arg(long, visible_alias = "st")] + pub start_time: Option, + + /// End time in HH:MM format + #[arg(long, visible_alias = "et")] + pub end_time: Option, + + /// Start date in YYYYMMDD format + #[arg(long, visible_alias = "sd")] + pub start_date: Option, + + /// End date in YYYYMMDD format + #[arg(long, visible_alias = "ed")] + pub end_date: Option, + + /// Use interval of length HH:MM + #[arg(long, visible_alias = "ex", conflicts_with_all = &["end_time", "end_date"])] + pub interval: Option, + + /// Scan length specification in whole seconds, overriding the scan flag + #[arg(long, visible_alias = "tl")] + pub scan_length: Option, + + /// Time interval to store in each grid record, in whole seconds + #[arg(short = 'i', long, value_parser, default_value = "120")] + pub record_interval: u32, + + /// Stereo channel identifier, either 'a' or 'b' + #[arg(long, visible_alias = "cn", value_parser)] + pub channel: Option, + + /// User-defined channel identifier for the output file only + #[arg(long, visible_alias = "cn_fix", conflicts_with = "channel")] + pub channel_fix: Option, + + /// Beams to exclude, as a comma-separated list + #[arg(long, visible_alias = "ebm", value_delimiter = ',', value_parser)] + pub exclude_beams: Option>, + + /// Minimum range gate + #[arg(long, visible_alias = "minrng")] + pub min_range_gate: Option, + + /// Maximum range gate + #[arg(long, visible_alias = "maxrng")] + pub max_range_gate: Option, + + /// Minimum slant range in km + #[arg(long, visible_alias = "minsrng")] + pub min_slant_range: Option, + + /// Maximum slant range in km + #[arg(long, visible_alias = "maxsrng")] + pub max_slant_range: Option, + + /// Filter weighting mode + #[arg(long, visible_alias = "fwgt", value_parser, default_value = "0")] + pub filter_weighting: i32, + + /// Maximum power (linear scale) + #[arg( + long, + visible_alias = "pmax", + value_parser, + default_value = "60", + requires = "op_param_flag" + )] + pub max_power: f32, + + /// Maximum velocity in m/s + #[arg( + long, + visible_alias = "vmax", + value_parser, + default_value = "2500", + requires = "op_param_flag" + )] + pub max_velocity: f32, + + /// Maximum spectral width in m/s + #[arg( + long, + visible_alias = "wmax", + value_parser, + default_value = "1000", + requires = "op_param_flag" + )] + pub max_spectral_width: f32, + + /// Maximum velocity error in m/s + #[arg( + long, + visible_alias = "vemax", + value_parser, + default_value = "200", + requires = "op_param_flag" + )] + pub max_velocity_error: f32, + + /// Minimum power (linear scale) + #[arg( + long, + visible_alias = "pmin", + value_parser, + default_value = "3", + requires = "op_param_flag" + )] + pub min_power: f32, + + /// Minimum velocity in m/s + #[arg( + long, + visible_alias = "vmin", + value_parser, + default_value = "35", + requires = "op_param_flag" + )] + pub min_velocity: f32, + + /// Minimum spectral width in m/s + #[arg( + long, + visible_alias = "wmin", + value_parser, + default_value = "10", + requires = "op_param_flag" + )] + pub min_spectral_width: f32, + + /// Minimum velocity error in m/s + #[arg( + long, + visible_alias = "vemin", + value_parser, + default_value = "0", + requires = "op_param_flag" + )] + pub min_velocity_error: f32, + + /// Altitude at which mapping is done in km + #[arg(long, visible_alias = "alt", value_parser, default_value = "300")] + pub altitude: f32, + + /// Maximum allowed frequency variation in Hz + #[arg(long, visible_alias = "fmax", value_parser, default_value = "500000")] + pub max_frequency_var: i32, + + /// Flag to disable boxcar median filtering + #[arg(long, visible_alias = "nav", action = clap::ArgAction::SetFalse)] + pub boxcar_filter_flag: bool, + + /// Flag to include data that exceeds limits + #[arg(long, visible_alias = "nlm", action = clap::ArgAction::SetTrue)] + pub no_limits_flag: bool, + + /// Flag to exclude data that doesn't match operating parameter requirements + #[arg(long, visible_alias = "nb", action = clap::ArgAction::SetTrue)] + pub op_param_flag: bool, + + /// Flag to exclude data with scan flag of -1 + #[arg(long, visible_alias = "ns", action = clap::ArgAction::SetTrue)] + pub exclude_neg_scan_flag: bool, + + /// Extended output, include power and width in output file + #[arg(long, visible_alias = "xtd", action = clap::ArgAction::SetTrue)] + pub extended_mode_flag: bool, + + /// If using a median filter, sort parameters independent of the velocity + #[arg(long, visible_alias = "isort", action = clap::ArgAction::SetTrue)] + pub sort_params_flag: bool, + + /// Exclude data marked as ground scatter + #[arg(long, visible_alias = "ion", default_value = "true", action = clap::ArgAction::SetTrue)] + pub ionosphere_only_flag: bool, + + /// Exclude data not marked as ground scatter + #[arg(long, visible_alias = "gs", action = clap::ArgAction::SetTrue, + conflicts_with = "ionosphere_only_flag")] + pub groundscatter_only_flag: bool, + + /// Do not exclude data based on scatter flag + #[arg(long, visible_alias = "both", action = clap::ArgAction::SetTrue, + conflicts_with_all = &["ionosphere_only_flag", "groundscatter_only_flag"])] + pub all_data_flag: bool, + + /// Use inertial reference frame + #[arg(long, visible_alias = "inertial", action = clap::ArgAction::SetTrue)] + pub inertial_frame_flag: bool, + + /// Map data using Chisham virtual height model + #[arg(long, visible_alias = "chisham", action = clap::ArgAction::SetTrue)] + pub chisham_flag: bool, + + /// Verbose mode + #[arg(short, long, action = clap::ArgAction::SetTrue)] + pub verbose: bool, +} + +/// Grids a list of fitacf files. +pub fn fit2grid_file(infiles: &[PathBuf], args: &GridArgs) -> Result, GridError> { + let mut grid_recs: Vec = vec![]; + + for infile in infiles.iter() { + if args.verbose { + println!("\nGridding file {}", infile.display()) + }; + let fitacf_records = FitacfRecord::read_file(infile.clone())?; + let mut grids = fit2grid(args, &fitacf_records)?; + grid_recs.append(&mut grids); + } + + Ok(grid_recs) +} + +/// Takes a list of fitacf records and converts them into grid records. +/// +/// This algorithm has many different options to tweak the behaviour. See [`GridArgs`] for more +/// information. +pub fn fit2grid(args: &GridArgs, fitacf_records: &[FitacfRecord]) -> Result, GridError> { + let mut grid_table = GridTable::default(); + + // If "filter_weighting_mode" greater than 0, decrement it by one + let mut filter_weighting_mode = args.filter_weighting; + if filter_weighting_mode > 0 { + filter_weighting_mode -= 1; + }; + + // Set GridTable groundscatter flag + grid_table.groundscatter = { + if args.groundscatter_only_flag { + 0 + } else if args.ionosphere_only_flag { + 1 + } else if args.all_data_flag { + 2 + } else { + return Err(GridError::BadArgs( + "Cannot interpret data exclusion flags [--ion, --gs, --both]".to_string(), + )); + } + }; + + // Set GridTable channel number + grid_table.channel = { + if let Some(c) = args.channel { + set_stereo_channel(c).unwrap_or(-1) // Determine stereo channel, either 'a' or 'b' + } else if let Some(c) = args.channel_fix { + set_fix_channel(c).unwrap_or(-1) // Determine appropriate channel for output file + } else { + 0 + } + }; + + // Store bounding thresholds for power, velocity, velocity error, and spectral width in GridTable + if !args.op_param_flag { + grid_table.min_power = args.min_power; + grid_table.min_velocity = args.min_velocity; + grid_table.min_spectral_width = args.min_spectral_width; + grid_table.min_velocity_error = args.min_velocity_error; + grid_table.max_power = args.max_power; + grid_table.max_velocity = args.max_velocity; + grid_table.max_spectral_width = args.max_spectral_width; + grid_table.max_velocity_error = args.max_velocity_error; + } + + // Initialize the size of the boxcar. Default 3 if median filtering being applied, 1 otherwise + let num_averages = if args.boxcar_filter_flag { 3 } else { 1 }; + + // Preallocate memory for a vector of records that will be boxcar filtered + let mut current_scans: Vec = Vec::with_capacity(num_averages as usize); + for _ in 0..num_averages { + current_scans.push(RadarScan::default()); + } + let mut found_record = false; + let mut index = 0; + let mut num_scans = 0; + let mut record_idx: Option; + let mut end_time: Option> = None; + let hdw_info: Option = None; + let mut records_for_file: Vec = vec![]; + let mut found_scan: bool; + + // Get the first scan from the file + match RadarScan::get_first_scan(fitacf_records, args.scan_length) { + Ok((x, num_read)) => { + current_scans[index] = x; + found_scan = true; + record_idx = Some(num_read); + } + Err(e) => { + return Err(GridError::InvalidFitacf(format!("Unable to get first scan from records - {e}"))) + } + }; + + let file_datetime = current_scans[index].start_time; + + // Determine the starting time for gridding based on the record and input options + let mut start_time = file_datetime; + if !found_record { + if let (None, None) = (&args.start_date, &args.start_time) { + start_time = current_scans[0].start_time; + found_record = true; + } else { + let date_string = match &args.start_date { + Some(d) => d.clone(), + None => current_scans[0].start_time.format("%Y%m%d").to_string(), + }; + + let time_string = match &args.start_time { + Some(t) => t.clone(), + // The None branch truncates back to the start of the minute + None => current_scans[0].start_time.format("%H:%M").to_string(), + }; + + start_time = NaiveDateTime::parse_from_str( + format!("{} {}", date_string, time_string).as_str(), + "%Y%m%d %H:%M", + ) + .map_err(|_| { + ProcdarnError::Timestamp( + "Unable to parse date and/or time from options or file".to_string(), + ) + })? + .and_utc(); + if args.verbose { + println!("start_time: {start_time}") + }; + // If applying boxcar median filter then we need to load data prior to the usual start + // time, so start_time needs to be adjusted + if num_averages > 1 { + match args.scan_length { + Some(x) => { + start_time -= TimeDelta::new(x as i64, 0).ok_or_else(|| { + ProcdarnError::Timestamp( + "Out of bounds duration when adjusting start_time".to_string(), + ) + })? + } + None => { + let td = current_scans[0].end_time - current_scans[0].start_time + + TimeDelta::seconds(15); + start_time -= td; + } + } + } + + // Find the first record which occurs after the grid start time, if any + if let Ok(Some((_, idx))) = fit_seek(fitacf_records, start_time) { + record_idx = Some(idx); + } else { + return Err(GridError::InvalidFitacf("Records end before requested start time".to_string())) + } + found_record = true; + + // If using scan flag, go to the next beginning of the next scan + if args.scan_length.is_none() { + if let Some(x) = record_idx { + let mut scan_flags: Vec = vec![]; + for rec in fitacf_records[x..].iter() { + let scn_flg = i16::try_from( + rec.get("scan") + .ok_or_else(|| { + GridError::InvalidFitacf("missing `scan` flag".to_string()) + })? + .clone(), + ) + .map_err(|_| { + GridError::InvalidFitacf("bad `scan` flag".to_string()) + })?; + scan_flags.push(scn_flg); + } + record_idx = Some( + scan_flags.iter().position(|&flg| flg == 1).ok_or_else(|| { + GridError::InvalidFitacf("No records with set `scan` flag".to_string()) + })? + x, + ); + } else { + return Err(GridError::BadArgs( + "No records match requested scan time".to_string(), + )); + } + } + + // Read the first full scan of data corresponding to grid start datetime + let first_idx = record_idx.unwrap_or(0); + if args.verbose { + println!("Gridding starting at record {first_idx}"); + } + if let Ok((scan, recs_read)) = + RadarScan::get_first_scan(&fitacf_records[first_idx..], args.scan_length) + { + found_scan = true; + current_scans[0] = scan; + record_idx = Some(record_idx.unwrap() + recs_read); + } else { + found_scan = false; + } + } + } + + if found_record { + if let Some(x) = &args.interval { + let dt = NaiveTime::parse_from_str(x, "%H:%M")?; + let dur = dt + - NaiveTime::from_hms_opt(0, 0, 0).ok_or_else(|| { + GridError::BadArgs( + "This should never happen, trying to make NaiveTime::from_hms(0, 0, 0)" + .to_string(), + ) + })?; + end_time = Some(start_time + dur); + } else { + end_time = match &args.end_time { + Some(t) => { + let time_string = t.clone(); + let date_string = match &args.end_date { + Some(d) => d.clone(), + None => current_scans[0].start_time.format("%Y%m%d").to_string(), + }; + NaiveDateTime::parse_from_str( + format!("{} {}", date_string, time_string).as_str(), + "%Y%m%d %H:%M", + ) + .map_err(|_| { + GridError::BadArgs( + "Unable to parse end date and/or time from options".to_string(), + ) + })? + .and_utc() + .into() + } + None => None, + }; + } + if num_averages != 1 && end_time.is_some() { + if let Some(x) = args.scan_length { + let dt = TimeDelta::seconds(x as i64); + end_time = Some(end_time.unwrap() + dt); + } else { + let td = current_scans[0].end_time - current_scans[0].start_time + + TimeDelta::seconds(15); + end_time = Some(end_time.unwrap() + td); + } + } + } + + let date = NaiveDate::from_ymd_opt(start_time.year(), start_time.month(), start_time.day()).unwrap().and_hms_opt(0, 0, 0).unwrap().and_utc(); + let mut aacgm_model = Aacgmv2::new(date).map_err(GridError::Aacgmv2)?; + num_scans += 1; + + // Grid all data until end of gridding time or end of file + while found_scan { + // Exclude scatter in beams listed in args.exclude_beams + if let Some(b) = &args.exclude_beams { + if args.verbose { + println!("excluding beams {:?}", &args.exclude_beams) + }; + current_scans[index].reset_beams(b)?; + } + + // Exclude data with scan flag == -1 if args.exclude_neg_scan_flag given + if args.exclude_neg_scan_flag { + if args.verbose { + println!("excluding data with negative scan flag") + }; + current_scans[index].exclude_outofscan(); + } + + // Exclude scatter in range gates below args.min_range_gate or above args.max_range_gate + if args.verbose { + println!( + "Excluding ranges outside [{:?}, {:?}] and slant range outside [{:?}, {:?}]", + args.min_range_gate, + args.max_range_gate, + args.min_slant_range, + args.max_slant_range + ); + } + + current_scans[index].exclude_range( + args.min_range_gate, + args.max_range_gate, + args.min_slant_range, + args.max_slant_range, + ); + + // Exclude groundscatter or ionospheric scatter, depending on the args given + if args.groundscatter_only_flag { + if args.verbose { + println!("excluding ionospheric scatter") + }; + current_scans[index].exclude_ionospheric_scatter(); + } else if args.ionosphere_only_flag { + if args.verbose { + println!("excluding ground scatter") + }; + current_scans[index].exclude_groundscatter(); + } + + // Exclude scatter outside power, velocity, spectral width, and velocity error bounds + if !args.op_param_flag { + if args.verbose { + println!("Excluding out of bounds") + }; + current_scans[index].exclude_outofbounds(&grid_table); + } + + // If enough scans have been loaded and args.no_limit not given, check to make sure the + // first range, range separation, and transmit frequency have not changed significantly + let mut passed_check = true; + if num_scans >= current_scans.capacity() + && !args.no_limits_flag + && filter_weighting_mode != -1 + { + passed_check = check_operational_params(¤t_scans, args.max_frequency_var); + } + + // If enough scans have been loaded, proceed with filtering and gridding + if passed_check && num_scans >= current_scans.capacity() { + let grid_record = match filter_weighting_mode { + -1 => current_scans[index].clone(), + _ => median_filter( + filter_weighting_mode, + current_scans.capacity() as u32, + index as i32, + 15, + args.sort_params_flag, + ¤t_scans, + )?, + }; + + // If not already done, load HdwInfo for radar + let hdw_params = match hdw_info { + Some(ref x) => x, + None => &HdwInfo::new(grid_record.station_id, start_time)?, + }; + + // Test whether the grid table should be written to file + if grid_table.test(&grid_record) { + // If GridTable good and grid record starts at or after start_time, write to file + if grid_table.start_time >= start_time { + if !args.verbose { + println!( + "Storing: {} {} pnts={}", + grid_table.start_time.format("%Y-%m-%d %H:%M:%S"), + grid_table.end_time.format("%H:%M:%S"), + grid_table.num_points_npnt + ); + } + records_for_file.push(grid_table.to_dmap_record(args.extended_mode_flag)?); + } + } + + // Map GridTable to equal-area grid in magnetic coordinates + grid_table.map( + &grid_record, + hdw_params, + &mut aacgm_model, + args.record_interval as i32, + args.inertial_frame_flag, + args.altitude, + args.chisham_flag, + )?; + } + + // Update index + index += 1; + if index >= current_scans.capacity() { + index = 0; + } + + // Get the next scan + let start_idx = record_idx.unwrap_or(0); + let scan_res = + RadarScan::get_first_scan(&fitacf_records[start_idx..], args.scan_length); + match scan_res { + Ok((new_scan, num_read)) => { + found_scan = true; + current_scans[index] = new_scan; + record_idx = Some(start_idx + num_read); + } + Err(ProcdarnError::ZeroRecords(_)) => { + found_scan = false; + record_idx = None; + } + Err(e) => Err(e)?, + }; + + // If scan starts after end_time, this file is done being gridded + if let Some(dt) = end_time { + if current_scans[index].start_time > dt { + break; + } + } + num_scans += 1; + } + + Ok(records_for_file) +} diff --git a/src/gridding/grid_table.rs b/src/gridding/grid_table.rs new file mode 100644 index 0000000..32fe60d --- /dev/null +++ b/src/gridding/grid_table.rs @@ -0,0 +1,641 @@ +use crate::gridding::grid::GridError; +use crate::utils::hdw::HdwInfo; +use crate::utils::rpos::{rpos_inv_mag, rpos_range_beam_azimuth_elevation}; +use crate::utils::scan::{RadarBeam, RadarScan}; +use chrono::{DateTime, Datelike, TimeDelta, Timelike, Utc}; +use dmap::formats::grid::GridRecord; +use dmap::record::Record; +use dmap::types::DmapField; +use indexmap::IndexMap; +use numpy::ndarray::array; +use numpy::ndarray::Array; +use std::f32::consts::PI; +use std::iter; +use aacgmv2_rs::aacgmv2::Aacgmv2; + +pub const GRID_REVISION_MAJOR: i32 = 2; +pub const GRID_REVISION_MINOR: i32 = 0; +pub const VELOCITY_ERROR_MIN: f32 = 100.0; // m/s +pub const POWER_LIN_ERROR_MIN: f32 = 1.0; // a.u. in linear scale +pub const WIDTH_LIN_ERROR_MIN: f32 = 1.0; // m/s + +pub const RADIUS_EARTH: f32 = 6371.2; // km + +#[derive(Clone, Debug, Default)] +pub struct GridBeam { + pub beam: i32, // bm in RST + pub first_range: i32, // frang in RST, km + pub range_sep: i32, // rsep in RST, km + pub rx_rise: i32, // rxrise in RST, microseconds? + pub num_ranges: i32, // nrang in RST + pub azimuth: Vec, // azm in RST, radians + pub slant_range: Vec, // srng in RST, km + pub ival: Vec, // ival in RST + pub index: Vec, // inx in RST +} + +#[derive(Copy, Clone, Debug, Default)] +pub struct GridPoint { + pub max: i32, // max in RST + pub count: i32, // cnt in RST + pub reference: i32, // ref in RST + pub magnetic_lat: f32, // mlat in RST + pub magnetic_lon: f32, // mlon in RST + pub azimuth: f32, // azm in RST, degrees + pub slant_range: f32, // srng in RST, km + pub velocity_median: f32, // vel.median in RST, m/s + pub velocity_median_north: f32, // vel.median_n in RST, m/s + pub velocity_median_east: f32, // vel.median_e in RST, m/s + pub velocity_stddev: f32, // vel.sd in RST, m/s + pub power_median: f32, // pwr.median in RST, a.u. in linear scale + pub power_stddev: f32, // pwr.sd in RST, a.u. in linear scale + pub spectral_width_median: f32, // wdt.median in RST, m/s + pub spectral_width_stddev: f32, // wdt.sd in RST, m/s +} +impl GridPoint { + pub fn clear(&mut self) { + self.azimuth = 0.0; + self.slant_range = 0.0; + self.velocity_median_north = 0.0; + self.velocity_median_east = 0.0; + self.velocity_stddev = 0.0; + self.power_median = 0.0; + self.power_stddev = 0.0; + self.spectral_width_median = 0.0; + self.spectral_width_stddev = 0.0; + self.count = 0; + } +} + +#[derive(Clone, Debug, Default)] +pub struct GridTable { + pub start_time: DateTime, // st_time in RST + pub end_time: DateTime, // ed_time in RST + pub channel: i16, // chn in RST + pub status: i32, // status in RST + pub station_id: i16, // st_id in RST + pub program_id: i32, // prog_id in RST + pub num_scans: i32, // nscan in RST + pub num_points_npnt: i32, // npnt in RST, number of grid points + pub freq: f32, // freq in RST + pub noise_mean: f32, // noise.mean in RST + pub noise_stddev: f32, // noise.sd in RST + pub groundscatter: i32, // gsct in RST + pub min_power: f32, // min[0] in RST, a.u. in linear scale + pub min_velocity: f32, // min[1] in RST, m/s + pub min_spectral_width: f32, // min[2] in RST, m/s + pub min_velocity_error: f32, // min[3] in RST, m/s + pub max_power: f32, // max[0] in RST, a.u. in linear scale + pub max_velocity: f32, // max[1] in RST, m/s + pub max_spectral_width: f32, // max[2] in RST, m/s + pub max_velocity_error: f32, // max[3] in RST, m/s + pub num_beams: i32, // bnum in RST + pub beams: Vec, // bm in RST + pub num_points_pnum: i32, // pnum in RST + pub points: Vec, // pnt in RST +} +impl GridTable { + /// Called GridTableZero in RST + pub fn clear(&mut self) { + for p in self.points.iter_mut() { + p.clear() + } + } + + /// Tests whether gridded data should be written to a file. + /// Modifies self, averaging measurements within grid cells if returning true. + /// + /// Called `GridTableTest` in RST. + pub fn test(&mut self, scan: &RadarScan) -> bool { + let time_micros = + (scan.start_time.timestamp_micros() + scan.end_time.timestamp_micros()) / 2; + + let time: DateTime = match DateTime::from_timestamp_micros(time_micros) { + Some(x) => { + x + } + None => return false, + }; + + if self.start_time == DateTime::::default() { + return false; + } + + if time <= self.end_time { + return false; + } + + self.num_points_npnt = 0; + + // Average values across all scans included in the grid table + let num_scans: &f32 = &(self.num_scans as f32); + self.freq /= num_scans; + self.noise_mean /= num_scans; + self.noise_stddev /= num_scans; + + for point in self.points.iter_mut() { + if point.count != 0 { + if point.count <= self.num_scans * point.max / 4 { + point.count = 0; + } else { + // Update the total number of grid points in the grid table + self.num_points_npnt += 1; + + // Calculate weighted mean of north/east velocity components + point.velocity_median_north /= &point.velocity_stddev; + point.velocity_median_east /= &point.velocity_stddev; + + // Calculate the magnitude of weighted mean velocity error + point.velocity_median = (point.velocity_median_north + * point.velocity_median_north + + point.velocity_median_east * point.velocity_median_east) + .sqrt(); + + // Calculate azimuth of weighted mean velocity vector + point.azimuth = point + .velocity_median_east + .atan2(point.velocity_median_north) + .to_degrees(); + + // Calculate average slant range of velocity vector + point.slant_range /= point.count as f32; + + // Calculate weighted mean of spectral width and power + point.spectral_width_median /= &point.spectral_width_stddev; + point.power_median /= &point.power_stddev; + + // Calculate standard deviation of velocity, power, and spectral width + point.velocity_stddev = 1.0 / &point.velocity_stddev.sqrt(); + point.spectral_width_stddev = 1.0 / &point.spectral_width_stddev.sqrt(); + point.power_stddev = 1.0 / &point.power_stddev.sqrt(); + } + } + } + self.status = 0; + true + } + + /// Returns the index of the pointer to a newly added grid cell in the structure + /// storing gridded radar data. + /// Called GridTableAddPoint in RST + pub fn add_point(&mut self) -> usize { + self.points.push(GridPoint::default()); + self.num_points_pnum += 1; + self.points.len() - 1 + } + + /// Returns the index of the point in the table whose reference number matches the input. + /// Called GridTableFindPoint in RST + pub fn find_point(&self, reference: i32) -> Option { + self.points.iter().position(|x| x.reference == reference) + } + + /// Adds a grid beam to the grid table. + /// Called GridTableAddBeam in RST + pub fn add_beam( + &mut self, + hdw: &HdwInfo, + aacgm_model: &mut Aacgmv2, + altitude: f32, + time: DateTime, + scan_beam: &RadarBeam, + chisham: bool, + ) -> Result { + let velocity_correction: f32 = + (2.0 * PI / 86400.0) * RADIUS_EARTH * 1000.0 * hdw.latitude.to_radians().cos(); + self.num_beams += 1; + + let mut grid_beam = GridBeam { + beam: scan_beam.beam, + first_range: scan_beam.first_range, + range_sep: scan_beam.range_sep, + rx_rise: scan_beam.rx_rise, + num_ranges: scan_beam.num_ranges, + ..Default::default() + }; + + for range in 0..grid_beam.num_ranges { + // Calculate geographic azimuth and elevation to scatter point + let result = rpos_range_beam_azimuth_elevation( + grid_beam.beam, + range, + time.year(), + hdw, + grid_beam.first_range as f32, + grid_beam.range_sep as f32, + grid_beam.rx_rise as f32, + altitude, + chisham, + )?; + let azimuth_geo = result.az; + + // Calculate magnetic latitude, longitude, azimuth, and slant range of scatter point + let (mut mag_loc, mut azimuth_mag, srng_mag) = rpos_inv_mag( + grid_beam.beam, + range, + time.year(), + hdw, + aacgm_model, + grid_beam.first_range as f32, + grid_beam.range_sep as f32, + grid_beam.rx_rise as f32, + altitude, + chisham, + )?; + + // Ensure magnetic azimuth and longitude between 0-360 degrees + if azimuth_mag < 0.0 { + azimuth_mag += 2.0 * PI; + } + if mag_loc.lon < 0.0 { + mag_loc.lon += 2.0 * PI as f64; + } + + // Calculate magnetic grid cell latitude, (e.g. 72.1->72.5, 57.8->57.5, etc) + let grid_lat = if mag_loc.lat > 0.0 { + mag_loc.lat.to_degrees().floor() as f32 + 0.5 + } else { + mag_loc.lat.to_degrees().floor() as f32 - 0.5 + }; + + // Calculate magnetic grid longitude spacing at grid latitude + let lon_spacing = (360.0 * grid_lat.abs().to_radians().cos() + 0.5).floor() / 360.0; + + // Calculate magnetic grid cell longitude + let grid_lon = + ((mag_loc.lon.to_degrees() as f32 * lon_spacing).floor() + 0.5) / lon_spacing; + + // Calculate reference number for cell + let reference = if mag_loc.lat > 0.0 { + (1000.0 * mag_loc.lat.to_degrees().floor() as f32 + + (mag_loc.lon.to_degrees() as f32 * lon_spacing).floor()) + as i32 + } else { + (-1000.0 * (-mag_loc.lat.to_degrees()).floor() as f32 + - (mag_loc.lon.to_degrees() as f32 * lon_spacing).floor()) + as i32 + }; + + // Find GridPoint corresponding to reference number for cell, make new GridPoint if none found + let index = match self.find_point(reference) { + Some(x) => x, + None => self.add_point(), + }; + let point = &mut self.points[index]; + + // Update the total number of range gates that map to GridPoint (GridPoint.max) + point.reference = reference; + point.max += 1; + + // Set magnetic lat/lon for GridPoint + point.magnetic_lat = grid_lat; + point.magnetic_lon = grid_lon; + + // Set index, magnetic azimuth, slant range, and inertial velocity correction factor of beam + grid_beam.index.push(index as i32); + grid_beam.azimuth.push(azimuth_mag); + grid_beam.slant_range.push(srng_mag); + grid_beam + .ival + .push(velocity_correction * (-azimuth_geo.sin() as f32)); + } + self.beams.push(grid_beam); + // Return index of beam number added to self + Ok((self.num_beams - 1) as usize) + } + + /// Find the index of the beam in the grid table whose beam number and operating parameters + /// match those of the input. + /// Called GridTableFindBeam in RST + pub fn find_beam(&self, beam: &RadarBeam) -> Option { + self.beams.iter().position(|x| { + x.beam == beam.beam + && x.first_range == beam.first_range + && x.range_sep == beam.range_sep + && x.num_ranges == beam.num_ranges + }) + } + + /// Maps radar scan data to an equal-area grid in magnetic coordinates. + /// Called GridTableMap in RST + pub fn map( + &mut self, + scan: &RadarScan, + hdw: &HdwInfo, + aacgm_model: &mut Aacgmv2, + tlen: i32, + iflg: bool, + altitude: f32, + chisham: bool, + ) -> Result<(), GridError> { + let time_micros = + (scan.start_time.timestamp_micros() + scan.end_time.timestamp_micros()) / 2; + let time = DateTime::from_timestamp_micros(time_micros).ok_or_else(|| { + GridError::InvalidFitacf("Invalid datetime for GridTable".to_string()) + })?; + if self.status == 0 { + // set to zero by self.test() + self.status = 1; + self.noise_mean = 0.0; + self.noise_stddev = 0.0; + self.freq = 0.0; + self.num_scans = 0; + self.clear(); + self.start_time = scan.start_time; + self.end_time = scan.start_time + TimeDelta::seconds(tlen as i64); + self.station_id = scan.station_id; + } + + for scan_beam in scan.beams.iter() { + if scan_beam.beam == -1 { + continue; + } + let beam_index = match self.find_beam(scan_beam) { + Some(i) => i, + None => self.add_beam(hdw, aacgm_model, altitude, time, scan_beam, chisham)?, + }; + let grid_beam = &self.beams[beam_index]; + + for range in 0..scan_beam.num_ranges as usize { + if scan_beam.scatter[range] == 0 { + continue; + } + + let velocity_error = scan_beam.cells[range] + .velocity_error + .max(VELOCITY_ERROR_MIN); + let power_lin_error = scan_beam.cells[range] + .power_lin_error + .max(POWER_LIN_ERROR_MIN); + let width_lin_error = scan_beam.cells[range] + .spectral_width_lin_error + .max(WIDTH_LIN_ERROR_MIN); + + // Get grid cell of radar beam/gate measurement + let grid_cell = &mut self.points[grid_beam.index[range] as usize]; + + // Add slant range of gate measurement + grid_cell.slant_range += grid_beam.slant_range[range]; + + if iflg { + grid_cell.velocity_median_north -= (scan_beam.cells[range].velocity + + grid_beam.ival[range]) + * grid_beam.azimuth[range].cos() + / (velocity_error * velocity_error); + grid_cell.velocity_median_east -= (scan_beam.cells[range].velocity + + grid_beam.ival[range]) + * grid_beam.azimuth[range].sin() + / (velocity_error * velocity_error); + } else { + grid_cell.velocity_median_north -= scan_beam.cells[range].velocity + * grid_beam.azimuth[range].cos() + / (velocity_error * velocity_error); + grid_cell.velocity_median_east -= scan_beam.cells[range].velocity + * grid_beam.azimuth[range].sin() + / (velocity_error * velocity_error); + } + + grid_cell.power_median += + scan_beam.cells[range].power_lin / (power_lin_error * power_lin_error); + grid_cell.spectral_width_median += + scan_beam.cells[range].spectral_width_lin / (width_lin_error * width_lin_error); + + grid_cell.velocity_stddev += 1.0 / (velocity_error * velocity_error); + grid_cell.power_stddev += 1.0 / (power_lin_error * power_lin_error); + grid_cell.spectral_width_stddev += 1.0 / (width_lin_error * width_lin_error); + grid_cell.count += 1; + } + } + + // TODO: Check if somehow all beams in scan not considered? + + let mut freq = 0.0; + let mut noise: f64 = 0.0; + let mut variance: f64 = 0.0; + let mut count: f64 = 0.0; + + for scan_beam in scan.beams.iter().filter(|beam| beam.beam != -1) { + self.program_id = scan_beam.program_id; + + // Sum the frequency and noise values + freq += scan_beam.freq as f32; + noise += scan_beam.noise as f64; + count += 1.0; + } + + // Average frequency and noise over all beams in scan + freq /= count as f32; + noise /= count; + + for scan_beam in scan.beams.iter().filter(|beam| beam.beam != -1) { + variance += (scan_beam.noise as f64 - noise) * (scan_beam.noise as f64 - noise); + } + self.noise_mean += noise as f32; + self.noise_stddev += (variance / count).sqrt() as f32; + self.freq += freq; + self.num_scans += 1; + + Ok(()) + } + + /// Converts the GridTable to a GridRecord for writing to file. + /// Equivalent to GridTableWrite in RST. + pub fn to_dmap_record(&self, extended_flag: bool) -> Result { + let mut grid_rec: IndexMap = IndexMap::new(); + + // Find the valid points in the grid + let valid_points: Vec<&GridPoint> = self.points.iter().filter(|&p| p.count > 0).collect(); + let num_points = valid_points.len(); + + // These vector fields require accessing the points of grid_table + let magnetic_lat: Vec = valid_points.iter().map(|&p| p.magnetic_lat).collect(); + let magnetic_lon = valid_points.iter().map(|&p| p.magnetic_lon).collect(); + let azimuth = valid_points.iter().map(|&p| p.azimuth).collect(); + let slant_range = valid_points.iter().map(|&p| p.slant_range).collect(); + let index: Vec = valid_points.iter().map(|&p| p.reference).collect(); + let velocity_median = valid_points.iter().map(|&p| p.velocity_median).collect(); + let velocity_stddev = valid_points.iter().map(|&p| p.velocity_stddev).collect(); + let power_median = valid_points.iter().map(|&p| p.power_median).collect(); + let power_stddev = valid_points.iter().map(|&p| p.power_stddev).collect(); + let spectral_width_median = valid_points + .iter() + .map(|&p| p.spectral_width_median) + .collect(); + let spectral_width_stddev: Vec = valid_points + .iter() + .map(|&p| p.spectral_width_stddev) + .collect(); + let station_ids: Vec = iter::repeat(self.station_id) + .take(valid_points.len()) + .collect(); + let channels: Vec = iter::repeat(self.channel) + .take(valid_points.len()) + .collect(); + + grid_rec.insert( + "start.year".to_string(), + (self.start_time.year() as i16).into(), + ); + grid_rec.insert( + "start.month".to_string(), + (self.start_time.month() as i16).into(), + ); + grid_rec.insert( + "start.day".to_string(), + (self.start_time.day() as i16).into(), + ); + grid_rec.insert( + "start.hour".to_string(), + (self.start_time.hour() as i16).into(), + ); + grid_rec.insert( + "start.minute".to_string(), + (self.start_time.minute() as i16).into(), + ); + grid_rec.insert( + "start.second".to_string(), + (self.start_time.second() as f64 + (self.start_time.nanosecond() as f64) * 1e-9).into(), + ); + grid_rec.insert("end.year".to_string(), (self.end_time.year() as i16).into()); + grid_rec.insert( + "end.month".to_string(), + (self.end_time.month() as i16).into(), + ); + grid_rec.insert("end.day".to_string(), (self.end_time.day() as i16).into()); + grid_rec.insert("end.hour".to_string(), (self.end_time.hour() as i16).into()); + grid_rec.insert( + "end.minute".to_string(), + (self.end_time.minute() as i16).into(), + ); + grid_rec.insert( + "end.second".to_string(), + (self.end_time.second() as f64 + (self.end_time.nanosecond() as f64) * 1e-9).into(), + ); + grid_rec.insert( + "stid".to_string(), + array![self.station_id].into_dyn().into(), + ); + grid_rec.insert( + "channel".to_string(), + array![self.channel].into_dyn().into(), + ); + grid_rec.insert( + "nvec".to_string(), + array![num_points as i16].into_dyn().into(), + ); + grid_rec.insert("freq".to_string(), array![self.freq].into_dyn().into()); + grid_rec.insert( + "major.revision".to_string(), + array![GRID_REVISION_MAJOR as i16].into_dyn().into(), + ); + grid_rec.insert( + "minor.revision".to_string(), + array![GRID_REVISION_MINOR as i16].into_dyn().into(), + ); + grid_rec.insert( + "program.id".to_string(), + array![self.program_id as i16].into_dyn().into(), + ); + grid_rec.insert( + "noise.mean".to_string(), + array![self.noise_mean].into_dyn().into(), + ); + grid_rec.insert( + "noise.sd".to_string(), + array![self.noise_stddev].into_dyn().into(), + ); + grid_rec.insert( + "gsct".to_string(), + array![self.groundscatter as i16].into_dyn().into(), + ); + grid_rec.insert( + "v.min".to_string(), + array![self.min_velocity].into_dyn().into(), + ); + grid_rec.insert( + "v.max".to_string(), + array![self.max_velocity].into_dyn().into(), + ); + grid_rec.insert( + "p.min".to_string(), + array![self.min_power].into_dyn().into(), + ); + grid_rec.insert( + "p.max".to_string(), + array![self.max_power].into_dyn().into(), + ); + grid_rec.insert( + "w.min".to_string(), + array![self.min_spectral_width].into_dyn().into(), + ); + grid_rec.insert( + "w.max".to_string(), + array![self.max_spectral_width].into_dyn().into(), + ); + grid_rec.insert( + "ve.min".to_string(), + array![self.min_velocity_error].into_dyn().into(), + ); + grid_rec.insert( + "ve.max".to_string(), + array![self.max_velocity_error].into_dyn().into(), + ); + if !magnetic_lat.is_empty() { + grid_rec.insert( + "vector.mlat".to_string(), + Array::from_vec(magnetic_lat).into_dyn().into(), + ); + grid_rec.insert( + "vector.mlon".to_string(), + Array::from_vec(magnetic_lon).into_dyn().into(), + ); + grid_rec.insert( + "vector.kvect".to_string(), + Array::from_vec(azimuth).into_dyn().into(), + ); + grid_rec.insert( + "vector.srng".to_string(), + Array::from_vec(slant_range).into_dyn().into(), + ); + grid_rec.insert( + "vector.stid".to_string(), + Array::from_vec(station_ids).into_dyn().into(), + ); + grid_rec.insert( + "vector.channel".to_string(), + Array::from_vec(channels).into_dyn().into(), + ); + grid_rec.insert( + "vector.index".to_string(), + Array::from_vec(index).into_dyn().into(), + ); + grid_rec.insert( + "vector.vel.median".to_string(), + Array::from_vec(velocity_median).into_dyn().into(), + ); + grid_rec.insert( + "vector.vel.sd".to_string(), + Array::from_vec(velocity_stddev).into_dyn().into(), + ); + if extended_flag { + grid_rec.insert( + "vector.pwr.median".to_string(), + Array::from_vec(power_median).into_dyn().into(), + ); + grid_rec.insert( + "vector.pwr.sd".to_string(), + Array::from_vec(power_stddev).into_dyn().into(), + ); + grid_rec.insert( + "vector.wdt.median".to_string(), + Array::from_vec(spectral_width_median).into_dyn().into(), + ); + grid_rec.insert( + "vector.wdt.sd".to_string(), + Array::from_vec(spectral_width_stddev).into_dyn().into(), + ); + } + } + GridRecord::new(&mut grid_rec).map_err(|e| e.into()) + } +} diff --git a/src/gridding/mod.rs b/src/gridding/mod.rs new file mode 100644 index 0000000..6998285 --- /dev/null +++ b/src/gridding/mod.rs @@ -0,0 +1,6 @@ +//! Gridding of FITACF data for multi-radar data integration. +//! +//! NOTE: Only `make_grid` has been implemented, not `combine_grid`. +pub(crate) mod filter; +pub mod grid; +pub(crate) mod grid_table; diff --git a/src/lib.rs b/src/lib.rs index 21bf58d..f0cce25 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,3 +1,209 @@ +//! Core SuperDARN Processing Tools +//! +//! [![github]](https://github.com/SuperDARNCanada/procdarn) +//! +//! [github]: https://img.shields.io/badge/github-8da0cb?style=for-the-badge&labelColor=555555&logo=github +//! +//!
+//! +//! This library also has a Python API using pyo3. +//! +//! This library is a re-implementation of the core tools from [SuperDARN's Radar Software Toolkit +//! (RST)](https://github.com/SuperDARN/rst). Currently, only two binaries from RST are implemented. +//! The goal is for the entire RAWACF -> FITACF -> GRID -> MAP +//! pipeline to be implemented. +//! +//! | RST binaries | `procdarn` function | +//! | ------------------- | ------------------- | +//! | `make_fit -fitacf3` | [`fitacf3`] | +//! | `make_grid` | [`fit2grid`] | +//! +//! The `procdarn` algorithms can be called with directly with data, or can be used to read data +//! from files and run it through the algorithms. + +use clap::Parser; +use dmap::error::DmapError; +use dmap::formats::rawacf::RawacfRecord; +use dmap::record::Record; +use dmap::types::DmapField; +use indexmap::IndexMap; +use itertools::{Either, Itertools}; +use pyo3::prelude::{PyAnyMethods, PyModule, PyModuleMethods}; +use pyo3::{pyfunction, pymodule, wrap_pyfunction, Bound, PyErr, PyResult, Python}; +use std::path::PathBuf; +use dmap::formats::fitacf::FitacfRecord; +use dmap::GridRecord; +use pyo3::types::PyDict; + pub mod error; pub mod fitting; -pub mod utils; +pub mod gridding; +mod utils; + +pub use crate::fitting::fitacf3::fitacf_v3::{fitacf3, Fitacf3Error}; +pub use crate::gridding::grid::{fit2grid, fit2grid_file, GridArgs, GridError}; + +/// Fits a list of RAWACF records into FITACF records using the FITACFv3 algorithm. +#[pyfunction] +#[pyo3(name = "fitacf3_recs")] +#[pyo3(text_signature = "(recs: list[dict], /)")] +fn fitacf3_py( + mut recs: Vec>, +) -> PyResult>> { + let (errors, formatted_recs): (Vec<_>, Vec<_>) = + recs.iter_mut() + .enumerate() + .partition_map(|(i, rec)| match RawacfRecord::try_from(rec) { + Err(e) => Either::Left((i, e)), + Ok(x) => Either::Right(x), + }); + if !errors.is_empty() { + Err(PyErr::from(DmapError::InvalidRecord(format!( + "Corrupted records: {errors:?}" + ))))? + } + let fitacf_recs = fitacf3(formatted_recs) + .map_err(PyErr::from)? + .into_iter() + .map(|rec| rec.inner()) + .collect(); + Ok(fitacf_recs) +} + +/// Fits a RAWACF file into a FITACF record using the FITACFv3 algorithm. +fn fitacf3_file(raw_file: PathBuf, fit_file: PathBuf) -> Result<(), Fitacf3Error> { + let rawacf_records = RawacfRecord::read_file(raw_file)?; + let fitacf_records = fitacf3(rawacf_records)?; + FitacfRecord::write_to_file(&fitacf_records, &fit_file, false)?; + Ok(()) +} + +/// Fits a RAWACF file into a FITACF record using the FITACFv3 algorithm. +#[pyfunction] +#[pyo3(name = "fitacf3_file")] +#[pyo3(text_signature = "(rawacf_file: str, fitacf_file: str, /)")] +fn fitacf3_file_py(raw_file: PathBuf, fit_file: PathBuf) -> PyResult<()> { + fitacf3_file(raw_file, fit_file)?; + Ok(()) +} + +#[derive(Parser, Debug)] +#[command(author, version, about, long_about = None)] +struct Fitacf3Args { + /// Rawacf file to fit + #[arg()] + infile: PathBuf, + + /// Output fitacf file path + #[arg()] + outfile: PathBuf, +} + +/// Fits a RAWACF file into a FITACF file using the FITACFv3 algorithm. +#[pyfunction] +#[pyo3(name = "raw2fit")] +fn fitacf3_cli(py: Python) -> PyResult<()> { + let argv = py + .import("sys")? + .getattr("argv")? + .extract::>()?; + let args = Fitacf3Args::parse_from(argv); + + let rawacf_records = RawacfRecord::read_file(args.infile)?; + let fitacf_records = fitacf3(rawacf_records)?; + + // Write to file + FitacfRecord::write_to_file(&fitacf_records, &args.outfile, false)?; + Ok(()) +} + + +/// Converts a list of FITACF records into GRID records. +#[pyfunction] +#[pyo3(name = "fit2grid_recs")] +#[pyo3(signature = (recs, /, **py_kwargs))] +#[pyo3(text_signature = "(recs: list[dict], /, **)")] +fn fit2grid_py( + mut recs: Vec>, + py_kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult>> { + let args = match py_kwargs { + Some(kwargs) => kwargs.extract()?, + None => GridArgs::parse_from(vec!["fit2grid"]), + }; + + let (errors, formatted_recs): (Vec<_>, Vec<_>) = + recs.iter_mut() + .enumerate() + .partition_map(|(i, rec)| match FitacfRecord::try_from(rec) { + Err(e) => Either::Left((i, e)), + Ok(x) => Either::Right(x), + }); + if !errors.is_empty() { + Err(PyErr::from(DmapError::InvalidRecord(format!( + "Corrupted records: {errors:?}" + ))))? + } + let grid_recs = fit2grid(&args, &formatted_recs) + .map_err(PyErr::from)? + .into_iter() + .map(|rec| rec.inner()) + .collect(); + Ok(grid_recs) +} + +/// Fits a list of FITACF files into a GRID file. +#[pyfunction] +#[pyo3(name = "fit2grid_file")] +#[pyo3(signature = (fitacf_files, grid_file, /, **py_kwargs))] +#[pyo3(text_signature = "(fitacf_files: list[str], grid_file: str, /, **kwargs)")] +fn fit2grid_file_py(fitacf_files: Vec, grid_file: PathBuf, py_kwargs: Option<&Bound<'_, PyDict>>) -> PyResult<()> { + let args = match py_kwargs { + Some(kwargs) => kwargs.extract()?, + None => GridArgs::parse_from(vec!["fit2grid"]), + }; + let grid_recs = fit2grid_file(&fitacf_files, &args)?; + GridRecord::write_to_file(&grid_recs, &grid_file, false)?; + Ok(()) +} + +#[derive(Parser, Debug)] +#[command(author, version, about, long_about = None)] +pub struct GridArgsCLI { + /// Output grid file path + pub outfile: PathBuf, + + /// Fitacf file(s) to grid + #[arg(num_args = 1.., last = true)] + pub infiles: Vec, + + #[command(flatten)] + pub grid_args: GridArgs, +} + +/// Converts a set of FITACF files into a GRID file. +#[pyfunction] +#[pyo3(name = "fit2grid")] +fn fit2grid_cli(py: Python) -> PyResult<()> { + let argv = py + .import("sys")? + .getattr("argv")? + .extract::>()?; + let args = GridArgsCLI::parse_from(argv); + let grid_records = fit2grid_file(&args.infiles, &args.grid_args)?; + GridRecord::write_to_file(&grid_records, &args.outfile, false)?; + Ok(()) +} + +/// Functions for SuperDARN data processing. +#[pymodule] +fn procdarn(m: &Bound<'_, PyModule>) -> PyResult<()> { + m.add_function(wrap_pyfunction!(fitacf3_py, m)?)?; + m.add_function(wrap_pyfunction!(fitacf3_file_py, m)?)?; + m.add_wrapped(wrap_pyfunction!(fitacf3_cli))?; + m.add_function(wrap_pyfunction!(fit2grid_py, m)?)?; + m.add_function(wrap_pyfunction!(fit2grid_file_py, m)?)?; + m.add_wrapped(wrap_pyfunction!(fit2grid_cli))?; + + Ok(()) +} diff --git a/src/main.rs b/src/main.rs deleted file mode 100644 index 48e8444..0000000 --- a/src/main.rs +++ /dev/null @@ -1,14 +0,0 @@ -use dmap::formats::{to_file, DmapRecord, RawacfRecord}; -use std::fs::File; - -fn main() { - let file = - File::open("tests/test_files/20210607.1801.00.cly.a.rawacf").expect("Test file not found"); - // let file = File::open(Path::new("tests/test_files/20160316.1945.01.rkn.iqdat")) - // .expect("Test file not found"); - // let file = File::open(Path::new("tests/test_files/20110214.map")) - // .expect("Test file not found"); - let contents = RawacfRecord::read_records(file).unwrap(); - - to_file("tests/test_files/temp.rawacf", &contents).unwrap(); -} diff --git a/src/utils/channel.rs b/src/utils/channel.rs new file mode 100644 index 0000000..d4c08ad --- /dev/null +++ b/src/utils/channel.rs @@ -0,0 +1,25 @@ +use crate::error::ProcdarnError; + +pub fn set_stereo_channel(channel_char: char) -> Result { + match channel_char { + 'a' => Ok(1), + 'b' => Ok(2), + _ => Err(ProcdarnError::Channel(format!( + "Invalid stereo channel {}", + channel_char + ))), + } +} + +pub fn set_fix_channel(channel_char: char) -> Result { + match channel_char { + 'a' => Ok(1), + 'b' => Ok(2), + 'c' => Ok(3), + 'd' => Ok(4), + _ => Err(ProcdarnError::Channel(format!( + "Invalid fix channel {}", + channel_char + ))), + } +} diff --git a/src/utils/coords.rs b/src/utils/coords.rs new file mode 100644 index 0000000..a225d67 --- /dev/null +++ b/src/utils/coords.rs @@ -0,0 +1,356 @@ +use crate::error::ProcdarnError; +use igrf::declination; +use std::f64::consts::PI; +use std::fmt::Display; +use aacgmv2_rs::aacgmv2::Aacgmv2; +use time::Date; + +#[derive(Copy, Clone, Debug, Default, PartialEq)] +pub struct GeocentricCoords { + /// Latitude (radians) + pub lat: f64, + /// Longitude (radians) + pub lon: f64, + /// Distance from the center of the Earth (km) + pub rad: f64, +} +impl GeocentricCoords { + /// Constructor, with `lat`, `lon` in radians. + pub fn new(lat: f64, lon: f64, rad: f64) -> GeocentricCoords { + GeocentricCoords { lat, lon, rad } + } + + /// Constructor with input `lat`, `lon` in degrees. + #[allow(dead_code)] + pub fn geo(lat: f64, lon: f64, rad: f64) -> GeocentricCoords { + GeocentricCoords::new(lat.to_radians(), lon.to_radians(), rad) + } + + /// Converts `self` to [`GeodeticCoords`]. The WGS84 Earth model is used. + pub fn to_geodetic(self) -> GeodeticCoords { + let semi_major_axis: f64 = 6378.137; + let flattening: f64 = 1.0 / 298.257223563; + let semi_minor_axis: f64 = semi_major_axis * (1.0 - flattening); + let second_eccentricity_squared: f64 = + (semi_major_axis * semi_major_axis) / (semi_minor_axis * semi_minor_axis) - 1.0; + + let gdlat = ((semi_major_axis * semi_major_axis) / (semi_minor_axis * semi_minor_axis) + * self.lat.tan()) + .atan(); + let gdlon = self.lon; + + let rho = semi_major_axis + / (1.0 + second_eccentricity_squared * self.lat.sin() * self.lat.sin()).sqrt(); + + GeodeticCoords::new(gdlat, gdlon, rho) + } + + /// Converts `self` to [`CartesianCoords`]. + pub fn to_cartesian(self) -> CartesianCoords { + let x = self.rad * self.lat.cos() * self.lon.cos(); + let y = self.rad * self.lat.cos() * self.lon.sin(); + let z = self.rad * self.lat.sin(); + CartesianCoords { x, y, z } + } + + /// Convert a [`CartesianCoords`] vector `v` centered at `self` into [`LocalCartesianCoords`]. + pub fn cartesian_to_local(&self, v: &CartesianCoords) -> LocalCartesianCoords { + // Rotate v about the z-axis by the longitude + let sx = self.lon.cos() * v.x + self.lon.sin() * v.y; + let sy = -self.lon.sin() * v.x + self.lon.cos() * v.y; + let sz = v.z; + + // Calculate the colatitude + let colat = PI / 2.0 - self.lat; + + // Rotate the vector about the east-axis by the colatitude + let tx = colat.cos() * sx - colat.sin() * sz; + let ty = sy; + let tz = colat.sin() * sx + colat.cos() * sz; + + LocalCartesianCoords::new(tx, ty, tz) + } + + /// Converts `self` into AACGMv2 coordinates. + /// + /// See https://superdarn.thayer.dartmouth.edu/aacgm.html and doi:10.1002/2014JA020264 + pub(crate) fn aacgmv2_convert(&self, model: &mut Aacgmv2) -> GeocentricCoords { + let new_coords = model.convert( + self.lat.to_degrees(), + self.lon.to_degrees(), + self.rad, + &aacgmv2_rs::Transform::GeodeticToAACGMv2, + &aacgmv2_rs::Method::Coeffs + ).expect("Aacgmv2::convert() failed"); + + GeocentricCoords { + lat: new_coords.0.to_radians(), + lon: new_coords.1.to_radians(), + rad: new_coords.2, + } + } + + /// Calculates the magnetic field at this location. + pub(crate) fn igrf_field(&self, date: Date) -> Result { + // Calculate the magnetic field vector in nT at the geocentric spherical cell position + let igrf_field = declination(self.lat.to_degrees(), self.lon.to_degrees(), self.rad, date)?; + + Ok(CartesianCoords::new( + igrf_field.x, + igrf_field.y, + igrf_field.z, + )) + } +} +impl Display for GeocentricCoords { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + f.write_fmt(format_args!( + "({}, {}, {})", + self.lat.to_degrees(), + self.lon.to_degrees(), + self.rad + )) + } +} + +#[derive(Copy, Clone, Debug, Default, PartialEq)] +pub struct GeodeticCoords { + /// Latitude (radians) + pub lat: f64, + /// Longitude (radians) + pub lon: f64, + /// Distance from the center of the Earth (km) + pub rad: f64, +} + +impl GeodeticCoords { + /// Constructor with input `lat`, `lon` in radians. + pub fn new(lat: f64, lon: f64, rad: f64) -> GeodeticCoords { + GeodeticCoords { lat, lon, rad } + } + /// Constructor with input `lat`, `lon` in degrees. + #[allow(dead_code)] + pub fn geo(lat: f64, lon: f64, rad: f64) -> GeodeticCoords { + GeodeticCoords::new(lat.to_radians(), lon.to_radians(), rad) + } + + /// Converts to [`GeocentricCoords`]. The WGS84 Earth model is used. + pub fn to_geocentric(self) -> GeocentricCoords { + let semi_major_axis: f64 = 6378.137; + let flattening: f64 = 1.0 / 298.257223563; + let semi_minor_axis: f64 = semi_major_axis * (1.0 - flattening); + let second_eccentricity_squared: f64 = + (semi_major_axis * semi_major_axis) / (semi_minor_axis * semi_minor_axis) - 1.0; + + let gclat = ((semi_minor_axis * semi_minor_axis) / (semi_major_axis * semi_major_axis) + * self.lat.tan()) + .atan(); + let mut gclon = self.lon; + if gclon.to_degrees() > 180.0 { + gclon -= 360.0_f64.to_radians(); + } + let rho = semi_major_axis + / (1.0 + second_eccentricity_squared * gclat.sin() * gclat.sin()).sqrt(); + + GeocentricCoords::new(gclat, gclon, rho) + } + + /// Corrects a vector `v` at `self` to account for the oblateness of the Earth. + pub(crate) fn correct_look_dir(&self, v: &mut LocalAngularCoords) { + let kxg = v.el.cos() * v.az.sin(); + let kyg = v.el.cos() * v.az.cos(); + let kzg = v.el.sin(); + + let point_gc = self.to_geocentric(); + let del = self.lat - point_gc.lat; + + let kxr = kxg; + let kyr = kyg * del.cos() + kzg * del.sin(); + let kzr = -kyg * del.sin() + kzg * del.cos(); + + v.az = kxr.atan2(kyr); + v.el = (kzr / (kxr * kxr + kyr * kyr).sqrt()).atan(); + } +} +impl Display for GeodeticCoords { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + f.write_fmt(format_args!( + "({}, {}, {})", + self.lat.to_degrees(), + self.lon.to_degrees(), + self.rad + )) + } +} + +#[derive(Copy, Clone, Debug, Default, PartialEq)] +pub struct CartesianCoords { + pub x: f64, + pub y: f64, + pub z: f64, +} +impl CartesianCoords { + pub fn new(x: f64, y: f64, z: f64) -> CartesianCoords { + CartesianCoords { x, y, z } + } + /// Scale to unit length. + pub fn norm(&mut self) { + let len = (self.x * self.x + self.y * self.y + self.z * self.z).sqrt(); + self.x /= len; + self.y /= len; + self.z /= len; + } +} +impl std::ops::Sub for CartesianCoords { + type Output = Self; + fn sub(self, rhs: Self) -> Self::Output { + Self { + x: self.x - rhs.x, + y: self.y - rhs.y, + z: self.z - rhs.z, + } + } +} +impl Display for CartesianCoords { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + f.write_fmt(format_args!("({}, {}, {})", self.x, self.y, self.z)) + } +} + +#[derive(Copy, Clone, Debug, Default, PartialEq)] +pub struct MagneticCoords { + pub lat: f64, + pub lon: f64, +} +impl MagneticCoords { + pub fn new(lat: f64, lon: f64) -> MagneticCoords { + MagneticCoords { lat, lon } + } +} + +#[derive(Copy, Clone, Debug, Default, PartialEq)] +pub struct LocalCartesianCoords { + /// Distance in km + pub south: f64, + /// Distance in km + pub east: f64, + /// Distance in km + pub up: f64, +} +impl LocalCartesianCoords { + pub fn new(south: f64, east: f64, up: f64) -> LocalCartesianCoords { + LocalCartesianCoords { south, east, up } + } + /// Scale to unit length. + pub fn norm(&mut self) { + let len = (self.south * self.south + self.east * self.east + self.up * self.up).sqrt(); + self.south /= len; + self.east /= len; + self.up /= len; + } +} + +#[derive(Copy, Clone, Debug, Default, PartialEq)] +pub struct LocalAngularCoords { + /// East of North (radians) + pub az: f64, + /// Up from horizon (radians) + pub el: f64, + /// Distance in km + pub range: f64, +} +impl LocalAngularCoords { + pub fn new(az: f64, el: f64, range: f64) -> LocalAngularCoords { + LocalAngularCoords { az, el, range } + } + /// Constructor with inputs in degrees. + #[allow(dead_code)] + pub fn from_degrees(az: f64, el: f64, range: f64) -> LocalAngularCoords { + LocalAngularCoords::new(az.to_radians(), el.to_radians(), range) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use approx::assert_relative_eq; + use chrono::NaiveDate; + + #[test] + fn test_geocentric_to_cartesian() { + let rel = 1e-7; + let p = GeocentricCoords::geo(69.519412, -133.91889, 6474.25015); + let res = p.to_cartesian(); + assert_relative_eq!(res.x, -1571.284220, max_relative = rel); + assert_relative_eq!(res.y, -1631.728799, max_relative = rel); + assert_relative_eq!(res.z, 6065.017892, max_relative = rel); + } + + #[test] + fn test_geodetic_to_geocentric() { + let rel = 1e-7; + let p = GeodeticCoords::geo(68.413, -133.769, 0.0); + let res = p.to_geocentric(); + assert_relative_eq!(res.lat.to_degrees(), 68.281017, max_relative = rel); + assert_relative_eq!(res.lon.to_degrees(), -133.769, max_relative = rel); + assert_relative_eq!(res.rad, 6359.668035, max_relative = rel); + } + + #[test] + fn test_geocentric_to_geodetic() { + let rel = 1e-9; + let p = GeocentricCoords::geo(69.519411986, -133.918890359, 0.0); + let res = p.to_geodetic(); + assert_relative_eq!(res.lat.to_degrees(), 69.645235706, max_relative = rel); + assert_relative_eq!(res.lon.to_degrees(), -133.918890359, max_relative = rel); + assert_relative_eq!(res.rad, 6359.358742609, max_relative = rel); + } + + #[test] + fn test_cartesian_to_local() { + let rel = 1e-9; + let loc = GeocentricCoords::geo(69.519411986, -133.918890359, 6474.25015); + let v = CartesianCoords::new(0.315016678, 0.376445908, 0.871236461); + let res = loc.cartesian_to_local(&v); + assert_relative_eq!(res.south, -0.763555664, max_relative = rel); + assert_relative_eq!(res.east, -0.034204109, max_relative = rel); + assert_relative_eq!(res.up, 0.644835504, max_relative = rel); + } + + #[test] + fn test_correct_look_dir() { + let rel = 1e-9; + let point = GeodeticCoords::geo(68.413, -133.769, 0.0); + let mut v = LocalAngularCoords::from_degrees(-2.429550020, 38.913774588, 0.0); + point.correct_look_dir(&mut v); + assert_relative_eq!(v.az.to_degrees(), -2.425048105, max_relative = rel); + assert_relative_eq!(v.el.to_degrees(), 38.781910399, max_relative = rel); + } + + #[test] + fn test_aacgmv2_convert() { + let rel = 1e-7; + + let point = GeocentricCoords::geo(69.917246, 226.029209, 114.891407); + let date = NaiveDate::from_ymd_opt(2025, 7, 12).unwrap().and_hms_opt(0, 0, 0).unwrap().and_utc(); + let mut model = Aacgmv2::new(date).unwrap(); + let mag_point = point.aacgmv2_convert(&mut model); + + assert_relative_eq!(mag_point.lat.to_degrees(), 72.507253, max_relative = rel); + assert_relative_eq!(mag_point.lon.to_degrees(), -81.359931, max_relative = rel); + assert_relative_eq!(mag_point.rad, 1.016164, max_relative = rel); + } + + #[test] + fn test_igrf_field() { + let rel = 1e-2; + let point = GeocentricCoords::geo(69.51941199, -133.91889036, 6474.25014983); + + let igrf_field = point + .igrf_field(Date::from_calendar_date(2025, time::Month::January, 1).unwrap()) + .unwrap(); + assert_relative_eq!(igrf_field.x, -7334.09740294, max_relative = rel); + assert_relative_eq!(igrf_field.y, 2496.73900915, max_relative = rel); + assert_relative_eq!(igrf_field.z, -53940.93134632, max_relative = rel); + } +} diff --git a/src/utils/hdw.rs b/src/utils/hdw.rs index c5cc3fb..e76020a 100644 --- a/src/utils/hdw.rs +++ b/src/utils/hdw.rs @@ -1,38 +1,60 @@ -use crate::error::BackscatterError; -use chrono::NaiveDateTime; +use chrono::{DateTime, NaiveDateTime, Utc}; use rust_embed::RustEmbed; use std::io::{BufRead, BufReader}; +use thiserror::Error; #[derive(RustEmbed)] #[folder = "target/hdw/"] struct Hdw; +#[derive(Error, Debug)] +pub enum HdwError { + /// Represents a file that does not follow the hdw file format + #[error("{0}")] + File(String), + + /// Represents trying to use a datetime that isn't covered by the hdw file + #[error("{0}")] + Datetime(String), + + /// Represents trying to find the hdw file for a non-existent radar + #[error("{0}")] + Station(i16), +} + +#[allow(dead_code)] #[derive(Debug)] pub struct HdwInfo { - pub station_id: i16, - pub valid_from: NaiveDateTime, - pub latitude: f32, - pub longitude: f32, - pub altitude: f32, - pub boresight: f32, - pub boresight_shift: f32, - pub beam_separation: f32, - pub velocity_sign: f32, - pub phase_sign: f32, - pub tdiff_a: f32, - pub tdiff_b: f32, - pub intf_offset_x: f32, - pub intf_offset_y: f32, - pub intf_offset_z: f32, - pub rx_rise_time: f32, - pub rx_atten_step: f32, - pub attenuation_stages: f32, - pub max_num_ranges: i16, - pub max_num_beams: i16, + pub station_id: i16, // stid in RST + pub valid_from: DateTime, // date, hr, mt, sc in RST + pub latitude: f32, // geolat in RST + pub longitude: f32, // geolon in RST + pub altitude: f32, // alt in RST + pub boresight: f32, // boresite in RST + pub boresight_shift: f32, // bmoff in RST + pub beam_separation: f32, // bmsep in RST + pub velocity_sign: f32, // vdir in RST + pub phase_sign: f32, // phidiff in RST + pub tdiff_a: f32, // tdiff[0] in RST + pub tdiff_b: f32, // tdiff[1] in RST + pub intf_offset_x: f32, // interfer[0] in RST + pub intf_offset_y: f32, // interfer[1] in RST + pub intf_offset_z: f32, // interfer[2] in RST + pub rx_rise_time: f32, // recrise in RST + pub rx_atten_step: f32, // atten in RST + pub attenuation_stages: f32, // maxatten in RST + pub max_num_ranges: i16, // maxrange in RST + pub max_num_beams: i16, // maxbeam in RST } impl HdwInfo { - pub fn new(station_id: i16, datetime: NaiveDateTime) -> Result { + /// Gets the hardware file information for a site at a particular time. + /// + /// # Errors + /// * If the `station_id` does not match the known sites + /// * If the hardware file does not have an entry applicable for the `datetime` + /// * If the hardware file is not properly formatted + pub fn new(station_id: i16, datetime: DateTime) -> Result { let site_name = match station_id { 209 => "ade", 208 => "adw", @@ -77,91 +99,118 @@ impl HdwInfo { 18 => "unw", 32 => "wal", 19 => "zho", - _ => Err(BackscatterError::new("Invalid station id"))?, + x => Err(HdwError::Station(x))?, }; - let hdw_file = Hdw::get(format!("hdw.dat.{}", site_name).as_str()).unwrap(); + let hdw_file = Hdw::get(format!("hdw.dat.{site_name}").as_str()) + .ok_or_else(|| HdwError::File(format!("No file named hdw.dat.{site_name}")))?; let mut hdw_params: Vec = vec![]; let reader = BufReader::new(hdw_file.data.as_ref()).lines(); for line in reader { - let line = - line.map_err(|_| BackscatterError::new("Unable to read line from hdw file"))?; + let line = line.map_err(|_| { + HdwError::File("Unable to read line from hdw file".to_string()) + })?; if !line.starts_with('#') { let elements: Vec<&str> = line.split_whitespace().collect(); let date = elements[2]; let time = elements[3]; let validity_date = NaiveDateTime::parse_from_str( - format!("{} {}", date, time).as_str(), + format!("{date} {time}").as_str(), "%Y%m%d %H:%M:%S", ) - .map_err(|_| BackscatterError::new("Unable to read station id from hdw file"))?; + .map_err(|_| { + HdwError::File("Unable to parse timeframe from hdw file".to_string()) + })? + .and_utc(); if datetime < validity_date { break; - } // + } hdw_params.push(HdwInfo { station_id: elements[0].parse::().map_err(|_| { - BackscatterError::new("Unable to read station id from hdw file") + HdwError::File("Unable to read station id from hdw file".to_string()) })?, valid_from: validity_date, latitude: elements[4].parse::().map_err(|_| { - BackscatterError::new("Unable to read latitude from hdw file") + HdwError::File("Unable to read latitude from hdw file".to_string()) })?, longitude: elements[5].parse::().map_err(|_| { - BackscatterError::new("Unable to read longitude from hdw file") + HdwError::File("Unable to read longitude from hdw file".to_string()) })?, altitude: elements[6].parse::().map_err(|_| { - BackscatterError::new("Unable to read altitude from hdw file") + HdwError::File("Unable to read altitude from hdw file".to_string()) })?, boresight: elements[7].parse::().map_err(|_| { - BackscatterError::new("Unable to read boresight from hdw file") + HdwError::File("Unable to read boresight from hdw file".to_string()) })?, boresight_shift: elements[8].parse::().map_err(|_| { - BackscatterError::new("Unable to read boresightshift from hdw file") + HdwError::File( + "Unable to read boresightshift from hdw file".to_string(), + ) })?, beam_separation: elements[9].parse::().map_err(|_| { - BackscatterError::new("Unable to read beam separation from hdw file") + HdwError::File( + "Unable to read beam separation from hdw file".to_string(), + ) })?, velocity_sign: elements[10].parse::().map_err(|_| { - BackscatterError::new("Unable to read velocity sign from hdw file") + HdwError::File( + "Unable to read velocity sign from hdw file".to_string(), + ) })?, phase_sign: elements[11].parse::().map_err(|_| { - BackscatterError::new("Unable to read phase sign from hdw file") + HdwError::File("Unable to read phase sign from hdw file".to_string()) })?, tdiff_a: elements[12].parse::().map_err(|_| { - BackscatterError::new("Unable to read tdiff A from hdw file") + HdwError::File("Unable to read tdiff A from hdw file".to_string()) })?, tdiff_b: elements[13].parse::().map_err(|_| { - BackscatterError::new("Unable to read tdiff B from hdw file") + HdwError::File("Unable to read tdiff B from hdw file".to_string()) })?, intf_offset_x: elements[14].parse::().map_err(|_| { - BackscatterError::new("Unable to read intf offset X from hdw file") + HdwError::File( + "Unable to read intf offset X from hdw file".to_string(), + ) })?, intf_offset_y: elements[15].parse::().map_err(|_| { - BackscatterError::new("Unable to read intf offset Y from hdw file") + HdwError::File( + "Unable to read intf offset Y from hdw file".to_string(), + ) })?, intf_offset_z: elements[16].parse::().map_err(|_| { - BackscatterError::new("Unable to read intf offset Z from hdw file") + HdwError::File( + "Unable to read intf offset Z from hdw file".to_string(), + ) })?, rx_rise_time: elements[17].parse::().map_err(|_| { - BackscatterError::new("Unable to read rx rise time from hdw file") + HdwError::File( + "Unable to read rx rise time from hdw file".to_string(), + ) })?, rx_atten_step: elements[18].parse::().map_err(|_| { - BackscatterError::new("Unable to read rx attenuation from hdw file") + HdwError::File( + "Unable to read rx attenuation from hdw file".to_string(), + ) })?, attenuation_stages: elements[19].parse::().map_err(|_| { - BackscatterError::new("Unable to attenuation stages from hdw file") + HdwError::File( + "Unable to read attenuation stages from hdw file".to_string(), + ) })?, max_num_ranges: elements[20].parse::().map_err(|_| { - BackscatterError::new("Unable to read max number of ranges from hdw file") + HdwError::File( + "Unable to read max number of ranges from hdw file".to_string(), + ) })?, max_num_beams: elements[21].parse::().map_err(|_| { - BackscatterError::new("Unable to read max number of beams from hdw file") + HdwError::File( + "Unable to read max number of beams from hdw file".to_string(), + ) })?, - }) + }); } } - hdw_params - .pop() - .ok_or_else(|| BackscatterError::new("No valid lines found in hdw file")) + hdw_params.pop().ok_or_else(|| { + HdwError::Datetime("No valid lines found in hdw file".to_string()) + }) } } diff --git a/src/utils/mod.rs b/src/utils/mod.rs index 094c59b..649c5e6 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -1 +1,8 @@ +pub mod channel; +pub mod coords; pub mod hdw; +pub mod rawacf; +pub mod rpos; +pub mod scan; +pub mod search; +pub mod sugar; diff --git a/src/utils/rawacf.rs b/src/utils/rawacf.rs new file mode 100644 index 0000000..fa64065 --- /dev/null +++ b/src/utils/rawacf.rs @@ -0,0 +1,236 @@ +use crate::error::ProcdarnError; +use crate::utils::hdw::HdwInfo; +use chrono::{DateTime, NaiveDate, Utc}; +use dmap::error::DmapError; +use dmap::formats::rawacf::RawacfRecord; +use dmap::Record; +use dmap::types::DmapField; +use numpy::ndarray::{Array1, Array2, Array3, ArrayD}; +use numpy::{Ix1, Ix2, Ix3}; + +pub(crate) struct Rawacf { + // Scalar fields + pub radar_revision_major: i8, + pub radar_revision_minor: i8, + pub origin_command: String, + pub cp: i16, + pub stid: i16, + pub time_yr: i16, + pub time_mo: i16, + pub time_dy: i16, + pub time_hr: i16, + pub time_mt: i16, + pub time_sc: i16, + pub time_us: i32, + pub txpow: i16, + pub nave: i16, + pub atten: i16, + pub lagfr: i16, + pub smsep: i16, + pub ercod: i16, + pub stat_agc: i16, + pub stat_lopwr: i16, + pub noise_search: f32, + pub noise_mean: f32, + pub channel: i16, + pub bmnum: i16, + pub bmazm: f32, + pub scan: i16, + pub offset: i16, + pub rxrise: i16, + pub intt_sc: i16, + pub intt_us: i32, + pub txpl: i16, + pub mpinc: i16, + pub mppul: i16, + pub mplgs: i16, + pub nrang: i16, + pub frang: i16, + pub rsep: i16, + pub xcf: i16, + pub tfreq: f32, // differs from RST, used as f32 in all calculations + pub mxpwr: i32, + pub lvmax: i32, + pub combf: String, + // pub rawacf_revision_major: i32, + // pub rawacf_revision_minor: i32, + // pub thr: f32, + + // Optional scalar fields + pub mplgexs: Option, + pub ifmode: Option, + + // Vector fields + pub ptab: Array1, + pub ltab: Array2, + pub pwr0: Array1, + pub slist: Array1, + pub acfd: Array3, + + // Optional vector fields + pub xcfd: Option>, +} +impl TryFrom<&RawacfRecord> for Rawacf { + type Error = DmapError; + fn try_from(value: &RawacfRecord) -> Result { + let scalar_getter = |key: &str| -> Result<&DmapField, DmapError> { + value + .get(key) + .ok_or_else(|| DmapError::InvalidScalar(key.to_string())) + }; + let opt_scalar_getter = |key: &str| -> Option<&DmapField> { value.get(key) }; + let vector_getter = |key: &str| -> Result<&DmapField, DmapError> { + value + .get(key) + .ok_or_else(|| DmapError::InvalidVector(key.to_string())) + }; + let opt_vector_getter = |key: &str| -> Option<&DmapField> { value.get(key) }; + Ok(Rawacf { + radar_revision_major: scalar_getter("radar.revision.major")?.clone().try_into()?, + radar_revision_minor: scalar_getter("radar.revision.minor")?.clone().try_into()?, + origin_command: scalar_getter("origin.command")?.clone().try_into()?, + cp: scalar_getter("cp")?.clone().try_into()?, + stid: scalar_getter("stid")?.clone().try_into()?, + time_yr: scalar_getter("time.yr")?.clone().try_into()?, + time_mo: scalar_getter("time.mo")?.clone().try_into()?, + time_dy: scalar_getter("time.dy")?.clone().try_into()?, + time_hr: scalar_getter("time.hr")?.clone().try_into()?, + time_mt: scalar_getter("time.mt")?.clone().try_into()?, + time_sc: scalar_getter("time.sc")?.clone().try_into()?, + time_us: scalar_getter("time.us")?.clone().try_into()?, + txpow: scalar_getter("txpow")?.clone().try_into()?, + nave: scalar_getter("nave")?.clone().try_into()?, + atten: scalar_getter("atten")?.clone().try_into()?, + lagfr: scalar_getter("lagfr")?.clone().try_into()?, + smsep: scalar_getter("smsep")?.clone().try_into()?, + ercod: scalar_getter("ercod")?.clone().try_into()?, + stat_agc: scalar_getter("stat.agc")?.clone().try_into()?, + stat_lopwr: scalar_getter("stat.lopwr")?.clone().try_into()?, + noise_search: scalar_getter("noise.search")?.clone().try_into()?, + noise_mean: scalar_getter("noise.mean")?.clone().try_into()?, + channel: scalar_getter("channel")?.clone().try_into()?, + bmnum: scalar_getter("bmnum")?.clone().try_into()?, + bmazm: scalar_getter("bmazm")?.clone().try_into()?, + scan: scalar_getter("scan")?.clone().try_into()?, + offset: scalar_getter("offset")?.clone().try_into()?, + rxrise: scalar_getter("rxrise")?.clone().try_into()?, + intt_sc: scalar_getter("intt.sc")?.clone().try_into()?, + intt_us: scalar_getter("intt.us")?.clone().try_into()?, + txpl: scalar_getter("txpl")?.clone().try_into()?, + mpinc: scalar_getter("mpinc")?.clone().try_into()?, + mppul: scalar_getter("mppul")?.clone().try_into()?, + mplgs: scalar_getter("mplgs")?.clone().try_into()?, + nrang: scalar_getter("nrang")?.clone().try_into()?, + frang: scalar_getter("frang")?.clone().try_into()?, + rsep: scalar_getter("rsep")?.clone().try_into()?, + xcf: scalar_getter("xcf")?.clone().try_into()?, + tfreq: scalar_getter("tfreq")?.clone().try_into()?, + mxpwr: scalar_getter("mxpwr")?.clone().try_into()?, + lvmax: scalar_getter("lvmax")?.clone().try_into()?, + combf: scalar_getter("combf")?.clone().try_into()?, + // rawacf_revision_major: scalar_getter("rawacf.revision.major")?.clone().try_into()?, + // rawacf_revision_minor: scalar_getter("rawacf.revision.minor")?.clone().try_into()?, + // thr: scalar_getter("thr")?.clone().try_into()?, + mplgexs: match opt_scalar_getter("mplgexs") { + Some(x) => Some(x.clone().try_into()?), + None => None, + }, + ifmode: match opt_scalar_getter("ifmode") { + Some(x) => Some(x.clone().try_into()?), + None => None, + }, + ptab: >>::try_into(vector_getter("ptab")?.clone())? + .into_dimensionality::() + .map_err(|e| { + DmapError::InvalidVector(format!("Unable to map ptab to 1D vector: {e}")) + })?, + ltab: >>::try_into(vector_getter("ltab")?.clone())? + .into_dimensionality::() + .map_err(|e| { + DmapError::InvalidVector(format!("Unable to map ltab to 2D vector: {e}")) + })?, + pwr0: >>::try_into(vector_getter("pwr0")?.clone())? + .into_dimensionality::() + .map_err(|e| { + DmapError::InvalidVector(format!("Unable to map pwr0 to 1D vector: {e}")) + })?, + slist: >>::try_into(vector_getter("slist")?.clone())? + .into_dimensionality::() + .map_err(|e| { + DmapError::InvalidVector(format!("Unable to map slist to 1D vector: {e}")) + })?, + acfd: >>::try_into(vector_getter("acfd")?.clone())? + .into_dimensionality::() + .map_err(|e| { + DmapError::InvalidVector(format!("Unable to map acfd to 3D vector: {e}")) + })?, + xcfd: match opt_vector_getter("xcfd") { + Some(x) => Some( + >>::try_into(x.clone())? + .into_dimensionality::() + .map_err(|e| { + DmapError::InvalidVector(format!( + "Unable to map xcfd to 3D vector: {e}" + )) + })?, + ), + None => None, + }, + }) + } +} + +pub(crate) fn get_datetime_stid(rec: &RawacfRecord) -> Result<(DateTime, i16), DmapError> { + let rec_date: NaiveDate = NaiveDate::from_ymd_opt( + rec.get("time.yr") + .ok_or_else(|| DmapError::InvalidScalar("Missing time.yr".to_string()))? + .clone() + .try_into()?, + rec.get("time.mo") + .ok_or_else(|| DmapError::InvalidScalar("Missing time.mo".to_string()))? + .clone() + .try_into()?, + rec.get("time.dy") + .ok_or_else(|| DmapError::InvalidScalar("Missing time.dy".to_string()))? + .clone() + .try_into()?, + ) + .ok_or_else(|| DmapError::InvalidRecord("Unable to parse date".to_string()))?; + let rec_datetime = rec_date + .and_hms_opt( + rec.get("time.hr") + .ok_or_else(|| DmapError::InvalidScalar("Missing time.hr".to_string()))? + .clone() + .try_into()?, + rec.get("time.mt") + .ok_or_else(|| DmapError::InvalidScalar("Missing time.mt".to_string()))? + .clone() + .try_into()?, + rec.get("time.sc") + .ok_or_else(|| DmapError::InvalidScalar("Missing time.sc".to_string()))? + .clone() + .try_into()?, + ) + .ok_or_else(|| DmapError::InvalidRecord("Unable to parse timestamp".to_string()))? + .and_utc(); + + let station_id: i16 = rec + .get("stid") + .ok_or_else(|| DmapError::InvalidScalar("Missing stid".to_string()))? + .clone() + .try_into()?; + + Ok((rec_datetime, station_id)) +} + +/// Gets the hardware file applicable to `rec` +/// +/// # Errors +/// * If the record contains invalid date/time fields or station ID field +/// * If the `stid` field in the `rec` does not match a known site +/// * If the hardware file for the site is improperly formatted +/// * If there is no applicable line in the hardware file for the date/time of the record +pub fn get_hdw(rec: &RawacfRecord) -> Result { + let (datetime, stid) = get_datetime_stid(rec)?; + HdwInfo::new(stid, datetime).map_err(std::convert::Into::into) +} diff --git a/src/utils/rpos.rs b/src/utils/rpos.rs new file mode 100644 index 0000000..810522e --- /dev/null +++ b/src/utils/rpos.rs @@ -0,0 +1,728 @@ +use crate::error::ProcdarnError; +use crate::gridding::grid::GridError; +use crate::gridding::grid_table::RADIUS_EARTH; +use crate::utils::coords::{GeocentricCoords, GeodeticCoords, LocalAngularCoords, MagneticCoords}; +use crate::utils::hdw::HdwInfo; +use std::f64::consts::PI; +use aacgmv2_rs::aacgmv2::Aacgmv2; +use time::Date; + +/// Calculates the slant range to a range gate in km. +/// +/// Called slant_range in cnvtcoord.c of RST +pub fn slant_range( + first_range: i32, + range_sep: i32, + rx_rise: f64, + range_edge: f64, + range_gate: i32, +) -> f64 { + // The next two lines truncate to integers, for some reason + let lag_to_first_range = (first_range * 20 / 3) as f64; // microseconds + let sample_separation = (range_sep * 20 / 3) as f64; // microseconds + + (lag_to_first_range - rx_rise + ((range_gate - 1) as f64 * sample_separation) + range_edge) + * 0.15 +} + +/// Calculates the coordinates from travelling along `look_dir` (ignoring elevation angle) from +/// `start`. +/// +/// Called fldpnt_sph in invmag.c of RST +fn fieldpoint_sphere(start: GeocentricCoords, look_dir: &LocalAngularCoords) -> GeocentricCoords { + let c_side = PI / 2.0 - start.lat; + + let a_angle = if look_dir.az > PI { + look_dir.az - 2.0 * PI + } else { + look_dir.az + }; + + let b_side = look_dir.range / start.rad; + let mut arg = b_side.cos() * c_side.cos() + b_side.sin() * c_side.sin() * a_angle.cos(); + + arg = arg.clamp(-1.0, 1.0); + + let a_side = arg.acos(); + arg = (b_side.cos() - a_side.cos() * c_side.cos()) / (a_side.sin() * c_side.sin()); + + arg = arg.clamp(-1.0, 1.0); + + let mut b_angle = arg.acos(); + if a_angle < 0.0 { + b_angle = -b_angle; + } + + let end_lat = PI / 2.0 - a_side; + let mut end_lon = start.lon + b_angle; + if end_lon < 0.0 { + end_lon += 2.0 * PI; + } else if end_lon > 2.0 * PI { + end_lon -= 2.0 * PI; + } + + GeocentricCoords::new(end_lat, end_lon, 0.0) +} + +/// Uses the Haversine formula to calculate bearing in radians East of North from `start` to `end`, +/// assuming a spherical Earth. +/// +/// Called fldpnt_azm in invmag.c of RST +fn fieldpoint_azimuth(start: &GeocentricCoords, end: &GeocentricCoords) -> f64 { + let a_side = PI / 2.0 - end.lat; + let c_side = PI / 2.0 - start.lat; + let b_angle = end.lon - start.lon; + + let mut arg = a_side.cos() * c_side.cos() + a_side.sin() * c_side.sin() * b_angle.cos(); + let b_side = arg.acos(); + + arg = (a_side.cos() - b_side.cos() * c_side.cos()) / (b_side.sin() * c_side.sin()); + let mut a_angle = arg.acos(); + + if b_angle < 0.0 { + a_angle = -a_angle; + } + + let mut bearing = a_angle; + if bearing.is_nan() { + bearing = 0.0; + } + + bearing +} + +/// Calculates the geocentric coordinates of a point located `direction` from `radar_location`. +/// +/// Called fldpnt in cnvtcoord.c of RST. +fn fieldpoint( + radar_location: &GeocentricCoords, + direction: &LocalAngularCoords, +) -> GeocentricCoords { + /* Convert from global spherical [lon, lat, rho] to global Cartesian [x, y, z] + * (rx,ry,rz: Earth centered) */ + let sin_colat = (PI / 2.0 - radar_location.lat).sin(); + let rx = radar_location.rad * sin_colat * radar_location.lon.cos(); + let ry = radar_location.rad * sin_colat * radar_location.lon.sin(); + let rz = radar_location.rad * (PI / 2.0 - radar_location.lat).cos(); + + /* Convert from local spherical (ral, rel, r) to local Cartesian + * (sx,sy,sz: south,east,up) */ + let mut sx = -direction.range * direction.el.cos() * direction.az.cos(); + let mut sy = direction.range * direction.el.cos() * direction.az.sin(); + let mut sz = direction.range * direction.el.sin(); + + /* Convert from local Cartesian to global Cartesian */ + let mut tx = + (PI / 2.0 - radar_location.lat).cos() * sx + (PI / 2.0 - radar_location.lat).sin() * sz; + let mut ty = sy; + let mut tz = + -(PI / 2.0 - radar_location.lat).sin() * sx + (PI / 2.0 - radar_location.lat).cos() * sz; + sx = radar_location.lon.cos() * tx - radar_location.lon.sin() * ty; + sy = radar_location.lon.sin() * tx + radar_location.lon.cos() * ty; + sz = tz; + + /* Find global Cartesian coordinates of new point by vector addition */ + tx = rx + sx; + ty = ry + sy; + tz = rz + sz; + + /* Convert from global Cartesian to global spherical */ + let frho = ((tx * tx) + (ty * ty) + (tz * tz)).sqrt(); + let flat = PI / 2.0 - (tz / frho).acos(); + let flon = { + if (tx == 0.0) && (ty == 0.0) { + 0.0 + } else { + ty.atan2(tx) + } + }; + + GeocentricCoords::new(flat, flon, frho) +} + +/// Calculate the geocentric coordinates of a radar field point using either the standard or +/// Chisham virtual height model. +/// +/// Called fldpnth in cnvtcoord.c of RST +fn fieldpoint_height( + point: &GeodeticCoords, + bearing_off_boresight: f64, + boresight_bearing: f64, + height: f64, + slant_range: f64, + chisham: bool, +) -> Result { + let mut virtual_height: f64; + if chisham { + if slant_range < 787.5 { + virtual_height = + 108.974 + 0.0191271 * slant_range + 6.68283e-5 * slant_range * slant_range; + } else if slant_range <= 2137.5 { + virtual_height = + 384.416 - 0.17864 * slant_range + 1.81405e-4 * slant_range * slant_range; + } else { + virtual_height = + 1098.28 - 0.354557 * slant_range + 9.39961e-5 * slant_range * slant_range; + } + if slant_range < 115.0 { + virtual_height = slant_range / 115.0 * 112.0; + } + } else { + if height <= 150.0 { + virtual_height = height; + } else { + if slant_range <= 600.0 { + virtual_height = 115.0; + } else if slant_range < 800.0 { + virtual_height = (slant_range - 600.0) / 200.0 * (height - 115.0) + 115.0; + } else { + virtual_height = height; + } + } + if slant_range < 150.0 { + virtual_height = (slant_range / 150.0) * 115.0; + } + } + + let radar_geo = point.to_geocentric(); + let radar_radius = radar_geo.rad; // Radius of Earth beneath point + let mut earth_rad_under_point = radar_radius; // Will update with calculations + let mut point_rho; + + // This will prevent elevation angle from being NaN later on + let range = if slant_range == 0.0 { 0.1 } else { slant_range }; + + let mut point_geoc = GeocentricCoords::default(); + let mut look_dir = LocalAngularCoords::new(0.0, 0.0, range); + + let mut point_height = virtual_height + 1.0; // Initialize to make the below loop a do-while loop + while (point_height - virtual_height).abs() > 0.5 { + point_rho = earth_rad_under_point + virtual_height; + + // Elevation angle relative to horizon [radians] + let angle_above_horizon = + ((point_rho * point_rho - radar_geo.rad * radar_geo.rad - range * range) + / (2.0 * radar_geo.rad * range)) + .asin(); + + // Need to calculate actual elevation angle for 1.5-hop propagation when using Chisham model + // for coning angle correction + + let xel = if chisham && range > 2137.5 { + let gamma = ((radar_geo.rad * radar_geo.rad + point_rho * point_rho - range * range) + / (2.0 * radar_geo.rad * point_rho)) + .acos(); + let beta = (radar_geo.rad * (gamma / 3.0).sin() / (range / 3.0)).asin(); + PI / 2.0 - beta - (gamma / 3.0) + } else { + angle_above_horizon + }; + + look_dir.el = xel; + + // Estimate the off-array-normal azimuth + let off_boresight_rad = bearing_off_boresight.to_radians(); + let boresight_bearing_rad = boresight_bearing.to_radians(); + let tan_azimuth = if off_boresight_rad.cos() * off_boresight_rad.cos() - look_dir.el.sin() * look_dir.el.sin() + < 0.0 + { + 1e32 + } else { + (off_boresight_rad.sin() * off_boresight_rad.sin() + / (off_boresight_rad.cos() * off_boresight_rad.cos() + - look_dir.el.sin() * look_dir.el.sin())) + .sqrt() + }; + + let azimuth = if off_boresight_rad > 0.0 { + tan_azimuth.atan() + } else { + -tan_azimuth.atan() + }; + + // Pointing azimuth in radians east of north + look_dir.az = azimuth + boresight_bearing_rad; + + // Adjust azimuth and elevation for oblateness of the Earth + point.correct_look_dir(&mut look_dir); + + // Adjust look dir to actual angle above horizon + look_dir.el = angle_above_horizon; + + // Obtain the geocentric coordinates of the field point + let point_geoc_new = fieldpoint(&radar_geo, &look_dir); + if point_geoc_new == point_geoc { + panic!("stagnation!!") + } else { + point_geoc = point_geoc_new; + } + + // Recalculate the radius of the Earth beneath the field point + let geodetic = point_geoc.to_geodetic(); + earth_rad_under_point = geodetic.rad; + + point_height = point_geoc.rad - earth_rad_under_point; + } + + Ok(point_geoc) +} + +/// Converts a gate/beam coordinate from the radar to [`GeocentricCoords`]. +/// +/// The height of the transformation is given by `height` - if it is less than 90, then it is +/// assumed to be the elevation angle from the radar. +/// +/// If center is not equal to zero, then the calculation is assumed to be for the center of the +/// cell, not the edge. +/// +/// Called `RPosGeo` in cnvtcoord.c of RST +fn rpos_geo( + center: bool, + beam_num: i32, + range_gate: i32, + hdw: &HdwInfo, + first_range: f32, + range_sep: f32, + rx_rise_time: f32, + altitude: f32, + chisham: bool, +) -> Result<(GeocentricCoords, f64), GridError> { + let mut beam_edge: f32 = 0.0; + let mut range_edge: f32 = 0.0; + + if !center { + beam_edge = -0.5 * hdw.beam_separation; + range_edge = -0.5 * range_sep * 20.0 / 3.0; + } + + let rx_rise = match rx_rise_time { + 0.0 => hdw.rx_rise_time, + _ => rx_rise_time, + }; + + let offset = hdw.max_num_beams as f32 / 2.0 - 0.5; + + // Calculate deviation from boresight in degrees + let psi = hdw.beam_separation * (beam_num as f32 - offset) + beam_edge + hdw.boresight_shift; + + // Calculate the slant range to the range gate in km + let distance = slant_range( + first_range as i32, + range_sep as i32, + rx_rise as f64, + range_edge as f64, + range_gate + 1, + ) as f32; + // If the input altitude is below 90, then it is actually an input elevation angle in degrees. + // If so, we calculate the field point height + let field_point_height = if altitude < 90.0 { + -RADIUS_EARTH + + ((RADIUS_EARTH * RADIUS_EARTH) + + 2.0 * distance * RADIUS_EARTH * altitude.to_radians().sin() + + distance * distance) + .sqrt() + } else { + altitude + }; + + // Calculate the geocentric coordinates of the field point + let result = fieldpoint_height( + &GeodeticCoords::new( + hdw.latitude.to_radians() as f64, + hdw.longitude.to_radians() as f64, + hdw.altitude as f64, + ), + psi as f64, + hdw.boresight as f64, + field_point_height as f64, + distance as f64, + chisham, + )?; + + Ok((result, distance as f64)) +} + +/// Calculates the look direction vector to a range/beam cell from the radar. +pub fn rpos_range_beam_azimuth_elevation( + beam: i32, + range: i32, + year: i32, + hdw: &HdwInfo, + first_range: f32, + range_sep: f32, + rx_rise: f32, + altitude: f32, + chisham: bool, +) -> Result { + let radar_loc_geod = GeodeticCoords::new( + hdw.latitude.to_radians() as f64, + hdw.longitude.to_radians() as f64, + 0.0, + ); + + let rx_rise_time = match rx_rise { + 0.0 => hdw.rx_rise_time, + _ => rx_rise, + }; + + // Convert center of range/beam cell to geocentric latitude/longitude/altitude + let (cell_geoc, slant_range) = rpos_geo( + true, + beam, + range, + hdw, + first_range, + range_sep, + rx_rise_time, + altitude, + chisham, + )?; + // Convert range/beam position from geocentric coordinates to global Cartesian coordinates + let cell_cartesian = cell_geoc.to_cartesian(); + + // Convert radar geocentric coordinates to global Cartesian coordinates + let radar_loc_cartesian = radar_loc_geod.to_geocentric().to_cartesian(); + + // Calculate vector from site to center of range/beam cell + let mut look_dir_cart = cell_cartesian - radar_loc_cartesian; + look_dir_cart.norm(); + + // Convert the radar->cell vector into local south/east/vertical coordinates + let mut look_dir_local = cell_geoc.cartesian_to_local(&look_dir_cart); + look_dir_local.norm(); + + // Calculate the magnetic field vector in nT at the geocentric spherical cell position + let date = Date::from_calendar_date(year, time::Month::January, 1) + .map_err(|_| ProcdarnError::Timestamp(format!("bad year: {year}")))?; + let mut b_field = cell_geoc.igrf_field(date)?; + b_field.norm(); + + // Calculate a new local vertical component such that the radar->cell vector becomes + // orthogonal to the local magnetic field at the cell position + look_dir_local.up = + -(b_field.x * look_dir_local.south + b_field.y * look_dir_local.east) / b_field.z; + look_dir_local.norm(); + + // Calculate the azimuth and elevation angles of the orthogonal radar->cell vector + let elevation = look_dir_local.up.atan2( + (look_dir_local.south * look_dir_local.south + look_dir_local.east * look_dir_local.east) + .sqrt(), + ); + let azimuth = look_dir_local.east.atan2(-look_dir_local.south); + + Ok(LocalAngularCoords::new(azimuth, elevation, slant_range)) +} + +/// Calculates the magnetic coordinates of a range/beam cell. +/// +/// Accounts for the virtual height using either the Chisham or standard virtual height models. +pub fn rpos_inv_mag( + beam: i32, + range: i32, + year: i32, + hdw: &HdwInfo, + aacgm_model: &mut Aacgmv2, + first_range: f32, + range_sep: f32, + rx_rise: f32, + altitude: f32, + chisham: bool, +) -> Result<(MagneticCoords, f32, f32), GridError> { + let site_location_geod = GeodeticCoords::new( + hdw.latitude.to_radians() as f64, + hdw.longitude.to_radians() as f64, + hdw.altitude as f64 + RADIUS_EARTH as f64, + ); + + let rx_rise_time = match rx_rise { + 0.0 => hdw.rx_rise_time, + _ => rx_rise, + }; + + // Convert center of range/beam cell to geocentric coordinates + let (cell_geoc, slant_range) = rpos_geo( + true, + beam, + range, + hdw, + first_range, + range_sep, + rx_rise_time, + altitude, + chisham, + )?; + + // Convert cell position from geocentric coordinates to global Cartesian coordinates + let cell_cartesian = cell_geoc.to_cartesian(); + + // Convert radar geocentric coordinates to global Cartesian coordinates + let radar_loc_cartesian = site_location_geod.to_geocentric().to_cartesian(); + + // Calculate vector from radar->cell + let mut del = cell_cartesian - radar_loc_cartesian; + del.norm(); + + // Convert the normalized vector from cartesian into local south/east/vertical coordinates + let mut local_del = cell_geoc.cartesian_to_local(&del); + local_del.norm(); + + // Calculate the magnetic field vector in nT at the geocentric spherical cell position + let date = Date::from_calendar_date(year, time::Month::January, 1) + .map_err(|_| ProcdarnError::Timestamp(format!("bad year: {year}")))?; + let mut b_field = cell_geoc.igrf_field(date)?; + b_field.norm(); + + // Calculate a new local vertical component such that the radar->cell vector becomes + // orthogonal to the magnetic field at the range/beam position + local_del.up = -(b_field.x * local_del.south + b_field.y * local_del.east) / b_field.z; + local_del.norm(); + + // Calculate the azimuth angle of the orthogonal radar->cell vector + let azimuth = local_del.east.atan2(-local_del.south); + + // Get geodetic coordinates of cell location + let cell_geod = cell_geoc.to_geodetic(); + + // Calculate virtual height of cell position + let virtual_height = cell_geoc.rad - cell_geod.rad; + + // Convert cell coordinates from geocentric to AACGM magnetic coordinates + let mut geoc_with_virtual_height = cell_geoc; + geoc_with_virtual_height.rad = virtual_height; + let mag_coords = geoc_with_virtual_height.aacgmv2_convert(aacgm_model); + + // Calculate new point given bearing from the radar + let mut pointing_loc = fieldpoint_sphere( + cell_geoc, + &LocalAngularCoords::new(azimuth, 0.0, range_sep as f64), + ); + pointing_loc.rad = virtual_height; + + // Convert new point into AACGM magnetic coordinates + let mut pointing_mag = pointing_loc.aacgmv2_convert(aacgm_model); + + // Make sure pointing_mag_lon lies between +/- 180 degrees + if pointing_mag.lon - mag_coords.lon > PI { + pointing_mag.lon -= 2.0 * PI; + } else if pointing_mag.lon - mag_coords.lon < -PI { + pointing_mag.lon += 2.0 * PI; + } + + // Calculate bearing (azimuth) to new point in magnetic coordinates + let azimuth = fieldpoint_azimuth(&mag_coords, &pointing_mag); + Ok(( + MagneticCoords::new(mag_coords.lat, mag_coords.lon), + azimuth as f32, + slant_range as f32, + )) +} + +#[cfg(test)] +mod tests { + use super::*; + use approx::assert_relative_eq; + use chrono::{NaiveDate, TimeZone}; + + #[test] + fn test_fieldpoint_sphere() { + let rel = 1e-8; + let start = GeocentricCoords::geo(69.51941199, -133.91889036, 6474.25014983); + let mut v = LocalAngularCoords::from_degrees(-2.56489722, 0.0, 45.0); + let res = fieldpoint_sphere(start, &v); + assert_relative_eq!(res.lat.to_degrees(), 69.91724618, max_relative = rel); + assert_relative_eq!(res.lon.to_degrees(), 226.02920893, max_relative = rel); + + let start = GeocentricCoords::geo(88.01854931, -3.04211092, 7205.35616109); + v.az = 134.27412781_f64.to_radians(); + println!("v: {v:?}"); + let res = fieldpoint_sphere(start, &v); + assert_relative_eq!(res.lat.to_degrees(), 87.75409320, max_relative = rel); + assert_relative_eq!(res.lon.to_degrees(), 3.51003980, max_relative = rel); + + let start = GeocentricCoords::geo(70.31822085, -70.16621865, 7178.67916230); + v.az = 115.48199583_f64.to_radians(); + let res = fieldpoint_sphere(start, &v); + assert_relative_eq!(res.lat.to_degrees(), 70.16115496, max_relative = rel); + assert_relative_eq!(res.lon.to_degrees(), 290.78917074, max_relative = rel); + } + + #[test] + fn test_fieldpoint_azimuth() { + let rel = 1e-7; + let start = GeocentricCoords::geo(72.13155188, -80.96482800, 0.0); + let end = GeocentricCoords::geo(72.50725310, -81.35993126, 0.0); + let res = fieldpoint_azimuth(&start, &end).to_degrees(); + assert_relative_eq!(res, -17.52508198, max_relative = rel); + + let start = GeocentricCoords::geo(80.31958289, 130.46130811, 0.0); + let end = GeocentricCoords::geo(80.01375090, 129.79161762, 0.0); + let res = fieldpoint_azimuth(&start, &end).to_degrees(); + assert_relative_eq!(res, -159.16561423, max_relative = rel); + + let start = GeocentricCoords::geo(87.46967400, -174.50635030, 0.0); + let end = GeocentricCoords::geo(87.37635121, -181.88776787, 0.0); + let res = fieldpoint_azimuth(&start, &end).to_degrees(); + assert_relative_eq!(res, -101.99796960, max_relative = rel); + } + + #[test] + fn test_fieldpoint_height() { + let rel = 1e-9; + let point = GeodeticCoords::geo(68.413, -133.769, 0.0); + let bearing_off_boresight = -24.3; + let boresight_bearing = 29.5; + let height = 300.0; + let slant_range = 180.0; + let chisham = true; + let res = fieldpoint_height( + &point, + bearing_off_boresight, + boresight_bearing, + height, + slant_range, + chisham, + ) + .unwrap(); + assert_relative_eq!(res.lat.to_degrees(), 69.519411986, max_relative = rel); + assert_relative_eq!(res.lon.to_degrees(), -133.918890359, max_relative = rel); + assert_relative_eq!(res.rad, 6474.250149832, max_relative = rel); + + let bearing_off_boresight = 21.06; + let slant_range = 810.0; + let res = fieldpoint_height( + &point, + bearing_off_boresight, + boresight_bearing, + height, + slant_range, + chisham, + ) + .unwrap(); + assert_relative_eq!(res.lat.to_degrees(), 71.497889906, max_relative = rel); + assert_relative_eq!(res.lon.to_degrees(), -117.673408068, max_relative = rel); + assert_relative_eq!(res.rad, 6717.634104790, max_relative = rel); + } + + #[test] + fn test_fieldpoint() { + let rel = 1e-9; + let radar_location = GeocentricCoords::geo(68.281017392, -133.769, 6359.668034912); + let direction = LocalAngularCoords::new( + -2.425048105_f64.to_radians(), + 38.913774588_f64.to_radians(), + 180.0, + ); + let res = fieldpoint(&radar_location, &direction); + assert_relative_eq!(res.lat.to_degrees(), 69.519411986, max_relative = rel); + assert_relative_eq!(res.lon.to_degrees(), -133.918890359, max_relative = rel); + assert_relative_eq!(res.rad, 6474.250149832, max_relative = rel); + } + + #[test] + fn test_rpos_range_beam_az_el() { + let rel = 1e-6; + let beam: i32 = 0; + let range: i32 = 0; + let year: i32 = 2025; + let hdw: &HdwInfo = &HdwInfo::new( + 64, + chrono::Utc.with_ymd_and_hms(2025, 7, 12, 0, 0, 0).unwrap(), + ) + .unwrap(); + let first_range: f32 = 180.0; + let range_sep: f32 = 45.0; + let rx_rise: f32 = 0.0; + let altitude: f32 = 300.0; + let chisham: bool = true; + let res = rpos_range_beam_azimuth_elevation( + beam, + range, + year, + hdw, + first_range, + range_sep, + rx_rise, + altitude, + chisham, + ) + .unwrap(); + assert_relative_eq!(res.az.to_degrees(), -2.564897223, max_relative = rel); + assert_relative_eq!(res.el.to_degrees(), 7.618535464, max_relative = 1e-2); + assert_relative_eq!(res.range, 180.0, max_relative = rel); + } + #[test] + fn test_rpos_geo() { + let rel = 1e-7; + let center = true; + let beam_num = 0; + let range_gate = 0; + let hdw = HdwInfo::new( + 64, + chrono::Utc.with_ymd_and_hms(2025, 7, 12, 0, 0, 0).unwrap(), + ) + .unwrap(); + let first_range = 180.0; + let range_sep = 45.0; + let rx_rise_time = 0.0; + let altitude = 300.0; + let chisham = true; + let (end, srng) = rpos_geo( + center, + beam_num, + range_gate, + &hdw, + first_range, + range_sep, + rx_rise_time, + altitude, + chisham, + ) + .unwrap(); + assert_relative_eq!(end.lat.to_degrees(), 69.51941199, max_relative = rel); + assert_relative_eq!(end.lon.to_degrees(), -133.91889036, max_relative = rel); + assert_relative_eq!(end.rad, 6474.25014983, max_relative = rel); + assert_relative_eq!(srng, 180.0, max_relative = rel); + } + + #[test] + fn test_rpos_inv_mag() { + let rel = 1e-5; + + let beam_num = 0; + let range_gate = 0; + let year = 2025; + let hdw = HdwInfo::new( + 64, + chrono::Utc.with_ymd_and_hms(2025, 7, 12, 0, 0, 0).unwrap(), + ) + .unwrap(); + let date = NaiveDate::from_ymd_opt(2025, 7, 12).unwrap().and_hms_opt(0, 0, 0).unwrap().and_utc(); + let mut aacgm_model = Aacgmv2::new(date).unwrap(); + let first_range = 180.0; + let range_sep = 45.0; + let rx_rise_time = 0.0; + let altitude = 300.0; + let chisham = true; + let (coords, azimuth, slant_range) = rpos_inv_mag( + beam_num, + range_gate, + year, + &hdw, + &mut aacgm_model, + first_range, + range_sep, + rx_rise_time, + altitude, + chisham, + ) + .unwrap(); + assert_relative_eq!(coords.lon.to_degrees(), -80.964827995, max_relative = rel); + assert_relative_eq!(coords.lat.to_degrees(), 72.131551884, max_relative = rel); + assert_relative_eq!( + azimuth.to_degrees() as f64, + -17.525081983, + max_relative = rel + ); + assert_relative_eq!(slant_range, 180.0); + } +} diff --git a/src/utils/scan.rs b/src/utils/scan.rs new file mode 100644 index 0000000..4756066 --- /dev/null +++ b/src/utils/scan.rs @@ -0,0 +1,480 @@ +use crate::error::ProcdarnError; +use crate::gridding::grid::GridError; +use crate::gridding::grid_table::GridTable; +use crate::utils::rpos::slant_range; +use crate::utils::sugar::get_datetime; +use chrono::{DateTime, TimeDelta, Utc}; +use dmap::formats::fitacf::FitacfRecord; +use dmap::Record; +use numpy::ndarray::{Array, ArrayD}; + +/// Data from one range gate from a single radar. +#[derive(Copy, Clone, Default, Debug, PartialEq)] +pub struct RadarCell { + pub groundscatter: i8, // gsct in RST + pub power_lag_zero: f32, // pwr0 in RST + pub power_error_lag_zero: f32, // pwr0_e in RST + pub velocity: f32, // v in RST + pub velocity_error: f32, // v_e in RST + pub spectral_width_lin: f32, // w_l in RST + pub spectral_width_lin_error: f32, // w_l_e in RST + pub power_lin: f32, // p_l in RST + pub power_lin_error: f32, // p_l_e in RST + pub phi_zero: f32, // phi0 in RST + pub elevation: f32, // elv in RST +} + +/// Data from one beam of a scan from one radar. +#[derive(Clone, Default, Debug, PartialEq)] +pub struct RadarBeam { + pub scan: i32, // scan in RST + pub beam: i32, // bm in RST + pub beam_azimuth: f32, // bmazm in RST + pub time: DateTime, // time in RST + pub program_id: i32, // cpid in RST + pub integration_time_s: i32, // intt.sc in RST + pub integration_time_us: i32, // intt.us in RST + pub num_averages: i32, // nave in RST + pub first_range: i32, // frang in RST + pub range_sep: i32, // rsep in RST + pub rx_rise: i32, // rxrise in RST + pub freq: i32, // freq in RST + pub noise: i32, // noise in RST + pub attenuation: i32, // atten in RST + pub channel: i32, // channel in RST + pub num_ranges: i32, // nrang in RST + pub scatter: Vec, // sct in RST + pub cells: Vec, // rng in RST +} + +/// Data from one scan from a single radar. +#[derive(Clone, Default, Debug, PartialEq)] +pub struct RadarScan { + pub station_id: i16, // stid in RST + pub version_major: i32, // version.major in RST + pub version_minor: i32, // version.minor in RST + pub start_time: DateTime, // st_time in RST + pub end_time: DateTime, // ed_time in RST + pub beams: Vec, // bm in RST +} +impl RadarScan { + /// Remove beams whose beam number is in beam_list + /// Called RadarScanResetBeam in RST + pub fn reset_beams(&mut self, beam_list: &[i32]) -> Result<(), GridError> { + // remove beams from self.beams that are in beam_list + self.beams = self + .beams + .clone() + .into_iter() + .filter(|beam| !beam_list.contains(&beam.beam)) + .collect(); + Ok(()) + } + + /// Called RadarScanAddBeam in RST + pub fn add_beam(&mut self, num_ranges: i32) { + self.beams.push(RadarBeam { + num_ranges, + scatter: vec![0; num_ranges as usize], + cells: vec![RadarCell::default(); num_ranges as usize], + ..Default::default() + }) + } + + /// Exclude beams that are not part of a scan. + /// Called exclude_outofscan in radarscan.c of RST. + pub fn exclude_outofscan(&mut self) { + self.beams = self + .beams + .clone() + .into_iter() + .filter(|beam| beam.scan >= 0) + .collect(); + } + + /// Read a full scan of data from a vector of FitacfRecords. If scan_length is `Some(x)`, will + /// grab the first records spanning `x` seconds. Otherwise, uses the scan flag in the FitacfRecords + /// to determine the end of the scan. + /// Called FitReadRadarScan in fitscan.c of RST. + pub fn get_first_scan( + fit_records: &[FitacfRecord], + scan_length: Option, + ) -> Result<(RadarScan, usize), ProcdarnError> { + if fit_records.is_empty() { + return Err(ProcdarnError::ZeroRecords("in `get_first_scan()`")); + } + let mut rec = &fit_records[0]; + let start_time = get_datetime(rec)?; + let mut scan: RadarScan = RadarScan { + station_id: i16::try_from( + rec.get("stid") + .ok_or(ProcdarnError::MissingField("stid"))? + .clone(), + ) + .map_err(|_| ProcdarnError::WrongType("stid"))?, + version_major: i32::try_from( + rec.get("radar.revision.major") + .ok_or(ProcdarnError::MissingField("radar.revision.major"))? + .clone(), + ) + .map_err(|_| ProcdarnError::WrongType("radar.revision.major"))?, + version_minor: i32::try_from( + rec.get("radar.revision.minor") + .ok_or(ProcdarnError::MissingField("radar.revision.minor"))? + .clone(), + ) + .map_err(|_| ProcdarnError::WrongType("radar.revision.minor"))?, + start_time: match scan_length { + Some(x) => { + let t = x as i64 * start_time.timestamp().div_euclid(x as i64); + DateTime::from_timestamp(t, 0) + .ok_or(ProcdarnError::Timestamp(format!("{t}")))? + } + None => start_time, + }, + ..Default::default() + }; + + let mut i = 0; + while i < fit_records.len() { + rec = &fit_records[i]; + + let mut beam = RadarBeam { + time: get_datetime(rec)?, + scan: i32::try_from( + rec.get("scan") + .ok_or(ProcdarnError::MissingField("scan"))? + .clone(), + ) + .map_err(|_| ProcdarnError::WrongType("scan"))?, + beam: i32::try_from( + rec.get("bmnum") + .ok_or(ProcdarnError::MissingField("bmnum"))? + .clone(), + ) + .map_err(|_| ProcdarnError::WrongType("bmnum"))?, + beam_azimuth: f32::try_from( + rec.get("bmazm") + .ok_or(ProcdarnError::MissingField("bmazm"))? + .clone(), + ) + .map_err(|_| ProcdarnError::WrongType("bmazm"))?, + program_id: i32::try_from( + rec.get("cp") + .ok_or(ProcdarnError::MissingField("cp"))? + .clone(), + ) + .map_err(|_| ProcdarnError::WrongType("cp"))?, + integration_time_s: i32::try_from( + rec.get("intt.sc") + .ok_or(ProcdarnError::MissingField("intt.sc"))? + .clone(), + ) + .map_err(|_| ProcdarnError::WrongType("intt.sc"))?, + integration_time_us: i32::try_from( + rec.get("intt.us") + .ok_or(ProcdarnError::MissingField("intt.us"))? + .clone(), + ) + .map_err(|_| ProcdarnError::WrongType("intt.us"))?, + num_averages: i32::try_from( + rec.get("nave") + .ok_or(ProcdarnError::MissingField("nave"))? + .clone(), + ) + .map_err(|_| ProcdarnError::WrongType("nave"))?, + first_range: i32::try_from( + rec.get("frang") + .ok_or(ProcdarnError::MissingField("frang"))? + .clone(), + ) + .map_err(|_| ProcdarnError::WrongType("frang"))?, + range_sep: i32::try_from( + rec.get("rsep") + .ok_or(ProcdarnError::MissingField("rsep"))? + .clone(), + ) + .map_err(|_| ProcdarnError::WrongType("rsep"))?, + rx_rise: i32::try_from( + rec.get("rxrise") + .ok_or(ProcdarnError::MissingField("rxrise"))? + .clone(), + ) + .map_err(|_| ProcdarnError::WrongType("rxrise"))?, + freq: i32::try_from( + rec.get("tfreq") + .ok_or(ProcdarnError::MissingField("tfreq"))? + .clone(), + ) + .map_err(|_| ProcdarnError::WrongType("tfreq"))?, + noise: f32::try_from( + rec.get("noise.sky") + .ok_or(ProcdarnError::MissingField("noise.sky"))? + .clone(), + ) + .map_err(|_| ProcdarnError::WrongType("noise.sky"))? + .floor() as i32, + attenuation: i32::try_from( + rec.get("atten") + .ok_or(ProcdarnError::MissingField("atten"))? + .clone(), + ) + .map_err(|_| ProcdarnError::WrongType("atten"))?, + channel: i32::try_from( + rec.get("channel") + .ok_or(ProcdarnError::MissingField("channel"))? + .clone(), + ) + .map_err(|_| ProcdarnError::WrongType("channel"))?, + num_ranges: i32::try_from( + rec.get("nrang") + .ok_or(ProcdarnError::MissingField("nrang"))? + .clone(), + ) + .map_err(|_| ProcdarnError::WrongType("nrang"))?, + ..Default::default() + }; + let groundscatter = match rec.get("gflg") { + Some(x) => { + ArrayD::try_from(x.clone()).map_err(|_| ProcdarnError::WrongType("gflg"))? + } + None => Array::zeros((0,)).into_dyn(), + }; + let power_lag_zero = ArrayD::try_from( + rec.get("pwr0") + .ok_or(ProcdarnError::MissingField("pwr0"))? + .clone(), + ) + .map_err(|_| ProcdarnError::WrongType("pwr0"))?; + let power_error_lag_zero = 0.0; + let velocity = match rec.get("v") { + Some(x) => { + ArrayD::try_from(x.clone()).map_err(|_| ProcdarnError::WrongType("v"))? + } + None => Array::zeros((0,)).into_dyn(), + }; + let power_lin = match rec.get("p_l") { + Some(x) => { + ArrayD::try_from(x.clone()).map_err(|_| ProcdarnError::WrongType("p_l"))? + } + None => Array::zeros((0,)).into_dyn(), + }; + let spectral_width_lin = match rec.get("w_l") { + Some(x) => { + ArrayD::try_from(x.clone()).map_err(|_| ProcdarnError::WrongType("w_l"))? + } + None => Array::zeros((0,)).into_dyn(), + }; + let velocity_error = match rec.get("v_e") { + Some(x) => { + ArrayD::try_from(x.clone()).map_err(|_| ProcdarnError::WrongType("v_e"))? + } + None => Array::zeros((0,)).into_dyn(), + }; + let quality_flag: ArrayD = match rec.get("qflg") { + Some(x) => { + ArrayD::try_from(x.clone()).map_err(|_| ProcdarnError::WrongType("qflg"))? + } + None => Array::zeros((0,)).into_dyn(), + }; + let slist = match rec.get("slist") { + Some(x) => { + ArrayD::try_from(x.clone()).map_err(|_| ProcdarnError::WrongType("slist"))? + } + None => Array::zeros((0,)).into_dyn(), + }; + let lag_zero_phi = rec.get("phi0"); + let elevation = rec.get("elv"); + for r in 0..beam.num_ranges as usize { + let slist_idx = slist.iter().position(|&x: &i16| x as usize == r); + let cell = match slist_idx { + Some(idx) => { + beam.scatter + .push(if quality_flag[idx] == 1 { 1 } else { 0 }); + let mut cell_tmp = RadarCell { + groundscatter: groundscatter[idx], + power_lag_zero: power_lag_zero[idx], + power_error_lag_zero, + velocity: velocity[idx], + power_lin: power_lin[idx], + spectral_width_lin: spectral_width_lin[idx], + velocity_error: velocity_error[idx], + ..Default::default() + }; + if let Some(x) = lag_zero_phi { + cell_tmp.phi_zero = ArrayD::try_from(x.clone()) + .map_err(|_| ProcdarnError::WrongType("phi0"))?[idx] + } + if let Some(x) = elevation { + cell_tmp.elevation = ArrayD::try_from(x.clone()) + .map_err(|_| ProcdarnError::WrongType("elv"))?[idx] + } + cell_tmp + } + None => { + beam.scatter.push(0); + RadarCell::default() + } + }; + + // Add the measurement (RadarCell) to the beam + beam.cells.push(cell); + } + + // Add the beam to the scan + scan.beams.push(beam); + + // Update the end time of the scan + scan.end_time = get_datetime(rec)?; + + // Increment the record index + i += 1; + + // Conditions for finding the end of the scan + match scan_length { + // If the scan has spanned longer than scan_length + Some(x) => { + if scan.end_time - scan.start_time >= TimeDelta::seconds(x as i64) { + break; + } + } + // If the next record is the start of a new scan + None => { + if i < fit_records.len() + && i8::try_from( + fit_records[i] + .get("scan") + .ok_or(ProcdarnError::MissingField("scan"))? + .clone(), + ) + .map_err(|_| ProcdarnError::WrongType("scan"))? + .abs() + == 1 + { + break; + } + } + } + } + Ok((scan, i)) + } + + /// Filters data in the scan based on optional min and max range gates or slant ranges. + /// Called exclude_range in make_grid.c of RST. + pub fn exclude_range( + &mut self, + min_range_gate: Option, + max_range_gate: Option, + min_slant_range: Option, + max_slant_range: Option, + ) { + let range_edge = 0; + for beam in self.beams.iter_mut().filter(|b| b.beam != -1) { + // If either min or max slant range given, then exclude data using slant range filters + if min_slant_range.is_some() || max_slant_range.is_some() { + for rg in 0..beam.num_ranges { + let range_slant = slant_range( + beam.first_range, + beam.range_sep, + beam.rx_rise as f64, + range_edge as f64, + rg + 1, + ) as f32; + match (min_slant_range, max_slant_range) { + (Some(min), Some(max)) => { + if min > range_slant || range_slant > max { + beam.scatter[rg as usize] = 0; + } + } + (Some(min), None) => { + if min > range_slant { + beam.scatter[rg as usize] = 0; + } + } + (None, Some(max)) => { + if range_slant > max { + beam.scatter[rg as usize] = 0; + } + } + (None, None) => {} + } + } + } else { + // Exclude data using range gate filters + match (min_range_gate, max_range_gate) { + (Some(min), Some(max)) => { + for scat in beam.scatter[..min].iter_mut() { + *scat = 0; + } + for scat in beam.scatter[max..].iter_mut() { + *scat = 0; + } + } + (Some(min), None) => { + for scat in beam.scatter[..min].iter_mut() { + *scat = 0; + } + } + (None, Some(max)) => { + for scat in beam.scatter[max..].iter_mut() { + *scat = 0; + } + } + (None, None) => {} + } + } + } + } + + /// Excludes ground scatter from the radar scan. + /// Called FilterBoundType in bound.c of RST + pub fn exclude_groundscatter(&mut self) { + for beam in self.beams.iter_mut() { + for rg in 0..beam.num_ranges as usize { + if beam.scatter[rg] == 0 { + continue; + } + if beam.cells[rg].groundscatter == 1 { + beam.scatter[rg] = 0; + } + } + } + } + + /// Excludes ionospheric scatter from the radar scan. + /// Called FilterBoundType in bound.c of RST + pub fn exclude_ionospheric_scatter(&mut self) { + for beam in self.beams.iter_mut() { + for rg in 0..beam.num_ranges as usize { + if beam.scatter[rg] == 0 { + continue; + } + if beam.cells[rg].groundscatter == 0 { + beam.scatter[rg] = 0; + } + } + } + } + + pub fn exclude_outofbounds(&mut self, grid_table: &GridTable) { + for beam in self.beams.iter_mut() { + for rg in 0..beam.num_ranges as usize { + if beam.scatter[rg] == 0 { + continue; + } + let cell = beam.cells[rg]; + let discard_cell = cell.velocity.abs() < grid_table.min_velocity + || cell.velocity.abs() > grid_table.max_velocity + || cell.power_lin < grid_table.min_power + || cell.power_lin > grid_table.max_power + || cell.spectral_width_lin < grid_table.min_spectral_width + || cell.spectral_width_lin > grid_table.max_spectral_width + || cell.velocity_error < grid_table.min_velocity_error + || cell.velocity_error > grid_table.max_velocity_error; + if discard_cell { + beam.scatter[rg] = 0; + } + } + } + } +} diff --git a/src/utils/search.rs b/src/utils/search.rs new file mode 100644 index 0000000..c3dd96c --- /dev/null +++ b/src/utils/search.rs @@ -0,0 +1,58 @@ +use crate::error::ProcdarnError; +use chrono::{DateTime, NaiveDate, Utc}; +use dmap::formats::fitacf::FitacfRecord; +use dmap::Record; + +/// Finds the first FitacfRecord in fitacf_records which occurs at or after date_time. +/// Called FitSeek/FitFSeek in RST +pub fn fit_seek( + fitacf_records: &[FitacfRecord], + date_time: DateTime, +) -> Result, ProcdarnError> { + let mut record_times: Vec> = vec![]; + for rec in fitacf_records.iter() { + let tstamp = NaiveDate::from_ymd_opt( + i32::try_from( + rec.get("time.yr") + .ok_or(ProcdarnError::MissingField("time.yr"))? + .clone(), + )?, + u32::try_from( + rec.get("time.mo") + .ok_or(ProcdarnError::MissingField("time.mo"))? + .clone(), + )?, + u32::try_from( + rec.get("time.dy") + .ok_or(ProcdarnError::MissingField("time.dy"))? + .clone(), + )?, + ) + .ok_or(ProcdarnError::Timestamp("could not parse date".to_string()))? + .and_hms_opt( + u32::try_from( + rec.get("time.hr") + .ok_or(ProcdarnError::MissingField("time.hr"))? + .clone(), + )?, + u32::try_from( + rec.get("time.mt") + .ok_or(ProcdarnError::MissingField("time.mt"))? + .clone(), + )?, + u32::try_from( + rec.get("time.sc") + .ok_or(ProcdarnError::MissingField("time.sc"))? + .clone(), + )?, + ) + .ok_or(ProcdarnError::Timestamp("could not parse time".to_string()))? + .and_utc(); + record_times.push(tstamp); + } + + match record_times.into_iter().position(|t| t >= date_time) { + Some(i) => Ok(Some((&fitacf_records[i], i))), + None => Ok(None), + } +} diff --git a/src/utils/sugar.rs b/src/utils/sugar.rs new file mode 100644 index 0000000..80a9532 --- /dev/null +++ b/src/utils/sugar.rs @@ -0,0 +1,50 @@ +use crate::error::ProcdarnError; +use chrono::{DateTime, NaiveDate, Utc}; +use dmap::formats::fitacf::FitacfRecord; +use dmap::Record; + +/// Gets the timestamp from a `FitacfRecord` +pub fn get_datetime(rec: &FitacfRecord) -> Result, ProcdarnError> { + let date = NaiveDate::from_ymd_opt( + rec.get("time.yr") + .ok_or(ProcdarnError::Timestamp("missing `time.yr`".to_string()))? + .clone() + .try_into()?, + rec.get("time.mo") + .ok_or(ProcdarnError::Timestamp("missing `time.mo`".to_string()))? + .clone() + .try_into()?, + rec.get("time.dy") + .ok_or(ProcdarnError::Timestamp("missing `time.dy`".to_string()))? + .clone() + .try_into()?, + ) + .ok_or(ProcdarnError::Timestamp( + "invalid ymd timestamp in record".to_string(), + ))?; + let dt = date + .and_hms_micro_opt( + rec.get("time.hr") + .ok_or(ProcdarnError::Timestamp("missing `time.hr`".to_string()))? + .clone() + .try_into()?, + rec.get("time.mt") + .ok_or(ProcdarnError::Timestamp("missing `time.mt`".to_string()))? + .clone() + .try_into()?, + rec.get("time.sc") + .ok_or(ProcdarnError::Timestamp("missing `time.sc`".to_string()))? + .clone() + .try_into()?, + rec.get("time.us") + .ok_or(ProcdarnError::Timestamp("missing `time.us`".to_string()))? + .clone() + .try_into()?, + ) + .ok_or(ProcdarnError::Timestamp( + "invalid hms_micro timestamp in record".to_string(), + ))? + .and_utc(); + + Ok(dt) +} diff --git a/tests/fit2grid.rs b/tests/fit2grid.rs new file mode 100644 index 0000000..6836d3c --- /dev/null +++ b/tests/fit2grid.rs @@ -0,0 +1,498 @@ +use approx::RelativeEq; +use assert_unordered::assert_eq_unordered; +use dmap::formats::grid::GridRecord; +use dmap::record::Record; +use dmap::types::{DmapField, DmapScalar, DmapVec}; +use itertools::enumerate; +use procdarn::gridding::grid::{fit2grid_file, GridArgs}; +use std::iter::zip; +use std::path::PathBuf; + +fn compare_grid_recs(left_recs: Vec, right_recs: Vec) { + let variable_fields = vec!["origin.time", "origin.command"]; + println!( + "left_recs: {:?}\nright_recs: {:?}", + left_recs[0], right_recs[0] + ); + assert_eq!(left_recs.len(), right_recs.len()); + for (i, (test_rec, rst_rec)) in enumerate(zip(left_recs.iter(), right_recs.iter())) { + assert_eq_unordered!(test_rec.keys(), rst_rec.keys()); + for k in test_rec.keys() { + if variable_fields.contains(&&**k) { + } else { + eprintln!("testing rec {i} field {k}"); + match test_rec.get(k) { + Some(DmapField::Vector(DmapVec::Float(x))) => { + assert!(rst_rec.get(k).is_some(), "Testing rec {i} {k}"); + if let Some(DmapField::Vector(DmapVec::Float(y))) = rst_rec.get(k) { + assert!( + x.map(|v| if v.is_nan() { -1_000_000.0 } else { *v }) + .relative_eq( + &y.map(|v| if v.is_nan() { -1_000_000.0 } else { *v }), + 1e-4, + 1e-4 + ), + "Testing rec {i} {k}: left == right\n\tleft: {x}\n\tright: {y}\n\tDiff: {}", + (x - y) / x, + ); + } + } + Some(DmapField::Scalar(DmapScalar::Float(x))) => { + assert!(rst_rec.get(k).is_some(), "Testing rec {i} {k}"); + if let Some(DmapField::Scalar(DmapScalar::Float(y))) = rst_rec.get(k) { + assert!(x.relative_eq(y, 1e-5, 1e-5), + "Testing rec {i} {k}: left == right\n\nleft: {x}\n\nright: {y}\n\nDiff: {}", + (x - y) / x + ); + } + } + Some(DmapField::Scalar(DmapScalar::Double(x))) => { + assert!(rst_rec.get(k).is_some(), "Testing rec {i} {k}"); + if let Some(DmapField::Scalar(DmapScalar::Double(y))) = rst_rec.get(k) { + assert!(x.relative_eq(y, 1e-5, 1e-5), + "Testing rec {i} {k}: left == right\n\nleft: {x}\n\nright: {y}\n\nDiff: {}", + (x - y) / x + ); + } + } + Some(_) => { + assert_eq!(test_rec.get(k), rst_rec.get(k), "Testing rec {i} {k}") + } + None => {} + } + } + } + } +} + +fn test_grid_with_args(args: &GridArgs, rst_args: Vec) { + let fitacf_files: Vec = glob::glob("tests/test_files/fitacfs/*inv*") + .expect("Could not find fitacf files") + .map(|x| x.expect("Could not read fitacf file")) + .collect(); + let fitacf_files = vec![fitacf_files[0].clone()]; + + // Create grid file + let grid_recs = fit2grid_file(&fitacf_files, args).expect("Unable to make grid from fitacf"); + + // Create grid file using same arguments in RST + let output = std::process::Command::new("make_grid") + .args(rst_args) + .output() + .expect("Failed to run make_grid"); + print!("{}", String::from_utf8_lossy(&output.stderr[..])); + let rst_records = + GridRecord::read_records(&output.stdout[..]).expect("Unable to read from stdout"); + + // Compare the two new grid files + compare_grid_recs(grid_recs, rst_records); +} + +fn init_test_env() -> (GridArgs, Vec) { + let fitacf_files: Vec = glob::glob("tests/test_files/fitacfs/*inv*") + .expect("Could not find fitacf files") + .map(|x| x.expect("Could not read fitacf file")) + .collect(); + let fitacf_files = vec![fitacf_files[0].clone()]; + + let args = GridArgs { + start_time: None, + end_time: None, + start_date: None, + end_date: None, + interval: None, + scan_length: None, + record_interval: 120, + channel: None, + channel_fix: None, + exclude_beams: None, + min_range_gate: None, + max_range_gate: None, + min_slant_range: None, + max_slant_range: None, + filter_weighting: 0, + max_power: 60.0, + max_velocity: 2500.0, + max_spectral_width: 1000.0, + max_velocity_error: 200.0, + min_power: 3.0, + min_velocity: 35.0, + min_spectral_width: 10.0, + min_velocity_error: 0.0, + altitude: 300.0, + max_frequency_var: 500000, + boxcar_filter_flag: true, + no_limits_flag: false, + op_param_flag: false, + exclude_neg_scan_flag: false, + extended_mode_flag: false, + sort_params_flag: false, + ionosphere_only_flag: true, + groundscatter_only_flag: false, + all_data_flag: false, + inertial_frame_flag: false, + chisham_flag: false, + verbose: false, + }; + + let mut rst_args = vec![]; + for infile in fitacf_files.iter() { + rst_args.push(infile.display().to_string()); + } + + (args, rst_args) +} + +#[test] +fn vanilla() { + let (args, rst_args) = init_test_env(); + test_grid_with_args(&args, rst_args.clone()); +} + +#[test] +fn sort_params() { + let (mut args, mut rst_args) = init_test_env(); + args.sort_params_flag = true; + rst_args.insert(0, "-isort".to_string()); + test_grid_with_args(&args, rst_args.clone()); +} + +#[test] +fn groundscatter_only() { + let (mut args, mut rst_args) = init_test_env(); + args.ionosphere_only_flag = false; + args.groundscatter_only_flag = true; + rst_args.insert(0, "-gs".to_string()); + test_grid_with_args(&args, rst_args.clone()); +} + +#[test] +fn ionosphere_only() { + let (mut args, mut rst_args) = init_test_env(); + args.ionosphere_only_flag = true; + rst_args.insert(0, "-ion".to_string()); + test_grid_with_args(&args, rst_args.clone()); +} + +#[test] +fn ion_and_gs() { + let (mut args, mut rst_args) = init_test_env(); + args.ionosphere_only_flag = false; + args.all_data_flag = true; + rst_args.insert(0, "-both".to_string()); + test_grid_with_args(&args, rst_args.clone()); +} + +#[test] +fn no_boxcar() { + let (mut args, mut rst_args) = init_test_env(); + args.boxcar_filter_flag = false; + rst_args.insert(0, "-nav".to_string()); + test_grid_with_args(&args, rst_args.clone()); +} + +#[test] +fn no_boxcar_short_duration() { + let (mut args, mut rst_args) = init_test_env(); + args.boxcar_filter_flag = false; + args.end_time = Some("01:00".to_string()); + args.start_time = Some("00:55".to_string()); + rst_args.insert(0, "-et".to_string()); + rst_args.insert(1, "01:00".to_string()); + rst_args.insert(0, "-st".to_string()); + rst_args.insert(1, "00:55".to_string()); + rst_args.insert(0, "-nav".to_string()); + test_grid_with_args(&args, rst_args); +} + +#[test] +fn scan_length() { + let (mut args, mut rst_args) = init_test_env(); + args.scan_length = Some(60); + rst_args.insert(0, "-tl".to_string()); + rst_args.insert(1, "60".to_string()); + test_grid_with_args(&args, rst_args); +} + +#[test] +fn start_time() { + let (mut args, mut rst_args) = init_test_env(); + args.start_time = Some("01:30".to_string()); + rst_args.insert(0, "-st".to_string()); + rst_args.insert(1, "01:30".to_string()); + test_grid_with_args(&args, rst_args); +} + +#[test] +fn end_time() { + let (mut args, mut rst_args) = init_test_env(); + args.end_time = Some("00:30".to_string()); + rst_args.insert(0, "-et".to_string()); + rst_args.insert(1, "00:30".to_string()); + test_grid_with_args(&args, rst_args); +} + +#[test] +fn start_and_end_time() { + let (mut args, mut rst_args) = init_test_env(); + args.end_time = Some("01:00".to_string()); + args.start_time = Some("00:55".to_string()); + rst_args.insert(0, "-et".to_string()); + rst_args.insert(1, "01:00".to_string()); + rst_args.insert(0, "-st".to_string()); + rst_args.insert(1, "00:55".to_string()); + test_grid_with_args(&args, rst_args); +} + +#[test] +fn start_end_and_chisham() { + let (mut args, mut rst_args) = init_test_env(); + args.end_time = Some("01:00".to_string()); + args.start_time = Some("00:55".to_string()); + args.chisham_flag = true; + rst_args.insert(0, "-et".to_string()); + rst_args.insert(1, "01:00".to_string()); + rst_args.insert(0, "-st".to_string()); + rst_args.insert(1, "00:55".to_string()); + rst_args.insert(0, "-chisham".to_string()); + test_grid_with_args(&args, rst_args); +} + +#[test] +fn start_end_and_range() { + let (mut args, mut rst_args) = init_test_env(); + args.end_time = Some("01:00".to_string()); + args.start_time = Some("00:50".to_string()); + args.min_range_gate = Some(10); + args.max_range_gate = Some(20); + rst_args.insert(0, "-et".to_string()); + rst_args.insert(1, "01:00".to_string()); + rst_args.insert(0, "-st".to_string()); + rst_args.insert(1, "00:50".to_string()); + rst_args.insert(0, "-minrng".to_string()); + rst_args.insert(1, "10".to_string()); + rst_args.insert(0, "-maxrng".to_string()); + rst_args.insert(1, "20".to_string()); + test_grid_with_args(&args, rst_args); +} + +#[test] +fn exclude_beams() { + let (mut args, mut rst_args) = init_test_env(); + args.exclude_beams = Some(vec![0, 7, 14]); + rst_args.insert(0, "-ebm".to_string()); + rst_args.insert(1, "0,7,14".to_string()); + test_grid_with_args(&args, rst_args); +} + +#[test] +fn interval() { + let (mut args, mut rst_args) = init_test_env(); + args.interval = Some("01:30".to_string()); + rst_args.insert(0, "-ex".to_string()); + rst_args.insert(1, "01:30".to_string()); + test_grid_with_args(&args, rst_args); +} + +#[test] +fn record_interval() { + let (mut args, mut rst_args) = init_test_env(); + args.record_interval = 30; + rst_args.insert(0, "-i".to_string()); + rst_args.insert(1, "30".to_string()); + rst_args.insert(2, "-vb".to_string()); + test_grid_with_args(&args, rst_args); +} + +#[test] +fn channel() { + let (mut args, mut rst_args) = init_test_env(); + args.channel = Some('a'); + rst_args.insert(0, "-cn".to_string()); + rst_args.insert(1, "a".to_string()); + test_grid_with_args(&args, rst_args); +} + +#[test] +fn channel_fix() { + let (mut args, mut rst_args) = init_test_env(); + args.channel_fix = Some('a'); + rst_args.insert(0, "-cn_fix".to_string()); + rst_args.insert(1, "a".to_string()); + test_grid_with_args(&args, rst_args); +} + +#[test] +fn min_range_gate() { + let (mut args, mut rst_args) = init_test_env(); + args.min_range_gate = Some(10); + rst_args.insert(0, "-minrng".to_string()); + rst_args.insert(1, "10".to_string()); + test_grid_with_args(&args, rst_args); +} + +#[test] +fn max_range_gate() { + let (mut args, mut rst_args) = init_test_env(); + args.max_range_gate = Some(20); + rst_args.insert(0, "-maxrng".to_string()); + rst_args.insert(1, "20".to_string()); + test_grid_with_args(&args, rst_args); +} + +#[test] +fn min_slant_range() { + let (mut args, mut rst_args) = init_test_env(); + args.min_slant_range = Some(500.0); + rst_args.insert(0, "-minsrng".to_string()); + rst_args.insert(1, "500".to_string()); + test_grid_with_args(&args, rst_args); +} + +#[test] +fn max_slant_range() { + let (mut args, mut rst_args) = init_test_env(); + args.max_slant_range = Some(500.0); + rst_args.insert(0, "-maxsrng".to_string()); + rst_args.insert(1, "500".to_string()); + test_grid_with_args(&args, rst_args); +} + +#[test] +fn filter_weighting() { + let (mut args, mut rst_args) = init_test_env(); + args.filter_weighting = 4; + rst_args.insert(0, "-fwgt".to_string()); + rst_args.insert(1, "4".to_string()); + test_grid_with_args(&args, rst_args); +} +#[test] +fn max_power() { + let (mut args, mut rst_args) = init_test_env(); + args.max_power = 30.0; + rst_args.insert(0, "-pmax".to_string()); + rst_args.insert(1, "30".to_string()); + test_grid_with_args(&args, rst_args); +} +#[test] +fn max_velocity() { + let (mut args, mut rst_args) = init_test_env(); + args.max_velocity = 1200.0; + rst_args.insert(0, "-vmax".to_string()); + rst_args.insert(1, "1200".to_string()); + test_grid_with_args(&args, rst_args); +} +#[test] +fn max_spectral_width() { + let (mut args, mut rst_args) = init_test_env(); + args.max_spectral_width = 500.0; + rst_args.insert(0, "-wmax".to_string()); + rst_args.insert(1, "500".to_string()); + test_grid_with_args(&args, rst_args); +} +#[test] +fn max_velocity_error() { + let (mut args, mut rst_args) = init_test_env(); + args.max_velocity_error = 250.0; + rst_args.insert(0, "-vemax".to_string()); + rst_args.insert(1, "250".to_string()); + test_grid_with_args(&args, rst_args); +} +#[test] +fn min_power() { + let (mut args, mut rst_args) = init_test_env(); + args.min_power = 4.5; + rst_args.insert(0, "-pmin".to_string()); + rst_args.insert(1, "4.5".to_string()); + test_grid_with_args(&args, rst_args); +} +#[test] +fn min_velocity() { + let (mut args, mut rst_args) = init_test_env(); + args.min_velocity = 50.0; + rst_args.insert(0, "-vmin".to_string()); + rst_args.insert(1, "50".to_string()); + test_grid_with_args(&args, rst_args); +} +#[test] +fn min_spectral_width() { + let (mut args, mut rst_args) = init_test_env(); + args.min_spectral_width = 30.0; + rst_args.insert(0, "-wmin".to_string()); + rst_args.insert(1, "30".to_string()); + test_grid_with_args(&args, rst_args); +} +#[test] +fn min_velocity_error() { + let (mut args, mut rst_args) = init_test_env(); + args.min_velocity_error = 15.0; + rst_args.insert(0, "-vemin".to_string()); + rst_args.insert(1, "15".to_string()); + test_grid_with_args(&args, rst_args); +} +#[test] +fn altitude() { + let (mut args, mut rst_args) = init_test_env(); + args.altitude = 250.0; + rst_args.insert(0, "-alt".to_string()); + rst_args.insert(1, "250".to_string()); + test_grid_with_args(&args, rst_args); +} +#[test] +fn max_frequency_var() { + let (mut args, mut rst_args) = init_test_env(); + args.max_frequency_var = 100000; + rst_args.insert(0, "-fmax".to_string()); + rst_args.insert(1, "100000".to_string()); + test_grid_with_args(&args, rst_args); +} +#[test] +fn no_limits_flag() { + let (mut args, mut rst_args) = init_test_env(); + args.no_limits_flag = true; + rst_args.insert(0, "-nlm".to_string()); + test_grid_with_args(&args, rst_args); +} +#[test] +fn op_param_flag() { + let (mut args, mut rst_args) = init_test_env(); + args.op_param_flag = true; + rst_args.insert(0, "-nb".to_string()); + test_grid_with_args(&args, rst_args); +} +#[test] +fn exclude_neg_scan_flag() { + let (mut args, mut rst_args) = init_test_env(); + args.exclude_neg_scan_flag = true; + rst_args.insert(0, "-ns".to_string()); + test_grid_with_args(&args, rst_args); +} +#[test] +fn extended_mode_flag() { + let (mut args, mut rst_args) = init_test_env(); + args.extended_mode_flag = true; + rst_args.insert(0, "-xtd".to_string()); + test_grid_with_args(&args, rst_args); +} +#[test] +fn sort_params_flag() { + let (mut args, mut rst_args) = init_test_env(); + args.sort_params_flag = true; + rst_args.insert(0, "-isort".to_string()); + test_grid_with_args(&args, rst_args); +} +#[test] +fn all_data_flag() { + let (mut args, mut rst_args) = init_test_env(); + args.all_data_flag = true; + args.ionosphere_only_flag = false; + rst_args.insert(0, "-both".to_string()); + test_grid_with_args(&args, rst_args); +} +#[test] +fn inertial_frame_flag() { + let (mut args, mut rst_args) = init_test_env(); + args.inertial_frame_flag = true; + rst_args.insert(0, "-inertial".to_string()); + test_grid_with_args(&args, rst_args); +} diff --git a/tests/fitacf.rs b/tests/fitacf.rs new file mode 100644 index 0000000..fadd067 --- /dev/null +++ b/tests/fitacf.rs @@ -0,0 +1,49 @@ +use assert_unordered::assert_eq_unordered; +use dmap::types::{DmapField, DmapVec}; +use itertools::enumerate; +use procdarn::fitting::fitacf3::fitacf_v3::fitacf3; +use std::iter::zip; +use dmap::{FitacfRecord, RawacfRecord, Record}; + +#[test] +fn test_fitacf3() { + // Create fitacf file from rawacf file + let rawacf = RawacfRecord::read_file("tests/test_files/test.rawacf".to_string()) + .expect("Could not read records"); + let fitacf_records = fitacf3(rawacf).expect("Unable to fit records"); + + // Compare to fitacf file generated by RST + let rst_records = FitacfRecord::read_file("tests/test_files/test.fitacf".to_string()) + .expect("Could not read test.fitacf records"); + let variable_fields = vec!["origin.time", "origin.command"]; + for (i, (test_rec, rst_rec)) in enumerate(zip(fitacf_records.iter(), rst_records.iter())) { + assert_eq_unordered!(test_rec.keys(), rst_rec.keys()); + for k in test_rec.keys() { + if variable_fields.contains(&&**k) { + } else { + match test_rec.get(k) { + Some(DmapField::Vector(DmapVec::Float(x))) => { + assert!(rst_rec.get(k).is_some(), "Testing rec {i} {k}"); + if let Some(DmapField::Vector(DmapVec::Float(y))) = rst_rec.get(k) { + assert!( + x.map(|v| if v.is_nan() { -1_000_000.0 } else { *v }) + .relative_eq( + &y.map(|v| if v.is_nan() { -1_000_000.0 } else { *v }), + 1e-5, + 1e-5 + ), + "Testing rec {i} {k}: left == right\n\nleft: {x}\n\nright: {y}\n\nDiff: {}\n\nslist: {:?}", + (x - y) / x, + test_rec.get(&"slist".to_string()) + ); + } + } + Some(_) => { + assert_eq!(test_rec.get(k), rst_rec.get(k), "Testing rec {i} {k}") + } + None => {} + } + } + } + } +} diff --git a/tests/test_files/large.rawacf b/tests/test_files/large.rawacf new file mode 100644 index 0000000..8f97c46 Binary files /dev/null and b/tests/test_files/large.rawacf differ diff --git a/tests/test_files/test.fitacf b/tests/test_files/test.fitacf index 9c6d534..171741b 100644 Binary files a/tests/test_files/test.fitacf and b/tests/test_files/test.fitacf differ diff --git a/tests/tests.rs b/tests/tests.rs deleted file mode 100644 index 5852663..0000000 --- a/tests/tests.rs +++ /dev/null @@ -1,40 +0,0 @@ -use backscatter_rs::fitting::fitacf3::fitacf_v3::fit_rawacf_record; -use backscatter_rs::utils::hdw::HdwInfo; -use chrono::NaiveDateTime; -use dmap::formats::{DmapRecord, FitacfRecord, RawacfRecord}; -use std::fs::{remove_file, File}; -use std::iter::zip; - -#[test] -fn test_fitacf3() { - // Create fitacf file from rawacf file - let file = File::open("tests/test_files/test.rawacf").expect("Test file not found"); - let rawacf = RawacfRecord::read_records(file).expect("Could not read records"); - let mut fitacf_records = vec![]; - - let rec = &rawacf[0]; - let file_datetime = NaiveDateTime::parse_from_str( - format!( - "{:4}{:0>2}{:0>2} {:0>2}:{:0>2}:{:0>2}", - rec.year, rec.month, rec.day, rec.hour, rec.minute, rec.second - ) - .as_str(), - "%Y%m%d %H:%M:%S", - ) - .expect("Unable to interpret record timestamp"); - let hdw = HdwInfo::new(rec.station_id, file_datetime).expect("Unable to read utils file"); - - for rec in rawacf { - fitacf_records.push(fit_rawacf_record(&rec, &hdw).expect("Could not fit record")); - } - - // Compare to fitacf file generated by RST - let fitacf_file = - File::open("tests/test_files/test.fitacf").expect("Could not open example fitacf file"); - let fitacf = - FitacfRecord::read_records(fitacf_file).expect("Could not read test.fitacf records"); - for (read_rec, written_rec) in zip(fitacf_records.iter(), fitacf.iter()) { - assert_eq!(read_rec, written_rec) - } - remove_file("tests/test_files/temp.fitacf").expect("Unable to delete file"); -}