diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..340ce963 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,13 @@ +* text=auto eol=lf + +*.png binary +*.jpg binary +*.jpeg binary +*.gif binary +*.webp binary +*.ico binary +*.pdf binary +*.woff binary +*.woff2 binary +*.ttf binary +*.otf binary \ No newline at end of file diff --git a/README.md b/README.md index bb1df5cb..d378807a 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,24 @@ starting point: (twelve BM1362 ASICs): a sister project from the 256 Foundation. An open-source hashboard designed to be driven by open firmware. +**Experimental** + +- **Intel BZM2 (Bonanza Mine 2) boards**: native Rust support for the + BZM2's 9-bit multidrop UART protocol with direct work dispatch, TDM + result handling, DTS/VS telemetry, PLL/DLL clock control, startup + calibration, and HTTP diagnostics. Targets the + [Satoshi Starter](https://github.com/Blockscale-Solutions/SatoshiStarter) + (1 ASIC) and related BZM2 board family. Functional and test-covered, + but board-specific bring-up is still maturing; not yet + production-ready. See [BZM2 docs](docs/bzm2/bzm2-port.md). Quick + start: + + ```bash + MUJINA_BZM2_SERIAL="/dev/ttyUSB0" \ + MUJINA_BZM2_BAUD="5000000" \ + cargo run -p mujina-miner --bin mujina-minerd + ``` + **Near-term targets** - Installable images for the Antminer S19 series @@ -221,6 +239,13 @@ GitHub. - [CPU Mining](docs/cpu-mining.md): the CPU backend in detail - [Container Image](docs/container.md): build and run Mujina as a container +- [BZM2 Port](docs/bzm2/bzm2-port.md): Intel BZM2 driver architecture, + with the [tuning planner](docs/bzm2/bzm2-pnp.md) and + [opcode grounding](docs/bzm2/bzm2-opcode-grounding.md). Hardware + reference (pinout/ball map, electrical, UART/TDM protocol, + integration guide) lives in the maintained + [bzm2-hwref](https://github.com/Blockscale-Solutions/bzm2-hwref) + repository (CC-BY-SA) ### Protocols @@ -252,6 +277,10 @@ GitHub. Foundation's first open-source Bitcoin mining hashboard - [Libreboard](https://github.com/256foundation/libreboard): 256 Foundation's open-source mining control board +- [Satoshi Starter](https://github.com/Blockscale-Solutions/SatoshiStarter): + open-source single-ASIC Intel BZM2 miner kit from Reckless Systems +- [bitaxeBIRDS](https://github.com/bitaxeorg/bitaxeBIRDS): four-ASIC + Intel BZM2 board from the Bitaxe community ## License diff --git a/docs/bzm2/bzm2-opcode-grounding.md b/docs/bzm2/bzm2-opcode-grounding.md new file mode 100644 index 00000000..c20f32b3 --- /dev/null +++ b/docs/bzm2/bzm2-opcode-grounding.md @@ -0,0 +1,82 @@ +# BZM2 Opcode And JTAG Grounding + +## Scope + +This note captures only behavior that is grounded in material available to this port: + +- the shipped Rust implementation in + [protocol.rs](../../mujina-miner/src/asic/bzm2/protocol.rs) and + [uart.rs](../../mujina-miner/src/asic/bzm2/uart.rs), including their + legacy wire-format tests +- the public Blockscale UART/TDM protocol reference in + [bzm2-hwref](https://github.com/Blockscale-Solutions/bzm2-hwref/blob/main/references/blockscale-uart-protocol-reference.md) + +The behavior below was originally derived from the legacy `bzmd` C source +(`uart.h`, `uart.c`, and `tests/test.c`). That source is not vendored in this +repository, so any claim that traces only to it is explicitly marked below as +unverified against the legacy source. + +Anything not evidenced by these sources is intentionally excluded from the Mujina port. + +## What The Legacy Source Proved + +The legacy `bzmd` source gives a concrete UART wire contract for these opcodes: + +- `WRITEJOB` +- `READRESULT` +- `WRITEREG` +- `READREG` +- `MULTICAST_WRITE` +- `DTS_VS` +- `LOOPBACK` +- `NOOP` + +Grounded request/response behavior, originally from the legacy `uart.c` and now verified against the shipped encoders and the public protocol reference: + +- `WRITEREG`: request is `len(2 LE) + header(4 BE) + count_minus_one + payload` +- `MULTICAST_WRITE`: same framing as `WRITEREG`, but opcode `0x4` +- `READREG`: request is fixed-length `8` byte frame with terminal target byte; direct response is `asic + opcode + payload` +- `READRESULT`: in TDM mode, result frame is `asic + opcode + 8-byte payload` +- `NOOP`: request is a 4-byte frame; response payload is 3 bytes +- `LOOPBACK`: request is `len + header + count_minus_one + payload`; response echoes `asic + opcode + payload` +- `DTS_VS`: in TDM mode, payload is 4 bytes for gen1 and 8 bytes for gen2 + +Grounded concurrency and parser behavior, originally from the legacy `uart.h`, `uart.c`, and `test.c`: + +- TDM parsing is byte-stream oriented and must resynchronize after unknown prefixes +- TDM `READREG` response size is caller-driven and tracked per ASIC +- one outstanding TDM register read per ASIC is the supported model +- one outstanding TDM noop per ASIC was reportedly the legacy model (unverified + against the legacy source); the Mujina parser treats TDM `NOOP` frames as + fixed-length and stateless, so the legacy restriction is not load-bearing +- broadcast register writes use `WRITEREG` with ASIC `0xFF`, not a separate broadcast opcode +- broadcast TDM register reads use no distinct opcode; the legacy mechanism of + layering them on top of `READREG` is unverified against the legacy source, + and the current controller exposes no broadcast TDM read helper + +## What Mujina Now Grounds + +Current Mujina BZM2 support in [protocol.rs](../../mujina-miner/src/asic/bzm2/protocol.rs) and [uart.rs](../../mujina-miner/src/asic/bzm2/uart.rs) is now explicitly locked to the legacy-tested UART behavior for: + +- `WRITEREG`, `READREG`, `WRITEJOB`, `MULTICAST_WRITE`, `READRESULT`, `NOOP`, `LOOPBACK`, `DTS_VS` +- gen1 and gen2 DTS/VS payload decoding +- partial-frame buffering and resynchronization after unknown byte prefixes +- legacy wire-format invariants for register, noop, and loopback command encoders + +## Deliberate Exclusions + +Not implemented from the docs side: + +- JTAG command transport +- JTAG IR/DR scan helpers +- any opcode semantics that cannot be traced to shipped UART code or tests + +PLL debug readout is no longer excluded: UART-register-based PLL and DLL +configuration, lock polling, and status readback are implemented in +`clock.rs` and surfaced through the `thread.rs` diagnostics path. + +Reason: + +- the shipped Rust implementation and its wire-format tests, plus the public + protocol reference, prove the UART mining/control path +- the repository-visible sources do not provide enough packet-level JTAG detail to implement anything defensible diff --git a/docs/bzm2/bzm2-pnp.md b/docs/bzm2/bzm2-pnp.md new file mode 100644 index 00000000..815bcc83 --- /dev/null +++ b/docs/bzm2/bzm2-pnp.md @@ -0,0 +1,138 @@ +# BZM2 PnP Calibration In Mujina + +This note captures the current BZM2 PnP state in Mujina, what the legacy `bzmd` implementation did, and what is now implemented in the Rust port. + +## Current Gap + +Before this change, Mujina's BZM2 support had: + +- UART work dispatch +- result parsing +- thermal and power safety shutdowns +- UART register access +- PLL and DLL control + +What it did not have was a native Mujina tuning planner for BZM2: + +- operating-class and performance-mode target selection +- parameter sweep generation +- initial voltage and frequency selection from site temperature +- saved operating point reuse checks +- retune decisions when measured throughput regresses +- domain-aware planning for hardware with multiple voltage domains +- per-ASIC or per-stack frequency fine-tuning around a target pass-rate window + +## Legacy `pnp.c` Behavior + +The following is background as recalled from the legacy design; it is +unverified against the legacy source and is not normative for the Rust port. + +The original C implementation reportedly mixed: + +- calibration search policy +- board and PSU policy +- persisted board calibration profiles +- per-ASIC telemetry accumulation +- per-engine pass-rate accounting +- platform-specific data collection and file I/O + +The reusable algorithmic parts, as recalled, are: + +- derive voltage, clock, and acceptance targets from operating class and performance mode +- derive initial voltage and clock from site thermal conditions +- broadcast a starting frequency +- sweep upward while respecting power and thermal guard rails +- tune back down on individual ASICs or stacks when pass rate falls outside the target window +- invalidate saved operating point when throughput regresses materially + +## Mujina Ported Behavior + +The new Rust module at +`mujina-miner/src/tuning/blockscale.rs` +implements the reusable planner without pulling board-MCU or PSU glue into the ASIC layer. + +Implemented: + +- operating-class target tables for: + - generic + - EarlyValidation + - ProductionValidation + - StackTunedA + - StackTunedB + - ExtendedHeadroom + - ExtendedHeadroomB +- search-space generation, modeled on the recalled legacy sweep behavior +- site-temperature-aware initial voltage and clock planning, modeled on the recalled legacy startup behavior +- saved operating point reuse vs. full retune decisions +- domain-aware voltage planning using explicit voltage-domain offsets and guards +- per-domain frequency planning using aggregated pass-rate, thermal, and power data +- per-ASIC fine-tuning with optional per-stack / per-PLL behavior + +## Efficiency Model + +The planner is structured to scale cleanly from a single ASIC to large chains. +It runs a single domain-ordered pass that, for each voltage domain: + +- aggregates that domain's metrics +- emits the domain-level plan +- emits that domain's per-ASIC adjustments + +That is O(domains x ASICs) overall, which keeps the planning work effectively +linear in ASIC count for the small domain counts on realistic hardware. + +For larger systems with multiple voltage domains, the planner prefers: + +- domain-level voltage decisions first +- domain-average frequency targets next +- per-ASIC or per-PLL corrections only where pass-rate or thermal data requires it + +That is materially more scalable than treating a 100-ASIC machine as 100 independent full-search problems. + +## Scope Boundary + +The planner is now wired into `Bzm2Board` startup so Mujina can: + +- execute a live pre-thread calibration phase +- persist applied calibration results as a saved operating point profile +- replay a compatible saved operating point profile directly on restart before falling back to retune +- collect live runtime tuning measurements during mining for: + - board throughput + - per-ASIC throughput + - per-ASIC average pass rate + - per-PLL throughput and pass rate + - per-domain measured rail voltage and power +- normalize saved-throughput comparisons and planned board hashrate against + actual active-engine capacity instead of assuming every ASIC still has the + default full map +- run the same planner against live runtime measurements during mining so the + board can continuously evaluate whether the current operating point is still + valid +- automatically promote saved operating point state from `pending` to + `validated` on the first monitor poll with no pending retune triggers +- automatically mark the saved operating point `invalidated`, with the trigger + reasons recorded and persisted, when persistent runtime retune triggers fire + +A saved operating point therefore starts `pending`, validates on the first +clean monitor poll, and is invalidated in place when retune triggers persist +past the trigger tracker's threshold. Replay compatibility rejects +`invalidated` profiles, so a restart after persistent triggers falls back to +live calibration instead of replaying a known-bad operating point, and the +fresh calibration persists a new `pending` profile. + +Engine-capacity inputs now come from, in order: + +- live pre-calibration engine discovery when enabled +- saved per-ASIC topology embedded in the saved operating point profile +- default BZM2 hole-map fallback when no better topology data is available + +That means tuning decisions can now distinguish between: + +- a slow ASIC that still has full engine capacity +- an ASIC that is throughput-limited because it has permanently missing engines + +What still remains outside the ASIC planner layer: + +- board-specific PSU ramp policy +- reimplementation of the legacy CSV/database layer + +Those pieces still belong above the ASIC planner, in board or daemon integration layers. diff --git a/docs/bzm2/bzm2-port.md b/docs/bzm2/bzm2-port.md new file mode 100644 index 00000000..929395e1 --- /dev/null +++ b/docs/bzm2/bzm2-port.md @@ -0,0 +1,392 @@ +# BZM2 Mujina Port + +## Architecture + +This port keeps BZM2 support inside Mujina rather than reviving the original split `cgminer` + `bzmd` process model. + +As unverified background, the legacy stack reportedly split responsibilities across two processes: + +- `cgminer` handling scheduling, pool interaction, and IPC to `bzmd` +- `bzmd` owning UART transport, job fanout, result validation, and board-management glue + +In Mujina, those responsibilities map cleanly onto existing abstractions: + +- `Daemon` can attach a configured `bzm2` board directly from serial-path configuration +- `Backplane` instantiates the board through the virtual-board registry +- `board::bzm2::Bzm2Board` opens serial transports and creates hash threads +- `asic::bzm2::Bzm2Thread` performs direct UART job dispatch, telemetry parsing, and share validation +- `board::power` provides reusable GPIO-reset and PMBus/I2C rail sequencing primitives + +A standalone Rust daemon is therefore not required for the mining path. + +## Bring-Up And Shutdown + +`Bzm2Board` now supports optional board-level power and reset sequencing through +the existing `VoltageStackBringupPlan`. + +The current generic integration path uses file-backed adapters: + +- rail setpoint files for coarse voltage application +- optional rail enable files for regulator enable or precharge control +- an optional reset file for ASIC reset assertion and release + +This keeps the implementation generic across custom Linux-based carriers without +hard-coding one vendor board layout or management MCU protocol. + +## Implemented Behavior + +The BZM2 Mujina thread now reimplements the core legacy data path and the generally reusable portions of the control path: + +- 20 x 12 logical engine grid with the four excluded engines from legacy code +- enhanced-mode 4-midstate dispatch per logical engine +- version-rolling micro-jobs in slots `0, 2, 4, 8` +- UART register writes for target bits, leading-zero threshold, and timestamp count +- TDM result parsing with sequence parity matching +- nonce correction via enhanced-mode nonce gap +- in-thread Bitcoin header reconstruction and share validation before scheduler submission +- UART opcode coverage for: + - `WRITEJOB` + - `WRITEREG` + - `READREG` + - `MULTICAST_WRITE` + - `READRESULT` + - `NOOP` + - `LOOPBACK` + - `DTS_VS` +- DTS/VS generation 1 and generation 2 frame decoding +- live DTS/VS gen2 hardware-fault handling that shuts down the hash thread on thermal or voltage fault indications +- reusable GPIO reset-line control through `GpioResetLine`/`FileGpioPin` in `board::power` +- reusable TPS546 PMBus rail control through `VoltageRegulator` +- reusable multi-rail bring-up and shutdown sequencing for single-rail, small-stack, and larger multi-stack designs +- UART-register-based PLL diagnostic/control flow for divider programming, enable/disable, lock polling, and readback +- UART-register-based DLL diagnostic/control flow for duty-cycle programming, enable/disable, lock polling, and fincon validation +- domain-aware BZM2 tuning planner documented in [bzm2-pnp.md](bzm2-pnp.md) for operating-class and performance-mode target selection, search-space generation, and per-domain plus per-ASIC tuning + +## Configuration + +Enable BZM2 by setting `MUJINA_BZM2_SERIAL` to one or more comma-separated serial device paths. + +Supported environment variables: + +- `MUJINA_BZM2_SERIAL`: required, comma-separated serial device paths +- `MUJINA_BZM2_SERIAL_PATHS`: alternate name for the same setting +- `MUJINA_BZM2_BAUD`: UART baud rate, default `5000000` +- `MUJINA_BZM2_TIMESTAMP_COUNT`: default `60` +- `MUJINA_BZM2_NONCE_GAP`: default `0x28` +- `MUJINA_BZM2_DISPATCH_MS`: redispatch interval in milliseconds, default `500` +- `MUJINA_BZM2_HASHRATE_THS`: nominal per-thread hashrate estimate, default `40` +- `MUJINA_BZM2_DTS_VS_GEN`: DTS/VS payload generation, `1` or `2`, default `2` +- `MUJINA_BZM2_ENUMERATE_CHAIN`: enable startup chain enumeration from the + documented default `ASIC_ID` +- `MUJINA_BZM2_AUTO_ENUMERATE`: alternate name for the same setting +- `MUJINA_BZM2_ENUM_START_ID`: first assigned runtime `ASIC_ID`, default `0` +- `MUJINA_BZM2_ENUM_MAX_ASICS_PER_BUS`: comma-separated per-bus enumeration + ceilings, default `100` per bus unless `MUJINA_BZM2_ASICS_PER_BUS` provides a + configured topology (any bus count greater than `1`), which is then used as + the ceiling +- `MUJINA_BZM2_ENABLE_BRINGUP`: enable startup and shutdown rail/reset + sequencing +- `MUJINA_BZM2_BRINGUP_ENABLE`: alternate name for the same setting +- `MUJINA_BZM2_RAIL_SET_PATHS`: comma-separated rail-control file paths +- `MUJINA_BZM2_RAIL_TARGET_VOLTS`: comma-separated target rail voltages for the + bring-up plan +- `MUJINA_BZM2_RAIL_WRITE_SCALES`: optional comma-separated scale factors used + when converting volts into the raw file value, for example `1000` for mV or + `1000000` for uV +- `MUJINA_BZM2_DOMAIN_RAIL_INDICES`: optional comma-separated mapping from + planner domain id to configured rail index; defaults to one-to-one + `domain_id -> rail_index` when omitted +- `MUJINA_BZM2_RAIL_ENABLE_PATHS`: optional comma-separated enable/control file + paths paired with the rail list +- `MUJINA_BZM2_RAIL_ENABLE_VALUES`: optional comma-separated values written to + the rail enable paths during rail initialization +- `MUJINA_BZM2_RAIL_VIN_PATHS`: optional comma-separated rail input-voltage + sensor files +- `MUJINA_BZM2_RAIL_VIN_SCALES`: optional scale factors for the rail input + voltage files +- `MUJINA_BZM2_RAIL_VOUT_PATHS`: optional comma-separated rail output-voltage + sensor files +- `MUJINA_BZM2_RAIL_VOUT_SCALES`: optional scale factors for the rail output + voltage files +- `MUJINA_BZM2_RAIL_CURRENT_PATHS`: optional comma-separated rail current sensor + files +- `MUJINA_BZM2_RAIL_CURRENT_SCALES`: optional scale factors for the rail current + files +- `MUJINA_BZM2_RAIL_POWER_PATHS`: optional comma-separated rail power sensor + files +- `MUJINA_BZM2_RAIL_POWER_SCALES`: optional scale factors for the rail power + files +- `MUJINA_BZM2_RAIL_TEMP_PATHS`: optional comma-separated rail regulator + temperature sensor files +- `MUJINA_BZM2_RAIL_TEMP_SCALES`: optional scale factors for the rail + regulator temperature files +- `MUJINA_BZM2_RESET_PATH`: optional reset-control file path +- `MUJINA_BZM2_RESET_ACTIVE_LOW`: whether the reset path is active-low, default + `true` +- `MUJINA_BZM2_BRINGUP_PRE_POWER_MS`: delay before rail initialization, default + `10` +- `MUJINA_BZM2_BRINGUP_POST_POWER_MS`: delay after the configured rail steps, + default `25` +- `MUJINA_BZM2_BRINGUP_RELEASE_RESET_MS`: delay after reset release, default + `25` + +The list above covers the core transport, enumeration, and bring-up settings +only. The driver reads many more `MUJINA_BZM2_*` variables than are listed +here, including calibration and PnP controls (for example +`MUJINA_BZM2_CALIBRATE`, `MUJINA_BZM2_OPERATING_CLASS`, and +`MUJINA_BZM2_PERFORMANCE_MODE`), runtime-retune tuning (for example +`MUJINA_BZM2_RUNTIME_RETUNE`), engine discovery, and host telemetry file +mappings (for example `MUJINA_BZM2_FAN_RPM_PATH` and +`MUJINA_BZM2_BOARD_TEMP_PATH`). `mujina-miner/src/board/bzm2/config.rs` is the +authoritative reference for the full variable set and its defaults, with the +host telemetry file mappings in the adjacent `telemetry.rs`. + +Startup enumeration notes: + +- this mode is intended for fresh chains where ASICs still answer on the + default `ASIC_ID` +- enumeration uses a bounded `NOOP` probe so the chain walk terminates cleanly + at the end of the bus +- if no default-id ASIC responds on startup, Mujina falls back to the configured + `MUJINA_BZM2_ASICS_PER_BUS` topology so warm-restart cases do not collapse to + zero ASICs + +Bring-up notes: + +- if bring-up is enabled, `Bzm2Board` applies the configured rail/reset + sequence before chain discovery, calibration, and hash-thread creation +- when the tuning planner produces per-domain voltage targets, `Bzm2Board` + now applies them onto the configured rail-control path before the PLL ramp + rather than treating them as advisory only +- saved operating-point replay now reapplies persisted per-domain voltages + before clock replay when the profile contains them +- if multiple domains are mapped onto one rail, the runtime requires them to + agree on the same target voltage; conflicting targets fail loudly instead of + applying an ambiguous setpoint +- on board shutdown, the same plan is used in reverse order to assert reset and + drive the configured rails back to `0` +- the runtime monitor now performs closed-loop checks against live rail + telemetry: measured per-domain voltage is compared to the planner's domain + targets and thermal drift is tracked, and triggers that persist across + monitor polls flag a pending retune and invalidate the saved operating + point; an in-place mid-run recalibration is still not performed, so a + flagged retune takes effect at the next startup, where the invalidated + profile is refused and the board recalibrates live + +If the optional rail telemetry files are configured, the board monitor also +publishes them into normal board state using stable names: + +- `rail0-input`, `rail1-input`, ... for input-side voltage snapshots +- `rail0-output`, `rail1-output`, ... for output-side voltage/current/power +- `rail0-regulator`, `rail1-regulator`, ... for regulator temperatures + +## API Telemetry + +When Gen2 `DTS_VS` frames are present on the UART path, Mujina now surfaces ASIC-internal telemetry through the normal board API state: + +- `BoardTelemetry.temperatures` +- `BoardTelemetry.powers` + +The values are named per serial bus and per ASIC so they can coexist with host-side sensor files: + +- temperature: `ttyUSB0-asic-2-dts` +- voltage channels: `ttyUSB0-asic-2-vs-ch0`, `ttyUSB0-asic-2-vs-ch1`, `ttyUSB0-asic-2-vs-ch2` + +Example JSON fragment: + +```json +{ + "temperatures": [ + { "name": "ttyUSB0-asic-2-dts", "temperature_c": 64.5 } + ], + "powers": [ + { "name": "ttyUSB0-asic-2-vs-ch0", "voltage_v": 0.78, "current_a": null, "power_w": null }, + { "name": "ttyUSB0-asic-2-vs-ch1", "voltage_v": 0.79, "current_a": null, "power_w": null }, + { "name": "ttyUSB0-asic-2-vs-ch2", "voltage_v": 0.77, "current_a": null, "power_w": null } + ] +} +``` + +Notes: + +- these ASIC-originated entries are merged into board state and do not replace host-file telemetry +- Celsius and voltage scaling follow the legacy `bzmd` DTS/VS conversion formulas +- Gen1 currently exposes voltage through this path, but not a Celsius temperature reading + +## On-Demand ASIC Sensor Queries + +Mujina now supports explicit DTS/VS query operations in addition to passive frame reporting. + +This is useful when: + +- the miner is idle and no passive DTS/VS traffic is arriving +- one ASIC is misbehaving and needs targeted inspection +- developers want a direct sensor read without enabling a full TDM watch session + +The query path is exposed through the HTTP API: + +- `POST /api/v0/boards/{name}/bzm2/dts-vs-query` + +The query path runs through the live BZM2 hash-thread actor so UART ownership remains correct. Queried frames are converted through the same telemetry code path used for passive DTS/VS reporting, so the returned values land in normal `BoardTelemetry` telemetry. + +Example HTTP request: + +```bash +curl -X POST http://127.0.0.1:3000/api/v0/boards/bzm2-0/bzm2/dts-vs-query \ + -H "Content-Type: application/json" \ + -d '{"thread_index":0,"asic":2}' +``` + +## On-Demand Engine Discovery + +Mujina also supports explicit per-ASIC engine-map discovery when the thread is +idle. + +This is useful when: + +- an ASIC is returning unstable shares and the default engine-hole assumption is + no longer trustworthy +- developers need to compare the live engine map against the historical default + BZM2 hole pattern +- operators want the discovered topology recorded in normal board API state + +The engine-discovery path runs through the live BZM2 hash-thread actor, just +like the DTS/VS query path, so UART ownership stays correct. Successful scans +update the live thread engine layout and `BoardTelemetry.asics` with: + +- `id` +- `thread_index` +- `serial_path` +- `discovered_engine_count` +- `missing_engines` + +After a successful idle-time scan, subsequent work dispatch and result +reconstruction use the discovered active-engine layout instead of the built-in +default four-hole map. + +HTTP API: + +- `POST /api/v0/boards/{name}/bzm2/discover-engines` + +Example HTTP request: + +```bash +curl -X POST http://127.0.0.1:3000/api/v0/boards/bzm2-0/bzm2/discover-engines \ + -H "Content-Type: application/json" \ + -d '{"thread_index":0,"asic":2,"tdm_prediv_raw":15,"tdm_counter":16,"timeout_ms":150}' +``` + +The response returns the refreshed `BoardTelemetry`, including the updated +`asics` topology entry for the queried ASIC. + +## API Diagnostics + +The board/API surface now exposes a first live UART diagnostics slice through +the BZM2 thread actor: + +- `POST /api/v0/boards/{name}/bzm2/noop` +- `POST /api/v0/boards/{name}/bzm2/loopback` +- `POST /api/v0/boards/{name}/bzm2/register-read` +- `POST /api/v0/boards/{name}/bzm2/register-write` +- `POST /api/v0/boards/{name}/bzm2/clock-report` + +These commands intentionally route through the live board-owned thread instead +of opening a second serial handle. That keeps UART ownership correct and avoids +silent contention with the mining path. + +Current safety boundary: + +- the target thread must be idle +- DTS/VS streaming must be inactive on that thread + +If either condition is false, the command is rejected rather than racing active +mining traffic or background telemetry frames. + +`clock-report` returns the same PLL/DLL status surface used by the low-level +UART diagnostics during development: + +- PLL enable register +- PLL misc register +- PLL enabled/locked bits +- DLL control2/control5 values +- DLL `coarsecon` +- DLL `fincon` +- DLL freeze-valid, lock, and fincon-valid state + +## Chain Summary + +The board/API surface also exposes the current BZM2 chain layout without +opening a second serial handle: + +- `GET /api/v0/boards/{name}/bzm2/chain-summary` + +The response summarizes: + +- current UART bus count +- serial path per bus +- global ASIC start/count per bus +- total ASIC count across the board +- whether the active operating point came from saved replay or live calibration +- current saved operating point validation state + +That gives operators a stable summary view of the active chain layout even when +the underlying board was discovered by startup enumeration rather than static +configuration alone. + +## Design Boundary + +As unverified background, the legacy `bzmd` board-power path reportedly mixes +three different concerns: + +- genuinely reusable sequencing concepts +- generic peripheral protocols like PMBus/I2C regulators and reset GPIOs +- highly board-specific MCU command sets, sysfs GPIO numbering, CAN PSU control, and platform wiring assumptions + +Only the first two belong in a generally applicable Mujina BZM2 implementation. + +Ported into Mujina: + +- generic reset assertion/deassertion +- generic ordered rail bring-up and shutdown +- generic PMBus/TPS546 voltage control and telemetry adapters +- ASIC-originated DTS/VS telemetry and fault handling + +Intentionally not ported verbatim (the legacy file attributions below are +unverified background, as recalled from the legacy stack): + +- the Intel board MCU command protocol, reportedly in `mcu.c` +- hard-coded board GPIO numbering and sysfs reset pulses, reportedly in `util.c` / `daemon.c` +- platform CAN PSU control, reportedly in `psu.c` +- board-specific fan and ambient-sensor plumbing that depends on the original + platform layout; generic file-backed fan and ambient/board temperature + telemetry is now provided through the `MUJINA_BZM2_FAN_*` and `*_TEMP_PATH` + variables, so only the platform-specific plumbing remains unported + +Those pieces should only be added behind a concrete Mujina board implementation when the target hardware actually uses them. + +## Current Limits + +Still not implemented from the broader legacy stack: + +- JTAG workflows from the standalone platform documents +- JTAG-only PLL debug sequences that are not represented in the shipped UART code +- in-place mid-run recalibration: retunes flagged by the runtime monitor take + effect at the next startup rather than being applied to a live board +- full manufacturing and diagnostics RPC parity + - beyond the current live API surface for: + - `NOOP` + - loopback + - register read/write + - clock report + - chain summary + - DTS/VS query + - engine discovery +- any board-MCU protocol that is specific to one carrier or backplane design + +This port currently implements the opcode surface that is evidenced in the legacy shipping UART path and not an inferred JTAG control plane. + + +See also: + +- [bzm2-opcode-grounding.md](bzm2-opcode-grounding.md) for the source-grounded opcode matrix and the current JTAG evidence boundary + diff --git a/mujina-miner/src/api/commands.rs b/mujina-miner/src/api/commands.rs index 83994f61..ef2fea3d 100644 --- a/mujina-miner/src/api/commands.rs +++ b/mujina-miner/src/api/commands.rs @@ -6,6 +6,8 @@ use anyhow::Result; use tokio::sync::oneshot; +use crate::api_client::types::{Bzm2ChainSummaryResponse, Bzm2ClockReportResponse}; + /// Commands from the API to the scheduler. pub enum SchedulerCommand { /// Pause job distribution to all threads. @@ -25,4 +27,71 @@ pub enum BoardCommand { percent: Option, reply: oneshot::Sender>, }, + + /// Trigger a DTS/VS (temperature/voltage sensor) query on a BZM2 + /// ASIC; results are published into the board's telemetry stream. + QueryBzm2DtsVs { + thread_index: usize, + asic: u8, + reply: oneshot::Sender>, + }, + + /// Send a NOOP to a BZM2 ASIC and return the 3-byte payload + /// (expected `b"BZ2"`). + QueryBzm2Noop { + thread_index: usize, + asic: u8, + reply: oneshot::Sender>, + }, + + /// Report the board's bus/ASIC layout and tuning status. + QueryBzm2ChainSummary { + reply: oneshot::Sender>, + }, + + /// Read PLL/DLL clock status registers from a BZM2 ASIC. + QueryBzm2ClockReport { + thread_index: usize, + asic: u8, + reply: oneshot::Sender>, + }, + + /// Echo a payload through a BZM2 ASIC's loopback path. + QueryBzm2Loopback { + thread_index: usize, + asic: u8, + payload: Vec, + reply: oneshot::Sender>>, + }, + + /// Read raw register bytes from a BZM2 engine address. + ReadBzm2Register { + thread_index: usize, + asic: u8, + engine_address: u16, + offset: u8, + count: u8, + reply: oneshot::Sender>>, + }, + + /// Write raw register bytes to a BZM2 engine address. + WriteBzm2Register { + thread_index: usize, + asic: u8, + engine_address: u16, + offset: u8, + value: Vec, + reply: oneshot::Sender>, + }, + + /// Run TDM engine-map discovery on a BZM2 ASIC (idle threads + /// only); results are published into the board's telemetry stream. + DiscoverBzm2Engines { + thread_index: usize, + asic: u8, + tdm_prediv_raw: u32, + tdm_counter: u8, + timeout_ms: Option, + reply: oneshot::Sender>, + }, } diff --git a/mujina-miner/src/api/registry.rs b/mujina-miner/src/api/registry.rs index c5325878..66b565ad 100644 --- a/mujina-miner/src/api/registry.rs +++ b/mujina-miner/src/api/registry.rs @@ -1,7 +1,9 @@ //! Dynamic board registration tracking. +use tokio::sync::{mpsc, watch}; + +use crate::api::commands::BoardCommand; use crate::api_client::types::BoardTelemetry; -use tokio::sync::watch; /// Dynamic collection of board registrations. /// @@ -23,23 +25,49 @@ impl BoardRegistry { self.boards.push(reg); } + /// Remove boards whose sender has been dropped (board disconnected). + fn prune_disconnected(&mut self) { + self.boards + .retain(|reg| reg.telemetry_rx.has_changed().is_ok()); + } + /// Snapshot all connected boards. /// /// Removes boards whose sender has been dropped (board disconnected) /// and returns the current state of each. pub fn boards(&mut self) -> Vec { - self.boards - .retain(|reg| reg.telemetry_rx.has_changed().is_ok()); + self.prune_disconnected(); self.boards .iter() .map(|reg| reg.telemetry_rx.borrow().clone()) .collect() } + + /// Snapshot a single connected board by name. + pub fn board(&mut self, name: &str) -> Option { + self.prune_disconnected(); + self.boards + .iter() + .find(|reg| reg.telemetry_rx.borrow().name == name) + .map(|reg| reg.telemetry_rx.borrow().clone()) + } + + /// Look up the command sender for a board by name. `None` if the + /// board is unknown or accepts no commands. + pub fn command_tx(&mut self, name: &str) -> Option> { + self.prune_disconnected(); + self.boards + .iter() + .find(|reg| reg.telemetry_rx.borrow().name == name) + .and_then(|reg| reg.command_tx.clone()) + } } /// A board's registration with the API server. pub struct BoardRegistration { pub telemetry_rx: watch::Receiver, + /// Sender for board commands. `None` if the board accepts no commands. + pub command_tx: Option>, } #[cfg(test)] @@ -57,7 +85,13 @@ mod tests { ..Default::default() }; let (tx, rx) = watch::channel(telemetry); - (tx, BoardRegistration { telemetry_rx: rx }) + ( + tx, + BoardRegistration { + telemetry_rx: rx, + command_tx: None, + }, + ) } #[test] @@ -109,4 +143,34 @@ mod tests { tx.send_modify(|s| s.model = "Updated".into()); assert_eq!(registry.boards()[0].model, "Updated"); } + + #[test] + fn returns_single_board_by_name() { + let mut registry = BoardRegistry::new(); + + let (_keep, reg) = make_board("board-a"); + registry.push(reg); + + assert_eq!(registry.board("board-a").unwrap().name, "board-a"); + assert!(registry.board("missing").is_none()); + } + + #[test] + fn returns_command_sender_for_named_board() { + use tokio::sync::mpsc; + + let mut registry = BoardRegistry::new(); + + let (_keep, mut reg) = make_board("board-a"); + let (cmd_tx, _cmd_rx) = mpsc::channel::(1); + reg.command_tx = Some(cmd_tx); + registry.push(reg); + + assert!(registry.command_tx("board-a").is_some()); + assert!(registry.command_tx("missing").is_none()); + + let (_keep_b, reg_b) = make_board("no-commands"); + registry.push(reg_b); + assert!(registry.command_tx("no-commands").is_none()); + } } diff --git a/mujina-miner/src/api/server.rs b/mujina-miner/src/api/server.rs index e7e0a16c..d8f34324 100644 --- a/mujina-miner/src/api/server.rs +++ b/mujina-miner/src/api/server.rs @@ -140,9 +140,17 @@ mod tests { use tower::ServiceExt; use super::*; - use crate::api::commands::SchedulerCommand; + use crate::api::commands::{BoardCommand, SchedulerCommand}; use crate::api::registry::BoardRegistration; - use crate::api_client::types::{BoardTelemetry, SourceTelemetry}; + use crate::api_client::types::{ + AsicState, BoardTelemetry, Bzm2BusSummary, Bzm2ChainSummaryResponse, + Bzm2ClockReportRequest, Bzm2ClockReportResponse, Bzm2DllClockStatus, Bzm2DtsVsQueryRequest, + Bzm2EngineDiscoveryRequest, Bzm2LoopbackRequest, Bzm2LoopbackResponse, Bzm2NoopRequest, + Bzm2NoopResponse, Bzm2PllClockStatus, Bzm2RegisterReadRequest, Bzm2RegisterReadResponse, + Bzm2RegisterWriteRequest, Bzm2RegisterWriteResponse, Bzm2SavedOperatingPointStatus, + Bzm2StartupPath, EngineCoordinate, SourceTelemetry, TemperatureSensor, + }; + use crate::types::Temperature; /// Test fixtures returned by the router builder. struct TestFixtures { @@ -166,7 +174,10 @@ mod tests { let mut board_senders = Vec::new(); for state in board_states { let (tx, rx) = watch::channel(state); - registry.push(BoardRegistration { telemetry_rx: rx }); + registry.push(BoardRegistration { + telemetry_rx: rx, + command_tx: None, + }); board_senders.push(tx); } @@ -362,4 +373,493 @@ mod tests { let (status, _body) = get(fixtures.router.clone(), "/api/v0/nope").await; assert_eq!(status, 404); } + + async fn post_json( + app: Router, + method: &str, + uri: &str, + body: &T, + ) -> (http::StatusCode, String) { + let req = Request::builder() + .method(method) + .uri(uri) + .header("content-type", "application/json") + .body(axum::body::Body::from(serde_json::to_vec(body).unwrap())) + .unwrap(); + let resp = app.oneshot(req).await.unwrap(); + let status = resp.status(); + let body = resp.into_body().collect().await.unwrap().to_bytes(); + (status, String::from_utf8(body.to_vec()).unwrap()) + } + + #[tokio::test] + async fn set_fan_target_round_trips_board_command() { + use crate::api::commands::BoardCommand; + use crate::api_client::types::{Fan, SetFanTargetRequest}; + + let (miner_tx, miner_rx) = watch::channel(MinerTelemetry::default()); + let (cmd_tx, _cmd_rx) = mpsc::channel::(16); + let mut registry = BoardRegistry::new(); + let (telemetry_tx, telemetry_rx) = watch::channel(BoardTelemetry { + name: "fan-board".into(), + model: "Test".into(), + ..Default::default() + }); + let (board_cmd_tx, mut board_cmd_rx) = mpsc::channel(1); + registry.push(BoardRegistration { + telemetry_rx, + command_tx: Some(board_cmd_tx), + }); + let router = build_router(miner_rx, Arc::new(Mutex::new(registry)), cmd_tx); + + // Clone for the task; the original must stay alive or the registry + // prunes the board before the handler's post-command re-read. + let telemetry_tx_for_command = telemetry_tx.clone(); + tokio::spawn(async move { + if let Some(BoardCommand::SetFanTarget { + board, + fan, + percent, + reply, + }) = board_cmd_rx.recv().await + { + assert_eq!(board, "fan-board"); + assert_eq!(fan, "fan0"); + assert_eq!(percent, Some(75)); + telemetry_tx_for_command.send_modify(|t| { + t.fans.push(Fan { + name: "fan0".into(), + rpm: None, + percent: None, + target_percent: percent, + }); + }); + let _ = reply.send(Ok(())); + } + }); + + let (status, body) = post_json( + router.clone(), + "PATCH", + "/api/v0/boards/fan-board/fans/fan0", + &SetFanTargetRequest { + target_percent: Some(75), + }, + ) + .await; + assert_eq!(status, 200); + let board: BoardTelemetry = serde_json::from_str(&body).unwrap(); + assert_eq!(board.fans[0].target_percent, Some(75)); + + // A board with no command channel answers 400. + let (_keep, no_cmd_rx) = watch::channel(BoardTelemetry { + name: "no-commands".into(), + model: "Test".into(), + ..Default::default() + }); + let (miner_tx2, miner_rx2) = watch::channel(MinerTelemetry::default()); + let (cmd_tx2, _cmd_rx2) = mpsc::channel::(16); + let mut registry2 = BoardRegistry::new(); + registry2.push(BoardRegistration { + telemetry_rx: no_cmd_rx, + command_tx: None, + }); + let router2 = build_router(miner_rx2, Arc::new(Mutex::new(registry2)), cmd_tx2); + let (status, _body) = post_json( + router2, + "PATCH", + "/api/v0/boards/no-commands/fans/fan0", + &SetFanTargetRequest { + target_percent: Some(50), + }, + ) + .await; + assert_eq!(status, 400); + + drop(miner_tx); + drop(miner_tx2); + drop(telemetry_tx); + } + + /// Build a router with one command-capable BZM2 test board, returning + /// the board's telemetry sender and command receiver. + fn build_bzm2_test_router( + channel_capacity: usize, + ) -> ( + Router, + watch::Sender, + mpsc::Receiver, + watch::Sender, + mpsc::Receiver, + ) { + let (miner_tx, miner_rx) = watch::channel(MinerTelemetry::default()); + let (cmd_tx, cmd_rx) = mpsc::channel::(16); + let mut registry = BoardRegistry::new(); + let (telemetry_tx, telemetry_rx) = watch::channel(BoardTelemetry { + name: "bzm2-test".into(), + model: "BZM2".into(), + ..Default::default() + }); + let (board_cmd_tx, board_cmd_rx) = mpsc::channel(channel_capacity); + registry.push(BoardRegistration { + telemetry_rx, + command_tx: Some(board_cmd_tx), + }); + let router = build_router(miner_rx, Arc::new(Mutex::new(registry)), cmd_tx); + (router, telemetry_tx, board_cmd_rx, miner_tx, cmd_rx) + } + + #[tokio::test] + async fn bzm2_query_endpoint_returns_refreshed_board_state() { + let (router, telemetry_tx, mut board_cmd_rx, _miner_tx, _cmd_rx) = + build_bzm2_test_router(1); + + // Clone for the task; the original must stay alive or the registry + // prunes the board before the handler's post-command re-read. + let telemetry_tx_for_command = telemetry_tx.clone(); + tokio::spawn(async move { + if let Some(BoardCommand::QueryBzm2DtsVs { + thread_index, + asic, + reply, + }) = board_cmd_rx.recv().await + { + assert_eq!(thread_index, 0); + assert_eq!(asic, 2); + telemetry_tx_for_command.send_modify(|state| { + state.temperatures.push(TemperatureSensor { + name: "ttyUSB0-asic-2-dts".into(), + temperature: Some(Temperature::from_celsius(64.5)), + }); + }); + let _ = reply.send(Ok(())); + } + }); + + let (status, body) = post_json( + router, + "POST", + "/api/v0/boards/bzm2-test/bzm2/dts-vs-query", + &Bzm2DtsVsQueryRequest { + thread_index: 0, + asic: 2, + }, + ) + .await; + + assert_eq!(status, 200); + let board: BoardTelemetry = serde_json::from_str(&body).unwrap(); + assert!(board.temperatures.iter().any(|sensor| { + sensor.name == "ttyUSB0-asic-2-dts" + && sensor.temperature.map(Temperature::as_degrees_c) == Some(64.5) + })); + } + + #[tokio::test] + async fn bzm2_diagnostic_endpoints_round_trip_payloads() { + let (router, _telemetry_tx, mut board_cmd_rx, _miner_tx, _cmd_rx) = + build_bzm2_test_router(4); + + tokio::spawn(async move { + while let Some(command) = board_cmd_rx.recv().await { + match command { + BoardCommand::QueryBzm2Noop { + thread_index, + asic, + reply, + } => { + assert_eq!(thread_index, 0); + assert_eq!(asic, 2); + let _ = reply.send(Ok(*b"BZ2")); + } + BoardCommand::QueryBzm2Loopback { + thread_index, + asic, + payload, + reply, + } => { + assert_eq!(thread_index, 0); + assert_eq!(asic, 2); + assert_eq!(payload, vec![0x01, 0x02, 0xaa, 0xbb]); + let _ = reply.send(Ok(payload)); + } + BoardCommand::ReadBzm2Register { + thread_index, + asic, + engine_address, + offset, + count, + reply, + } => { + assert_eq!(thread_index, 0); + assert_eq!(asic, 2); + assert_eq!(engine_address, 0x0fff); + assert_eq!(offset, 0x12); + assert_eq!(count, 4); + let _ = reply.send(Ok(vec![0x11, 0x22, 0x33, 0x44])); + } + BoardCommand::WriteBzm2Register { + thread_index, + asic, + engine_address, + offset, + value, + reply, + } => { + assert_eq!(thread_index, 0); + assert_eq!(asic, 2); + assert_eq!(engine_address, 0x0fff); + assert_eq!(offset, 0x12); + assert_eq!(value, vec![0xde, 0xad, 0xbe, 0xef]); + let _ = reply.send(Ok(())); + } + _ => {} + } + } + }); + + let (status, body) = post_json( + router.clone(), + "POST", + "/api/v0/boards/bzm2-test/bzm2/noop", + &Bzm2NoopRequest { + thread_index: 0, + asic: 2, + }, + ) + .await; + assert_eq!(status, 200); + let noop: Bzm2NoopResponse = serde_json::from_str(&body).unwrap(); + assert_eq!(noop.payload_hex, "425a32"); + + let (status, body) = post_json( + router.clone(), + "POST", + "/api/v0/boards/bzm2-test/bzm2/loopback", + &Bzm2LoopbackRequest { + thread_index: 0, + asic: 2, + payload_hex: "0102aabb".into(), + }, + ) + .await; + assert_eq!(status, 200); + let loopback: Bzm2LoopbackResponse = serde_json::from_str(&body).unwrap(); + assert_eq!(loopback.payload_hex, "0102aabb"); + + let (status, body) = post_json( + router.clone(), + "POST", + "/api/v0/boards/bzm2-test/bzm2/register-read", + &Bzm2RegisterReadRequest { + thread_index: 0, + asic: 2, + engine_address: 0x0fff, + offset: 0x12, + count: 4, + }, + ) + .await; + assert_eq!(status, 200); + let readback: Bzm2RegisterReadResponse = serde_json::from_str(&body).unwrap(); + assert_eq!(readback.value_hex, "11223344"); + + let (status, body) = post_json( + router, + "POST", + "/api/v0/boards/bzm2-test/bzm2/register-write", + &Bzm2RegisterWriteRequest { + thread_index: 0, + asic: 2, + engine_address: 0x0fff, + offset: 0x12, + value_hex: "deadbeef".into(), + }, + ) + .await; + assert_eq!(status, 200); + let write_ack: Bzm2RegisterWriteResponse = serde_json::from_str(&body).unwrap(); + assert_eq!(write_ack.bytes_written, 4); + } + + #[tokio::test] + async fn bzm2_chain_summary_endpoint_returns_live_layout() { + let (router, _telemetry_tx, mut board_cmd_rx, _miner_tx, _cmd_rx) = + build_bzm2_test_router(1); + + tokio::spawn(async move { + if let Some(BoardCommand::QueryBzm2ChainSummary { reply }) = board_cmd_rx.recv().await { + let _ = reply.send(Ok(Bzm2ChainSummaryResponse { + total_asics: 6, + startup_path: Some(Bzm2StartupPath::SavedReplay), + saved_operating_point_status: Some(Bzm2SavedOperatingPointStatus::Validated), + buses: vec![ + Bzm2BusSummary { + thread_index: 0, + serial_path: "/dev/ttyUSB0".into(), + asic_start: 0, + asic_count: 2, + }, + Bzm2BusSummary { + thread_index: 1, + serial_path: "/dev/ttyUSB1".into(), + asic_start: 2, + asic_count: 4, + }, + ], + })); + } + }); + + let (status, body) = get(router, "/api/v0/boards/bzm2-test/bzm2/chain-summary").await; + assert_eq!(status, 200); + let summary: Bzm2ChainSummaryResponse = serde_json::from_str(&body).unwrap(); + assert_eq!(summary.total_asics, 6); + assert_eq!(summary.startup_path, Some(Bzm2StartupPath::SavedReplay)); + assert_eq!( + summary.saved_operating_point_status, + Some(Bzm2SavedOperatingPointStatus::Validated) + ); + assert_eq!(summary.buses.len(), 2); + assert_eq!(summary.buses[1].serial_path, "/dev/ttyUSB1"); + assert_eq!(summary.buses[1].asic_start, 2); + assert_eq!(summary.buses[1].asic_count, 4); + } + + #[tokio::test] + async fn bzm2_clock_report_endpoint_returns_payload() { + let (router, _telemetry_tx, mut board_cmd_rx, _miner_tx, _cmd_rx) = + build_bzm2_test_router(1); + + tokio::spawn(async move { + if let Some(BoardCommand::QueryBzm2ClockReport { + thread_index, + asic, + reply, + }) = board_cmd_rx.recv().await + { + assert_eq!(thread_index, 0); + assert_eq!(asic, 2); + let _ = reply.send(Ok(Bzm2ClockReportResponse { + asic, + pll0: Bzm2PllClockStatus { + enable_register: 0x0000_0005, + misc_register: 0x0000_0012, + enabled: true, + locked: true, + }, + pll1: Bzm2PllClockStatus { + enable_register: 0x0000_0001, + misc_register: 0x0000_001a, + enabled: true, + locked: false, + }, + dll0: Bzm2DllClockStatus { + control2: 0x04, + control5: 0x07, + coarsecon: 0x03, + fincon: 0x9c, + freeze_valid: false, + locked: true, + fincon_valid: true, + }, + dll1: Bzm2DllClockStatus { + control2: 0x06, + control5: 0x03, + coarsecon: 0x02, + fincon: 0x10, + freeze_valid: true, + locked: true, + fincon_valid: true, + }, + })); + } + }); + + let (status, body) = post_json( + router, + "POST", + "/api/v0/boards/bzm2-test/bzm2/clock-report", + &Bzm2ClockReportRequest { + thread_index: 0, + asic: 2, + }, + ) + .await; + assert_eq!(status, 200); + let report: Bzm2ClockReportResponse = serde_json::from_str(&body).unwrap(); + assert_eq!(report.asic, 2); + assert_eq!(report.pll0.enable_register, 0x0000_0005); + assert!(report.pll0.locked); + assert!(!report.pll1.locked); + assert_eq!(report.dll0.fincon, 0x9c); + assert!(report.dll1.freeze_valid); + } + + #[tokio::test] + async fn bzm2_engine_discovery_endpoint_returns_refreshed_board_state() { + let (router, telemetry_tx, mut board_cmd_rx, _miner_tx, _cmd_rx) = + build_bzm2_test_router(1); + + // Clone for the task; the original must stay alive or the registry + // prunes the board before the handler's post-command re-read. + let telemetry_tx_for_command = telemetry_tx.clone(); + tokio::spawn(async move { + if let Some(BoardCommand::DiscoverBzm2Engines { + thread_index, + asic, + tdm_prediv_raw, + tdm_counter, + timeout_ms, + reply, + }) = board_cmd_rx.recv().await + { + assert_eq!(thread_index, 0); + assert_eq!(asic, 2); + assert_eq!(tdm_prediv_raw, 0x0f); + assert_eq!(tdm_counter, 16); + assert_eq!(timeout_ms, Some(150)); + telemetry_tx_for_command.send_modify(|state| { + state.asics.push(AsicState { + id: 2, + thread_index: Some(0), + serial_path: Some("/dev/ttyUSB0".into()), + discovered_engine_count: Some(236), + missing_engines: vec![ + EngineCoordinate { row: 3, col: 7 }, + EngineCoordinate { row: 5, col: 11 }, + ], + }); + }); + let _ = reply.send(Ok(())); + } + }); + + let (status, body) = post_json( + router, + "POST", + "/api/v0/boards/bzm2-test/bzm2/discover-engines", + &Bzm2EngineDiscoveryRequest { + thread_index: 0, + asic: 2, + tdm_prediv_raw: 0x0f, + tdm_counter: 16, + timeout_ms: Some(150), + }, + ) + .await; + + assert_eq!(status, 200); + let board: BoardTelemetry = serde_json::from_str(&body).unwrap(); + assert!(board.asics.iter().any(|asic| { + asic.id == 2 + && asic.thread_index == Some(0) + && asic.discovered_engine_count == Some(236) + && asic.missing_engines + == vec![ + EngineCoordinate { row: 3, col: 7 }, + EngineCoordinate { row: 5, col: 11 }, + ] + })); + } } diff --git a/mujina-miner/src/api/v0.rs b/mujina-miner/src/api/v0.rs index 5778fe5d..03217b5a 100644 --- a/mujina-miner/src/api/v0.rs +++ b/mujina-miner/src/api/v0.rs @@ -13,10 +13,14 @@ use std::time::Duration; use tokio::sync::oneshot; use utoipa_axum::{router::OpenApiRouter, routes}; -use super::commands::SchedulerCommand; +use super::commands::{BoardCommand, SchedulerCommand}; use super::server::SharedState; use crate::api_client::types::{ - BoardTelemetry, MinerPatchRequest, MinerTelemetry, SourceTelemetry, + BoardTelemetry, Bzm2ChainSummaryResponse, Bzm2ClockReportRequest, Bzm2ClockReportResponse, + Bzm2DtsVsQueryRequest, Bzm2EngineDiscoveryRequest, Bzm2LoopbackRequest, Bzm2LoopbackResponse, + Bzm2NoopRequest, Bzm2NoopResponse, Bzm2RegisterReadRequest, Bzm2RegisterReadResponse, + Bzm2RegisterWriteRequest, Bzm2RegisterWriteResponse, MinerPatchRequest, MinerTelemetry, + SetFanTargetRequest, SourceTelemetry, }; /// Build the v0 API routes with OpenAPI metadata. @@ -26,6 +30,15 @@ pub fn routes() -> OpenApiRouter { .routes(routes!(get_miner, patch_miner)) .routes(routes!(get_boards)) .routes(routes!(get_board)) + .routes(routes!(set_fan_target)) + .routes(routes!(query_bzm2_dts_vs)) + .routes(routes!(query_bzm2_noop)) + .routes(routes!(query_bzm2_loopback)) + .routes(routes!(read_bzm2_register)) + .routes(routes!(write_bzm2_register)) + .routes(routes!(query_bzm2_clock_report)) + .routes(routes!(get_bzm2_chain_summary)) + .routes(routes!(discover_bzm2_engines)) .routes(routes!(get_sources)) .routes(routes!(get_source)) } @@ -132,9 +145,505 @@ async fn get_board( .board_registry .lock() .unwrap_or_else(|e| e.into_inner()) - .boards() - .into_iter() - .find(|b| b.name == name) + .board(&name) + .map(Json) + .ok_or(StatusCode::NOT_FOUND) +} + +/// Set a fan's target duty cycle on a board, or return it to automatic +/// control. +#[utoipa::path( + patch, + path = "/boards/{name}/fans/{fan}", + tag = "boards", + params( + ("name" = String, Path, description = "Board name"), + ("fan" = String, Path, description = "Fan name"), + ), + request_body = SetFanTargetRequest, + responses( + (status = OK, description = "Updated board telemetry", body = BoardTelemetry), + (status = NOT_FOUND, description = "Board not found"), + (status = BAD_REQUEST, description = "Board accepts no commands"), + (status = INTERNAL_SERVER_ERROR, description = "Command channel error"), + ), +)] +async fn set_fan_target( + State(state): State, + Path((name, fan)): Path<(String, String)>, + Json(req): Json, +) -> Result, StatusCode> { + let (board_exists, command_tx) = { + let mut registry = state + .board_registry + .lock() + .unwrap_or_else(|e| e.into_inner()); + (registry.board(&name).is_some(), registry.command_tx(&name)) + }; + if !board_exists { + return Err(StatusCode::NOT_FOUND); + } + let Some(command_tx) = command_tx else { + return Err(StatusCode::BAD_REQUEST); + }; + + let (tx, rx) = oneshot::channel(); + command_tx + .send(BoardCommand::SetFanTarget { + board: name.clone(), + fan, + percent: req.target_percent, + reply: tx, + }) + .await + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + // Result layers: timeout / channel-closed / command-error. + let Ok(Ok(Ok(()))) = tokio::time::timeout(Duration::from_secs(5), rx).await else { + return Err(StatusCode::INTERNAL_SERVER_ERROR); + }; + + state + .board_registry + .lock() + .unwrap_or_else(|e| e.into_inner()) + .board(&name) + .map(Json) + .ok_or(StatusCode::NOT_FOUND) +} + +/// Trigger an explicit BZM2 DTS/VS query and return the refreshed board state. +#[utoipa::path( + post, + path = "/boards/{name}/bzm2/dts-vs-query", + tag = "boards", + params( + ("name" = String, Path, description = "Board name"), + ), + request_body = Bzm2DtsVsQueryRequest, + responses( + (status = OK, description = "Refreshed board details", body = BoardTelemetry), + (status = BAD_REQUEST, description = "Board does not support BZM2 telemetry queries"), + (status = NOT_FOUND, description = "Board not found"), + (status = INTERNAL_SERVER_ERROR, description = "Board command failed"), + ), +)] +async fn query_bzm2_dts_vs( + State(state): State, + Path(name): Path, + Json(req): Json, +) -> Result, StatusCode> { + let (board_exists, command_tx) = { + let mut registry = state + .board_registry + .lock() + .unwrap_or_else(|e| e.into_inner()); + (registry.board(&name).is_some(), registry.command_tx(&name)) + }; + if !board_exists { + return Err(StatusCode::NOT_FOUND); + } + let Some(command_tx) = command_tx else { + return Err(StatusCode::BAD_REQUEST); + }; + + let (tx, rx) = oneshot::channel(); + command_tx + .send(BoardCommand::QueryBzm2DtsVs { + thread_index: req.thread_index, + asic: req.asic, + reply: tx, + }) + .await + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + let Ok(Ok(Ok(()))) = tokio::time::timeout(Duration::from_secs(5), rx).await else { + return Err(StatusCode::INTERNAL_SERVER_ERROR); + }; + + state + .board_registry + .lock() + .unwrap_or_else(|e| e.into_inner()) + .board(&name) + .map(Json) + .ok_or(StatusCode::NOT_FOUND) +} + +/// Trigger a live BZM2 NOOP diagnostic through a board-owned UART thread. +#[utoipa::path( + post, + path = "/boards/{name}/bzm2/noop", + tag = "boards", + params( + ("name" = String, Path, description = "Board name"), + ), + request_body = Bzm2NoopRequest, + responses( + (status = OK, description = "NOOP response payload", body = Bzm2NoopResponse), + (status = BAD_REQUEST, description = "Board does not support BZM2 diagnostics"), + (status = NOT_FOUND, description = "Board not found"), + (status = INTERNAL_SERVER_ERROR, description = "Board command failed"), + ), +)] +async fn query_bzm2_noop( + State(state): State, + Path(name): Path, + Json(req): Json, +) -> Result, StatusCode> { + let (board_exists, command_tx) = { + let mut registry = state + .board_registry + .lock() + .unwrap_or_else(|e| e.into_inner()); + (registry.board(&name).is_some(), registry.command_tx(&name)) + }; + if !board_exists { + return Err(StatusCode::NOT_FOUND); + } + let Some(command_tx) = command_tx else { + return Err(StatusCode::BAD_REQUEST); + }; + + let (tx, rx) = oneshot::channel(); + command_tx + .send(BoardCommand::QueryBzm2Noop { + thread_index: req.thread_index, + asic: req.asic, + reply: tx, + }) + .await + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + let Ok(Ok(Ok(payload))) = tokio::time::timeout(Duration::from_secs(5), rx).await else { + return Err(StatusCode::INTERNAL_SERVER_ERROR); + }; + + Ok(Json(Bzm2NoopResponse { + payload_hex: hex::encode(payload), + })) +} + +/// Trigger a live BZM2 loopback diagnostic through a board-owned UART thread. +#[utoipa::path( + post, + path = "/boards/{name}/bzm2/loopback", + tag = "boards", + params( + ("name" = String, Path, description = "Board name"), + ), + request_body = Bzm2LoopbackRequest, + responses( + (status = OK, description = "Loopback response payload", body = Bzm2LoopbackResponse), + (status = BAD_REQUEST, description = "Board does not support BZM2 diagnostics or request payload is invalid"), + (status = NOT_FOUND, description = "Board not found"), + (status = INTERNAL_SERVER_ERROR, description = "Board command failed"), + ), +)] +async fn query_bzm2_loopback( + State(state): State, + Path(name): Path, + Json(req): Json, +) -> Result, StatusCode> { + let payload = decode_hex_payload(&req.payload_hex)?; + let (board_exists, command_tx) = { + let mut registry = state + .board_registry + .lock() + .unwrap_or_else(|e| e.into_inner()); + (registry.board(&name).is_some(), registry.command_tx(&name)) + }; + if !board_exists { + return Err(StatusCode::NOT_FOUND); + } + let Some(command_tx) = command_tx else { + return Err(StatusCode::BAD_REQUEST); + }; + + let (tx, rx) = oneshot::channel(); + command_tx + .send(BoardCommand::QueryBzm2Loopback { + thread_index: req.thread_index, + asic: req.asic, + payload, + reply: tx, + }) + .await + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + let Ok(Ok(Ok(payload))) = tokio::time::timeout(Duration::from_secs(5), rx).await else { + return Err(StatusCode::INTERNAL_SERVER_ERROR); + }; + + Ok(Json(Bzm2LoopbackResponse { + payload_hex: hex::encode(payload), + })) +} + +/// Perform a live BZM2 register read through a board-owned UART thread. +#[utoipa::path( + post, + path = "/boards/{name}/bzm2/register-read", + tag = "boards", + params( + ("name" = String, Path, description = "Board name"), + ), + request_body = Bzm2RegisterReadRequest, + responses( + (status = OK, description = "Register payload", body = Bzm2RegisterReadResponse), + (status = BAD_REQUEST, description = "Board does not support BZM2 diagnostics"), + (status = NOT_FOUND, description = "Board not found"), + (status = INTERNAL_SERVER_ERROR, description = "Board command failed"), + ), +)] +async fn read_bzm2_register( + State(state): State, + Path(name): Path, + Json(req): Json, +) -> Result, StatusCode> { + let (board_exists, command_tx) = { + let mut registry = state + .board_registry + .lock() + .unwrap_or_else(|e| e.into_inner()); + (registry.board(&name).is_some(), registry.command_tx(&name)) + }; + if !board_exists { + return Err(StatusCode::NOT_FOUND); + } + let Some(command_tx) = command_tx else { + return Err(StatusCode::BAD_REQUEST); + }; + + let (tx, rx) = oneshot::channel(); + command_tx + .send(BoardCommand::ReadBzm2Register { + thread_index: req.thread_index, + asic: req.asic, + engine_address: req.engine_address, + offset: req.offset, + count: req.count, + reply: tx, + }) + .await + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + let Ok(Ok(Ok(value))) = tokio::time::timeout(Duration::from_secs(5), rx).await else { + return Err(StatusCode::INTERNAL_SERVER_ERROR); + }; + + Ok(Json(Bzm2RegisterReadResponse { + value_hex: hex::encode(value), + })) +} + +/// Perform a live BZM2 register write through a board-owned UART thread. +#[utoipa::path( + post, + path = "/boards/{name}/bzm2/register-write", + tag = "boards", + params( + ("name" = String, Path, description = "Board name"), + ), + request_body = Bzm2RegisterWriteRequest, + responses( + (status = OK, description = "Register write acknowledgement", body = Bzm2RegisterWriteResponse), + (status = BAD_REQUEST, description = "Board does not support BZM2 diagnostics or request payload is invalid"), + (status = NOT_FOUND, description = "Board not found"), + (status = INTERNAL_SERVER_ERROR, description = "Board command failed"), + ), +)] +async fn write_bzm2_register( + State(state): State, + Path(name): Path, + Json(req): Json, +) -> Result, StatusCode> { + let value = decode_hex_payload(&req.value_hex)?; + let bytes_written = value.len(); + let (board_exists, command_tx) = { + let mut registry = state + .board_registry + .lock() + .unwrap_or_else(|e| e.into_inner()); + (registry.board(&name).is_some(), registry.command_tx(&name)) + }; + if !board_exists { + return Err(StatusCode::NOT_FOUND); + } + let Some(command_tx) = command_tx else { + return Err(StatusCode::BAD_REQUEST); + }; + + let (tx, rx) = oneshot::channel(); + command_tx + .send(BoardCommand::WriteBzm2Register { + thread_index: req.thread_index, + asic: req.asic, + engine_address: req.engine_address, + offset: req.offset, + value, + reply: tx, + }) + .await + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + let Ok(Ok(Ok(()))) = tokio::time::timeout(Duration::from_secs(5), rx).await else { + return Err(StatusCode::INTERNAL_SERVER_ERROR); + }; + + Ok(Json(Bzm2RegisterWriteResponse { bytes_written })) +} + +fn decode_hex_payload(raw: &str) -> Result, StatusCode> { + hex::decode(raw.trim()).map_err(|_| StatusCode::BAD_REQUEST) +} + +/// Return a live BZM2 clock report through a board-owned UART thread. +#[utoipa::path( + post, + path = "/boards/{name}/bzm2/clock-report", + tag = "boards", + params( + ("name" = String, Path, description = "Board name"), + ), + request_body = Bzm2ClockReportRequest, + responses( + (status = OK, description = "Clock status payload", body = Bzm2ClockReportResponse), + (status = BAD_REQUEST, description = "Board does not support BZM2 diagnostics"), + (status = NOT_FOUND, description = "Board not found"), + (status = INTERNAL_SERVER_ERROR, description = "Board command failed"), + ), +)] +async fn query_bzm2_clock_report( + State(state): State, + Path(name): Path, + Json(req): Json, +) -> Result, StatusCode> { + let (board_exists, command_tx) = { + let mut registry = state + .board_registry + .lock() + .unwrap_or_else(|e| e.into_inner()); + (registry.board(&name).is_some(), registry.command_tx(&name)) + }; + if !board_exists { + return Err(StatusCode::NOT_FOUND); + } + let Some(command_tx) = command_tx else { + return Err(StatusCode::BAD_REQUEST); + }; + + let (tx, rx) = oneshot::channel(); + command_tx + .send(BoardCommand::QueryBzm2ClockReport { + thread_index: req.thread_index, + asic: req.asic, + reply: tx, + }) + .await + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + let Ok(Ok(Ok(report))) = tokio::time::timeout(Duration::from_secs(5), rx).await else { + return Err(StatusCode::INTERNAL_SERVER_ERROR); + }; + + Ok(Json(report)) +} + +/// Return the current BZM2 chain summary for a live board. +#[utoipa::path( + get, + path = "/boards/{name}/bzm2/chain-summary", + tag = "boards", + params( + ("name" = String, Path, description = "Board name"), + ), + responses( + (status = OK, description = "Current BZM2 chain summary", body = Bzm2ChainSummaryResponse), + (status = BAD_REQUEST, description = "Board does not support BZM2 chain summary"), + (status = NOT_FOUND, description = "Board not found"), + (status = INTERNAL_SERVER_ERROR, description = "Board command failed"), + ), +)] +async fn get_bzm2_chain_summary( + State(state): State, + Path(name): Path, +) -> Result, StatusCode> { + let (board_exists, command_tx) = { + let mut registry = state + .board_registry + .lock() + .unwrap_or_else(|e| e.into_inner()); + (registry.board(&name).is_some(), registry.command_tx(&name)) + }; + if !board_exists { + return Err(StatusCode::NOT_FOUND); + } + let Some(command_tx) = command_tx else { + return Err(StatusCode::BAD_REQUEST); + }; + + let (tx, rx) = oneshot::channel(); + command_tx + .send(BoardCommand::QueryBzm2ChainSummary { reply: tx }) + .await + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + let Ok(Ok(Ok(summary))) = tokio::time::timeout(Duration::from_secs(5), rx).await else { + return Err(StatusCode::INTERNAL_SERVER_ERROR); + }; + + Ok(Json(summary)) +} + +/// Trigger an explicit BZM2 engine-discovery scan and return the refreshed board state. +#[utoipa::path( + post, + path = "/boards/{name}/bzm2/discover-engines", + tag = "boards", + params( + ("name" = String, Path, description = "Board name"), + ), + request_body = Bzm2EngineDiscoveryRequest, + responses( + (status = OK, description = "Refreshed board details", body = BoardTelemetry), + (status = BAD_REQUEST, description = "Board does not support BZM2 engine discovery"), + (status = NOT_FOUND, description = "Board not found"), + (status = INTERNAL_SERVER_ERROR, description = "Board command failed"), + ), +)] +async fn discover_bzm2_engines( + State(state): State, + Path(name): Path, + Json(req): Json, +) -> Result, StatusCode> { + let (board_exists, command_tx) = { + let mut registry = state + .board_registry + .lock() + .unwrap_or_else(|e| e.into_inner()); + (registry.board(&name).is_some(), registry.command_tx(&name)) + }; + if !board_exists { + return Err(StatusCode::NOT_FOUND); + } + let Some(command_tx) = command_tx else { + return Err(StatusCode::BAD_REQUEST); + }; + + let (tx, rx) = oneshot::channel(); + command_tx + .send(BoardCommand::DiscoverBzm2Engines { + thread_index: req.thread_index, + asic: req.asic, + tdm_prediv_raw: req.tdm_prediv_raw, + tdm_counter: req.tdm_counter, + timeout_ms: req.timeout_ms, + reply: tx, + }) + .await + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + let Ok(Ok(Ok(()))) = tokio::time::timeout(Duration::from_secs(5), rx).await else { + return Err(StatusCode::INTERNAL_SERVER_ERROR); + }; + + state + .board_registry + .lock() + .unwrap_or_else(|e| e.into_inner()) + .board(&name) .map(Json) .ok_or(StatusCode::NOT_FOUND) } diff --git a/mujina-miner/src/api_client/types.rs b/mujina-miner/src/api_client/types.rs index 6042ba0b..ceeda72b 100644 --- a/mujina-miner/src/api_client/types.rs +++ b/mujina-miner/src/api_client/types.rs @@ -33,6 +33,12 @@ pub struct BoardTelemetry { pub temperatures: Vec, pub powers: Vec, pub threads: Vec, + /// Per-ASIC topology/diagnostics state (multi-ASIC boards only). + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub asics: Vec, + /// BZM2 runtime tuning state (BZM2 boards only). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub bzm2_tuning: Option, } /// Fan status. @@ -74,6 +80,172 @@ pub struct ThreadTelemetry { pub is_active: bool, } +/// Per-ASIC runtime topology or diagnostics state. +#[derive(Clone, Debug, Default, Deserialize, Serialize, ToSchema)] +pub struct AsicState { + pub id: u8, + #[serde(skip_serializing_if = "Option::is_none")] + pub thread_index: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub serial_path: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub discovered_engine_count: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub missing_engines: Vec, +} + +/// Physical engine coordinate on one ASIC. +#[derive(Clone, Debug, Deserialize, Serialize, ToSchema, PartialEq, Eq)] +pub struct EngineCoordinate { + pub row: u8, + pub col: u8, +} + +/// BZM2 runtime tuning measurements derived from live mining operation. +#[derive(Clone, Debug, Default, Deserialize, Serialize, ToSchema)] +pub struct Bzm2TuningState { + #[serde(skip_serializing_if = "Option::is_none")] + pub board_throughput_hs: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub reuse_saved_operating_point: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub needs_retune: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub desired_voltage_mv: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub desired_clock_mhz: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub desired_accept_ratio: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub retune_pending: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub retune_reasons: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub saved_operating_point_status: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub saved_operating_point_reasons: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub planner_notes: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub domains: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub asics: Vec, +} + +/// Validation status of a saved BZM2 operating point. +#[derive(Clone, Copy, Debug, Default, Deserialize, Serialize, ToSchema, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum Bzm2SavedOperatingPointStatus { + #[default] + Pending, + Validated, + Invalidated, +} + +/// How a BZM2 board reached its current operating point at startup. +#[derive(Clone, Copy, Debug, Deserialize, Serialize, ToSchema, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum Bzm2StartupPath { + SavedReplay, + LiveCalibration, +} + +/// Per-domain live tuning measurement. +#[derive(Clone, Debug, Deserialize, Serialize, ToSchema)] +pub struct Bzm2DomainTuningState { + pub domain_id: u16, + #[serde(skip_serializing_if = "Option::is_none")] + pub rail_index: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub target_voltage_mv: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub measured_voltage_mv: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub measured_power_w: Option, +} + +/// Per-PLL live tuning measurement. +#[derive(Clone, Debug, Deserialize, Serialize, ToSchema)] +pub struct Bzm2PllTuningState { + pub pll_index: u8, + #[serde(skip_serializing_if = "Option::is_none")] + pub frequency_mhz: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub throughput_hs: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub pass_rate: Option, +} + +/// Per-ASIC live tuning measurement. +#[derive(Clone, Debug, Deserialize, Serialize, ToSchema)] +pub struct Bzm2AsicTuningState { + pub id: u8, + #[serde(skip_serializing_if = "Option::is_none")] + pub thread_index: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub active_engine_count: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub throughput_hs: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub average_pass_rate: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub scheduler_share_count: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub plls: Vec, +} + +/// Per-bus BZM2 chain layout summary. +#[derive(Clone, Debug, Deserialize, Serialize, ToSchema)] +pub struct Bzm2BusSummary { + pub thread_index: usize, + pub serial_path: String, + pub asic_start: u16, + pub asic_count: u16, +} + +/// Current BZM2 chain summary for a live board. +#[derive(Clone, Debug, Deserialize, Serialize, ToSchema)] +pub struct Bzm2ChainSummaryResponse { + pub total_asics: u16, + #[serde(skip_serializing_if = "Option::is_none")] + pub startup_path: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub saved_operating_point_status: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub buses: Vec, +} + +/// One PLL status block in a BZM2 clock report. +#[derive(Clone, Debug, Deserialize, Serialize, ToSchema)] +pub struct Bzm2PllClockStatus { + pub enable_register: u32, + pub misc_register: u32, + pub enabled: bool, + pub locked: bool, +} + +/// One DLL status block in a BZM2 clock report. +#[derive(Clone, Debug, Deserialize, Serialize, ToSchema)] +pub struct Bzm2DllClockStatus { + pub control2: u8, + pub control5: u8, + pub coarsecon: u8, + pub fincon: u8, + pub freeze_valid: bool, + pub locked: bool, + pub fincon_valid: bool, +} + +/// Response body for a live BZM2 clock-report query. +#[derive(Clone, Debug, Deserialize, Serialize, ToSchema)] +pub struct Bzm2ClockReportResponse { + pub asic: u8, + pub pll0: Bzm2PllClockStatus, + pub pll1: Bzm2PllClockStatus, + pub dll0: Bzm2DllClockStatus, + pub dll1: Bzm2DllClockStatus, +} + /// Writable fields for `PATCH /api/v0/miner`. /// /// All fields are optional; only those present in the request body are @@ -92,6 +264,118 @@ pub struct SetFanTargetRequest { pub target_percent: Option, } +/// Request body for an explicit BZM2 ASIC DTS/VS query. +#[derive(Clone, Debug, Deserialize, Serialize, ToSchema)] +pub struct Bzm2DtsVsQueryRequest { + /// Index of the BZM2 UART thread/bus to query. + pub thread_index: usize, + /// ASIC id on that UART bus. + pub asic: u8, +} + +/// Request body for an explicit BZM2 ASIC engine-discovery scan. +#[derive(Clone, Debug, Deserialize, Serialize, ToSchema)] +pub struct Bzm2EngineDiscoveryRequest { + /// Index of the BZM2 UART thread/bus to query. + pub thread_index: usize, + /// ASIC id on that UART bus. + pub asic: u8, + /// Raw TDM pre-divider value written into `LOCAL_REG_UART_TDM_CTL`. + pub tdm_prediv_raw: u32, + /// TDM counter value written into `LOCAL_REG_UART_TDM_CTL`. + pub tdm_counter: u8, + /// Optional per-engine probe timeout in milliseconds. + #[serde(skip_serializing_if = "Option::is_none")] + pub timeout_ms: Option, +} + +/// Request body for a live BZM2 NOOP diagnostic query. +#[derive(Clone, Debug, Deserialize, Serialize, ToSchema)] +pub struct Bzm2NoopRequest { + /// Index of the BZM2 UART thread/bus to query. + pub thread_index: usize, + /// ASIC id on that UART bus. + pub asic: u8, +} + +/// Response body for a live BZM2 NOOP diagnostic query. +#[derive(Clone, Debug, Deserialize, Serialize, ToSchema)] +pub struct Bzm2NoopResponse { + /// Hex-encoded three-byte NOOP payload returned by the ASIC. + pub payload_hex: String, +} + +/// Request body for a live BZM2 loopback diagnostic query. +#[derive(Clone, Debug, Deserialize, Serialize, ToSchema)] +pub struct Bzm2LoopbackRequest { + /// Index of the BZM2 UART thread/bus to query. + pub thread_index: usize, + /// ASIC id on that UART bus. + pub asic: u8, + /// Hex-encoded payload to round-trip through the ASIC loopback opcode. + pub payload_hex: String, +} + +/// Response body for a live BZM2 loopback diagnostic query. +#[derive(Clone, Debug, Deserialize, Serialize, ToSchema)] +pub struct Bzm2LoopbackResponse { + /// Hex-encoded payload returned by the ASIC. + pub payload_hex: String, +} + +/// Request body for a live BZM2 register read. +#[derive(Clone, Debug, Deserialize, Serialize, ToSchema)] +pub struct Bzm2RegisterReadRequest { + /// Index of the BZM2 UART thread/bus to query. + pub thread_index: usize, + /// ASIC id on that UART bus. + pub asic: u8, + /// Engine or local-register address. + pub engine_address: u16, + /// Register offset within the selected engine or local block. + pub offset: u8, + /// Number of bytes to read. + pub count: u8, +} + +/// Response body for a live BZM2 register read. +#[derive(Clone, Debug, Deserialize, Serialize, ToSchema)] +pub struct Bzm2RegisterReadResponse { + /// Hex-encoded register payload. + pub value_hex: String, +} + +/// Request body for a live BZM2 register write. +#[derive(Clone, Debug, Deserialize, Serialize, ToSchema)] +pub struct Bzm2RegisterWriteRequest { + /// Index of the BZM2 UART thread/bus to query. + pub thread_index: usize, + /// ASIC id on that UART bus. + pub asic: u8, + /// Engine or local-register address. + pub engine_address: u16, + /// Register offset within the selected engine or local block. + pub offset: u8, + /// Hex-encoded bytes to write. + pub value_hex: String, +} + +/// Response body for a live BZM2 register write. +#[derive(Clone, Debug, Deserialize, Serialize, ToSchema)] +pub struct Bzm2RegisterWriteResponse { + /// Number of bytes written to the requested register. + pub bytes_written: usize, +} + +/// Request body for a live BZM2 clock-report query. +#[derive(Clone, Debug, Deserialize, Serialize, ToSchema)] +pub struct Bzm2ClockReportRequest { + /// Index of the BZM2 UART thread/bus to query. + pub thread_index: usize, + /// ASIC id on that UART bus. + pub asic: u8, +} + /// Job source telemetry. #[derive(Clone, Debug, Default, Deserialize, Serialize, ToSchema)] pub struct SourceTelemetry { diff --git a/mujina-miner/src/asic/bzm2/clock.rs b/mujina-miner/src/asic/bzm2/clock.rs new file mode 100644 index 00000000..a59d7cc3 --- /dev/null +++ b/mujina-miner/src/asic/bzm2/clock.rs @@ -0,0 +1,606 @@ +use std::time::Duration; + +use tokio::time::{Instant, sleep}; + +use crate::transport::{SerialReader, SerialWriter}; + +use super::uart::{Bzm2UartController, Bzm2UartError}; + +const REF_CLK_MHZ: f32 = 50.0; +const REF_DIVIDER: u8 = 1; +const POST2_DIVIDER: u8 = 0; + +const LOCAL_REG_PLL_POSTDIV: u8 = 0x10; +const LOCAL_REG_PLL_FBDIV: u8 = 0x11; +const LOCAL_REG_PLL_ENABLE: u8 = 0x12; +const LOCAL_REG_PLL_MISC: u8 = 0x13; +const LOCAL_REG_PLL1_POSTDIV: u8 = 0x1a; +const LOCAL_REG_PLL1_FBDIV: u8 = 0x1b; +const LOCAL_REG_PLL1_ENABLE: u8 = 0x1c; +const LOCAL_REG_PLL1_MISC: u8 = 0x1d; + +const LOCAL_REG_CKDCCR_2_0: u8 = 0x56; +const LOCAL_REG_CKDCCR_3_0: u8 = 0x57; +const LOCAL_REG_CKDCCR_4_0: u8 = 0x58; +const LOCAL_REG_CKDCCR_5_0: u8 = 0x59; +const LOCAL_REG_CKDLLR_0_0: u8 = 0x5a; +const LOCAL_REG_CKDLLR_1_0: u8 = 0x5b; +const LOCAL_REG_CKDCCR_2_1: u8 = 0x5e; +const LOCAL_REG_CKDCCR_3_1: u8 = 0x5f; +const LOCAL_REG_CKDCCR_4_1: u8 = 0x60; +const LOCAL_REG_CKDCCR_5_1: u8 = 0x61; +const LOCAL_REG_CKDLLR_0_1: u8 = 0x62; +const LOCAL_REG_CKDLLR_1_1: u8 = 0x63; + +pub struct Bzm2ClockController { + uart: Bzm2UartController, +} + +impl Bzm2ClockController { + pub fn new(reader: SerialReader, writer: SerialWriter) -> Self { + Self { + uart: Bzm2UartController::new(reader, writer), + } + } + + pub fn from_uart(uart: Bzm2UartController) -> Self { + Self { uart } + } + + pub fn into_uart(self) -> Bzm2UartController { + self.uart + } + + pub async fn program_pll( + &mut self, + asic: u8, + pll: Bzm2Pll, + config: Bzm2PllConfig, + ) -> Result<(), Bzm2ClockError> { + let (postdiv_reg, fbdiv_reg, _, _) = pll.register_block(); + self.uart + .write_local_reg_u32(asic, fbdiv_reg, config.feedback_divider as u32) + .await?; + self.uart + .write_local_reg_u32(asic, postdiv_reg, config.packed_post_divider) + .await?; + sleep(Duration::from_millis(1)).await; + Ok(()) + } + + pub async fn enable_pll(&mut self, asic: u8, pll: Bzm2Pll) -> Result<(), Bzm2ClockError> { + let (_, _, enable_reg, _) = pll.register_block(); + self.uart.write_local_reg_u32(asic, enable_reg, 1).await?; + Ok(()) + } + + pub async fn disable_pll(&mut self, asic: u8, pll: Bzm2Pll) -> Result<(), Bzm2ClockError> { + let (_, _, enable_reg, _) = pll.register_block(); + self.uart.write_local_reg_u32(asic, enable_reg, 0).await?; + Ok(()) + } + + pub async fn set_pll_frequency( + &mut self, + asic: u8, + pll: Bzm2Pll, + frequency_mhz: f32, + post1_divider: u8, + ) -> Result { + let config = Bzm2PllConfig::from_target_frequency(frequency_mhz, post1_divider)?; + self.program_pll(asic, pll, config).await?; + Ok(config) + } + + pub async fn broadcast_pll_frequency( + &mut self, + pll: Bzm2Pll, + frequency_mhz: f32, + post1_divider: u8, + ) -> Result { + let config = Bzm2PllConfig::from_target_frequency(frequency_mhz, post1_divider)?; + let (postdiv_reg, fbdiv_reg, _, _) = pll.register_block(); + self.uart + .broadcast_local_reg_u32(fbdiv_reg, config.feedback_divider as u32) + .await?; + self.uart + .broadcast_local_reg_u32(postdiv_reg, config.packed_post_divider) + .await?; + sleep(Duration::from_millis(1)).await; + Ok(config) + } + + pub async fn broadcast_enable_pll(&mut self, pll: Bzm2Pll) -> Result<(), Bzm2ClockError> { + let (_, _, enable_reg, _) = pll.register_block(); + self.uart.broadcast_local_reg_u32(enable_reg, 1).await?; + Ok(()) + } + + pub async fn broadcast_disable_pll(&mut self, pll: Bzm2Pll) -> Result<(), Bzm2ClockError> { + let (_, _, enable_reg, _) = pll.register_block(); + self.uart.broadcast_local_reg_u32(enable_reg, 0).await?; + Ok(()) + } + + pub async fn wait_for_pll_lock( + &mut self, + asic: u8, + pll: Bzm2Pll, + timeout: Duration, + poll_interval: Duration, + ) -> Result { + let (_, _, enable_reg, _) = pll.register_block(); + let start = Instant::now(); + + loop { + let last_enable = self.uart.read_local_reg_u32(asic, enable_reg).await?; + let status = self.read_pll_status(asic, pll).await?; + if status.locked { + return Ok(status); + } + if start.elapsed() >= timeout { + return Err(Bzm2ClockError::PllLockTimeout { + asic, + pll, + last_enable, + }); + } + sleep(poll_interval).await; + } + } + + pub async fn configure_and_lock_pll( + &mut self, + asic: u8, + pll: Bzm2Pll, + frequency_mhz: f32, + post1_divider: u8, + timeout: Duration, + ) -> Result<(Bzm2PllConfig, Bzm2PllStatus), Bzm2ClockError> { + let config = self + .set_pll_frequency(asic, pll, frequency_mhz, post1_divider) + .await?; + self.enable_pll(asic, pll).await?; + let status = self + .wait_for_pll_lock(asic, pll, timeout, Duration::from_millis(100)) + .await?; + Ok((config, status)) + } + + pub async fn read_pll_status( + &mut self, + asic: u8, + pll: Bzm2Pll, + ) -> Result { + let (_, _, enable_reg, misc_reg) = pll.register_block(); + let enable = self.uart.read_local_reg_u32(asic, enable_reg).await?; + let misc = self.uart.read_local_reg_u32(asic, misc_reg).await?; + Ok(Bzm2PllStatus { + pll, + enable_register: enable, + misc_register: misc, + enabled: (enable & 0x1) != 0, + locked: (enable & 0x4) != 0, + }) + } + + pub async fn program_dll( + &mut self, + asic: u8, + dll: Bzm2Dll, + config: Bzm2DllConfig, + ) -> Result<(), Bzm2ClockError> { + let (control2_reg, control3_reg, control4_reg, _, _) = dll.registers(); + self.uart + .write_local_reg_u8(asic, control3_reg, config.nde_dll & 0x1f) + .await?; + self.uart + .write_local_reg_u8(asic, control4_reg, config.nde_clk & 0x1f) + .await?; + self.uart + .write_local_reg_u8(asic, control2_reg, config.control2()) + .await?; + sleep(Duration::from_millis(1)).await; + Ok(()) + } + + pub async fn set_dll_duty_cycle( + &mut self, + asic: u8, + dll: Bzm2Dll, + duty_cycle: u8, + ) -> Result { + let config = Bzm2DllConfig::from_duty_cycle(duty_cycle)?; + self.program_dll(asic, dll, config).await?; + Ok(config) + } + + pub async fn enable_dll(&mut self, asic: u8, dll: Bzm2Dll) -> Result<(), Bzm2ClockError> { + let (_, _, _, control5_reg, _) = dll.registers(); + let value = self.uart.read_local_reg_u8(asic, control5_reg).await?; + self.uart + .write_local_reg_u8(asic, control5_reg, value | 0x1) + .await?; + let value = self.uart.read_local_reg_u8(asic, control5_reg).await?; + self.uart + .write_local_reg_u8(asic, control5_reg, value | (0x1 << 2)) + .await?; + Ok(()) + } + + pub async fn disable_dll(&mut self, asic: u8, dll: Bzm2Dll) -> Result<(), Bzm2ClockError> { + let (_, _, _, control5_reg, _) = dll.registers(); + self.uart.write_local_reg_u8(asic, control5_reg, 0).await?; + Ok(()) + } + + pub async fn wait_for_dll_lock( + &mut self, + asic: u8, + dll: Bzm2Dll, + timeout: Duration, + poll_interval: Duration, + ) -> Result { + let (control2_reg, _, _, control5_reg, _) = dll.registers(); + let control2 = self.uart.read_local_reg_u8(asic, control2_reg).await?; + if (control2 & 0x2) != 0 { + sleep(Duration::from_millis(10)).await; + return self.read_dll_status(asic, dll).await; + } + + let start = Instant::now(); + + loop { + let last_control = self.uart.read_local_reg_u8(asic, control5_reg).await?; + let status = self.read_dll_status(asic, dll).await?; + if status.locked { + return Ok(status); + } + if start.elapsed() >= timeout { + return Err(Bzm2ClockError::DllLockTimeout { + asic, + dll, + last_control, + }); + } + sleep(poll_interval).await; + } + } + + pub async fn ensure_dll_fincon_valid( + &mut self, + asic: u8, + dll: Bzm2Dll, + ) -> Result { + let status = self.read_dll_status(asic, dll).await?; + if !status.fincon_valid { + return Err(Bzm2ClockError::InvalidDllFincon { + asic, + dll, + fincon: status.fincon, + }); + } + Ok(status) + } + + pub async fn configure_and_lock_dll( + &mut self, + asic: u8, + dll: Bzm2Dll, + duty_cycle: u8, + timeout: Duration, + ) -> Result<(Bzm2DllConfig, Bzm2DllStatus), Bzm2ClockError> { + let config = self.set_dll_duty_cycle(asic, dll, duty_cycle).await?; + self.enable_dll(asic, dll).await?; + self.wait_for_dll_lock(asic, dll, timeout, Duration::from_millis(10)) + .await?; + let status = self.ensure_dll_fincon_valid(asic, dll).await?; + Ok((config, status)) + } + + pub async fn read_dll_status( + &mut self, + asic: u8, + dll: Bzm2Dll, + ) -> Result { + let (control2_reg, _, _, control5_reg, coarse_reg) = dll.registers(); + let control2 = self.uart.read_local_reg_u8(asic, control2_reg).await?; + let control5 = self.uart.read_local_reg_u8(asic, control5_reg).await?; + let coarse_raw = self.uart.read_local_reg_u8(asic, coarse_reg).await?; + let fincon = self + .uart + .read_local_reg_u8(asic, dll.fincon_register()) + .await?; + + Ok(Bzm2DllStatus { + dll, + control2, + control5, + coarsecon: (coarse_raw >> 5) & 0x7, + fincon, + freeze_valid: (control2 & 0x2) != 0, + locked: (control5 & 0x2) != 0, + fincon_valid: fincon_is_valid(fincon), + }) + } + + pub async fn debug_report(&mut self, asic: u8) -> Result { + Ok(Bzm2ClockDebugReport { + asic, + pll0: self.read_pll_status(asic, Bzm2Pll::Pll0).await?, + pll1: self.read_pll_status(asic, Bzm2Pll::Pll1).await?, + dll0: self.read_dll_status(asic, Bzm2Dll::Dll0).await?, + dll1: self.read_dll_status(asic, Bzm2Dll::Dll1).await?, + }) + } +} + +#[derive(Debug, thiserror::Error)] +pub enum Bzm2ClockError { + #[error(transparent)] + Uart(#[from] Bzm2UartError), + + #[error("invalid desired PLL frequency {0} MHz")] + InvalidFrequency(f32), + + #[error("invalid PLL post divider {0}")] + InvalidPostDivider(u8), + + #[error("unsupported DLL duty cycle {0}; supported values are 25, 50, 55, 60, 75")] + InvalidDllDutyCycle(u8), + + #[error( + "PLL {pll:?} on ASIC {asic} did not lock before timeout; last enable value {last_enable:#x}" + )] + PllLockTimeout { + asic: u8, + pll: Bzm2Pll, + last_enable: u32, + }, + + #[error( + "DLL {dll:?} on ASIC {asic} did not lock before timeout; last control value {last_control:#x}" + )] + DllLockTimeout { + asic: u8, + dll: Bzm2Dll, + last_control: u8, + }, + + #[error("DLL {dll:?} on ASIC {asic} reported invalid fincon {fincon:#x}")] + InvalidDllFincon { asic: u8, dll: Bzm2Dll, fincon: u8 }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Bzm2Pll { + Pll0, + Pll1, +} + +impl Bzm2Pll { + pub(crate) fn register_block(self) -> (u8, u8, u8, u8) { + match self { + Self::Pll0 => ( + LOCAL_REG_PLL_POSTDIV, + LOCAL_REG_PLL_FBDIV, + LOCAL_REG_PLL_ENABLE, + LOCAL_REG_PLL_MISC, + ), + Self::Pll1 => ( + LOCAL_REG_PLL1_POSTDIV, + LOCAL_REG_PLL1_FBDIV, + LOCAL_REG_PLL1_ENABLE, + LOCAL_REG_PLL1_MISC, + ), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Bzm2Dll { + Dll0, + Dll1, +} + +impl Bzm2Dll { + pub(crate) fn registers(self) -> (u8, u8, u8, u8, u8) { + match self { + Self::Dll0 => ( + LOCAL_REG_CKDCCR_2_0, + LOCAL_REG_CKDCCR_3_0, + LOCAL_REG_CKDCCR_4_0, + LOCAL_REG_CKDCCR_5_0, + LOCAL_REG_CKDLLR_0_0, + ), + Self::Dll1 => ( + LOCAL_REG_CKDCCR_2_1, + LOCAL_REG_CKDCCR_3_1, + LOCAL_REG_CKDCCR_4_1, + LOCAL_REG_CKDCCR_5_1, + LOCAL_REG_CKDLLR_0_1, + ), + } + } + + pub(crate) fn fincon_register(self) -> u8 { + match self { + Self::Dll0 => LOCAL_REG_CKDLLR_1_0, + Self::Dll1 => LOCAL_REG_CKDLLR_1_1, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct Bzm2PllConfig { + pub frequency_mhz: f32, + pub post1_divider: u8, + pub ref_divider: u8, + pub post2_divider: u8, + pub feedback_divider: u16, + pub packed_post_divider: u32, +} + +impl Bzm2PllConfig { + pub fn from_target_frequency( + frequency_mhz: f32, + post1_divider: u8, + ) -> Result { + if !frequency_mhz.is_finite() || frequency_mhz <= 0.0 { + return Err(Bzm2ClockError::InvalidFrequency(frequency_mhz)); + } + if post1_divider > 7 { + return Err(Bzm2ClockError::InvalidPostDivider(post1_divider)); + } + + let feedback = REF_DIVIDER as f32 + * (post1_divider as f32 + 1.0) + * (POST2_DIVIDER as f32 + 1.0) + * frequency_mhz + / REF_CLK_MHZ; + let feedback_divider = round_legacy(feedback); + let packed_post_divider = (1u32 << 12) + | ((POST2_DIVIDER as u32) << 9) + | ((post1_divider as u32) << 6) + | REF_DIVIDER as u32; + + Ok(Self { + frequency_mhz, + post1_divider, + ref_divider: REF_DIVIDER, + post2_divider: POST2_DIVIDER, + feedback_divider, + packed_post_divider, + }) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Bzm2DllConfig { + pub duty_cycle: u8, + pub nde_dll: u8, + pub nde_clk: u8, + pub npi_clk: u8, + pub pibypb: u8, + pub dllfreeze: u8, +} + +impl Bzm2DllConfig { + pub fn from_duty_cycle(duty_cycle: u8) -> Result { + let mut config = Self { + duty_cycle, + nde_dll: 0x1f, + nde_clk: 0x0f, + npi_clk: 0x0, + pibypb: 1, + dllfreeze: 0, + }; + + match duty_cycle { + 50 => {} + 75 => config.nde_clk = 0x17, + 60 => { + config.nde_dll = 0x1d; + config.nde_clk = 0x11; + } + 55 => { + config.nde_dll = 0x1d; + config.nde_clk = 0x0f; + config.npi_clk = 0x4; + } + 25 => config.nde_clk = 0x07, + _ => return Err(Bzm2ClockError::InvalidDllDutyCycle(duty_cycle)), + } + + Ok(config) + } + + fn control2(self) -> u8 { + ((self.npi_clk & 0x7) << 3) | ((self.pibypb & 0x1) << 2) | ((self.dllfreeze & 0x1) << 1) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Bzm2PllStatus { + pub pll: Bzm2Pll, + pub enable_register: u32, + pub misc_register: u32, + pub enabled: bool, + pub locked: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Bzm2DllStatus { + pub dll: Bzm2Dll, + pub control2: u8, + pub control5: u8, + pub coarsecon: u8, + pub fincon: u8, + pub freeze_valid: bool, + pub locked: bool, + pub fincon_valid: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Bzm2ClockDebugReport { + pub asic: u8, + pub pll0: Bzm2PllStatus, + pub pll1: Bzm2PllStatus, + pub dll0: Bzm2DllStatus, + pub dll1: Bzm2DllStatus, +} + +pub(crate) fn fincon_is_valid(fincon: u8) -> bool { + !matches!(fincon & 0xf0, 0xf0 | 0x00) && !matches!(fincon & 0xe0, 0xe0 | 0x00) +} + +fn round_legacy(value: f32) -> u16 { + let truncated = value as u16; + if value - truncated as f32 > 0.5 { + truncated.saturating_add(1) + } else { + truncated + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn pll_divider_rounding_matches_legacy_formula() { + let config = Bzm2PllConfig::from_target_frequency(625.0, 0).unwrap(); + assert_eq!(config.ref_divider, 1); + assert_eq!(config.post1_divider, 0); + assert_eq!(config.post2_divider, 0); + assert_eq!(config.feedback_divider, 12); + assert_eq!(config.packed_post_divider, 0x1001); + } + + #[test] + fn pll_divider_rounding_uses_half_down_legacy_behavior() { + assert_eq!(round_legacy(12.49), 12); + assert_eq!(round_legacy(12.50), 12); + assert_eq!(round_legacy(12.51), 13); + } + + #[test] + fn dll_duty_cycle_matches_legacy_presets() { + let duty_55 = Bzm2DllConfig::from_duty_cycle(55).unwrap(); + assert_eq!(duty_55.nde_dll, 0x1d); + assert_eq!(duty_55.nde_clk, 0x0f); + assert_eq!(duty_55.npi_clk, 0x4); + + let duty_75 = Bzm2DllConfig::from_duty_cycle(75).unwrap(); + assert_eq!(duty_75.nde_dll, 0x1f); + assert_eq!(duty_75.nde_clk, 0x17); + assert_eq!(duty_75.npi_clk, 0x0); + } + + #[test] + fn fincon_validation_matches_legacy_rules() { + assert!(fincon_is_valid(0x9c)); + assert!(!fincon_is_valid(0xf4)); + assert!(!fincon_is_valid(0x0f)); + assert!(!fincon_is_valid(0xe1)); + } +} diff --git a/mujina-miner/src/asic/bzm2/mod.rs b/mujina-miner/src/asic/bzm2/mod.rs new file mode 100644 index 00000000..8c606d91 --- /dev/null +++ b/mujina-miner/src/asic/bzm2/mod.rs @@ -0,0 +1,18 @@ +pub mod clock; +pub mod protocol; +pub mod thread; +pub mod uart; + +pub use clock::{ + Bzm2ClockController, Bzm2ClockDebugReport, Bzm2ClockError, Bzm2Dll, Bzm2DllConfig, + Bzm2DllStatus, Bzm2Pll, Bzm2PllConfig, Bzm2PllStatus, +}; +pub use protocol::Bzm2EngineLayout; +pub use thread::{ + Bzm2AsicRuntimeMetrics, Bzm2PllRuntimeMetrics, Bzm2Thread, Bzm2ThreadConfig, Bzm2ThreadHandle, + Bzm2ThreadRuntimeMetrics, +}; +pub use uart::{ + BROADCAST_GROUP_ASIC, Bzm2DiscoveredEngineMap, Bzm2DtsVsConfig, Bzm2EngineCoordinate, + Bzm2UartController, Bzm2UartError, DEFAULT_ASIC_ID, DEFAULT_DTS_VS_QUERY_TIMEOUT, NOTCH_REG, +}; diff --git a/mujina-miner/src/asic/bzm2/protocol.rs b/mujina-miner/src/asic/bzm2/protocol.rs new file mode 100644 index 00000000..6d91ae84 --- /dev/null +++ b/mujina-miner/src/asic/bzm2/protocol.rs @@ -0,0 +1,776 @@ +use std::collections::{HashMap, HashSet}; + +pub const OPCODE_UART_WRITEJOB: u8 = 0x0; +pub const OPCODE_UART_READRESULT: u8 = 0x1; +pub const OPCODE_UART_WRITEREG: u8 = 0x2; +pub const OPCODE_UART_READREG: u8 = 0x3; +pub const OPCODE_UART_MULTICAST_WRITE: u8 = 0x4; +pub const OPCODE_UART_DTS_VS: u8 = 0x0d; +pub const OPCODE_UART_LOOPBACK: u8 = 0x0e; +pub const OPCODE_UART_NOOP: u8 = 0x0f; + +pub const BROADCAST_ASIC: u8 = 0xff; +pub const TARGET_BYTE: u8 = 0x08; + +pub const ENGINE_REG_TARGET: u8 = 0x44; +pub const ENGINE_REG_START_NONCE: u8 = 0x3c; +pub const ENGINE_REG_TIMESTAMP_COUNT: u8 = 0x48; +pub const ENGINE_REG_ZEROS_TO_FIND: u8 = 0x49; +pub const ENGINE_REG_END_NONCE: u8 = 0x40; + +pub const DEFAULT_TIMESTAMP_COUNT: u8 = 60; +pub const DEFAULT_NONCE_GAP: u32 = 0x28; +pub const DEFAULT_BOARD_END_NONCE: u32 = 0xffff_ffff; +pub const LOGICAL_ENGINE_ROWS: u8 = 20; +pub const LOGICAL_ENGINE_COLS: u8 = 12; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DtsVsGeneration { + Gen1, + Gen2, +} + +impl DtsVsGeneration { + pub fn from_env_value(raw: &str) -> Option { + match raw.trim() { + "1" | "gen1" | "GEN1" => Some(Self::Gen1), + "2" | "gen2" | "GEN2" => Some(Self::Gen2), + _ => None, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TdmResultFrame { + pub asic: u8, + pub engine_address: u16, + pub status: u8, + pub nonce: u32, + pub sequence_id: u8, + pub reported_time: u8, +} + +impl TdmResultFrame { + pub fn row(self) -> u8 { + (self.engine_address & 0x3f) as u8 + } + + pub fn col(self) -> u8 { + (self.engine_address >> 6) as u8 + } + + pub fn logical_engine_id(self) -> Option { + logical_engine_id(self.row(), self.col()) + } + + pub fn nonce_valid(self) -> bool { + (self.status & 0x8) != 0 + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TdmRegisterFrame { + pub asic: u8, + pub data: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TdmNoopFrame { + pub asic: u8, + pub data: [u8; 3], +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TdmDtsVsGen1Frame { + pub asic: u8, + pub voltage: u16, + pub voltage_enabled: bool, + pub thermal_tune_code: u8, + pub thermal_validity: bool, + pub thermal_enabled: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TdmDtsVsGen2Frame { + pub asic: u8, + pub ch0_voltage: u16, + pub ch1_voltage: u16, + pub ch2_voltage: u16, + pub voltage_shutdown_status: bool, + pub voltage_enabled: bool, + pub thermal_tune_code: u16, + pub thermal_trip_status: bool, + pub thermal_fault: bool, + pub thermal_validity: bool, + pub thermal_enabled: bool, + pub voltage_fault: bool, + pub dll0_lock: bool, + pub dll1_lock: bool, + pub pll_lock: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TdmDtsVsFrame { + Gen1(TdmDtsVsGen1Frame), + Gen2(TdmDtsVsGen2Frame), +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum TdmFrame { + Result(TdmResultFrame), + Register(TdmRegisterFrame), + DtsVs(TdmDtsVsFrame), + Noop(TdmNoopFrame), +} + +pub struct TdmFrameParser { + dts_vs_generation: DtsVsGeneration, + buffer: Vec, + expected_read_lengths: HashMap, +} + +impl Default for TdmFrameParser { + fn default() -> Self { + Self::new(DtsVsGeneration::Gen2) + } +} + +impl TdmFrameParser { + pub fn new(dts_vs_generation: DtsVsGeneration) -> Self { + Self { + dts_vs_generation, + buffer: Vec::new(), + expected_read_lengths: HashMap::new(), + } + } + + pub fn expect_read_register_bytes(&mut self, asic: u8, count: usize) { + self.expected_read_lengths.insert(asic, count); + } + + pub fn push(&mut self, bytes: &[u8]) -> Vec { + self.buffer.extend_from_slice(bytes); + + let mut frames = Vec::new(); + let mut cursor = 0usize; + + while self.buffer.len().saturating_sub(cursor) >= 2 { + let asic = self.buffer[cursor]; + let opcode = self.buffer[cursor + 1]; + + // Resync heuristic: an id this large in the asic position is almost + // always line noise in the supported small-id chains, so skip it as + // a stray byte. DTS/VS frames are exempt: they carry the + // thermal-trip / voltage-fault bits that drive the protective + // shutdown and must never be silently dropped by an ad-hoc id bound + // (an operator can also legitimately push ids >= 100 via + // MUJINA_BZM2_ENUM_START_ID). A fault frame that reaches the handler + // is logged loudly there before any shutdown action is taken. + if asic >= 100 && opcode != OPCODE_UART_DTS_VS { + cursor += 1; + continue; + } + + match opcode { + OPCODE_UART_READRESULT => { + if self.buffer.len().saturating_sub(cursor) < 10 { + break; + } + + let payload = &self.buffer[cursor + 2..cursor + 10]; + let header = u16::from_be_bytes([payload[0], payload[1]]); + let engine_address = header & 0x0fff; + let status = (header >> 12) as u8; + let nonce = u32::from_le_bytes(payload[2..6].try_into().unwrap()); + let sequence_id = payload[6]; + let reported_time = payload[7]; + + frames.push(TdmFrame::Result(TdmResultFrame { + asic, + engine_address, + status, + nonce, + sequence_id, + reported_time, + })); + cursor += 10; + } + OPCODE_UART_READREG => { + let Some(&count) = self.expected_read_lengths.get(&asic) else { + // No READREG was requested for this id, so these bytes are + // stray/noise (the long-lived streaming parser never sets + // an expected read length). Resync one byte forward like + // the unknown-opcode arm instead of breaking: a bare + // READREG-looking prefix sitting at cursor 0 would + // otherwise wedge framing forever and grow `self.buffer` + // without bound on every subsequent push. + cursor += 1; + continue; + }; + if self.buffer.len().saturating_sub(cursor) < 2 + count { + break; + } + + frames.push(TdmFrame::Register(TdmRegisterFrame { + asic, + data: self.buffer[cursor + 2..cursor + 2 + count].to_vec(), + })); + self.expected_read_lengths.remove(&asic); + cursor += 2 + count; + } + OPCODE_UART_DTS_VS => { + let payload_len = match self.dts_vs_generation { + DtsVsGeneration::Gen1 => 4, + DtsVsGeneration::Gen2 => 8, + }; + if self.buffer.len().saturating_sub(cursor) < 2 + payload_len { + break; + } + + let payload = &self.buffer[cursor + 2..cursor + 2 + payload_len]; + let frame = match self.dts_vs_generation { + DtsVsGeneration::Gen1 => { + TdmDtsVsFrame::Gen1(parse_dts_vs_gen1(asic, payload)) + } + DtsVsGeneration::Gen2 => { + TdmDtsVsFrame::Gen2(parse_dts_vs_gen2(asic, payload)) + } + }; + frames.push(TdmFrame::DtsVs(frame)); + cursor += 2 + payload_len; + } + OPCODE_UART_NOOP => { + if self.buffer.len().saturating_sub(cursor) < 5 { + break; + } + let data = self.buffer[cursor + 2..cursor + 5].try_into().unwrap(); + frames.push(TdmFrame::Noop(TdmNoopFrame { asic, data })); + cursor += 5; + } + _ => { + cursor += 1; + } + } + } + + if cursor > 0 { + self.buffer.drain(..cursor); + } + + frames + } +} + +#[derive(Default)] +pub struct TdmResultParser { + inner: TdmFrameParser, +} + +impl TdmResultParser { + pub fn push(&mut self, bytes: &[u8]) -> Vec { + self.inner + .push(bytes) + .into_iter() + .filter_map(|frame| match frame { + TdmFrame::Result(result) => Some(result), + _ => None, + }) + .collect() + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Bzm2EngineLayout { + active_coordinates: Vec<(u8, u8)>, + logical_ids_by_address: HashMap, +} + +impl Bzm2EngineLayout { + pub fn from_active_coordinates(coords: I) -> Self + where + I: IntoIterator, + { + let mut active_coordinates = coords + .into_iter() + .filter(|(row, col)| *row < LOGICAL_ENGINE_ROWS && *col < LOGICAL_ENGINE_COLS) + .collect::>(); + active_coordinates.sort_by_key(|(row, col)| (*col, *row)); + active_coordinates.dedup(); + + let logical_ids_by_address = active_coordinates + .iter() + .enumerate() + .map(|(logical_id, (row, col))| (logical_engine_address(*row, *col), logical_id as u16)) + .collect(); + + Self { + active_coordinates, + logical_ids_by_address, + } + } + + pub fn active_coordinates(&self) -> &[(u8, u8)] { + &self.active_coordinates + } + + pub fn active_engine_count(&self) -> usize { + self.active_coordinates.len() + } + + pub fn logical_engine_id(&self, row: u8, col: u8) -> Option { + self.logical_engine_id_from_address(logical_engine_address(row, col)) + } + + pub fn logical_engine_id_from_address(&self, engine_address: u16) -> Option { + self.logical_ids_by_address.get(&engine_address).copied() + } +} + +impl Default for Bzm2EngineLayout { + fn default() -> Self { + Self::from_active_coordinates(default_engine_coordinates()) + } +} + +pub fn encode_write_register(asic: u8, engine_address: u16, offset: u8, value: &[u8]) -> Vec { + let mut bytes = Vec::with_capacity(7 + value.len()); + let header = ((asic as u32) << 24) + | ((OPCODE_UART_WRITEREG as u32) << 20) + | ((engine_address as u32) << 8) + | offset as u32; + + bytes.extend_from_slice(&((7 + value.len()) as u16).to_le_bytes()); + bytes.extend_from_slice(&header.to_be_bytes()); + bytes.push((value.len() as u8).saturating_sub(1)); + bytes.extend_from_slice(value); + bytes +} + +pub fn encode_multicast_write(asic: u8, group: u16, offset: u8, value: &[u8]) -> Vec { + let mut bytes = Vec::with_capacity(7 + value.len()); + let header = ((asic as u32) << 24) + | ((OPCODE_UART_MULTICAST_WRITE as u32) << 20) + | ((group as u32) << 8) + | offset as u32; + + bytes.extend_from_slice(&((7 + value.len()) as u16).to_le_bytes()); + bytes.extend_from_slice(&header.to_be_bytes()); + bytes.push((value.len() as u8).saturating_sub(1)); + bytes.extend_from_slice(value); + bytes +} + +pub fn encode_read_register(asic: u8, engine_address: u16, offset: u8, count: u8) -> Vec { + let mut bytes = Vec::with_capacity(8); + let header = ((asic as u32) << 24) + | ((OPCODE_UART_READREG as u32) << 20) + | ((engine_address as u32) << 8) + | offset as u32; + + bytes.extend_from_slice(&8u16.to_le_bytes()); + bytes.extend_from_slice(&header.to_be_bytes()); + bytes.push(count.saturating_sub(1)); + bytes.push(TARGET_BYTE); + bytes +} + +pub fn encode_write_job( + asic: u8, + engine_address: u16, + midstate: &[u8; 32], + merkle_root_residue: u32, + ntime: u32, + sequence_id: u8, + job_control: u8, +) -> Vec { + let mut bytes = Vec::with_capacity(48); + let header = ((asic as u32) << 24) + | ((OPCODE_UART_WRITEJOB as u32) << 20) + | ((engine_address as u32) << 8) + | 41u32; + + bytes.extend_from_slice(&(48u16).to_le_bytes()); + bytes.extend_from_slice(&header.to_be_bytes()); + bytes.extend_from_slice(midstate); + bytes.extend_from_slice(&merkle_root_residue.to_le_bytes()); + bytes.extend_from_slice(&ntime.to_le_bytes()); + bytes.push(sequence_id); + bytes.push(job_control); + bytes +} + +pub fn encode_read_result_command(asic: u8) -> Vec { + let mut bytes = Vec::with_capacity(4); + let header = ((asic as u16) << 8) | ((OPCODE_UART_READRESULT as u16) << 4); + bytes.extend_from_slice(&4u16.to_le_bytes()); + bytes.extend_from_slice(&header.to_be_bytes()); + bytes +} + +pub fn encode_noop(asic: u8) -> Vec { + let mut bytes = Vec::with_capacity(4); + let header = ((asic as u16) << 8) | ((OPCODE_UART_NOOP as u16) << 4); + bytes.extend_from_slice(&4u16.to_le_bytes()); + bytes.extend_from_slice(&header.to_be_bytes()); + bytes +} + +pub fn encode_loopback(asic: u8, data: &[u8]) -> Vec { + let mut bytes = Vec::with_capacity(5 + data.len()); + let header = ((asic as u16) << 8) | ((OPCODE_UART_LOOPBACK as u16) << 4); + bytes.extend_from_slice(&((5 + data.len()) as u16).to_le_bytes()); + bytes.extend_from_slice(&header.to_be_bytes()); + bytes.push((data.len() as u8).saturating_sub(1)); + bytes.extend_from_slice(data); + bytes +} + +pub fn logical_engine_address(row: u8, col: u8) -> u16 { + ((col as u16) << 6) | row as u16 +} + +pub fn logical_engine_id(row: u8, col: u8) -> Option { + if row >= LOGICAL_ENGINE_ROWS || col >= LOGICAL_ENGINE_COLS { + return None; + } + if default_excluded_engines().contains(&(row, col)) { + return None; + } + + let excluded = default_excluded_engines(); + let mut id = 0u16; + for c in 0..LOGICAL_ENGINE_COLS { + for r in 0..LOGICAL_ENGINE_ROWS { + if excluded.contains(&(r, c)) { + continue; + } + if r == row && c == col { + return Some(id); + } + id += 1; + } + } + + None +} + +pub fn default_excluded_engines() -> HashSet<(u8, u8)> { + HashSet::from([(0, 4), (0, 5), (19, 5), (19, 11)]) +} + +pub fn physical_engine_coordinates() -> Vec<(u8, u8)> { + let mut coords = Vec::new(); + for col in 0..LOGICAL_ENGINE_COLS { + for row in 0..LOGICAL_ENGINE_ROWS { + coords.push((row, col)); + } + } + coords +} + +pub fn default_engine_coordinates() -> Vec<(u8, u8)> { + let excluded = default_excluded_engines(); + let mut coords = Vec::new(); + for col in 0..LOGICAL_ENGINE_COLS { + for row in 0..LOGICAL_ENGINE_ROWS { + if excluded.contains(&(row, col)) { + continue; + } + coords.push((row, col)); + } + } + coords +} + +pub fn leading_zero_threshold(target: bitcoin::pow::Target) -> u8 { + let bytes = target.to_be_bytes(); + let mut zeros = 0u8; + + 'outer: for byte in bytes { + if byte == 0 { + zeros = zeros.saturating_add(8); + continue; + } + + for bit in (0..8).rev() { + if (byte & (1 << bit)) == 0 { + zeros = zeros.saturating_add(1); + } else { + break 'outer; + } + } + break; + } + + zeros.clamp(32, 64) +} + +fn parse_dts_vs_gen1(asic: u8, payload: &[u8]) -> TdmDtsVsGen1Frame { + let raw = u32::from_be_bytes(payload.try_into().unwrap()); + let bytes = raw.to_le_bytes(); + let voltage = (((bytes[1] & 0x07) as u16) << 8) | bytes[0] as u16; + + TdmDtsVsGen1Frame { + asic, + voltage, + voltage_enabled: (bytes[1] & 0x80) != 0, + thermal_tune_code: bytes[2], + thermal_validity: (bytes[3] & 0x40) != 0, + thermal_enabled: (bytes[3] & 0x80) != 0, + } +} + +fn parse_dts_vs_gen2(asic: u8, payload: &[u8]) -> TdmDtsVsGen2Frame { + let raw = u64::from_be_bytes(payload.try_into().unwrap()); + let bytes = raw.to_le_bytes(); + + TdmDtsVsGen2Frame { + asic, + ch0_voltage: (((bytes[2] & 0x3f) as u16) << 8) | bytes[3] as u16, + ch1_voltage: ((bytes[4] as u16) << 6) | ((bytes[5] & 0x3f) as u16), + ch2_voltage: (((bytes[7] & 0x0f) as u16) << 10) + | ((bytes[6] as u16) << 2) + | (((bytes[5] >> 6) & 0x03) as u16), + voltage_shutdown_status: (bytes[2] & 0x40) != 0, + voltage_enabled: (bytes[2] & 0x80) != 0, + thermal_tune_code: (((bytes[0] & 0x0f) as u16) << 8) | bytes[1] as u16, + thermal_trip_status: (bytes[0] & 0x10) != 0, + thermal_fault: (bytes[0] & 0x20) != 0, + thermal_validity: (bytes[0] & 0x40) != 0, + thermal_enabled: (bytes[0] & 0x80) != 0, + voltage_fault: (bytes[7] & 0x10) != 0, + dll0_lock: (bytes[7] & 0x20) != 0, + dll1_lock: (bytes[7] & 0x40) != 0, + pll_lock: (bytes[7] & 0x80) != 0, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn write_register_encoder_matches_legacy_wire_format() { + let encoded = encode_write_register(0x12, 0x0345, 0x67, &[0x78, 0x56, 0x34, 0x12]); + assert_eq!( + encoded, + vec![ + 0x0b, 0x00, 0x12, 0x23, 0x45, 0x67, 0x03, 0x78, 0x56, 0x34, 0x12 + ] + ); + } + + #[test] + fn read_register_encoder_matches_legacy_wire_format() { + let encoded = encode_read_register(0x12, 0x0345, 0x67, 4); + assert_eq!( + encoded, + vec![0x08, 0x00, 0x12, 0x33, 0x45, 0x67, 0x03, TARGET_BYTE] + ); + } + + #[test] + fn noop_and_loopback_encoders_match_legacy_wire_format() { + assert_eq!(encode_noop(0x12), vec![0x04, 0x00, 0x12, 0xf0]); + assert_eq!( + encode_loopback(0x12, &[0xaa, 0xbb, 0xcc]), + vec![0x08, 0x00, 0x12, 0xe0, 0x02, 0xaa, 0xbb, 0xcc] + ); + } + + #[test] + fn parser_decodes_tdm_result() { + let mut parser = TdmResultParser::default(); + let frame = [ + 0x02, + OPCODE_UART_READRESULT, + 0x41, + 0x23, + 0x78, + 0x56, + 0x34, + 0x12, + 0x05, + 0x09, + ]; + + let parsed = parser.push(&frame); + assert_eq!(parsed.len(), 1); + assert_eq!(parsed[0].asic, 0x02); + assert_eq!(parsed[0].status, 0x4); + assert_eq!(parsed[0].engine_address, 0x0123); + assert_eq!(parsed[0].nonce, 0x1234_5678); + assert_eq!(parsed[0].sequence_id, 0x05); + assert_eq!(parsed[0].reported_time, 0x09); + } + + #[test] + fn parser_decodes_gen2_dts_vs() { + let mut parser = TdmFrameParser::new(DtsVsGeneration::Gen2); + let raw = [ + 0x02, + OPCODE_UART_DTS_VS, + 0xD5, + 0xAB, + 0x34, + 0x12, + 0x45, + 0x96, + 0xA9, + 0xF7, + ]; + let parsed = parser.push(&raw); + assert_eq!(parsed.len(), 1); + match &parsed[0] { + TdmFrame::DtsVs(TdmDtsVsFrame::Gen2(frame)) => { + assert_eq!(frame.asic, 0x02); + assert_eq!(frame.thermal_tune_code, 0x07A9); + assert!(frame.thermal_trip_status); + assert!(frame.thermal_fault); + assert!(frame.thermal_validity); + assert!(frame.thermal_enabled); + assert_eq!(frame.ch0_voltage, 0x1645); + assert!(!frame.voltage_shutdown_status); + assert!(frame.voltage_enabled); + assert_eq!(frame.ch1_voltage, 0x04B4); + assert_eq!(frame.ch2_voltage, 0x16AC); + assert!(frame.voltage_fault); + assert!(!frame.dll0_lock); + assert!(frame.dll1_lock); + assert!(frame.pll_lock); + } + other => panic!("unexpected frame: {other:?}"), + } + } + + #[test] + fn parser_decodes_gen1_dts_vs() { + let mut parser = TdmFrameParser::new(DtsVsGeneration::Gen1); + let raw = [0x02, OPCODE_UART_DTS_VS, 0x91, 0xab, 0xcd, 0x45]; + let parsed = parser.push(&raw); + assert_eq!(parsed.len(), 1); + match &parsed[0] { + TdmFrame::DtsVs(TdmDtsVsFrame::Gen1(frame)) => { + assert_eq!(frame.asic, 0x02); + assert_eq!(frame.voltage, 0x545); + assert!(frame.voltage_enabled); + assert_eq!(frame.thermal_tune_code, 0xab); + assert!(!frame.thermal_validity); + assert!(frame.thermal_enabled); + } + other => panic!("unexpected frame: {other:?}"), + } + } + + #[test] + fn parser_decodes_readreg_and_noop() { + let mut parser = TdmFrameParser::new(DtsVsGeneration::Gen2); + parser.expect_read_register_bytes(0x03, 4); + let parsed = parser.push(&[ + 0x03, + OPCODE_UART_READREG, + 0x78, + 0x56, + 0x34, + 0x12, + 0x01, + OPCODE_UART_NOOP, + 0xaa, + 0xbb, + 0xcc, + ]); + assert_eq!(parsed.len(), 2); + match &parsed[0] { + TdmFrame::Register(frame) => assert_eq!(frame.data, vec![0x78, 0x56, 0x34, 0x12]), + other => panic!("unexpected frame: {other:?}"), + } + match &parsed[1] { + TdmFrame::Noop(frame) => assert_eq!(frame.data, [0xaa, 0xbb, 0xcc]), + other => panic!("unexpected frame: {other:?}"), + } + } + + #[test] + fn parser_keeps_dts_vs_trip_frame_from_high_asic_id() { + let mut parser = TdmFrameParser::new(DtsVsGeneration::Gen2); + // A Gen2 DTS/VS frame from asic id 100 with the thermal-trip bit set. + // The `asic >= 100` resync heuristic must NOT drop it: silently + // discarding it would disable over-temp protection for that device. + let parsed = parser.push(&[100, OPCODE_UART_DTS_VS, 0, 0, 0, 0, 0, 0, 0, 0x10]); + assert_eq!(parsed.len(), 1); + match &parsed[0] { + TdmFrame::DtsVs(TdmDtsVsFrame::Gen2(frame)) => { + assert_eq!(frame.asic, 100); + assert!(frame.thermal_trip_status); + } + other => panic!("unexpected frame: {other:?}"), + } + } + + #[test] + fn parser_resyncs_after_unknown_prefix_and_partial_frames() { + let mut parser = TdmFrameParser::new(DtsVsGeneration::Gen2); + parser.expect_read_register_bytes(0x03, 4); + + let first = parser.push(&[0xfe, 0xaa, 0x03, OPCODE_UART_READREG, 0x78, 0x56]); + assert!(first.is_empty()); + + let second = parser.push(&[0x34, 0x12, 0x01, OPCODE_UART_NOOP, 0xaa, 0xbb, 0xcc]); + assert_eq!(second.len(), 2); + match &second[0] { + TdmFrame::Register(frame) => assert_eq!(frame.data, vec![0x78, 0x56, 0x34, 0x12]), + other => panic!("unexpected frame: {other:?}"), + } + match &second[1] { + TdmFrame::Noop(frame) => assert_eq!(frame.data, [0xaa, 0xbb, 0xcc]), + other => panic!("unexpected frame: {other:?}"), + } + } + + #[test] + fn parser_bounds_buffer_and_recovers_on_readreg_without_pending_count() { + let mut parser = TdmFrameParser::new(DtsVsGeneration::Gen2); + + // A READREG-looking prefix that never has a pending read length is pure + // line noise on the long-lived streaming parser. Before the fix this hit + // a no-count `break` at cursor 0, wedging framing and growing the buffer + // by two bytes on every push. Feed it many times and confirm the buffer + // stays bounded rather than accumulating ~2000 bytes. + for _ in 0..1000 { + assert!(parser.push(&[0x02, OPCODE_UART_READREG]).is_empty()); + } + assert!( + parser.buffer.len() <= 4, + "buffer grew unbounded on malformed READREG noise: {} bytes", + parser.buffer.len() + ); + + // Framing is not wedged: a subsequent well-formed NOOP frame is still + // decoded once the noise resyncs forward. + let recovered = parser.push(&[0x05, OPCODE_UART_NOOP, 0xaa, 0xbb, 0xcc]); + assert_eq!(recovered.len(), 1); + match &recovered[0] { + TdmFrame::Noop(frame) => assert_eq!(frame.data, [0xaa, 0xbb, 0xcc]), + other => panic!("unexpected frame: {other:?}"), + } + } + + #[test] + fn command_encoders_cover_all_uart_opcodes() { + let writereg = encode_write_register(1, 2, 3, &[0x44]); + let writejob = encode_write_job(1, 2, &[0u8; 32], 4, 5, 6, 7); + let readreg = encode_read_register(1, 2, 3, 4); + let multicast = encode_multicast_write(1, 2, 3, &[0x44]); + let readresult = encode_read_result_command(1); + let noop = encode_noop(1); + let loopback = encode_loopback(1, &[0xaa, 0xbb]); + + assert_eq!(writereg[3] >> 4, OPCODE_UART_WRITEREG); + assert_eq!(writejob[3] >> 4, OPCODE_UART_WRITEJOB); + assert_eq!(readreg[3] >> 4, OPCODE_UART_READREG); + assert_eq!(multicast[3] >> 4, OPCODE_UART_MULTICAST_WRITE); + assert_eq!(readresult[3] >> 4, OPCODE_UART_READRESULT); + assert_eq!(noop[3] >> 4, OPCODE_UART_NOOP); + assert_eq!(loopback[3] >> 4, OPCODE_UART_LOOPBACK); + } +} diff --git a/mujina-miner/src/asic/bzm2/thread.rs b/mujina-miner/src/asic/bzm2/thread.rs new file mode 100644 index 00000000..8b0a48e3 --- /dev/null +++ b/mujina-miner/src/asic/bzm2/thread.rs @@ -0,0 +1,2289 @@ +use std::collections::{BTreeMap, HashMap}; +use std::path::Path; +use std::sync::{Arc, RwLock}; +use std::time::{Duration, Instant}; + +use async_trait::async_trait; +use bitcoin::block::Header as BlockHeader; +use bitcoin::consensus::serialize; +use bitcoin::hashes::{HashEngine, sha256}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::sync::{mpsc, oneshot}; + +use crate::asic::hash_thread::{ + HashTask, HashThread, HashThreadCapabilities, HashThreadError, HashThreadEvent, + HashThreadPowerReading, HashThreadStatus, HashThreadTelemetryUpdate, + HashThreadTemperatureReading, Share, +}; +use crate::job_source::{GeneralPurposeBits, MerkleRootKind}; +use crate::tracing::prelude::*; +use crate::transport::serial::{SerialControl, SerialReader, SerialWriter}; +use crate::types::{Difficulty, HashRate, HashrateEstimator, Work}; + +use super::clock::{ + Bzm2ClockDebugReport, Bzm2Dll, Bzm2DllStatus, Bzm2Pll, Bzm2PllStatus, fincon_is_valid, +}; +use super::protocol::{ + self, BROADCAST_ASIC, Bzm2EngineLayout, DEFAULT_BOARD_END_NONCE, DEFAULT_NONCE_GAP, + DEFAULT_TIMESTAMP_COUNT, DtsVsGeneration, ENGINE_REG_END_NONCE, ENGINE_REG_START_NONCE, + ENGINE_REG_TARGET, ENGINE_REG_TIMESTAMP_COUNT, ENGINE_REG_ZEROS_TO_FIND, OPCODE_UART_LOOPBACK, + OPCODE_UART_NOOP, OPCODE_UART_READREG, TdmDtsVsFrame, TdmFrame, TdmFrameParser, + encode_loopback, encode_noop, encode_read_register, encode_write_job, encode_write_register, + leading_zero_threshold, logical_engine_address, +}; +use super::uart::{ + Bzm2DiscoveredEngineMap, Bzm2DtsVsConfig, DEFAULT_DTS_VS_QUERY_TIMEOUT, + configure_dts_vs_stream, discover_engine_map_stream, +}; + +const RUNTIME_MEASUREMENT_WINDOW: Duration = Duration::from_secs(5 * 60); +// Legacy source treats rows 0-9 as the bottom stack (PLL0) and rows 10-19 as +// the top stack (PLL1). +const PLL_STACK_SPLIT_ROW: u8 = 10; +const TIMESTAMP_COUNT_AUTO_CLOCK_UNGATE: u8 = 0x80; +/// Bound for a single diagnostic UART read. The actor is one task, so a silent +/// or short-answering chip must not wedge it (including a pending `Shutdown`) on +/// an unbounded `read_exact`; on expiry the diagnostic fails instead of hanging. +const DIAGNOSTIC_READ_TIMEOUT: Duration = Duration::from_secs(2); + +pub struct Bzm2Thread { + name: String, + command_tx: mpsc::Sender, + event_rx: Option>, + capabilities: HashThreadCapabilities, + status: Arc>, +} + +impl Bzm2Thread { + pub fn new( + name: String, + reader: SerialReader, + writer: SerialWriter, + control: SerialControl, + config: Bzm2ThreadConfig, + ) -> Self { + let (command_tx, command_rx) = mpsc::channel(16); + let (event_tx, event_rx) = mpsc::channel(64); + let status = Arc::new(RwLock::new(HashThreadStatus::default())); + let status_clone = Arc::clone(&status); + + tokio::spawn(async move { + bzm2_thread_actor( + command_rx, + event_tx, + status_clone, + reader, + writer, + control, + config, + ) + .await; + }); + + Self { + name, + command_tx, + event_rx: Some(event_rx), + capabilities: HashThreadCapabilities::default(), + status, + } + } + + pub fn shutdown_handle(&self) -> Bzm2ThreadHandle { + Bzm2ThreadHandle { + command_tx: self.command_tx.clone(), + } + } +} + +#[async_trait] +impl HashThread for Bzm2Thread { + fn name(&self) -> &str { + &self.name + } + + fn capabilities(&self) -> &HashThreadCapabilities { + &self.capabilities + } + + async fn configure(&mut self) -> anyhow::Result<()> { + self.command_tx + .send(ThreadCommand::Configure) + .await + .map_err(|_| HashThreadError::ChannelClosed("command channel closed".into()))?; + Ok(()) + } + + async fn update_task(&mut self, new_task: HashTask) -> anyhow::Result> { + let (response_tx, response_rx) = oneshot::channel(); + self.command_tx + .send(ThreadCommand::UpdateTask { + new_task, + response_tx, + }) + .await + .map_err(|_| HashThreadError::ChannelClosed("command channel closed".into()))?; + response_rx + .await + .map_err(|_| HashThreadError::WorkAssignmentFailed("thread dropped response".into()))? + .map_err(Into::into) + } + + async fn replace_task(&mut self, new_task: HashTask) -> anyhow::Result> { + let (response_tx, response_rx) = oneshot::channel(); + self.command_tx + .send(ThreadCommand::ReplaceTask { + new_task, + response_tx, + }) + .await + .map_err(|_| HashThreadError::ChannelClosed("command channel closed".into()))?; + response_rx + .await + .map_err(|_| HashThreadError::WorkAssignmentFailed("thread dropped response".into()))? + .map_err(Into::into) + } + + async fn go_idle(&mut self) -> anyhow::Result> { + let (response_tx, response_rx) = oneshot::channel(); + self.command_tx + .send(ThreadCommand::GoIdle { response_tx }) + .await + .map_err(|_| HashThreadError::ChannelClosed("command channel closed".into()))?; + response_rx + .await + .map_err(|_| HashThreadError::WorkAssignmentFailed("thread dropped response".into()))? + .map_err(Into::into) + } + + fn take_event_receiver(&mut self) -> Option> { + self.event_rx.take() + } + + fn status(&self) -> HashThreadStatus { + self.status.read().unwrap().clone() + } +} + +#[derive(Debug, Clone)] +pub struct Bzm2ThreadConfig { + pub serial_path: String, + pub baud_rate: u32, + pub timestamp_count: u8, + pub nonce_gap: u32, + pub dispatch_interval: Duration, + pub nominal_hashrate_ths: f64, + pub dts_vs_generation: DtsVsGeneration, +} + +impl Bzm2ThreadConfig { + pub fn new(serial_path: String, baud_rate: u32) -> Self { + Self { + serial_path, + baud_rate, + timestamp_count: DEFAULT_TIMESTAMP_COUNT, + nonce_gap: DEFAULT_NONCE_GAP, + dispatch_interval: Duration::from_millis(500), + nominal_hashrate_ths: 40.0, + dts_vs_generation: DtsVsGeneration::Gen2, + } + } +} + +#[derive(Debug, Clone, Default, PartialEq)] +pub struct Bzm2PllRuntimeMetrics { + pub throughput_hs: Option, + pub scheduler_share_count: u64, +} + +#[derive(Debug, Clone, Default, PartialEq)] +pub struct Bzm2AsicRuntimeMetrics { + pub asic: u8, + pub throughput_hs: Option, + pub scheduler_share_count: u64, + pub plls: [Bzm2PllRuntimeMetrics; 2], +} + +#[derive(Debug, Clone, Default, PartialEq)] +pub struct Bzm2ThreadRuntimeMetrics { + pub throughput_hs: Option, + pub asics: Vec, +} + +#[derive(Clone)] +pub struct Bzm2ThreadHandle { + command_tx: mpsc::Sender, +} + +impl Bzm2ThreadHandle { + pub fn shutdown(&self) { + let _ = self.command_tx.try_send(ThreadCommand::Shutdown); + } + + pub async fn noop(&self, asic: u8) -> Result<[u8; 3], HashThreadError> { + let (response_tx, response_rx) = oneshot::channel(); + self.command_tx + .send(ThreadCommand::QueryNoop { asic, response_tx }) + .await + .map_err(|_| HashThreadError::ChannelClosed("command channel closed".into()))?; + response_rx + .await + .map_err(|_| HashThreadError::DiagnosticsFailed("thread dropped response".into()))? + } + + pub async fn loopback(&self, asic: u8, payload: Vec) -> Result, HashThreadError> { + let (response_tx, response_rx) = oneshot::channel(); + self.command_tx + .send(ThreadCommand::QueryLoopback { + asic, + payload, + response_tx, + }) + .await + .map_err(|_| HashThreadError::ChannelClosed("command channel closed".into()))?; + response_rx + .await + .map_err(|_| HashThreadError::DiagnosticsFailed("thread dropped response".into()))? + } + + pub async fn read_register( + &self, + asic: u8, + engine_address: u16, + offset: u8, + count: u8, + ) -> Result, HashThreadError> { + let (response_tx, response_rx) = oneshot::channel(); + self.command_tx + .send(ThreadCommand::ReadRegister { + asic, + engine_address, + offset, + count, + response_tx, + }) + .await + .map_err(|_| HashThreadError::ChannelClosed("command channel closed".into()))?; + response_rx + .await + .map_err(|_| HashThreadError::DiagnosticsFailed("thread dropped response".into()))? + } + + pub async fn write_register( + &self, + asic: u8, + engine_address: u16, + offset: u8, + value: Vec, + ) -> Result<(), HashThreadError> { + let (response_tx, response_rx) = oneshot::channel(); + self.command_tx + .send(ThreadCommand::WriteRegister { + asic, + engine_address, + offset, + value, + response_tx, + }) + .await + .map_err(|_| HashThreadError::ChannelClosed("command channel closed".into()))?; + response_rx + .await + .map_err(|_| HashThreadError::DiagnosticsFailed("thread dropped response".into()))? + } + + pub async fn query_dts_vs( + &self, + asic: u8, + ) -> Result { + let (response_tx, response_rx) = oneshot::channel(); + self.command_tx + .send(ThreadCommand::QueryDtsVs { asic, response_tx }) + .await + .map_err(|_| HashThreadError::ChannelClosed("command channel closed".into()))?; + response_rx + .await + .map_err(|_| HashThreadError::TelemetryQueryFailed("thread dropped response".into()))? + } + + pub async fn clock_report(&self, asic: u8) -> Result { + let (response_tx, response_rx) = oneshot::channel(); + self.command_tx + .send(ThreadCommand::QueryClockReport { asic, response_tx }) + .await + .map_err(|_| HashThreadError::ChannelClosed("command channel closed".into()))?; + response_rx + .await + .map_err(|_| HashThreadError::DiagnosticsFailed("thread dropped response".into()))? + } + + pub async fn discover_engine_map( + &self, + asic: u8, + tdm_prediv_raw: u32, + tdm_counter: u8, + timeout: Duration, + ) -> Result { + let (response_tx, response_rx) = oneshot::channel(); + self.command_tx + .send(ThreadCommand::DiscoverEngineMap { + asic, + tdm_prediv_raw, + tdm_counter, + timeout, + response_tx, + }) + .await + .map_err(|_| HashThreadError::ChannelClosed("command channel closed".into()))?; + response_rx + .await + .map_err(|_| HashThreadError::DiagnosticsFailed("thread dropped response".into()))? + } + + pub async fn runtime_metrics(&self) -> Result { + let (response_tx, response_rx) = oneshot::channel(); + self.command_tx + .send(ThreadCommand::QueryRuntimeMetrics { response_tx }) + .await + .map_err(|_| HashThreadError::ChannelClosed("command channel closed".into()))?; + response_rx + .await + .map_err(|_| HashThreadError::DiagnosticsFailed("thread dropped response".into()))? + } +} + +struct PllRuntimeMeasurement { + estimator: HashrateEstimator, + scheduler_share_count: u64, +} + +impl PllRuntimeMeasurement { + fn new() -> Self { + Self { + estimator: HashrateEstimator::new(RUNTIME_MEASUREMENT_WINDOW), + scheduler_share_count: 0, + } + } + + fn record_at(&mut self, at: Instant, work: Work) { + self.estimator.record_at(at, work); + self.scheduler_share_count = self.scheduler_share_count.saturating_add(1); + } + + fn snapshot_at(&mut self, now: Instant) -> Bzm2PllRuntimeMetrics { + Bzm2PllRuntimeMetrics { + throughput_hs: self + .estimator + .settled_hashrate() + .map(u64::from) + .or_else(|| { + self.estimator + .has_samples() + .then(|| u64::from(self.estimator.hashrate_at(now))) + }), + scheduler_share_count: self.scheduler_share_count, + } + } +} + +struct AsicRuntimeMeasurement { + estimator: HashrateEstimator, + scheduler_share_count: u64, + plls: [PllRuntimeMeasurement; 2], +} + +impl AsicRuntimeMeasurement { + fn new() -> Self { + Self { + estimator: HashrateEstimator::new(RUNTIME_MEASUREMENT_WINDOW), + scheduler_share_count: 0, + plls: [PllRuntimeMeasurement::new(), PllRuntimeMeasurement::new()], + } + } + + fn record_at(&mut self, at: Instant, pll_index: usize, work: Work) { + self.estimator.record_at(at, work); + self.scheduler_share_count = self.scheduler_share_count.saturating_add(1); + self.plls[pll_index].record_at(at, work); + } + + fn snapshot_at(&mut self, now: Instant, asic: u8) -> Bzm2AsicRuntimeMetrics { + Bzm2AsicRuntimeMetrics { + asic, + throughput_hs: self + .estimator + .settled_hashrate() + .map(u64::from) + .or_else(|| { + self.estimator + .has_samples() + .then(|| u64::from(self.estimator.hashrate_at(now))) + }), + scheduler_share_count: self.scheduler_share_count, + plls: [self.plls[0].snapshot_at(now), self.plls[1].snapshot_at(now)], + } + } +} + +struct ThreadRuntimeMeasurementState { + estimator: HashrateEstimator, + asics: BTreeMap, +} + +impl ThreadRuntimeMeasurementState { + fn new() -> Self { + Self { + estimator: HashrateEstimator::new(RUNTIME_MEASUREMENT_WINDOW), + asics: BTreeMap::new(), + } + } + + fn record_at(&mut self, at: Instant, asic: u8, row: u8, work: Work) { + let pll_index = pll_index_for_row(row); + self.estimator.record_at(at, work); + self.asics + .entry(asic) + .or_insert_with(AsicRuntimeMeasurement::new) + .record_at(at, pll_index, work); + } + + fn snapshot_at(&mut self, now: Instant) -> Bzm2ThreadRuntimeMetrics { + Bzm2ThreadRuntimeMetrics { + throughput_hs: self + .estimator + .settled_hashrate() + .map(u64::from) + .or_else(|| { + self.estimator + .has_samples() + .then(|| u64::from(self.estimator.hashrate_at(now))) + }), + asics: self + .asics + .iter_mut() + .map(|(&asic, measurement)| measurement.snapshot_at(now, asic)) + .collect(), + } + } + + fn current_hashrate( + &mut self, + now: Instant, + is_active: bool, + nominal_hashrate_ths: f64, + ) -> HashRate { + if !is_active { + return HashRate::default(); + } + + let measured = self.estimator.settled_hashrate().or_else(|| { + self.estimator + .has_samples() + .then(|| self.estimator.hashrate_at(now)) + }); + match measured { + Some(hashrate) if !hashrate.is_zero() => hashrate, + _ => HashRate::from_terahashes(nominal_hashrate_ths), + } + } +} + +#[derive(Debug)] +enum ThreadCommand { + /// Declare expected hashrate and ready the thread for work + Configure, + + UpdateTask { + new_task: HashTask, + response_tx: oneshot::Sender, HashThreadError>>, + }, + ReplaceTask { + new_task: HashTask, + response_tx: oneshot::Sender, HashThreadError>>, + }, + GoIdle { + response_tx: oneshot::Sender, HashThreadError>>, + }, + QueryNoop { + asic: u8, + response_tx: oneshot::Sender>, + }, + QueryLoopback { + asic: u8, + payload: Vec, + response_tx: oneshot::Sender, HashThreadError>>, + }, + QueryClockReport { + asic: u8, + response_tx: oneshot::Sender>, + }, + ReadRegister { + asic: u8, + engine_address: u16, + offset: u8, + count: u8, + response_tx: oneshot::Sender, HashThreadError>>, + }, + WriteRegister { + asic: u8, + engine_address: u16, + offset: u8, + value: Vec, + response_tx: oneshot::Sender>, + }, + QueryDtsVs { + asic: u8, + response_tx: oneshot::Sender>, + }, + DiscoverEngineMap { + asic: u8, + tdm_prediv_raw: u32, + tdm_counter: u8, + timeout: Duration, + response_tx: oneshot::Sender>, + }, + QueryRuntimeMetrics { + response_tx: oneshot::Sender>, + }, + Shutdown, +} + +#[derive(Clone)] +struct EngineDispatch { + task: HashTask, + merkle_root: bitcoin::TxMerkleNode, + versions: [bitcoin::block::Version; 4], + base_sequence: u8, +} + +async fn bzm2_thread_actor( + mut command_rx: mpsc::Receiver, + event_tx: mpsc::Sender, + status: Arc>, + mut reader: SerialReader, + mut writer: SerialWriter, + control: SerialControl, + config: Bzm2ThreadConfig, +) { + if let Err(err) = control.set_baud_rate(config.baud_rate) { + warn!(path = %config.serial_path, error = %err, "Failed to set BZM2 baud rate"); + } + + let _ = event_tx + .send(HashThreadEvent::StatusUpdate(snapshot_status(&status))) + .await; + + let mut engine_layout = Bzm2EngineLayout::default(); + let mut parser = TdmFrameParser::new(config.dts_vs_generation); + let mut current_task: Option = None; + let mut engine_dispatches: HashMap = HashMap::new(); + let mut base_sequence: u8 = 0; + let mut dispatch_tick = tokio::time::interval(config.dispatch_interval); + dispatch_tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + let mut ntime_tick = tokio::time::interval(Duration::from_secs(1)); + ntime_tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + let mut status_tick = tokio::time::interval(Duration::from_secs(5)); + status_tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + let mut read_buf = [0u8; 4096]; + let mut dts_vs_configured = false; + let mut runtime_measurements = ThreadRuntimeMeasurementState::new(); + + loop { + tokio::select! { + Some(command) = command_rx.recv() => { + match command { + ThreadCommand::Configure => { + // Nameplate chain rate; a rough stand-in until the + // board layer supplies a calibration-derived figure. + let expected = HashRate::from_terahashes(config.nominal_hashrate_ths); + if event_tx.send(HashThreadEvent::ExpectedHashRate(expected)).await.is_err() { + debug!("Event channel closed during configure"); + } + } + + ThreadCommand::UpdateTask { new_task, response_tx } => { + let old = current_task.replace(new_task); + if let Some(ref task) = current_task { + if let Err(err) = dispatch_task_to_board( + &mut writer, + task, + base_sequence, + &engine_layout, + &mut engine_dispatches, + &config, + ).await { + let _ = response_tx.send(Err(err)); + continue; + } + base_sequence = base_sequence.wrapping_add(1); + set_active(&status, true, config.nominal_hashrate_ths); + refresh_status_hashrate( + &status, + &mut runtime_measurements, + config.nominal_hashrate_ths, + ); + let _ = event_tx.send(HashThreadEvent::StatusUpdate(snapshot_status(&status))).await; + } + let _ = response_tx.send(Ok(old)); + } + ThreadCommand::ReplaceTask { new_task, response_tx } => { + engine_dispatches.clear(); + let old = current_task.replace(new_task); + if let Some(ref task) = current_task { + if let Err(err) = dispatch_task_to_board( + &mut writer, + task, + base_sequence, + &engine_layout, + &mut engine_dispatches, + &config, + ).await { + let _ = response_tx.send(Err(err)); + continue; + } + base_sequence = base_sequence.wrapping_add(1); + set_active(&status, true, config.nominal_hashrate_ths); + refresh_status_hashrate( + &status, + &mut runtime_measurements, + config.nominal_hashrate_ths, + ); + let _ = event_tx.send(HashThreadEvent::StatusUpdate(snapshot_status(&status))).await; + } + let _ = response_tx.send(Ok(old)); + } + ThreadCommand::GoIdle { response_tx } => { + engine_dispatches.clear(); + let old = current_task.take(); + set_active(&status, false, config.nominal_hashrate_ths); + refresh_status_hashrate( + &status, + &mut runtime_measurements, + config.nominal_hashrate_ths, + ); + let _ = event_tx.send(HashThreadEvent::StatusUpdate(snapshot_status(&status))).await; + let _ = response_tx.send(Ok(old)); + } + ThreadCommand::QueryNoop { asic, response_tx } => { + let result = run_idle_uart_diagnostic( + current_task.is_some(), + dts_vs_configured, + || async { + query_noop(&mut reader, &mut writer, asic).await + }, + ) + .await; + let _ = response_tx.send(result); + } + ThreadCommand::QueryLoopback { + asic, + payload, + response_tx, + } => { + let result = run_idle_uart_diagnostic( + current_task.is_some(), + dts_vs_configured, + || async { + query_loopback(&mut reader, &mut writer, asic, &payload).await + }, + ) + .await; + let _ = response_tx.send(result); + } + ThreadCommand::QueryClockReport { asic, response_tx } => { + let result = run_idle_uart_diagnostic( + current_task.is_some(), + dts_vs_configured, + || async { query_clock_report(&mut reader, &mut writer, asic).await }, + ) + .await; + let _ = response_tx.send(result); + } + ThreadCommand::ReadRegister { + asic, + engine_address, + offset, + count, + response_tx, + } => { + let result = run_idle_uart_diagnostic( + current_task.is_some(), + dts_vs_configured, + || async { + read_register( + &mut reader, + &mut writer, + asic, + engine_address, + offset, + count, + ) + .await + }, + ) + .await; + let _ = response_tx.send(result); + } + ThreadCommand::WriteRegister { + asic, + engine_address, + offset, + value, + response_tx, + } => { + let result = run_idle_uart_diagnostic( + current_task.is_some(), + dts_vs_configured, + || async { + write_register( + &mut writer, + asic, + engine_address, + offset, + &value, + ) + .await + }, + ) + .await; + let _ = response_tx.send(result); + } + ThreadCommand::QueryDtsVs { asic, response_tx } => { + let result = query_dts_vs_telemetry( + asic, + &mut reader, + &mut writer, + &mut parser, + &engine_dispatches, + &engine_layout, + &config, + &status, + &event_tx, + &mut runtime_measurements, + &mut dts_vs_configured, + ).await; + let _ = response_tx.send(result); + } + ThreadCommand::DiscoverEngineMap { + asic, + tdm_prediv_raw, + tdm_counter, + timeout, + response_tx, + } => { + if current_task.is_some() { + let _ = response_tx.send(Err(HashThreadError::DiagnosticsFailed( + "BZM2 engine discovery requires the thread to be idle".into(), + ))); + continue; + } + let result = discover_engine_map_stream( + &mut reader, + &mut writer, + asic, + tdm_prediv_raw, + tdm_counter, + timeout, + ) + .await + .inspect(|discovery| { + engine_layout = Bzm2EngineLayout::from_active_coordinates( + discovery.present.iter().map(|engine| (engine.row, engine.col)), + ); + }) + .map_err(|err| HashThreadError::DiagnosticsFailed(err.to_string())); + let _ = response_tx.send(result); + } + ThreadCommand::QueryRuntimeMetrics { response_tx } => { + let _ = response_tx.send(Ok(runtime_measurements.snapshot_at(Instant::now()))); + } + ThreadCommand::Shutdown => break, + } + } + read_result = reader.read(&mut read_buf) => { + match read_result { + Ok(0) => break, + Ok(n) => { + let mut should_shutdown = false; + for frame in parser.push(&read_buf[..n]) { + match frame { + TdmFrame::Result(frame) => { + handle_result_frame( + &frame, + &engine_dispatches, + &engine_layout, + &config, + &status, + &event_tx, + &mut runtime_measurements, + ) + .await; + } + TdmFrame::DtsVs(frame) => { + should_shutdown = handle_dts_vs_frame(&frame, &config, &status, &event_tx).await; + if should_shutdown { + break; + } + } + TdmFrame::Register(_) | TdmFrame::Noop(_) => {} + } + } + if should_shutdown { + break; + } + } + Err(err) => { + error!(path = %config.serial_path, error = %err, "BZM2 serial read failed"); + record_hardware_error(&status); + break; + } + } + } + _ = dispatch_tick.tick(), if current_task.is_some() => { + if let Some(ref task) = current_task { + match dispatch_task_to_board( + &mut writer, + task, + base_sequence, + &engine_layout, + &mut engine_dispatches, + &config, + ).await { + Ok(()) => { + base_sequence = base_sequence.wrapping_add(1); + } + Err(err) => { + error!(path = %config.serial_path, error = %err, "BZM2 dispatch failed"); + record_hardware_error(&status); + } + } + } + } + _ = ntime_tick.tick(), if current_task.is_some() => { + if let Some(ref mut task) = current_task { + task.ntime = task.ntime.wrapping_add(1); + } + } + _ = status_tick.tick() => { + refresh_status_hashrate( + &status, + &mut runtime_measurements, + config.nominal_hashrate_ths, + ); + let _ = event_tx.send(HashThreadEvent::StatusUpdate(snapshot_status(&status))).await; + } + } + } + + set_active(&status, false, config.nominal_hashrate_ths); + refresh_status_hashrate( + &status, + &mut runtime_measurements, + config.nominal_hashrate_ths, + ); + let _ = event_tx + .send(HashThreadEvent::StatusUpdate(snapshot_status(&status))) + .await; +} + +async fn handle_dts_vs_frame( + frame: &TdmDtsVsFrame, + config: &Bzm2ThreadConfig, + status: &Arc>, + event_tx: &mpsc::Sender, +) -> bool { + if let Some(update) = build_dts_vs_telemetry_update(frame, config) { + if let Some(reading) = update.temperatures.first() { + set_temperature(status, reading.temperature_c); + } + let _ = event_tx + .send(HashThreadEvent::TelemetryUpdate(update)) + .await; + } + + match frame { + TdmDtsVsFrame::Gen1(frame) => { + trace!( + path = %config.serial_path, + asic = frame.asic, + voltage = frame.voltage, + voltage_enabled = frame.voltage_enabled, + thermal_tune_code = frame.thermal_tune_code, + thermal_validity = frame.thermal_validity, + thermal_enabled = frame.thermal_enabled, + "BZM2 DTS/VS telemetry frame" + ); + false + } + TdmDtsVsFrame::Gen2(frame) => { + trace!( + path = %config.serial_path, + asic = frame.asic, + thermal_trip = frame.thermal_trip_status, + thermal_fault = frame.thermal_fault, + voltage_fault = frame.voltage_fault, + voltage_shutdown = frame.voltage_shutdown_status, + thermal_tune_code = frame.thermal_tune_code, + ch0_voltage = frame.ch0_voltage, + ch1_voltage = frame.ch1_voltage, + ch2_voltage = frame.ch2_voltage, + "BZM2 DTS/VS gen2 telemetry frame" + ); + + // Validity gate: a fault/trip bit is only authoritative when the + // sensor that produced it is enabled. All of the trip / fault / + // validity / enabled bits live in a single payload byte, so a + // stray or mis-framed frame can set one at random; acting on that + // unconditionally lets line noise on a shared bus force a permanent, + // unrecoverable shutdown (DoS-by-noise). The sensor-enable bits + // reflect host configuration (see `configure_dts_vs_stream`) and are + // set on every genuine frame, so a real over-temp still stops + // immediately -- no debounce, no delayed protection. + // + // Residual risk: the DTS/VS frame carries no checksum, so a single + // noise frame that happens to set BOTH the enable bit and a fault + // bit in the same byte can still trip. This gate removes the far + // more probable single-bit case; closing it fully would need a + // protocol-level integrity field the silicon does not provide. + let thermal_shutdown = + frame.thermal_enabled && (frame.thermal_trip_status || frame.thermal_fault); + let voltage_shutdown = + frame.voltage_enabled && (frame.voltage_fault || frame.voltage_shutdown_status); + if thermal_shutdown || voltage_shutdown { + warn!( + path = %config.serial_path, + asic = frame.asic, + thermal_trip = frame.thermal_trip_status, + thermal_fault = frame.thermal_fault, + voltage_fault = frame.voltage_fault, + voltage_shutdown = frame.voltage_shutdown_status, + "BZM2 hardware fault reported by DTS/VS frame" + ); + record_hardware_error(status); + return true; + } + false + } + } +} + +async fn run_idle_uart_diagnostic( + thread_active: bool, + dts_vs_configured: bool, + operation: F, +) -> Result +where + F: FnOnce() -> Fut, + Fut: std::future::Future>, +{ + if thread_active { + return Err(HashThreadError::DiagnosticsFailed( + "BZM2 UART diagnostics require the thread to be idle".into(), + )); + } + if dts_vs_configured { + return Err(HashThreadError::DiagnosticsFailed( + "BZM2 UART diagnostics require DTS/VS streaming to be inactive".into(), + )); + } + operation().await +} + +async fn write_register( + writer: &mut SerialWriter, + asic: u8, + engine_address: u16, + offset: u8, + value: &[u8], +) -> Result<(), HashThreadError> { + writer + .write_all(&encode_write_register(asic, engine_address, offset, value)) + .await + .map_err(|err| HashThreadError::DiagnosticsFailed(err.to_string()))?; + writer + .flush() + .await + .map_err(|err| HashThreadError::DiagnosticsFailed(err.to_string()))?; + Ok(()) +} + +async fn read_local_reg_u8( + reader: &mut SerialReader, + writer: &mut SerialWriter, + asic: u8, + offset: u8, +) -> Result { + let value = read_register(reader, writer, asic, super::uart::NOTCH_REG, offset, 1).await?; + value + .first() + .copied() + .ok_or_else(|| HashThreadError::DiagnosticsFailed("short local register response".into())) +} + +async fn read_local_reg_u32( + reader: &mut SerialReader, + writer: &mut SerialWriter, + asic: u8, + offset: u8, +) -> Result { + let value = read_register(reader, writer, asic, super::uart::NOTCH_REG, offset, 4).await?; + let bytes: [u8; 4] = value + .as_slice() + .try_into() + .map_err(|_| HashThreadError::DiagnosticsFailed("short local register response".into()))?; + Ok(u32::from_le_bytes(bytes)) +} + +async fn read_register( + reader: &mut SerialReader, + writer: &mut SerialWriter, + asic: u8, + engine_address: u16, + offset: u8, + count: u8, +) -> Result, HashThreadError> { + let request = encode_read_register(asic, engine_address, offset, count); + writer + .write_all(&request) + .await + .map_err(|err| HashThreadError::DiagnosticsFailed(err.to_string()))?; + writer + .flush() + .await + .map_err(|err| HashThreadError::DiagnosticsFailed(err.to_string()))?; + + let expected = count as usize + 2; + let mut response = vec![0u8; expected]; + read_exact_diagnostic(reader, &mut response).await?; + validate_response_header(asic, OPCODE_UART_READREG, &response)?; + Ok(response[2..].to_vec()) +} + +async fn read_pll_status( + reader: &mut SerialReader, + writer: &mut SerialWriter, + asic: u8, + pll: Bzm2Pll, +) -> Result { + let (_, _, enable_reg, misc_reg) = pll.register_block(); + let enable = read_local_reg_u32(reader, writer, asic, enable_reg).await?; + let misc = read_local_reg_u32(reader, writer, asic, misc_reg).await?; + Ok(Bzm2PllStatus { + pll, + enable_register: enable, + misc_register: misc, + enabled: (enable & 0x1) != 0, + locked: (enable & 0x4) != 0, + }) +} + +async fn read_dll_status( + reader: &mut SerialReader, + writer: &mut SerialWriter, + asic: u8, + dll: Bzm2Dll, +) -> Result { + let (control2_reg, _, _, control5_reg, coarse_reg) = dll.registers(); + let control2 = read_local_reg_u8(reader, writer, asic, control2_reg).await?; + let control5 = read_local_reg_u8(reader, writer, asic, control5_reg).await?; + let coarse_raw = read_local_reg_u8(reader, writer, asic, coarse_reg).await?; + let fincon = read_local_reg_u8(reader, writer, asic, dll.fincon_register()).await?; + + Ok(Bzm2DllStatus { + dll, + control2, + control5, + coarsecon: (coarse_raw >> 5) & 0x7, + fincon, + freeze_valid: (control2 & 0x2) != 0, + locked: (control5 & 0x2) != 0, + fincon_valid: fincon_is_valid(fincon), + }) +} + +async fn query_clock_report( + reader: &mut SerialReader, + writer: &mut SerialWriter, + asic: u8, +) -> Result { + Ok(Bzm2ClockDebugReport { + asic, + pll0: read_pll_status(reader, writer, asic, Bzm2Pll::Pll0).await?, + pll1: read_pll_status(reader, writer, asic, Bzm2Pll::Pll1).await?, + dll0: read_dll_status(reader, writer, asic, Bzm2Dll::Dll0).await?, + dll1: read_dll_status(reader, writer, asic, Bzm2Dll::Dll1).await?, + }) +} + +async fn query_noop( + reader: &mut SerialReader, + writer: &mut SerialWriter, + asic: u8, +) -> Result<[u8; 3], HashThreadError> { + let request = encode_noop(asic); + writer + .write_all(&request) + .await + .map_err(|err| HashThreadError::DiagnosticsFailed(err.to_string()))?; + writer + .flush() + .await + .map_err(|err| HashThreadError::DiagnosticsFailed(err.to_string()))?; + + let mut response = [0u8; 5]; + read_exact_diagnostic(reader, &mut response).await?; + validate_response_header(asic, OPCODE_UART_NOOP, &response)?; + Ok(response[2..5].try_into().unwrap()) +} + +async fn query_loopback( + reader: &mut SerialReader, + writer: &mut SerialWriter, + asic: u8, + payload: &[u8], +) -> Result, HashThreadError> { + let request = encode_loopback(asic, payload); + writer + .write_all(&request) + .await + .map_err(|err| HashThreadError::DiagnosticsFailed(err.to_string()))?; + writer + .flush() + .await + .map_err(|err| HashThreadError::DiagnosticsFailed(err.to_string()))?; + + let expected = payload.len() + 2; + let mut response = vec![0u8; expected]; + read_exact_diagnostic(reader, &mut response).await?; + validate_response_header(asic, OPCODE_UART_LOOPBACK, &response)?; + Ok(response[2..].to_vec()) +} + +async fn read_exact_diagnostic( + reader: &mut SerialReader, + buf: &mut [u8], +) -> Result<(), HashThreadError> { + tokio::time::timeout(DIAGNOSTIC_READ_TIMEOUT, reader.read_exact(buf)) + .await + .map_err(|_| { + HashThreadError::DiagnosticsFailed(format!( + "timed out after {} ms waiting for UART response", + DIAGNOSTIC_READ_TIMEOUT.as_millis() + )) + })? + .map_err(|err| HashThreadError::DiagnosticsFailed(err.to_string()))?; + Ok(()) +} + +fn validate_response_header( + expected_asic: u8, + expected_opcode: u8, + response: &[u8], +) -> Result<(), HashThreadError> { + if response.len() < 2 { + return Err(HashThreadError::DiagnosticsFailed(format!( + "short UART response: expected at least 2 bytes, got {}", + response.len() + ))); + } + let actual_asic = response[0]; + let actual_opcode = response[1]; + if actual_asic != expected_asic || actual_opcode != expected_opcode { + return Err(HashThreadError::DiagnosticsFailed(format!( + "unexpected UART response header: expected asic {expected_asic:#x} opcode {expected_opcode:#x}, got asic {actual_asic:#x} opcode {actual_opcode:#x}" + ))); + } + Ok(()) +} + +// Takes the actor's working state piecemeal; bundling it into a struct would +// just relocate the argument list without simplifying the call site. +#[allow(clippy::too_many_arguments)] +async fn query_dts_vs_telemetry( + asic: u8, + reader: &mut SerialReader, + writer: &mut SerialWriter, + parser: &mut TdmFrameParser, + engine_dispatches: &HashMap, + engine_layout: &Bzm2EngineLayout, + config: &Bzm2ThreadConfig, + status: &Arc>, + event_tx: &mpsc::Sender, + runtime_measurements: &mut ThreadRuntimeMeasurementState, + dts_vs_configured: &mut bool, +) -> Result { + if !*dts_vs_configured { + configure_dts_vs_stream(writer, reader, &Bzm2DtsVsConfig::default()) + .await + .map_err(|err| HashThreadError::TelemetryQueryFailed(err.to_string()))?; + *dts_vs_configured = true; + } + + let deadline = tokio::time::Instant::now() + DEFAULT_DTS_VS_QUERY_TIMEOUT; + let mut read_buf = [0u8; 512]; + loop { + let now = tokio::time::Instant::now(); + if now >= deadline { + return Err(HashThreadError::TelemetryQueryFailed(format!( + "timed out waiting for DTS/VS frame from ASIC {asic:#x}" + ))); + } + let remaining = deadline.saturating_duration_since(now); + let read = tokio::time::timeout(remaining, reader.read(&mut read_buf)) + .await + .map_err(|_| { + HashThreadError::TelemetryQueryFailed(format!( + "timed out waiting for DTS/VS frame from ASIC {asic:#x}" + )) + }) + .and_then(|result| { + result.map_err(|err| HashThreadError::TelemetryQueryFailed(err.to_string())) + })?; + if read == 0 { + return Err(HashThreadError::TelemetryQueryFailed( + "serial stream closed while waiting for DTS/VS data".into(), + )); + } + + for frame in parser.push(&read_buf[..read]) { + match frame { + TdmFrame::Result(frame) => { + handle_result_frame( + &frame, + engine_dispatches, + engine_layout, + config, + status, + event_tx, + runtime_measurements, + ) + .await; + } + TdmFrame::DtsVs(frame) => { + let frame_asic = dts_vs_frame_asic(&frame); + let update = + build_dts_vs_telemetry_update(&frame, config).ok_or_else(|| { + HashThreadError::TelemetryQueryFailed( + "failed to build DTS/VS telemetry update".into(), + ) + })?; + let should_shutdown = + handle_dts_vs_frame(&frame, config, status, event_tx).await; + if should_shutdown { + return Err(HashThreadError::TelemetryQueryFailed( + "DTS/VS query observed a hardware fault".into(), + )); + } + if frame_asic == asic { + return Ok(update); + } + } + TdmFrame::Register(_) | TdmFrame::Noop(_) => {} + } + } + } +} + +async fn dispatch_task_to_board( + writer: &mut SerialWriter, + task: &HashTask, + base_sequence: u8, + engine_layout: &Bzm2EngineLayout, + engine_dispatches: &mut HashMap, + config: &Bzm2ThreadConfig, +) -> Result<(), HashThreadError> { + let merkle_root = match &task.template.merkle_root { + MerkleRootKind::Fixed(root) => *root, + MerkleRootKind::Computed(_) => { + let en2 = task.en2.as_ref().ok_or_else(|| { + HashThreadError::WorkAssignmentFailed( + "BZM2 requires extranonce2 for computed merkle root".into(), + ) + })?; + task.template.compute_merkle_root(en2).map_err(|err| { + HashThreadError::WorkAssignmentFailed(format!( + "BZM2 merkle root computation failed: {err}" + )) + })? + } + }; + + let versions = compute_micro_versions(task); + let midstates = versions.map(|version| compute_midstate(task, merkle_root, version)); + let header_bytes = serialize(&BlockHeader { + version: versions[0], + prev_blockhash: task.template.prev_blockhash, + merkle_root, + time: task.ntime, + bits: task.template.bits, + nonce: 0, + }); + let merkle_root_residue = u32::from_le_bytes(header_bytes[64..68].try_into().unwrap()); + let lead_zeros = leading_zero_threshold(task.share_target).saturating_sub(32); + let timestamp_count = config.timestamp_count | TIMESTAMP_COUNT_AUTO_CLOCK_UNGATE; + let bits = task.template.bits.to_consensus(); + let start_nonce = 0u32; + let end_nonce = DEFAULT_BOARD_END_NONCE; + + for &(row, col) in engine_layout.active_coordinates() { + let engine_address = logical_engine_address(row, col); + + writer + .write_all(&encode_write_register( + BROADCAST_ASIC, + engine_address, + ENGINE_REG_ZEROS_TO_FIND, + &[lead_zeros], + )) + .await + .map_err(|err| { + HashThreadError::WorkAssignmentFailed(format!("Failed to write lead zeros: {err}")) + })?; + + writer + .write_all(&encode_write_register( + BROADCAST_ASIC, + engine_address, + ENGINE_REG_TIMESTAMP_COUNT, + &[timestamp_count], + )) + .await + .map_err(|err| { + HashThreadError::WorkAssignmentFailed(format!( + "Failed to write timestamp count: {err}" + )) + })?; + + writer + .write_all(&encode_write_register( + BROADCAST_ASIC, + engine_address, + ENGINE_REG_TARGET, + &bits.to_le_bytes(), + )) + .await + .map_err(|err| { + HashThreadError::WorkAssignmentFailed(format!("Failed to write target bits: {err}")) + })?; + + writer + .write_all(&encode_write_register( + BROADCAST_ASIC, + engine_address, + ENGINE_REG_START_NONCE, + &start_nonce.to_le_bytes(), + )) + .await + .map_err(|err| { + HashThreadError::WorkAssignmentFailed(format!("Failed to write start nonce: {err}")) + })?; + + writer + .write_all(&encode_write_register( + BROADCAST_ASIC, + engine_address, + ENGINE_REG_END_NONCE, + &end_nonce.to_le_bytes(), + )) + .await + .map_err(|err| { + HashThreadError::WorkAssignmentFailed(format!("Failed to write end nonce: {err}")) + })?; + + let seq_start = (base_sequence % 2) * 4; + for (micro_job_id, midstate) in midstates.iter().enumerate() { + let job_control = if micro_job_id == 3 { 3 } else { 0 }; + writer + .write_all(&encode_write_job( + BROADCAST_ASIC, + engine_address, + midstate, + merkle_root_residue, + task.ntime, + seq_start + micro_job_id as u8, + job_control, + )) + .await + .map_err(|err| { + HashThreadError::WorkAssignmentFailed(format!("Failed to write job: {err}")) + })?; + } + + engine_dispatches.insert( + engine_layout.logical_engine_id(row, col).unwrap(), + EngineDispatch { + task: task.clone(), + merkle_root, + versions, + base_sequence, + }, + ); + } + + Ok(()) +} + +async fn handle_result_frame( + frame: &protocol::TdmResultFrame, + engine_dispatches: &HashMap, + engine_layout: &Bzm2EngineLayout, + config: &Bzm2ThreadConfig, + status: &Arc>, + event_tx: &mpsc::Sender, + runtime_measurements: &mut ThreadRuntimeMeasurementState, +) { + let Some((share, target_diff, engine_id)) = + reconstruct_share_from_result(frame, engine_dispatches, engine_layout, config) + else { + return; + }; + + let share_tx = { + let dispatch = engine_dispatches + .get(&engine_id) + .expect("dispatch must exist for reconstructed share"); + dispatch.task.share_tx.clone() + }; + + runtime_measurements.record_at(Instant::now(), frame.asic, frame.row(), share.expected_work); + refresh_status_hashrate(status, runtime_measurements, config.nominal_hashrate_ths); + + if share_tx.send(share.clone()).await.is_ok() { + let snapshot = { + let mut lock = status.write().unwrap(); + lock.chip_shares_found += 1; + lock.clone() + }; + let _ = event_tx.send(HashThreadEvent::StatusUpdate(snapshot)).await; + } + + trace!( + engine_id, + seq = frame.sequence_id, + nonce = format!("{:#010x}", share.nonce), + hash = %share.hash, + hash_diff = %Difficulty::from_hash(&share.hash), + target_diff = %target_diff, + "BZM2 share accepted" + ); +} + +fn reconstruct_share_from_result( + frame: &protocol::TdmResultFrame, + engine_dispatches: &HashMap, + engine_layout: &Bzm2EngineLayout, + config: &Bzm2ThreadConfig, +) -> Option<(Share, Difficulty, u16)> { + if !frame.nonce_valid() { + return None; + } + + let engine_id = engine_layout.logical_engine_id(frame.row(), frame.col())?; + let dispatch = engine_dispatches.get(&engine_id)?; + + let hardware_base_sequence = frame.sequence_id / 4; + if (dispatch.base_sequence % 2) != hardware_base_sequence { + return None; + } + + let micro_job_id = (frame.sequence_id % 4) as usize; + let version = dispatch.versions[micro_job_id]; + let ntime_offset = u32::from(config.timestamp_count.saturating_sub(frame.reported_time)); + let ntime = dispatch.task.ntime.wrapping_add(ntime_offset); + let nonce = frame.nonce.wrapping_sub(config.nonce_gap); + + let header = BlockHeader { + version, + prev_blockhash: dispatch.task.template.prev_blockhash, + merkle_root: dispatch.merkle_root, + time: ntime, + bits: dispatch.task.template.bits, + nonce, + }; + let hash = header.block_hash(); + + if !dispatch.task.share_target.is_met_by(hash) { + return None; + } + + Some(( + Share { + nonce, + hash, + version, + ntime, + extranonce2: dispatch.task.en2, + expected_work: dispatch.task.share_target.to_work(), + }, + Difficulty::from_target(dispatch.task.share_target), + engine_id, + )) +} + +fn compute_micro_versions(task: &HashTask) -> [bitcoin::block::Version; 4] { + let candidates = [0u16, 2, 4, 8]; + let mut versions = [task.template.version.base(); 4]; + + for (slot, candidate) in candidates.into_iter().enumerate() { + let gp_bits = GeneralPurposeBits::new(candidate.to_be_bytes()); + versions[slot] = task + .template + .version + .apply_gp_bits(&gp_bits) + .unwrap_or_else(|_| task.template.version.base()); + } + + versions +} + +fn compute_midstate( + task: &HashTask, + merkle_root: bitcoin::TxMerkleNode, + version: bitcoin::block::Version, +) -> [u8; 32] { + let header_bytes = serialize(&BlockHeader { + version, + prev_blockhash: task.template.prev_blockhash, + merkle_root, + time: task.ntime, + bits: task.template.bits, + nonce: 0, + }); + + let mut engine = sha256::HashEngine::default(); + engine.input(&header_bytes[..64]); + engine.midstate().to_byte_array() +} + +fn snapshot_status(status: &Arc>) -> HashThreadStatus { + status.read().unwrap().clone() +} + +fn set_active(status: &Arc>, is_active: bool, nominal_hashrate_ths: f64) { + let mut lock = status.write().unwrap(); + lock.is_active = is_active; + lock.hashrate = if is_active { + HashRate::from_terahashes(nominal_hashrate_ths) + } else { + HashRate::default() + }; +} + +fn refresh_status_hashrate( + status: &Arc>, + runtime_measurements: &mut ThreadRuntimeMeasurementState, + nominal_hashrate_ths: f64, +) { + let now = Instant::now(); + let mut lock = status.write().unwrap(); + lock.hashrate = + runtime_measurements.current_hashrate(now, lock.is_active, nominal_hashrate_ths); +} + +fn record_hardware_error(status: &Arc>) { + let mut lock = status.write().unwrap(); + lock.hardware_errors = lock.hardware_errors.saturating_add(1); +} + +fn set_temperature(status: &Arc>, temperature_c: Option) { + let mut lock = status.write().unwrap(); + lock.temperature_c = temperature_c; +} + +fn build_dts_vs_telemetry_update( + frame: &TdmDtsVsFrame, + config: &Bzm2ThreadConfig, +) -> Option { + let prefix = sensor_prefix(&config.serial_path); + match frame { + TdmDtsVsFrame::Gen1(frame) => Some(HashThreadTelemetryUpdate { + temperatures: Vec::new(), + powers: vec![HashThreadPowerReading { + name: format!("{prefix}-asic-{}-vs", frame.asic), + voltage_v: frame + .voltage_enabled + .then(|| legacy_tune_code_to_voltage_v(frame.voltage)), + current_a: None, + power_w: None, + }], + }), + TdmDtsVsFrame::Gen2(frame) => Some(HashThreadTelemetryUpdate { + temperatures: vec![HashThreadTemperatureReading { + name: format!("{prefix}-asic-{}-dts", frame.asic), + temperature_c: (frame.thermal_enabled && frame.thermal_validity) + .then(|| legacy_tune_code_to_temperature_c(frame.thermal_tune_code)), + }], + powers: vec![ + HashThreadPowerReading { + name: format!("{prefix}-asic-{}-vs-ch0", frame.asic), + voltage_v: frame + .voltage_enabled + .then(|| legacy_tune_code_to_voltage_v(frame.ch0_voltage)), + current_a: None, + power_w: None, + }, + HashThreadPowerReading { + name: format!("{prefix}-asic-{}-vs-ch1", frame.asic), + voltage_v: frame + .voltage_enabled + .then(|| legacy_tune_code_to_voltage_v(frame.ch1_voltage)), + current_a: None, + power_w: None, + }, + HashThreadPowerReading { + name: format!("{prefix}-asic-{}-vs-ch2", frame.asic), + voltage_v: frame + .voltage_enabled + .then(|| legacy_tune_code_to_voltage_v(frame.ch2_voltage)), + current_a: None, + power_w: None, + }, + ], + }), + } +} + +fn dts_vs_frame_asic(frame: &TdmDtsVsFrame) -> u8 { + match frame { + TdmDtsVsFrame::Gen1(frame) => frame.asic, + TdmDtsVsFrame::Gen2(frame) => frame.asic, + } +} + +fn sensor_prefix(serial_path: &str) -> String { + Path::new(serial_path) + .file_name() + .and_then(|value| value.to_str()) + .filter(|value| !value.is_empty()) + .unwrap_or(serial_path) + .chars() + .map(|ch| if ch.is_ascii_alphanumeric() { ch } else { '-' }) + .collect() +} + +fn legacy_tune_code_to_temperature_c(tune_code: u16) -> f32 { + let resolution_power = 4096.0_f32; + -293.8 + 631.8 * ((tune_code as f32) - (2048.0 / resolution_power)) / 4096.0 +} + +fn legacy_tune_code_to_voltage_v(tune_code: u16) -> f32 { + let resolution_power = 16384.0_f32; + 0.4 * 0.7067 * (6.0 * (tune_code as f32) / 16384.0 - 3.0 / resolution_power - 1.0) +} + +fn pll_index_for_row(row: u8) -> usize { + if row < PLL_STACK_SPLIT_ROW { 0 } else { 1 } +} + +#[cfg(all(test, unix))] +mod tests { + use super::*; + use crate::job_source::{GeneralPurposeBits, JobTemplate, VersionTemplate}; + use crate::transport::{SerialConfig, SerialStream}; + use bitcoin::hashes::Hash; + use bitcoin::pow::Target; + use nix::pty::openpty; + use std::collections::HashMap as StdHashMap; + use std::os::unix::io::IntoRawFd; + use tokio::sync::mpsc as tokio_mpsc; + + fn test_task() -> HashTask { + let share_target = Target::from_be_bytes([ + 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, + ]); + let template = Arc::new(JobTemplate { + id: "bzm2-test".into(), + prev_blockhash: bitcoin::BlockHash::all_zeros(), + version: VersionTemplate::new( + bitcoin::block::Version::from_consensus(0x2000_0000), + GeneralPurposeBits::full(), + ) + .unwrap(), + bits: bitcoin::pow::CompactTarget::from_consensus(0x1d00ffff), + share_target, + time: 1_700_000_000, + merkle_root: MerkleRootKind::Fixed(bitcoin::TxMerkleNode::all_zeros()), + }); + let (share_tx, _share_rx) = tokio_mpsc::channel(4); + HashTask { + template, + en2_range: None, + en2: None, + share_target, + ntime: 1_700_000_000, + share_tx, + } + } + + #[test] + fn midstate_changes_with_micro_job_versions() { + let task = test_task(); + let merkle_root = bitcoin::TxMerkleNode::all_zeros(); + let versions = compute_micro_versions(&task); + let a = compute_midstate(&task, merkle_root, versions[0]); + let b = compute_midstate(&task, merkle_root, versions[1]); + assert_ne!(a, b); + } + + #[tokio::test] + async fn dispatch_writes_expected_packet_fanout() { + let pty = openpty(None, None).unwrap(); + let writer_side = + SerialStream::from_fd(pty.master.into_raw_fd(), SerialConfig::default()).unwrap(); + let reader_side = + SerialStream::from_fd(pty.slave.into_raw_fd(), SerialConfig::default()).unwrap(); + let (_reader_a, mut writer, _control_a) = writer_side.split(); + let (mut reader, _writer_b, _control_b) = reader_side.split(); + + let task = test_task(); + let mut engine_dispatches = StdHashMap::new(); + let config = Bzm2ThreadConfig::new("/dev/null".into(), 5_000_000); + let engine_coords = vec![(0, 0), (0, 1)]; + let engine_layout = Bzm2EngineLayout::from_active_coordinates(engine_coords.clone()); + + dispatch_task_to_board( + &mut writer, + &task, + 1, + &engine_layout, + &mut engine_dispatches, + &config, + ) + .await + .unwrap(); + + let expected_bytes_per_engine = 8 + 8 + 11 + 11 + 11 + (48 * 4); + let expected_total = expected_bytes_per_engine * engine_coords.len(); + let deadline = tokio::time::Instant::now() + Duration::from_millis(250); + let mut buf = vec![0u8; 512]; + let mut bytes = Vec::with_capacity(expected_total); + while bytes.len() < expected_total { + let remaining = deadline.saturating_duration_since(tokio::time::Instant::now()); + assert!( + !remaining.is_zero(), + "timed out before collecting the full dispatch stream" + ); + let n = tokio::time::timeout(remaining, reader.read(&mut buf)) + .await + .unwrap() + .unwrap(); + if n == 0 { + break; + } + bytes.extend_from_slice(&buf[..n]); + } + + assert_eq!(bytes.len(), expected_total); + assert_eq!(engine_dispatches.len(), engine_coords.len()); + + let packet_lengths = bytes + .chunks_exact(expected_bytes_per_engine) + .map(|chunk| { + [ + u16::from_le_bytes([chunk[0], chunk[1]]) as usize, + u16::from_le_bytes([chunk[8], chunk[9]]) as usize, + u16::from_le_bytes([chunk[16], chunk[17]]) as usize, + u16::from_le_bytes([chunk[27], chunk[28]]) as usize, + u16::from_le_bytes([chunk[38], chunk[39]]) as usize, + ] + }) + .collect::>(); + assert!( + packet_lengths + .iter() + .all(|lens| *lens == [8, 8, 11, 11, 11]) + ); + + for chunk in bytes.chunks_exact(expected_bytes_per_engine) { + assert_eq!( + chunk[7], + leading_zero_threshold(task.share_target).saturating_sub(32) + ); + assert_eq!(chunk[15], 0x80 | DEFAULT_TIMESTAMP_COUNT); + assert_eq!( + &chunk[23..27], + &task.template.bits.to_consensus().to_le_bytes() + ); + assert_eq!(&chunk[34..38], &0u32.to_le_bytes()); + assert_eq!(&chunk[45..49], &DEFAULT_BOARD_END_NONCE.to_le_bytes()); + } + + let last_packet_start = bytes.len() - 48; + assert_eq!( + u16::from_le_bytes([bytes[last_packet_start], bytes[last_packet_start + 1]]) as usize, + 48 + ); + assert_eq!(bytes[last_packet_start + 46], 7); + assert_eq!(bytes[last_packet_start + 47], 3); + } + + #[tokio::test] + async fn parsed_uart_frame_emits_share_and_status_event() { + let mut task = test_task(); + let merkle_root = bitcoin::TxMerkleNode::all_zeros(); + let versions = compute_micro_versions(&task); + let engine_layout = Bzm2EngineLayout::default(); + let (row, col) = protocol::default_engine_coordinates()[0]; + let engine_id = engine_layout.logical_engine_id(row, col).unwrap(); + let nonce = 0; + let expected_hash = bitcoin::block::Header { + version: versions[0], + prev_blockhash: task.template.prev_blockhash, + merkle_root, + time: task.ntime, + bits: task.template.bits, + nonce, + } + .block_hash(); + task.share_target = Difficulty::from_hash(&expected_hash).to_target(); + + let mut engine_dispatches = StdHashMap::new(); + engine_dispatches.insert( + engine_id, + EngineDispatch { + task: task.clone(), + merkle_root, + versions, + base_sequence: 0, + }, + ); + + let config = Bzm2ThreadConfig::new("/dev/null".into(), 5_000_000); + let status = Arc::new(RwLock::new(HashThreadStatus { + hashrate: HashRate::from_terahashes(40.0), + is_active: true, + ..Default::default() + })); + let mut runtime_measurements = ThreadRuntimeMeasurementState::new(); + let (event_tx, mut event_rx) = tokio_mpsc::channel(4); + let (share_tx, mut share_rx) = tokio_mpsc::channel(4); + engine_dispatches.get_mut(&engine_id).unwrap().task.share_tx = share_tx; + + let engine_address = protocol::logical_engine_address(row, col); + let header = ((0x8u16) << 12) | engine_address; + let mut raw = Vec::with_capacity(10); + raw.push(0); + raw.push(protocol::OPCODE_UART_READRESULT); + raw.extend_from_slice(&header.to_be_bytes()); + raw.extend_from_slice(&(nonce + DEFAULT_NONCE_GAP).to_le_bytes()); + raw.push(0); + raw.push(DEFAULT_TIMESTAMP_COUNT); + + let mut parser = protocol::TdmResultParser::default(); + let frames = parser.push(&raw); + assert_eq!(frames.len(), 1); + assert!( + reconstruct_share_from_result(&frames[0], &engine_dispatches, &engine_layout, &config) + .is_some() + ); + + handle_result_frame( + &frames[0], + &engine_dispatches, + &engine_layout, + &config, + &status, + &event_tx, + &mut runtime_measurements, + ) + .await; + + let share = tokio::time::timeout(Duration::from_millis(250), share_rx.recv()) + .await + .unwrap() + .unwrap(); + assert_eq!(share.nonce, nonce); + assert_eq!(share.ntime, task.ntime); + assert_eq!(share.version, versions[0]); + assert_eq!(share.hash, expected_hash); + + let status_update = tokio::time::timeout(Duration::from_millis(250), event_rx.recv()) + .await + .unwrap() + .unwrap(); + match status_update { + HashThreadEvent::StatusUpdate(snapshot) => { + assert!(snapshot.is_active); + assert_eq!(snapshot.chip_shares_found, 1); + assert!(u64::from(snapshot.hashrate) > 0); + } + other => panic!("unexpected event: {:?}", other), + } + } + + #[test] + fn runtime_metrics_track_per_asic_and_per_pll_throughput() { + let base = Instant::now(); + let mut runtime = ThreadRuntimeMeasurementState::new(); + let work = bitcoin::pow::Work::from_le_bytes({ + let mut bytes = [0u8; 32]; + bytes[..8].copy_from_slice(&1_000u64.to_le_bytes()); + bytes + }); + + runtime.record_at(base, 2, 0, work); + runtime.record_at(base + Duration::from_secs(10), 2, 12, work); + runtime.record_at(base + Duration::from_secs(20), 2, 12, work); + let snapshot = runtime.snapshot_at(base + Duration::from_secs(20)); + + assert_eq!(snapshot.asics.len(), 1); + let asic = &snapshot.asics[0]; + assert_eq!(asic.asic, 2); + assert_eq!(asic.scheduler_share_count, 3); + assert_eq!(asic.throughput_hs, Some(150)); + assert_eq!(asic.plls[0].scheduler_share_count, 1); + assert_eq!(asic.plls[0].throughput_hs, Some(50)); + assert_eq!(asic.plls[1].scheduler_share_count, 2); + assert_eq!(asic.plls[1].throughput_hs, Some(200)); + } + #[tokio::test] + async fn gen2_dts_vs_emits_api_telemetry() { + let config = Bzm2ThreadConfig::new("/dev/ttyUSB0".into(), 5_000_000); + let status = Arc::new(RwLock::new(HashThreadStatus::default())); + let (event_tx, mut event_rx) = tokio_mpsc::channel(4); + let frame = TdmDtsVsFrame::Gen2(protocol::TdmDtsVsGen2Frame { + asic: 2, + ch0_voltage: 0x1645, + ch1_voltage: 0x04B4, + ch2_voltage: 0x16AC, + voltage_shutdown_status: false, + voltage_enabled: true, + thermal_tune_code: 0x07A9, + thermal_trip_status: false, + thermal_fault: false, + thermal_validity: true, + thermal_enabled: true, + voltage_fault: false, + dll0_lock: false, + dll1_lock: true, + pll_lock: true, + }); + + let should_shutdown = handle_dts_vs_frame(&frame, &config, &status, &event_tx).await; + assert!(!should_shutdown); + + let update = event_rx.recv().await.unwrap(); + match update { + HashThreadEvent::TelemetryUpdate(update) => { + assert_eq!(update.temperatures.len(), 1); + assert_eq!(update.temperatures[0].name, "ttyUSB0-asic-2-dts"); + let temp = update.temperatures[0].temperature_c.unwrap(); + assert!((temp - legacy_tune_code_to_temperature_c(0x07A9)).abs() < 0.01); + + assert_eq!(update.powers.len(), 3); + assert_eq!(update.powers[0].name, "ttyUSB0-asic-2-vs-ch0"); + assert!( + (update.powers[0].voltage_v.unwrap() - legacy_tune_code_to_voltage_v(0x1645)) + .abs() + < 0.0001 + ); + assert_eq!(update.powers[1].name, "ttyUSB0-asic-2-vs-ch1"); + assert_eq!(update.powers[2].name, "ttyUSB0-asic-2-vs-ch2"); + } + other => panic!("unexpected event: {other:?}"), + } + + let snapshot = status.read().unwrap().clone(); + assert!( + (snapshot.temperature_c.unwrap() - legacy_tune_code_to_temperature_c(0x07A9)).abs() + < 0.01 + ); + } + + #[tokio::test] + async fn gen2_dts_vs_unvalidated_trip_bit_does_not_shut_down() { + // A single noisy DTS/VS frame whose only meaningful content is a set + // thermal-trip bit, with the thermal sensor NOT enabled. This is line + // noise, not a credible over-temp, and must not take the thread + // permanently offline. + let config = Bzm2ThreadConfig::new("/dev/null".into(), 5_000_000); + let status = Arc::new(RwLock::new(HashThreadStatus::default())); + let (event_tx, _event_rx) = tokio_mpsc::channel(4); + + let mut parser = TdmFrameParser::new(protocol::DtsVsGeneration::Gen2); + let frame = match parser + .push(&[ + 0x00, + protocol::OPCODE_UART_DTS_VS, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0x10, + ]) + .into_iter() + .next() + { + Some(TdmFrame::DtsVs(frame)) => frame, + other => panic!("expected a DTS/VS frame, got {other:?}"), + }; + + // Premise: the trip bit decoded true, but the sensor is not enabled. + match frame { + TdmDtsVsFrame::Gen2(gen2) => { + assert!(gen2.thermal_trip_status); + assert!(!gen2.thermal_enabled); + } + other => panic!("expected a gen2 frame, got {other:?}"), + } + + let should_shutdown = handle_dts_vs_frame(&frame, &config, &status, &event_tx).await; + assert!( + !should_shutdown, + "an unvalidated trip bit from a disabled sensor must not shut the thread down" + ); + assert_eq!(status.read().unwrap().hardware_errors, 0); + } + + #[tokio::test] + async fn gen2_dts_vs_fault_shuts_down_live_thread() { + let pty = openpty(None, None).unwrap(); + let thread_side = + SerialStream::from_fd(pty.master.into_raw_fd(), SerialConfig::default()).unwrap(); + let host_side = + SerialStream::from_fd(pty.slave.into_raw_fd(), SerialConfig::default()).unwrap(); + let (reader, writer, control) = thread_side.split(); + let (_host_reader, mut host_writer, _host_control) = host_side.split(); + + let mut config = Bzm2ThreadConfig::new("/dev/null".into(), 5_000_000); + config.dts_vs_generation = protocol::DtsVsGeneration::Gen2; + let mut thread = Bzm2Thread::new("BZM2 test".into(), reader, writer, control, config); + let mut event_rx = thread.take_event_receiver().unwrap(); + + let initial = tokio::time::timeout(Duration::from_millis(250), event_rx.recv()) + .await + .unwrap() + .unwrap(); + assert!(matches!(initial, HashThreadEvent::StatusUpdate(_))); + + host_writer + .write_all(&[ + 0x00, + protocol::OPCODE_UART_DTS_VS, + 0xD5, + 0xAB, + 0x34, + 0x12, + 0x45, + 0x96, + 0xA9, + 0xF7, + ]) + .await + .unwrap(); + + let mut saw_fault_status = false; + let closed = tokio::time::timeout(Duration::from_secs(1), async { + while let Some(event) = event_rx.recv().await { + if let HashThreadEvent::StatusUpdate(status) = event { + if !status.is_active && status.hardware_errors >= 1 { + saw_fault_status = true; + } + } + } + }) + .await; + + assert!( + closed.is_ok(), + "thread should exit after DTS/VS hardware fault" + ); + assert!( + saw_fault_status, + "thread should publish a final faulted status" + ); + } + + #[tokio::test] + async fn diagnostic_read_times_out_and_actor_stays_responsive() { + let pty = openpty(None, None).unwrap(); + let thread_side = + SerialStream::from_fd(pty.master.into_raw_fd(), SerialConfig::default()).unwrap(); + let host_side = + SerialStream::from_fd(pty.slave.into_raw_fd(), SerialConfig::default()).unwrap(); + let (reader, writer, control) = thread_side.split(); + let (mut host_reader, mut host_writer, _host_control) = host_side.split(); + + let config = Bzm2ThreadConfig::new("/dev/null".into(), 5_000_000); + let mut thread = Bzm2Thread::new("BZM2 test".into(), reader, writer, control, config); + let handle = thread.shutdown_handle(); + let mut event_rx = thread.take_event_receiver().unwrap(); + + // Drain the initial status update so we know the actor is up. + let initial = tokio::time::timeout(Duration::from_millis(250), event_rx.recv()) + .await + .unwrap() + .unwrap(); + assert!(matches!(initial, HashThreadEvent::StatusUpdate(_))); + + // Host reads the NOOP request, replies with a single (short) byte, then + // goes silent. Without a bounded read the actor's read_exact would block + // forever, wedging every other command including Shutdown. + let emulator = tokio::spawn(async move { + let mut request = [0u8; 4]; + host_reader.read_exact(&mut request).await.unwrap(); + host_writer.write_all(&[0x02]).await.unwrap(); + tokio::time::sleep(Duration::from_secs(5)).await; + drop(host_writer); + }); + + // QueryNoop must resolve to an error within the diagnostic timeout. + let result = tokio::time::timeout(Duration::from_secs(4), handle.noop(2)).await; + assert!(result.is_ok(), "QueryNoop hung past the diagnostic timeout"); + assert!( + result.unwrap().is_err(), + "QueryNoop should fail on a short, stalled response" + ); + + // The actor is still responsive: Shutdown is honored and the event + // stream closes. + handle.shutdown(); + let closed = tokio::time::timeout(Duration::from_secs(2), async { + while event_rx.recv().await.is_some() {} + }) + .await; + assert!( + closed.is_ok(), + "actor did not honor Shutdown after the diagnostic read timed out" + ); + + emulator.abort(); + } + + #[test] + fn reconstructs_share_from_matching_result_frame() { + let mut task = test_task(); + + let merkle_root = bitcoin::TxMerkleNode::all_zeros(); + let versions = compute_micro_versions(&task); + let engine_layout = Bzm2EngineLayout::default(); + let (row, col) = protocol::default_engine_coordinates()[0]; + let engine_id = engine_layout.logical_engine_id(row, col).unwrap(); + let nonce = 0; + let expected_hash = bitcoin::block::Header { + version: versions[0], + prev_blockhash: task.template.prev_blockhash, + merkle_root, + time: task.ntime, + bits: task.template.bits, + nonce, + } + .block_hash(); + task.share_target = Difficulty::from_hash(&expected_hash).to_target(); + let frame = protocol::TdmResultFrame { + asic: 0, + engine_address: protocol::logical_engine_address(row, col), + status: 0x8, + nonce: nonce + DEFAULT_NONCE_GAP, + sequence_id: 0, + reported_time: DEFAULT_TIMESTAMP_COUNT, + }; + + let mut engine_dispatches = StdHashMap::new(); + engine_dispatches.insert( + engine_id, + EngineDispatch { + task: task.clone(), + merkle_root, + versions, + base_sequence: 0, + }, + ); + + let config = Bzm2ThreadConfig::new("/dev/null".into(), 5_000_000); + let (share, target_diff, reconstructed_engine_id) = + reconstruct_share_from_result(&frame, &engine_dispatches, &engine_layout, &config) + .unwrap(); + + assert_eq!(reconstructed_engine_id, engine_id); + assert_eq!(share.nonce, nonce); + assert_eq!(share.ntime, task.ntime); + assert_eq!(share.version, versions[0]); + assert_eq!( + share.hash, + bitcoin::block::Header { + version: versions[0], + prev_blockhash: task.template.prev_blockhash, + merkle_root, + time: task.ntime, + bits: task.template.bits, + nonce, + } + .block_hash() + ); + assert_eq!(target_diff, Difficulty::from_target(task.share_target)); + } + + #[test] + fn runtime_engine_layout_compresses_logical_ids_after_missing_engines() { + let layout = Bzm2EngineLayout::from_active_coordinates([(0, 0), (0, 6), (19, 10)]); + + assert_eq!(layout.active_engine_count(), 3); + assert_eq!(layout.logical_engine_id(0, 0), Some(0)); + assert_eq!(layout.logical_engine_id(0, 6), Some(1)); + assert_eq!(layout.logical_engine_id(19, 10), Some(2)); + assert_eq!(layout.logical_engine_id(0, 1), None); + } + + #[tokio::test] + async fn dispatch_uses_runtime_engine_layout() { + let pty = openpty(None, None).unwrap(); + let writer_side = + SerialStream::from_fd(pty.master.into_raw_fd(), SerialConfig::default()).unwrap(); + let reader_side = + SerialStream::from_fd(pty.slave.into_raw_fd(), SerialConfig::default()).unwrap(); + let (_reader_a, mut writer, _control_a) = writer_side.split(); + let (mut reader, _writer_b, _control_b) = reader_side.split(); + + let task = test_task(); + let mut engine_dispatches = StdHashMap::new(); + let config = Bzm2ThreadConfig::new("/dev/null".into(), 5_000_000); + let engine_layout = Bzm2EngineLayout::from_active_coordinates([(0, 0), (19, 10)]); + + dispatch_task_to_board( + &mut writer, + &task, + 1, + &engine_layout, + &mut engine_dispatches, + &config, + ) + .await + .unwrap(); + + let mut buf = vec![0u8; 512]; + let mut bytes = Vec::new(); + let deadline = tokio::time::Instant::now() + Duration::from_millis(250); + let expected_bytes_per_engine = 8 + 8 + 11 + 11 + 11 + (48 * 4); + while bytes.len() < expected_bytes_per_engine * engine_layout.active_engine_count() { + let remaining = deadline.saturating_duration_since(tokio::time::Instant::now()); + let n = tokio::time::timeout(remaining, reader.read(&mut buf)) + .await + .unwrap() + .unwrap(); + if n == 0 { + break; + } + bytes.extend_from_slice(&buf[..n]); + } + + let first_engine = logical_engine_address(0, 0); + let second_engine = logical_engine_address(19, 10); + let touched_engines = bytes + .chunks_exact(expected_bytes_per_engine) + .map(|chunk| u32::from_be_bytes([chunk[2], chunk[3], chunk[4], chunk[5]])) + .map(|header| ((header >> 8) & 0x0fff) as u16) + .collect::>(); + assert!(touched_engines.contains(&first_engine)); + assert!(touched_engines.contains(&second_engine)); + assert!(!touched_engines.contains(&logical_engine_address(0, 1))); + assert_eq!(engine_dispatches.len(), 2); + assert!(engine_dispatches.contains_key(&0)); + assert!(engine_dispatches.contains_key(&1)); + } +} diff --git a/mujina-miner/src/asic/bzm2/uart.rs b/mujina-miner/src/asic/bzm2/uart.rs new file mode 100644 index 00000000..c8eb5952 --- /dev/null +++ b/mujina-miner/src/asic/bzm2/uart.rs @@ -0,0 +1,1140 @@ +use std::time::Duration; + +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::time::{Instant, timeout}; + +use crate::transport::{SerialReader, SerialWriter}; + +use super::protocol::{ + BROADCAST_ASIC, DtsVsGeneration, ENGINE_REG_END_NONCE, OPCODE_UART_LOOPBACK, OPCODE_UART_NOOP, + OPCODE_UART_READREG, TdmDtsVsFrame, TdmFrame, TdmFrameParser, encode_loopback, + encode_multicast_write, encode_noop, encode_read_register, encode_write_job, + encode_write_register, logical_engine_address, physical_engine_coordinates, +}; + +pub const NOTCH_REG: u16 = 0x0fff; +pub const BROADCAST_GROUP_ASIC: u8 = BROADCAST_ASIC; +pub const DEFAULT_DTS_VS_QUERY_TIMEOUT: Duration = Duration::from_secs(2); +pub const DEFAULT_ASIC_ID: u8 = 0xfa; +pub const DEFAULT_NOOP_PROBE_TIMEOUT: Duration = Duration::from_millis(100); + +const LOCAL_REG_ASIC_ID: u8 = 0x0b; +const LOCAL_REG_UART_TDM_CTL: u8 = 0x07; +const LOCAL_REG_SLOW_CLK_DIV: u8 = 0x08; +const LOCAL_REG_UART_TX: u8 = 0x0a; +const LOCAL_REG_SENS_TDM_GAP_CNT: u8 = 0x2d; +const LOCAL_REG_DTS_SRST_PD: u8 = 0x2e; +const LOCAL_REG_DTS_CFG: u8 = 0x2f; +const LOCAL_REG_TEMPSENSOR_TUNE_CODE: u8 = 0x30; +const LOCAL_REG_SENSOR_THRS_CNT: u8 = 0x3c; +const LOCAL_REG_SENSOR_CLK_DIV: u8 = 0x3d; +const LOCAL_REG_VSENSOR_SRST_PD: u8 = 0x3e; +const LOCAL_REG_VSENSOR_CFG: u8 = 0x3f; +const LOCAL_REG_VOLTAGE_SENSOR_ENABLE: u8 = 0x40; +const LOCAL_REG_BANDGAP: u8 = 0x45; + +const THERMAL_SENSOR_RESOLUTION: u8 = 12; +const THERMAL_SENSOR_MODE: u8 = 0; +const VOLTAGE_SENSOR_RESOLUTION: u8 = 14; +const VOLTAGE_SENSOR_CONVERSION_MODE: u8 = 1; +const VOLTAGE_SENSOR_MODE: u8 = 0; +const DISCOVERED_ENGINE_END_NONCE: u32 = 0xffff_fffe; + +/// Low-level BZM2 UART control surface. +/// +/// This controller wraps the legacy BZM2 UART framing in a small, explicit API. +/// It is intended for board bring-up, ASIC diagnostics, and developer tooling. +/// The methods are organized around the three routing modes exposed by the ASIC: +/// +/// - unicast: target one ASIC and one register space address +/// - multicast: target an ASIC and one engine group row +/// - broadcast: target all ASICs on a bus via ASIC id `0xff` +/// +/// Typical usage patterns: +/// +/// ```rust,no_run +/// # async fn demo(mut uart: mujina_miner::asic::bzm2::Bzm2UartController) -> Result<(), Box> { +/// use mujina_miner::asic::bzm2::{Bzm2Pll, Bzm2UartController, NOTCH_REG}; +/// +/// // Unicast: write one ASIC-local register. +/// uart.write_local_reg_u32(0x02, 0x12, 1).await?; +/// +/// // Broadcast: push one local register update to every ASIC on the UART bus. +/// uart.broadcast_local_reg_u32(0x07, 0x1).await?; +/// +/// // Multicast: update all engines in one row group on one ASIC. +/// uart.multicast_write_reg_u8(0x02, 7, 0x49, 60).await?; +/// # Ok(()) } +/// ``` +pub struct Bzm2UartController { + reader: SerialReader, + writer: SerialWriter, +} + +impl Bzm2UartController { + pub fn new(reader: SerialReader, writer: SerialWriter) -> Self { + Self { reader, writer } + } + + pub async fn write_register( + &mut self, + asic: u8, + engine_address: u16, + offset: u8, + value: &[u8], + ) -> Result<(), Bzm2UartError> { + self.writer + .write_all(&encode_write_register(asic, engine_address, offset, value)) + .await?; + self.writer.flush().await?; + Ok(()) + } + + pub async fn write_register_u8( + &mut self, + asic: u8, + engine_address: u16, + offset: u8, + value: u8, + ) -> Result<(), Bzm2UartError> { + self.write_register(asic, engine_address, offset, &[value]) + .await + } + + pub async fn write_register_u32( + &mut self, + asic: u8, + engine_address: u16, + offset: u8, + value: u32, + ) -> Result<(), Bzm2UartError> { + self.write_register(asic, engine_address, offset, &value.to_le_bytes()) + .await + } + + pub async fn write_local_reg_u8( + &mut self, + asic: u8, + offset: u8, + value: u8, + ) -> Result<(), Bzm2UartError> { + self.write_register_u8(asic, NOTCH_REG, offset, value).await + } + + pub async fn write_local_reg_u32( + &mut self, + asic: u8, + offset: u8, + value: u32, + ) -> Result<(), Bzm2UartError> { + self.write_register_u32(asic, NOTCH_REG, offset, value) + .await + } + + pub async fn broadcast_local_reg_u8( + &mut self, + offset: u8, + value: u8, + ) -> Result<(), Bzm2UartError> { + self.write_local_reg_u8(BROADCAST_ASIC, offset, value).await + } + + pub async fn broadcast_local_reg_u32( + &mut self, + offset: u8, + value: u32, + ) -> Result<(), Bzm2UartError> { + self.write_local_reg_u32(BROADCAST_ASIC, offset, value) + .await + } + + /// Program the next ASIC still responding on the default chain id. + pub async fn assign_default_asic_id(&mut self, new_id: u8) -> Result<(), Bzm2UartError> { + self.write_local_reg_u32(DEFAULT_ASIC_ID, LOCAL_REG_ASIC_ID, new_id as u32) + .await + } + + pub async fn multicast_write_register( + &mut self, + asic: u8, + group: u16, + offset: u8, + value: &[u8], + ) -> Result<(), Bzm2UartError> { + self.writer + .write_all(&encode_multicast_write(asic, group, offset, value)) + .await?; + self.writer.flush().await?; + Ok(()) + } + + pub async fn multicast_write_reg_u8( + &mut self, + asic: u8, + group: u16, + offset: u8, + value: u8, + ) -> Result<(), Bzm2UartError> { + self.multicast_write_register(asic, group, offset, &[value]) + .await + } + + pub async fn read_register( + &mut self, + asic: u8, + engine_address: u16, + offset: u8, + count: u8, + ) -> Result, Bzm2UartError> { + let request = encode_read_register(asic, engine_address, offset, count); + self.writer.write_all(&request).await?; + self.writer.flush().await?; + + let expected = count as usize + 2; + let mut response = vec![0u8; expected]; + self.reader.read_exact(&mut response).await?; + validate_response_header(asic, OPCODE_UART_READREG, &response)?; + Ok(response[2..].to_vec()) + } + + pub async fn read_register_u8( + &mut self, + asic: u8, + engine_address: u16, + offset: u8, + ) -> Result { + Ok(self.read_register(asic, engine_address, offset, 1).await?[0]) + } + + pub async fn read_register_u32( + &mut self, + asic: u8, + engine_address: u16, + offset: u8, + ) -> Result { + let data = self.read_register(asic, engine_address, offset, 4).await?; + Ok(u32::from_le_bytes(data.try_into().unwrap())) + } + + pub async fn read_local_reg_u8(&mut self, asic: u8, offset: u8) -> Result { + self.read_register_u8(asic, NOTCH_REG, offset).await + } + + pub async fn read_local_reg_u32(&mut self, asic: u8, offset: u8) -> Result { + self.read_register_u32(asic, NOTCH_REG, offset).await + } + + pub async fn noop(&mut self, asic: u8) -> Result<[u8; 3], Bzm2UartError> { + let request = encode_noop(asic); + self.writer.write_all(&request).await?; + self.writer.flush().await?; + + let mut response = [0u8; 5]; + self.reader.read_exact(&mut response).await?; + validate_response_header(asic, OPCODE_UART_NOOP, &response)?; + Ok(response[2..5].try_into().unwrap()) + } + + pub async fn noop_with_timeout( + &mut self, + asic: u8, + wait: Duration, + ) -> Result<[u8; 3], Bzm2UartError> { + let request = encode_noop(asic); + self.writer.write_all(&request).await?; + self.writer.flush().await?; + + let mut response = [0u8; 5]; + timeout(wait, self.reader.read_exact(&mut response)) + .await + .map_err(|_| Bzm2UartError::NoopTimeout { + asic, + timeout_ms: wait.as_millis().min(u128::from(u64::MAX)) as u64, + })??; + validate_response_header(asic, OPCODE_UART_NOOP, &response)?; + Ok(response[2..5].try_into().unwrap()) + } + + pub async fn verify_noop_bz2(&mut self, asic: u8) -> Result<(), Bzm2UartError> { + let data = self.noop(asic).await?; + if data == *b"BZ2" { + Ok(()) + } else { + Err(Bzm2UartError::UnexpectedNoopPayload { asic, data }) + } + } + + pub async fn verify_noop_bz2_with_timeout( + &mut self, + asic: u8, + wait: Duration, + ) -> Result<(), Bzm2UartError> { + let data = self.noop_with_timeout(asic, wait).await?; + if data == *b"BZ2" { + Ok(()) + } else { + Err(Bzm2UartError::UnexpectedNoopPayload { asic, data }) + } + } + + /// Enumerate a fresh chain by assigning ids to devices that still answer on + /// the documented default ASIC id `0xFA`. + pub async fn enumerate_chain( + &mut self, + max_asics: u8, + start_id: u8, + ) -> Result, Bzm2UartError> { + self.enumerate_chain_with_timeout(max_asics, start_id, DEFAULT_NOOP_PROBE_TIMEOUT) + .await + } + + /// Enumerate a fresh chain using a bounded NOOP probe so the walk can stop + /// cleanly when the last default-id device has been assigned. + pub async fn enumerate_chain_with_timeout( + &mut self, + max_asics: u8, + start_id: u8, + probe_timeout: Duration, + ) -> Result, Bzm2UartError> { + let mut assigned = Vec::new(); + for offset in 0..max_asics { + let next_id = start_id.saturating_add(offset); + if self + .verify_noop_bz2_with_timeout(DEFAULT_ASIC_ID, probe_timeout) + .await + .is_err() + { + break; + } + self.assign_default_asic_id(next_id).await?; + // Bound the post-assignment verify with the same probe timeout: a + // device that passed the timed probe on 0xfa but then fails to echo + // on its new id (it died, an id collision garbled the reply, or the + // assignment half-took) must not wedge enumeration — and board init + // with it — on an unbounded read_exact. + self.verify_noop_bz2_with_timeout(next_id, probe_timeout) + .await?; + assigned.push(next_id); + } + Ok(assigned) + } + + pub async fn set_tdm_enabled( + &mut self, + prediv_raw: u32, + counter: u8, + enable: bool, + ) -> Result<(), Bzm2UartError> { + set_tdm_enabled_stream(&mut self.writer, prediv_raw, counter, enable).await + } + + pub async fn enable_tdm(&mut self, prediv_raw: u32, counter: u8) -> Result<(), Bzm2UartError> { + self.set_tdm_enabled(prediv_raw, counter, true).await + } + + pub async fn disable_tdm(&mut self, prediv_raw: u32, counter: u8) -> Result<(), Bzm2UartError> { + self.set_tdm_enabled(prediv_raw, counter, false).await + } + + pub async fn read_register_tdm_sync( + &mut self, + asic: u8, + engine_address: u16, + offset: u8, + count: u8, + wait: Duration, + ) -> Result, Bzm2UartError> { + read_register_tdm_sync_stream( + &mut self.reader, + &mut self.writer, + asic, + engine_address, + offset, + count, + wait, + ) + .await + } + + pub async fn detect_engine( + &mut self, + asic: u8, + row: u8, + col: u8, + wait: Duration, + ) -> Result { + detect_engine_stream(&mut self.reader, &mut self.writer, asic, row, col, wait).await + } + + pub async fn discover_engine_map( + &mut self, + asic: u8, + tdm_prediv_raw: u32, + tdm_counter: u8, + wait: Duration, + ) -> Result { + discover_engine_map_stream( + &mut self.reader, + &mut self.writer, + asic, + tdm_prediv_raw, + tdm_counter, + wait, + ) + .await + } + + pub async fn loopback(&mut self, asic: u8, payload: &[u8]) -> Result, Bzm2UartError> { + let request = encode_loopback(asic, payload); + self.writer.write_all(&request).await?; + self.writer.flush().await?; + + let expected = payload.len() + 2; + let mut response = vec![0u8; expected]; + self.reader.read_exact(&mut response).await?; + validate_response_header(asic, OPCODE_UART_LOOPBACK, &response)?; + Ok(response[2..].to_vec()) + } + + // Mirrors the write-job wire format field for field; a parameter struct + // would obscure the correspondence with the opcode layout. + #[allow(clippy::too_many_arguments)] + pub async fn write_job( + &mut self, + asic: u8, + engine_address: u16, + midstate: &[u8; 32], + merkle_root_residue: u32, + ntime: u32, + sequence_id: u8, + job_control: u8, + ) -> Result<(), Bzm2UartError> { + self.writer + .write_all(&encode_write_job( + asic, + engine_address, + midstate, + merkle_root_residue, + ntime, + sequence_id, + job_control, + )) + .await?; + self.writer.flush().await?; + Ok(()) + } + + /// Enable DTS/VS reporting using the legacy local-register sequence. + pub async fn enable_dts_vs(&mut self, config: Bzm2DtsVsConfig) -> Result<(), Bzm2UartError> { + configure_dts_vs_stream(&mut self.writer, &mut self.reader, &config).await + } + + /// Read the next DTS/VS frame for a specific ASIC after ensuring DTS/VS is enabled. + pub async fn query_dts_vs( + &mut self, + asic: u8, + generation: DtsVsGeneration, + config: Bzm2DtsVsConfig, + timeout: Duration, + ) -> Result { + self.enable_dts_vs(config).await?; + read_dts_vs_frame_stream(&mut self.reader, generation, asic, timeout).await + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Bzm2DtsVsConfig { + pub tdm_interval: u8, + pub thermal_trip_c: i32, + pub voltage_ch0_shutdown_mv: u32, + pub voltage_ch1_shutdown_mv: u32, +} + +impl Default for Bzm2DtsVsConfig { + fn default() -> Self { + Self { + tdm_interval: 1, + thermal_trip_c: 115, + voltage_ch0_shutdown_mv: 500, + voltage_ch1_shutdown_mv: 500, + } + } +} + +#[derive(Debug, thiserror::Error)] +pub enum Bzm2UartError { + #[error("serial I/O failed: {0}")] + Io(#[from] std::io::Error), + + #[error("short UART response: expected {expected} bytes, got {actual}")] + ShortResponse { expected: usize, actual: usize }, + + #[error( + "unexpected UART response header: expected asic {expected_asic:#x} opcode {expected_opcode:#x}, got asic {actual_asic:#x} opcode {actual_opcode:#x}" + )] + UnexpectedHeader { + expected_asic: u8, + expected_opcode: u8, + actual_asic: u8, + actual_opcode: u8, + }, + + #[error("unexpected NOOP payload from ASIC {asic:#x}: {data:02x?}")] + UnexpectedNoopPayload { asic: u8, data: [u8; 3] }, + + #[error("timed out waiting for NOOP response from ASIC {asic:#x} after {timeout_ms} ms")] + NoopTimeout { asic: u8, timeout_ms: u64 }, + + #[error("timed out waiting for DTS/VS frame from ASIC {asic:#x}")] + DtsVsTimeout { asic: u8 }, + + #[error( + "timed out waiting for TDM register response from ASIC {asic:#x} engine {engine_address:#05x} offset {offset:#04x}" + )] + TdmRegisterTimeout { + asic: u8, + engine_address: u16, + offset: u8, + }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub struct Bzm2EngineCoordinate { + pub row: u8, + pub col: u8, + pub engine_address: u16, +} + +impl Bzm2EngineCoordinate { + pub fn new(row: u8, col: u8) -> Self { + Self { + row, + col, + engine_address: logical_engine_address(row, col), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Bzm2DiscoveredEngineMap { + pub asic: u8, + pub present: Vec, + pub missing: Vec, +} + +impl Bzm2DiscoveredEngineMap { + pub fn present_count(&self) -> usize { + self.present.len() + } + + pub fn missing_count(&self) -> usize { + self.missing.len() + } +} + +pub async fn configure_dts_vs_stream( + writer: &mut SerialWriter, + reader: &mut SerialReader, + config: &Bzm2DtsVsConfig, +) -> Result<(), Bzm2UartError> { + // Enable thermal and voltage sensor messages on the UART TDM path. + write_local_reg_u32_raw(writer, BROADCAST_ASIC, LOCAL_REG_UART_TX, 0x0f).await?; + + // Legacy reference clock setup: 50 MHz reference, 6.25 MHz sensor clocks. + let slow_clk_div = 2u32; + let sensor_clk_div = 8u32; + write_local_reg_u32_raw(writer, BROADCAST_ASIC, LOCAL_REG_SLOW_CLK_DIV, slow_clk_div).await?; + write_local_reg_u32_raw( + writer, + BROADCAST_ASIC, + LOCAL_REG_SENSOR_CLK_DIV, + (sensor_clk_div << 5) | sensor_clk_div, + ) + .await?; + write_local_reg_u32_raw(writer, BROADCAST_ASIC, LOCAL_REG_DTS_SRST_PD, 1 << 8).await?; + write_local_reg_u32_raw( + writer, + BROADCAST_ASIC, + LOCAL_REG_SENS_TDM_GAP_CNT, + config.tdm_interval as u32, + ) + .await?; + + let cfg0_ts_resolution = match THERMAL_SENSOR_RESOLUTION { + 10 => 1, + 8 => 2, + _ => 0, + }; + write_local_reg_u32_raw( + writer, + BROADCAST_ASIC, + LOCAL_REG_DTS_CFG, + ((cfg0_ts_resolution as u32) << 5) | THERMAL_SENSOR_MODE as u32, + ) + .await?; + + let thermal_threshold_cnt = 10u32; + let voltage_ch0_threshold_cnt = 10u32; + write_local_reg_u32_raw( + writer, + BROADCAST_ASIC, + LOCAL_REG_SENSOR_THRS_CNT, + (thermal_threshold_cnt << 16) | voltage_ch0_threshold_cnt, + ) + .await?; + + let thermal_trip_code = legacy_temperature_c_to_tune_code(config.thermal_trip_c); + write_local_reg_u32_raw( + writer, + BROADCAST_ASIC, + LOCAL_REG_TEMPSENSOR_TUNE_CODE, + 0x8001 | (thermal_trip_code << 1), + ) + .await?; + + let bandgap = read_local_reg_u32_raw(reader, writer, BROADCAST_ASIC, LOCAL_REG_BANDGAP).await?; + write_local_reg_u32_raw( + writer, + BROADCAST_ASIC, + LOCAL_REG_BANDGAP, + (bandgap & !0x0f) | 0x03, + ) + .await?; + + write_local_reg_u32_raw(writer, BROADCAST_ASIC, LOCAL_REG_VSENSOR_SRST_PD, 1 << 8).await?; + + let cfg0_vs_resolution = match VOLTAGE_SENSOR_RESOLUTION { + 12 => 1, + 10 => 2, + 8 => 3, + _ => 0, + }; + let gap_cnt = 8u32; + write_local_reg_u32_raw( + writer, + BROADCAST_ASIC, + LOCAL_REG_VSENSOR_CFG, + (gap_cnt << 28) + | ((VOLTAGE_SENSOR_CONVERSION_MODE as u32) << 24) + | ((cfg0_vs_resolution as u32) << 5) + | VOLTAGE_SENSOR_MODE as u32, + ) + .await?; + + write_local_reg_u32_raw( + writer, + BROADCAST_ASIC, + LOCAL_REG_VOLTAGE_SENSOR_ENABLE, + (legacy_voltage_mv_to_tune_code(config.voltage_ch1_shutdown_mv) << 16) + | (legacy_voltage_mv_to_tune_code(config.voltage_ch0_shutdown_mv) << 1) + | 1, + ) + .await?; + + Ok(()) +} + +pub async fn set_tdm_enabled_stream( + writer: &mut SerialWriter, + prediv_raw: u32, + counter: u8, + enable: bool, +) -> Result<(), Bzm2UartError> { + write_local_reg_u32_raw( + writer, + BROADCAST_ASIC, + LOCAL_REG_UART_TDM_CTL, + encode_tdm_control(prediv_raw, counter, enable), + ) + .await +} + +pub async fn read_register_tdm_sync_stream( + reader: &mut SerialReader, + writer: &mut SerialWriter, + asic: u8, + engine_address: u16, + offset: u8, + count: u8, + wait: Duration, +) -> Result, Bzm2UartError> { + let request = encode_read_register(asic, engine_address, offset, count); + writer.write_all(&request).await?; + writer.flush().await?; + + let deadline = Instant::now() + wait; + let mut parser = TdmFrameParser::default(); + parser.expect_read_register_bytes(asic, count as usize); + let mut read_buf = [0u8; 256]; + + loop { + let now = Instant::now(); + if now >= deadline { + return Err(Bzm2UartError::TdmRegisterTimeout { + asic, + engine_address, + offset, + }); + } + let remaining = deadline.saturating_duration_since(now); + let read = timeout(remaining, reader.read(&mut read_buf)) + .await + .map_err(|_| Bzm2UartError::TdmRegisterTimeout { + asic, + engine_address, + offset, + })??; + if read == 0 { + return Err(Bzm2UartError::ShortResponse { + expected: 1, + actual: 0, + }); + } + for frame in parser.push(&read_buf[..read]) { + if let TdmFrame::Register(frame) = frame + && frame.asic == asic + { + return Ok(frame.data); + } + } + } +} + +pub async fn detect_engine_stream( + reader: &mut SerialReader, + writer: &mut SerialWriter, + asic: u8, + row: u8, + col: u8, + wait: Duration, +) -> Result { + let engine_address = logical_engine_address(row, col); + let data = read_register_tdm_sync_stream( + reader, + writer, + asic, + engine_address, + ENGINE_REG_END_NONCE, + 4, + wait, + ) + .await?; + let end_nonce = u32::from_le_bytes(data.try_into().unwrap()); + Ok(end_nonce == DISCOVERED_ENGINE_END_NONCE) +} + +pub async fn discover_engine_map_stream( + reader: &mut SerialReader, + writer: &mut SerialWriter, + asic: u8, + tdm_prediv_raw: u32, + tdm_counter: u8, + wait: Duration, +) -> Result { + set_tdm_enabled_stream(writer, tdm_prediv_raw, tdm_counter, true).await?; + + let result = async { + let mut present = Vec::new(); + let mut missing = Vec::new(); + + for (row, col) in physical_engine_coordinates() { + let coordinate = Bzm2EngineCoordinate::new(row, col); + if detect_engine_stream(reader, writer, asic, row, col, wait).await? { + present.push(coordinate); + } else { + missing.push(coordinate); + } + } + + Ok(Bzm2DiscoveredEngineMap { + asic, + present, + missing, + }) + } + .await; + + let disable_result = set_tdm_enabled_stream(writer, tdm_prediv_raw, tdm_counter, false).await; + match (result, disable_result) { + (Ok(map), Ok(())) => Ok(map), + (Err(err), _) => Err(err), + (Ok(_), Err(err)) => Err(err), + } +} + +pub async fn read_dts_vs_frame_stream( + reader: &mut SerialReader, + generation: DtsVsGeneration, + asic: u8, + timeout: Duration, +) -> Result { + let deadline = tokio::time::Instant::now() + timeout; + let mut parser = TdmFrameParser::new(generation); + let mut read_buf = [0u8; 256]; + + loop { + let now = tokio::time::Instant::now(); + if now >= deadline { + return Err(Bzm2UartError::DtsVsTimeout { asic }); + } + let remaining = deadline.saturating_duration_since(now); + let read = tokio::time::timeout(remaining, reader.read(&mut read_buf)) + .await + .map_err(|_| Bzm2UartError::DtsVsTimeout { asic })??; + if read == 0 { + return Err(Bzm2UartError::ShortResponse { + expected: 1, + actual: 0, + }); + } + for frame in parser.push(&read_buf[..read]) { + if let TdmFrame::DtsVs(frame) = frame { + let frame_asic = match frame { + TdmDtsVsFrame::Gen1(gen1) => gen1.asic, + TdmDtsVsFrame::Gen2(gen2) => gen2.asic, + }; + if frame_asic == asic { + return Ok(frame); + } + } + } + } +} + +async fn write_local_reg_u32_raw( + writer: &mut SerialWriter, + asic: u8, + offset: u8, + value: u32, +) -> Result<(), Bzm2UartError> { + writer + .write_all(&encode_write_register( + asic, + NOTCH_REG, + offset, + &value.to_le_bytes(), + )) + .await?; + writer.flush().await?; + Ok(()) +} + +async fn read_local_reg_u32_raw( + reader: &mut SerialReader, + writer: &mut SerialWriter, + asic: u8, + offset: u8, +) -> Result { + let request = encode_read_register(asic, NOTCH_REG, offset, 4); + writer.write_all(&request).await?; + writer.flush().await?; + + let mut response = [0u8; 6]; + reader.read_exact(&mut response).await?; + validate_response_header(asic, OPCODE_UART_READREG, &response)?; + Ok(u32::from_le_bytes(response[2..6].try_into().unwrap())) +} + +fn legacy_temperature_c_to_tune_code(temperature_c: i32) -> u32 { + let resolution_power = match THERMAL_SENSOR_RESOLUTION { + 10 => 1024.0_f32, + 8 => 256.0_f32, + _ => 4096.0_f32, + }; + (2048.0 / resolution_power + 4096.0 * (temperature_c as f32 + 293.8) / 631.8) as u32 +} + +fn legacy_voltage_mv_to_tune_code(voltage_mv: u32) -> u32 { + let resolution_power = match VOLTAGE_SENSOR_RESOLUTION { + 12 => 4096.0_f32, + 10 => 1024.0_f32, + 8 => 256.0_f32, + _ => 16384.0_f32, + }; + ((16384.0 / 6.0) * (2.5 * voltage_mv as f32 / 706.7 + 3.0 / resolution_power + 1.0)) as u32 +} + +fn encode_tdm_control(prediv_raw: u32, counter: u8, enable: bool) -> u32 { + (prediv_raw << 9) | ((counter as u32) << 1) | u32::from(enable) +} + +fn validate_response_header( + expected_asic: u8, + expected_opcode: u8, + response: &[u8], +) -> Result<(), Bzm2UartError> { + if response.len() < 2 { + return Err(Bzm2UartError::ShortResponse { + expected: 2, + actual: response.len(), + }); + } + + let actual_asic = response[0]; + let actual_opcode = response[1]; + if actual_asic != expected_asic || actual_opcode != expected_opcode { + return Err(Bzm2UartError::UnexpectedHeader { + expected_asic, + expected_opcode, + actual_asic, + actual_opcode, + }); + } + + Ok(()) +} + +#[cfg(all(test, unix))] +mod tests { + use super::*; + use std::fs; + use std::io::{Read, Write}; + use std::os::fd::AsRawFd; + + use nix::pty::openpty; + + use crate::transport::SerialStream; + + #[test] + fn response_header_validation_accepts_matching_unicast_response() { + validate_response_header(0x12, OPCODE_UART_READREG, &[0x12, OPCODE_UART_READREG]).unwrap(); + } + + #[test] + fn response_header_validation_rejects_mismatched_response() { + let err = validate_response_header(0x12, OPCODE_UART_NOOP, &[0x13, OPCODE_UART_LOOPBACK]) + .unwrap_err(); + assert!(matches!( + err, + Bzm2UartError::UnexpectedHeader { + expected_asic: 0x12, + expected_opcode: OPCODE_UART_NOOP, + actual_asic: 0x13, + actual_opcode: OPCODE_UART_LOOPBACK, + } + )); + } + + #[test] + fn legacy_temperature_query_threshold_matches_legacy_formula_family() { + assert_eq!(legacy_temperature_c_to_tune_code(115), 2650); + } + + #[test] + fn legacy_voltage_query_threshold_matches_legacy_formula_family() { + assert_eq!(legacy_voltage_mv_to_tune_code(500), 7561); + } + + #[test] + fn default_asic_id_matches_legacy_value() { + assert_eq!(DEFAULT_ASIC_ID, 0xfa); + } + + #[test] + fn default_noop_probe_timeout_is_bounded() { + assert_eq!(DEFAULT_NOOP_PROBE_TIMEOUT, Duration::from_millis(100)); + } + + #[test] + fn discovered_engine_map_counts_entries() { + let map = Bzm2DiscoveredEngineMap { + asic: 2, + present: vec![ + Bzm2EngineCoordinate::new(0, 0), + Bzm2EngineCoordinate::new(1, 0), + ], + missing: vec![Bzm2EngineCoordinate::new(0, 4)], + }; + + assert_eq!(map.present_count(), 2); + assert_eq!(map.missing_count(), 1); + } + + #[tokio::test] + async fn read_register_tdm_sync_decodes_engine_response() { + let pty = openpty(None, None).unwrap(); + let master = pty.master; + let slave = pty.slave; + let serial_path = fs::read_link(format!("/proc/self/fd/{}", slave.as_raw_fd())) + .unwrap() + .to_string_lossy() + .into_owned(); + let engine_address = logical_engine_address(3, 4); + + let emulator = std::thread::spawn(move || { + let mut file = fs::File::from(master); + let mut request = [0u8; 8]; + file.read_exact(&mut request).unwrap(); + assert_eq!( + request.to_vec(), + encode_read_register(2, engine_address, ENGINE_REG_END_NONCE, 4) + ); + file.write_all(&[2, OPCODE_UART_READREG, 0xfe, 0xff, 0xff, 0xff]) + .unwrap(); + file.flush().unwrap(); + std::thread::sleep(Duration::from_millis(20)); + }); + + let stream = SerialStream::new(&serial_path, 5_000_000).unwrap(); + let (reader, writer, _control) = stream.split(); + let mut uart = Bzm2UartController::new(reader, writer); + let data = uart + .read_register_tdm_sync( + 2, + engine_address, + ENGINE_REG_END_NONCE, + 4, + Duration::from_millis(100), + ) + .await + .unwrap(); + assert_eq!(data, vec![0xfe, 0xff, 0xff, 0xff]); + + emulator.join().unwrap(); + } + + #[tokio::test] + async fn discover_engine_map_scans_physical_coordinates() { + let pty = openpty(None, None).unwrap(); + let master = pty.master; + let slave = pty.slave; + let serial_path = fs::read_link(format!("/proc/self/fd/{}", slave.as_raw_fd())) + .unwrap() + .to_string_lossy() + .into_owned(); + let present = std::collections::BTreeSet::from([(0u8, 0u8), (19u8, 10u8)]); + let prediv = 0x0f; + let counter = 16; + + let emulator = std::thread::spawn(move || { + let mut file = fs::File::from(master); + let expected_enable = encode_write_register( + BROADCAST_ASIC, + NOTCH_REG, + LOCAL_REG_UART_TDM_CTL, + &encode_tdm_control(prediv, counter, true).to_le_bytes(), + ); + let mut enable_request = vec![0u8; expected_enable.len()]; + file.read_exact(&mut enable_request).unwrap(); + assert_eq!(enable_request, expected_enable); + + for (row, col) in physical_engine_coordinates() { + let mut request = [0u8; 8]; + file.read_exact(&mut request).unwrap(); + assert_eq!( + request.to_vec(), + encode_read_register( + 1, + logical_engine_address(row, col), + ENGINE_REG_END_NONCE, + 4 + ) + ); + let value = if present.contains(&(row, col)) { + DISCOVERED_ENGINE_END_NONCE + } else { + 0 + }; + let mut response = vec![1, OPCODE_UART_READREG]; + response.extend_from_slice(&value.to_le_bytes()); + file.write_all(&response).unwrap(); + file.flush().unwrap(); + } + + let expected_disable = encode_write_register( + BROADCAST_ASIC, + NOTCH_REG, + LOCAL_REG_UART_TDM_CTL, + &encode_tdm_control(prediv, counter, false).to_le_bytes(), + ); + let mut disable_request = vec![0u8; expected_disable.len()]; + file.read_exact(&mut disable_request).unwrap(); + assert_eq!(disable_request, expected_disable); + std::thread::sleep(Duration::from_millis(20)); + }); + + let stream = SerialStream::new(&serial_path, 5_000_000).unwrap(); + let (reader, writer, _control) = stream.split(); + let mut uart = Bzm2UartController::new(reader, writer); + let discovery = uart + .discover_engine_map(1, prediv, counter, Duration::from_millis(100)) + .await + .unwrap(); + + assert_eq!(discovery.present_count(), 2); + assert_eq!( + discovery.missing_count(), + physical_engine_coordinates().len() - 2 + ); + assert_eq!( + discovery.present, + vec![ + Bzm2EngineCoordinate::new(0, 0), + Bzm2EngineCoordinate::new(19, 10) + ] + ); + + emulator.join().unwrap(); + } + + #[tokio::test] + async fn enumerate_chain_bounds_post_assignment_verify() { + let pty = openpty(None, None).unwrap(); + let master = pty.master; + let slave = pty.slave; + let serial_path = fs::read_link(format!("/proc/self/fd/{}", slave.as_raw_fd())) + .unwrap() + .to_string_lossy() + .into_owned(); + + let emulator = std::thread::spawn(move || { + let mut file = fs::File::from(master); + + // First timed NOOP probe on the default id -> answer "BZ2" (ok). + let mut probe = [0u8; 4]; + file.read_exact(&mut probe).unwrap(); + assert_eq!(probe.to_vec(), encode_noop(DEFAULT_ASIC_ID)); + file.write_all(&[DEFAULT_ASIC_ID, OPCODE_UART_NOOP, b'B', b'Z', b'2']) + .unwrap(); + file.flush().unwrap(); + + // Accept the assign-id write... + let assign_len = encode_write_register( + DEFAULT_ASIC_ID, + NOTCH_REG, + LOCAL_REG_ASIC_ID, + &7u32.to_le_bytes(), + ) + .len(); + let mut assign = vec![0u8; assign_len]; + file.read_exact(&mut assign).unwrap(); + + // ...then go silent for the post-assignment verify probe. + let mut verify = [0u8; 4]; + let _ = file.read_exact(&mut verify); + std::thread::sleep(Duration::from_millis(300)); + }); + + let stream = SerialStream::new(&serial_path, 5_000_000).unwrap(); + let (reader, writer, _control) = stream.split(); + let mut uart = Bzm2UartController::new(reader, writer); + + // Without the post-assign timeout this call never returns; assert it + // completes with a bounded timeout error well inside the wall clock. + let result = tokio::time::timeout( + Duration::from_secs(2), + uart.enumerate_chain_with_timeout(4, 7, Duration::from_millis(100)), + ) + .await; + assert!( + result.is_ok(), + "enumerate_chain hung on the post-assign verify" + ); + assert!(matches!( + result.unwrap(), + Err(Bzm2UartError::NoopTimeout { asic: 7, .. }) + )); + + emulator.join().unwrap(); + } +} diff --git a/mujina-miner/src/asic/hash_thread.rs b/mujina-miner/src/asic/hash_thread.rs index bf4f2636..6be56026 100644 --- a/mujina-miner/src/asic/hash_thread.rs +++ b/mujina-miner/src/asic/hash_thread.rs @@ -76,6 +76,29 @@ pub struct HashThreadStatus { pub is_active: bool, } +/// Temperature reading reported by a hash thread. +#[derive(Debug, Clone, PartialEq)] +pub struct HashThreadTemperatureReading { + pub name: String, + pub temperature_c: Option, +} + +/// Voltage/current/power reading reported by a hash thread. +#[derive(Debug, Clone, PartialEq)] +pub struct HashThreadPowerReading { + pub name: String, + pub voltage_v: Option, + pub current_a: Option, + pub power_w: Option, +} + +/// Telemetry update reported by a hash thread. +#[derive(Debug, Clone, PartialEq, Default)] +pub struct HashThreadTelemetryUpdate { + pub temperatures: Vec, + pub powers: Vec, +} + /// Events emitted by HashThreads back to the scheduler. /// /// When a thread shuts down (USB unplug, fault, user request, etc.), it closes @@ -106,8 +129,38 @@ pub enum HashThreadEvent { /// /// Emitted after `configure()` and whenever the expectation changes. ExpectedHashRate(HashRate), + + /// Additional telemetry update + TelemetryUpdate(HashThreadTelemetryUpdate), } +/// Error types for HashThread operations. +#[derive(Debug, thiserror::Error)] +pub enum HashThreadError { + #[error("Thread has been shut down")] + ThreadOffline, + + #[error("Channel closed: {0}")] + ChannelClosed(String), + + #[error("Work assignment failed: {0}")] + WorkAssignmentFailed(String), + + #[error("Preemption failed: {0}")] + PreemptionFailed(String), + + #[error("Telemetry query failed: {0}")] + TelemetryQueryFailed(String), + + #[error("Diagnostics failed: {0}")] + DiagnosticsFailed(String), + + #[error("Shutdown timeout")] + ShutdownTimeout, + + #[error("Chip initialization failed: {0}")] + InitializationFailed(String), +} // --------------------------------------------------------------------------- // Hardware abstraction traits for hash threads // --------------------------------------------------------------------------- diff --git a/mujina-miner/src/asic/mod.rs b/mujina-miner/src/asic/mod.rs index 6d8a347a..4cb84451 100644 --- a/mujina-miner/src/asic/mod.rs +++ b/mujina-miner/src/asic/mod.rs @@ -1,4 +1,5 @@ pub mod bm13xx; +pub mod bzm2; pub mod hash_thread; /// Information about a chip diff --git a/mujina-miner/src/backplane.rs b/mujina-miner/src/backplane.rs index 44c30612..26e91ffa 100644 --- a/mujina-miner/src/backplane.rs +++ b/mujina-miner/src/backplane.rs @@ -139,10 +139,14 @@ impl Backplane { info, threads, telemetry_rx, + command_tx, shutdown, } = conn; - let registration = BoardRegistration { telemetry_rx }; + let registration = BoardRegistration { + telemetry_rx, + command_tx, + }; if let Err(e) = self.board_reg_tx.send(registration).await { error!( board = %info.model, @@ -290,6 +294,41 @@ impl Backplane { } } + Ok(()) + } + /// Attach a configured board directly without going through a synthetic transport. + pub async fn attach_configured_board( + &mut self, + device_type: &str, + device_id: String, + ) -> Result<()> { + let Some(descriptor) = self.virtual_registry.find(device_type) else { + error!(device_type = %device_type, "No configured board descriptor found"); + return Ok(()); + }; + + info!( + board = descriptor.name, + device_type = %device_type, + device_id = %device_id, + "Configured board attached." + ); + + let conn = match (descriptor.create_fn)().await { + Ok(conn) => conn, + Err(e) => { + error!( + board = descriptor.name, + device_type = %device_type, + error = %e, + "Failed to create configured board" + ); + return Ok(()); + } + }; + + self.start_board(device_id, conn).await; + Ok(()) } } diff --git a/mujina-miner/src/board/bitaxe.rs b/mujina-miner/src/board/bitaxe.rs index 2108e665..03b3769b 100644 --- a/mujina-miner/src/board/bitaxe.rs +++ b/mujina-miner/src/board/bitaxe.rs @@ -228,6 +228,7 @@ async fn create_from_usb(device: UsbDeviceInfo) -> Result { info, threads, telemetry_rx, + command_tx: None, shutdown: Some(shutdown), }) } @@ -419,6 +420,7 @@ impl Bitaxe { }, ], threads: Vec::new(), // TODO: populate from hash thread telemetry + ..Default::default() }); // Periodic log diff --git a/mujina-miner/src/board/bzm2/bringup.rs b/mujina-miner/src/board/bzm2/bringup.rs new file mode 100644 index 00000000..567a5bbd --- /dev/null +++ b/mujina-miner/src/board/bzm2/bringup.rs @@ -0,0 +1,557 @@ +//! Power-rail bring-up, reset sequencing, and voltage/frequency application for the BZM2 board. + +use std::collections::BTreeMap; +use std::env; +use std::time::Duration; + +use crate::api_client::types::{Bzm2StartupPath, PowerMeasurement, TemperatureSensor}; +use crate::asic::bzm2::{Bzm2ClockController, Bzm2Pll}; +use crate::board::power::{ + FileGpioPin, FilePowerRail, GpioResetLine, PowerRail, VoltageStackBringupPlan, VoltageStackStep, +}; +use crate::tracing::prelude::*; +use crate::transport::SerialStream; +use crate::types::Temperature; + +use super::calibration::{ + Bzm2BusLayout, Bzm2PersistedCalibrationProfile, store_applied_operating_state, +}; +use super::config::{ + DEFAULT_BOARD_TEMP_SCALE, DEFAULT_BRINGUP_POST_POWER_MS, DEFAULT_BRINGUP_PRE_POWER_MS, + DEFAULT_BRINGUP_RELEASE_RESET_MS, DEFAULT_CALIBRATION_REPLAY_FREQ_MHZ, DEFAULT_CURRENT_SCALE, + DEFAULT_POWER_SCALE, DEFAULT_VOLTAGE_SCALE, average_f32, env_csv_strings_any, env_flag_any, + env_flag_default_any, env_var_any, parse_csv_numbers, parse_csv_numbers_any, +}; +use super::telemetry::{Bzm2TelemetrySnapshot, SensorSpec, sensor_specs_from_env}; +use super::{BoardError, Bzm2Board}; + +#[derive(Debug, Clone)] +pub struct Bzm2BringupConfig { + pub enabled: bool, + pub rail_set_paths: Vec, + pub rail_write_scales: Vec, + pub domain_rail_indices: Vec, + pub rail_enable_paths: Vec, + pub rail_enable_values: Vec, + pub rail_vin: Vec, + pub rail_vout: Vec, + pub rail_current: Vec, + pub rail_power: Vec, + pub rail_temperature: Vec, + pub reset_path: Option, + pub reset_active_low: bool, + pub plan: VoltageStackBringupPlan, +} + +impl Default for Bzm2BringupConfig { + fn default() -> Self { + Self { + enabled: false, + rail_set_paths: Vec::new(), + rail_write_scales: Vec::new(), + domain_rail_indices: Vec::new(), + rail_enable_paths: Vec::new(), + rail_enable_values: Vec::new(), + rail_vin: Vec::new(), + rail_vout: Vec::new(), + rail_current: Vec::new(), + rail_power: Vec::new(), + rail_temperature: Vec::new(), + reset_path: None, + reset_active_low: true, + plan: VoltageStackBringupPlan { + pre_power_delay: Duration::from_millis(DEFAULT_BRINGUP_PRE_POWER_MS), + post_power_delay: Duration::from_millis(DEFAULT_BRINGUP_POST_POWER_MS), + release_reset_delay: Duration::from_millis(DEFAULT_BRINGUP_RELEASE_RESET_MS), + ..Default::default() + }, + } + } +} + +impl Bzm2BringupConfig { + pub(super) fn from_env() -> Self { + let rail_set_paths = env_csv_strings_any(&[ + "MUJINA_BZM2_RAIL_SET_PATHS", + "MUJINA_BZM2_BRINGUP_RAIL_SET_PATHS", + ]); + let rail_target_volts = parse_csv_numbers::("MUJINA_BZM2_RAIL_TARGET_VOLTS") + .or_else(|| parse_csv_numbers::("MUJINA_BZM2_BRINGUP_RAIL_TARGET_VOLTS")) + .unwrap_or_default(); + let rail_write_scales = parse_csv_numbers::("MUJINA_BZM2_RAIL_WRITE_SCALES") + .or_else(|| parse_csv_numbers::("MUJINA_BZM2_BRINGUP_RAIL_WRITE_SCALES")) + .unwrap_or_default(); + let domain_rail_indices = + parse_csv_numbers_any::(&["MUJINA_BZM2_DOMAIN_RAIL_INDICES"]) + .unwrap_or_default(); + let rail_enable_paths = env_csv_strings_any(&[ + "MUJINA_BZM2_RAIL_ENABLE_PATHS", + "MUJINA_BZM2_BRINGUP_RAIL_ENABLE_PATHS", + ]); + let rail_enable_values = env_csv_strings_any(&[ + "MUJINA_BZM2_RAIL_ENABLE_VALUES", + "MUJINA_BZM2_BRINGUP_RAIL_ENABLE_VALUES", + ]); + let rail_vin = sensor_specs_from_env( + &["MUJINA_BZM2_RAIL_VIN_PATHS"], + &["MUJINA_BZM2_RAIL_VIN_SCALES"], + DEFAULT_VOLTAGE_SCALE, + ); + let rail_vout = sensor_specs_from_env( + &["MUJINA_BZM2_RAIL_VOUT_PATHS"], + &["MUJINA_BZM2_RAIL_VOUT_SCALES"], + DEFAULT_VOLTAGE_SCALE, + ); + let rail_current = sensor_specs_from_env( + &["MUJINA_BZM2_RAIL_CURRENT_PATHS"], + &["MUJINA_BZM2_RAIL_CURRENT_SCALES"], + DEFAULT_CURRENT_SCALE, + ); + let rail_power = sensor_specs_from_env( + &["MUJINA_BZM2_RAIL_POWER_PATHS"], + &["MUJINA_BZM2_RAIL_POWER_SCALES"], + DEFAULT_POWER_SCALE, + ); + let rail_temperature = sensor_specs_from_env( + &["MUJINA_BZM2_RAIL_TEMP_PATHS"], + &["MUJINA_BZM2_RAIL_TEMP_SCALES"], + DEFAULT_BOARD_TEMP_SCALE, + ); + let reset_path = env_var_any(&["MUJINA_BZM2_RESET_PATH", "MUJINA_BZM2_BRINGUP_RESET_PATH"]); + let enabled = env_flag_any(&["MUJINA_BZM2_ENABLE_BRINGUP", "MUJINA_BZM2_BRINGUP_ENABLE"]) + || !rail_set_paths.is_empty() + || reset_path.is_some(); + + let mut plan = VoltageStackBringupPlan { + assert_reset_before_power: env_flag_default_any( + &["MUJINA_BZM2_ASSERT_RESET_BEFORE_POWER"], + true, + ), + pre_power_delay: Duration::from_millis( + env::var("MUJINA_BZM2_BRINGUP_PRE_POWER_MS") + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or(DEFAULT_BRINGUP_PRE_POWER_MS), + ), + post_power_delay: Duration::from_millis( + env::var("MUJINA_BZM2_BRINGUP_POST_POWER_MS") + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or(DEFAULT_BRINGUP_POST_POWER_MS), + ), + release_reset_delay: Duration::from_millis( + env::var("MUJINA_BZM2_BRINGUP_RELEASE_RESET_MS") + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or(DEFAULT_BRINGUP_RELEASE_RESET_MS), + ), + ..Default::default() + }; + plan.steps = rail_set_paths + .iter() + .enumerate() + .filter_map(|(index, _)| { + rail_target_volts + .get(index) + .or_else(|| rail_target_volts.last()) + .copied() + .map(|voltage| VoltageStackStep { + rail_index: index, + voltage, + settle_for: Duration::ZERO, + }) + }) + .collect(); + + Self { + enabled, + rail_set_paths, + rail_write_scales, + domain_rail_indices, + rail_enable_paths, + rail_enable_values, + rail_vin, + rail_vout, + rail_current, + rail_power, + rail_temperature, + reset_path, + reset_active_low: env_flag_default_any(&["MUJINA_BZM2_RESET_ACTIVE_LOW"], true), + plan, + } + } + + pub(super) fn rail_index_for_domain(&self, domain_id: u16) -> Option { + self.domain_rail_indices + .get(domain_id as usize) + .copied() + .or_else(|| { + let fallback = domain_id as usize; + (fallback < self.rail_set_paths.len()).then_some(fallback) + }) + } + + pub(super) fn has_telemetry(&self) -> bool { + !self.rail_vin.is_empty() + || !self.rail_vout.is_empty() + || !self.rail_current.is_empty() + || !self.rail_power.is_empty() + || !self.rail_temperature.is_empty() + } + + pub(super) fn snapshot_telemetry(&self) -> Bzm2TelemetrySnapshot { + let rail_count = [ + self.rail_set_paths.len(), + self.rail_vin.len(), + self.rail_vout.len(), + self.rail_current.len(), + self.rail_power.len(), + self.rail_temperature.len(), + ] + .into_iter() + .max() + .unwrap_or(0); + + let mut temperatures = Vec::new(); + let mut powers = Vec::new(); + for index in 0..rail_count { + let vin = self.rail_vin.get(index).and_then(SensorSpec::read); + let vout = self.rail_vout.get(index).and_then(SensorSpec::read); + let current = self.rail_current.get(index).and_then(SensorSpec::read); + let power = self + .rail_power + .get(index) + .and_then(SensorSpec::read) + .or_else(|| vout.zip(current).map(|(v, c)| v * c)); + let temperature_c = self.rail_temperature.get(index).and_then(SensorSpec::read); + + if let Some(temperature_c) = temperature_c { + temperatures.push(TemperatureSensor { + name: format!("rail{}-regulator", index), + temperature: Some(Temperature::from_celsius(temperature_c)), + }); + } + if vin.is_some() { + powers.push(PowerMeasurement { + name: format!("rail{}-input", index), + voltage_v: vin, + current_a: None, + power_w: None, + }); + } + if vout.is_some() || current.is_some() || power.is_some() { + powers.push(PowerMeasurement { + name: format!("rail{}-output", index), + voltage_v: vout, + current_a: current, + power_w: power, + }); + } + } + + Bzm2TelemetrySnapshot { + fans: Vec::new(), + temperatures, + powers, + trip_reason: None, + } + } + fn build_rails(&self) -> Vec { + self.rail_set_paths + .iter() + .enumerate() + .map(|(index, path)| { + let write_scale = *self + .rail_write_scales + .get(index) + .or_else(|| self.rail_write_scales.last()) + .unwrap_or(&1.0); + let mut rail = FilePowerRail::new(path.clone(), write_scale); + if let Some(enable_path) = self + .rail_enable_paths + .get(index) + .or_else(|| self.rail_enable_paths.last()) + { + let enable_value = self + .rail_enable_values + .get(index) + .or_else(|| self.rail_enable_values.last()) + .cloned() + .unwrap_or_else(|| "1".into()); + rail = rail.with_enable(enable_path.clone(), enable_value); + } + rail + }) + .collect() + } + + fn build_reset_line(&self) -> Option> { + self.reset_path.as_ref().map(|path| { + GpioResetLine::new( + FileGpioPin::new(path.clone(), "1", "0"), + self.reset_active_low, + ) + }) + } +} + +impl Bzm2Board { + pub(super) async fn apply_bringup_sequence(&mut self) -> Result<(), BoardError> { + if self.bringup_applied || !self.config.bringup.enabled { + return Ok(()); + } + + let mut rails = self.config.bringup.build_rails(); + let mut reset_line = self.config.bringup.build_reset_line(); + self.config + .bringup + .plan + .apply(&mut rails, reset_line.as_mut()) + .await + .map_err(|err| { + BoardError::InitializationFailed(format!("BZM2 bring-up sequence failed: {err}")) + })?; + self.bringup_applied = true; + Ok(()) + } + + pub(super) async fn apply_shutdown_sequence(&mut self) -> Result<(), BoardError> { + if !self.bringup_applied || !self.config.bringup.enabled { + return Ok(()); + } + + let mut rails = self.config.bringup.build_rails(); + let mut reset_line = self.config.bringup.build_reset_line(); + self.config + .bringup + .plan + .shutdown(&mut rails, reset_line.as_mut()) + .await + .map_err(|err| { + BoardError::HardwareControl(format!("BZM2 shutdown sequence failed: {err}")) + })?; + self.bringup_applied = false; + Ok(()) + } + + pub(super) async fn apply_saved_operating_point( + &self, + bus_layouts: &[Bzm2BusLayout], + profile: &Bzm2PersistedCalibrationProfile, + ) -> Result<(), BoardError> { + self.apply_domain_voltage_map(&profile.saved_state.per_domain_voltage_mv) + .await?; + for bus in bus_layouts { + if bus.asic_count == 0 { + continue; + } + let initial_frequencies = [0usize, 1usize].map(|pll_index| { + average_f32( + (bus.asic_start..bus.asic_start + bus.asic_count) + .filter_map(|asic_id| profile.saved_state.per_asic_pll_mhz.get(&asic_id)) + .map(|frequencies| frequencies[pll_index]), + ) + .unwrap_or(DEFAULT_CALIBRATION_REPLAY_FREQ_MHZ) + }); + self.apply_bus_frequency_map( + bus, + initial_frequencies, + &profile.saved_state.per_asic_pll_mhz, + ) + .await?; + } + store_applied_operating_state( + &self.applied_operating_state, + &profile.saved_state.per_domain_voltage_mv, + &profile.saved_state.per_asic_pll_mhz, + Some(profile.saved_state.clone()), + Some(Bzm2StartupPath::SavedReplay), + Some(profile.saved_operating_point_status), + &profile.saved_operating_point_reasons, + ); + Ok(()) + } + + pub(super) async fn apply_domain_voltage_map( + &self, + per_domain_voltage_mv: &BTreeMap, + ) -> Result<(), BoardError> { + if per_domain_voltage_mv.is_empty() { + return Ok(()); + } + if self.config.bringup.rail_set_paths.is_empty() { + warn!( + board = %self.config.device_id(), + ?per_domain_voltage_mv, + "planner produced per-domain voltages, but no BZM2 rail control path is configured" + ); + return Ok(()); + } + + let mut rail_targets_mv = BTreeMap::::new(); + for (&domain_id, &voltage_mv) in per_domain_voltage_mv { + let rail_index = self + .config + .bringup + .rail_index_for_domain(domain_id) + .ok_or_else(|| { + BoardError::HardwareControl(format!( + "BZM2 domain {domain_id} has no mapped rail index" + )) + })?; + if rail_index >= self.config.bringup.rail_set_paths.len() { + return Err(BoardError::HardwareControl(format!( + "BZM2 domain {domain_id} mapped to rail {rail_index}, but only {} rail set paths are configured", + self.config.bringup.rail_set_paths.len() + ))); + } + match rail_targets_mv.entry(rail_index) { + std::collections::btree_map::Entry::Vacant(entry) => { + entry.insert(voltage_mv); + } + std::collections::btree_map::Entry::Occupied(entry) + if *entry.get() != voltage_mv => + { + return Err(BoardError::HardwareControl(format!( + "BZM2 rail {rail_index} received conflicting domain voltages: {}mV vs {}mV", + entry.get(), + voltage_mv + ))); + } + std::collections::btree_map::Entry::Occupied(_) => {} + } + } + + let mut rails = self.config.bringup.build_rails(); + for (rail_index, voltage_mv) in rail_targets_mv { + let rail = rails.get_mut(rail_index).ok_or_else(|| { + BoardError::HardwareControl(format!( + "BZM2 rail {rail_index} is missing from configured rail controls" + )) + })?; + rail.set_voltage(voltage_mv as f32 / 1000.0) + .await + .map_err(|err| { + BoardError::HardwareControl(format!( + "Failed to apply BZM2 domain voltage {voltage_mv}mV on rail {rail_index}: {err}" + )) + })?; + } + Ok(()) + } + + pub(super) async fn apply_frequency_map( + &self, + bus_layouts: &[Bzm2BusLayout], + initial_frequencies_mhz: [f32; 2], + per_asic_pll_mhz: &BTreeMap, + ) -> Result<(), BoardError> { + for bus in bus_layouts { + self.apply_bus_frequency_map(bus, initial_frequencies_mhz, per_asic_pll_mhz) + .await?; + } + Ok(()) + } + + pub(super) async fn apply_bus_frequency_map( + &self, + bus: &Bzm2BusLayout, + initial_frequencies_mhz: [f32; 2], + per_asic_pll_mhz: &BTreeMap, + ) -> Result<(), BoardError> { + if bus.asic_count == 0 { + return Ok(()); + } + let stream = SerialStream::new(&bus.serial_path, self.config.baud_rate).map_err(|err| { + BoardError::InitializationFailed(format!( + "Failed to open BZM2 calibration transport {}: {}", + bus.serial_path, err + )) + })?; + let (reader, writer, _control) = stream.split(); + let mut clock = Bzm2ClockController::new(reader, writer); + + for (pll, frequency_mhz) in [Bzm2Pll::Pll0, Bzm2Pll::Pll1] + .into_iter() + .zip(initial_frequencies_mhz) + { + clock + .broadcast_pll_frequency( + pll, + frequency_mhz, + self.config.calibration.pll_post1_divider, + ) + .await + .map_err(|err| calibration_error(&bus.serial_path, err))?; + clock + .broadcast_enable_pll(pll) + .await + .map_err(|err| calibration_error(&bus.serial_path, err))?; + } + + if !self.config.calibration.skip_lock_check { + for local_asic in 0..bus.asic_count { + for pll in [Bzm2Pll::Pll0, Bzm2Pll::Pll1] { + clock + .wait_for_pll_lock( + local_asic as u8, + pll, + self.config.calibration.lock_timeout, + self.config.calibration.lock_poll_interval, + ) + .await + .map_err(|err| calibration_error(&bus.serial_path, err))?; + } + } + } + + for asic_id in bus.asic_start..bus.asic_start + bus.asic_count { + let Some(frequencies_mhz) = per_asic_pll_mhz.get(&asic_id) else { + continue; + }; + let local_asic = bus + .local_asic_id(asic_id) + .expect("bus layout must contain loop asic id"); + for (index, frequency_mhz) in frequencies_mhz.iter().enumerate() { + let pll = if index == 0 { + Bzm2Pll::Pll0 + } else { + Bzm2Pll::Pll1 + }; + clock + .set_pll_frequency( + local_asic, + pll, + *frequency_mhz, + self.config.calibration.pll_post1_divider, + ) + .await + .map_err(|err| calibration_error(&bus.serial_path, err))?; + clock + .enable_pll(local_asic, pll) + .await + .map_err(|err| calibration_error(&bus.serial_path, err))?; + if !self.config.calibration.skip_lock_check { + clock + .wait_for_pll_lock( + local_asic, + pll, + self.config.calibration.lock_timeout, + self.config.calibration.lock_poll_interval, + ) + .await + .map_err(|err| calibration_error(&bus.serial_path, err))?; + } + } + } + + Ok(()) + } +} + +fn calibration_error(serial_path: &str, err: impl std::fmt::Display) -> BoardError { + BoardError::InitializationFailed(format!( + "BZM2 calibration failed on {}: {}", + serial_path, err + )) +} diff --git a/mujina-miner/src/board/bzm2/calibration.rs b/mujina-miner/src/board/bzm2/calibration.rs new file mode 100644 index 00000000..233c4c98 --- /dev/null +++ b/mujina-miner/src/board/bzm2/calibration.rs @@ -0,0 +1,1074 @@ +//! Chain enumeration, calibration planner I/O, and operating-point persistence for the BZM2 board. + +use std::collections::BTreeMap; +use std::fs; +use std::path::Path; +use std::sync::{Arc, Mutex}; + +use serde::{Deserialize, Serialize}; + +use crate::api_client::types::{Bzm2SavedOperatingPointStatus, Bzm2StartupPath}; +use crate::asic::bzm2::{Bzm2DiscoveredEngineMap, Bzm2UartController}; +use crate::tracing::prelude::*; +use crate::transport::SerialStream; +use crate::tuning::blockscale::{ + Bzm2AsicMeasurement, Bzm2AsicTopology, Bzm2BoardCalibrationInput, Bzm2CalibrationConstraints, + Bzm2CalibrationPlanner, Bzm2DomainMeasurement, Bzm2SavedEngineCoordinate, + Bzm2SavedEngineTopology, Bzm2SavedOperatingPoint, Bzm2VoltageDomain, +}; + +use super::config::{ + Bzm2CalibrationConfig, DEFAULT_CALIBRATION_SITE_TEMP_C, DEFAULT_ENUMERATION_MAX_ASICS_PER_BUS, + average_u32, operating_class_name, performance_mode_name, +}; +use super::telemetry::{ + publish_discovered_engine_map, publish_saved_engine_topology, snapshot_input_power, + snapshot_temperature, +}; +use super::{BoardError, Bzm2Board}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct Bzm2BusLayout { + pub(super) serial_path: String, + pub(super) asic_start: u16, + pub(super) asic_count: u16, +} + +impl Bzm2BusLayout { + pub(super) fn contains(&self, global_asic_id: u16) -> bool { + global_asic_id >= self.asic_start && global_asic_id < self.asic_start + self.asic_count + } + + pub(super) fn global_asic_id(&self, local_asic_id: u8) -> Option { + (u16::from(local_asic_id) < self.asic_count) + .then_some(self.asic_start + u16::from(local_asic_id)) + } + + pub(super) fn local_asic_id(&self, global_asic_id: u16) -> Option { + self.contains(global_asic_id) + .then_some((global_asic_id - self.asic_start) as u8) + } +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub(super) struct Bzm2PersistedCalibrationProfile { + pub(super) schema_version: u32, + #[serde(alias = "board_bin")] + pub(super) operating_class: String, + #[serde(alias = "strategy")] + pub(super) performance_mode: String, + pub(super) asics_per_bus: Vec, + pub(super) pll_post1_divider: u8, + #[serde(default)] + pub(super) saved_operating_point_status: Bzm2SavedOperatingPointStatus, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub(super) saved_operating_point_reasons: Vec, + #[serde(alias = "calibration")] + pub(super) saved_state: Bzm2SavedOperatingPoint, +} + +impl Bzm2PersistedCalibrationProfile { + const SCHEMA_VERSION: u32 = 1; + + fn is_compatible( + &self, + calibration: &Bzm2CalibrationConfig, + bus_layouts: &[Bzm2BusLayout], + ) -> bool { + self.schema_version == Self::SCHEMA_VERSION + && self.saved_operating_point_status != Bzm2SavedOperatingPointStatus::Invalidated + && self.operating_class == operating_class_name(calibration.operating_class) + && self.performance_mode == performance_mode_name(calibration.performance_mode) + && self.pll_post1_divider == calibration.pll_post1_divider + && self.asics_per_bus + == bus_layouts + .iter() + .map(|bus| bus.asic_count) + .collect::>() + && self.saved_state.per_asic_pll_mhz.len() + == bus_layouts + .iter() + .map(|bus| bus.asic_count as usize) + .sum::() + } +} + +#[derive(Debug, Clone)] +pub(super) struct Bzm2LoadedCalibrationProfile { + pub(super) persisted: Option, + pub(super) saved_state: Bzm2SavedOperatingPoint, +} + +#[derive(Debug, Clone, Default)] +pub(super) struct Bzm2AppliedOperatingState { + pub(super) per_domain_voltage_mv: BTreeMap, + pub(super) per_asic_pll_mhz: BTreeMap, + pub(super) saved_operating_point: Option, + pub(super) startup_path: Option, + pub(super) saved_operating_point_status: Option, + pub(super) saved_operating_point_reasons: Vec, +} + +impl Bzm2Board { + pub(super) async fn resolve_bus_layouts(&self) -> Result, BoardError> { + let configured = build_bus_layouts( + &self.config.serial_paths, + &self.config.calibration.asics_per_bus, + ); + if !self.config.enumeration.enabled { + return Ok(configured); + } + + let discovered = self.enumerate_bus_layouts().await?; + if should_fallback_to_configured_bus_layouts(&discovered, &configured) { + warn!( + board = %self.config.device_id(), + "BZM2 startup enumeration found no ASICs on the default id; falling back to configured bus topology" + ); + return Ok(configured); + } + + Ok(discovered) + } + + async fn enumerate_bus_layouts(&self) -> Result, BoardError> { + let mut counts = Vec::with_capacity(self.config.serial_paths.len()); + + for (index, serial_path) in self.config.serial_paths.iter().enumerate() { + let max_asics = *self + .config + .enumeration + .max_asics_per_bus + .get(index) + .or_else(|| self.config.enumeration.max_asics_per_bus.last()) + .unwrap_or(&DEFAULT_ENUMERATION_MAX_ASICS_PER_BUS); + let max_asics = max_asics.min(u8::MAX as u16) as u8; + + let stream = SerialStream::new(serial_path, self.config.baud_rate).map_err(|err| { + BoardError::InitializationFailed(format!( + "Failed to open BZM2 enumeration transport {}: {}", + serial_path, err + )) + })?; + let (reader, writer, _control) = stream.split(); + let mut uart = Bzm2UartController::new(reader, writer); + let assigned = uart + .enumerate_chain(max_asics, self.config.enumeration.start_id) + .await + .map_err(|err| { + BoardError::InitializationFailed(format!( + "BZM2 startup enumeration failed on {}: {}", + serial_path, err + )) + })?; + counts.push(assigned.len() as u16); + info!( + board = %self.config.device_id(), + serial_path, + asic_count = assigned.len(), + "BZM2 startup enumeration completed" + ); + } + + Ok(build_discovered_bus_layouts( + &self.config.serial_paths, + &counts, + )) + } + + pub(super) async fn execute_live_calibration( + &self, + bus_layouts: &[Bzm2BusLayout], + ) -> Result<(), BoardError> { + let calibration = &self.config.calibration; + if !calibration.enabled { + return Ok(()); + } + + let total_asics = bus_layouts + .iter() + .map(|layout| layout.asic_count as usize) + .sum::(); + if total_asics == 0 { + return Ok(()); + } + + let loaded_profile = + load_saved_operating_point_profile(calibration.profile_path.as_deref()) + .map_err(BoardError::InitializationFailed)?; + if calibration.apply_saved_operating_point + && !calibration.force_retune + && let Some(profile) = loaded_profile + .as_ref() + .and_then(|loaded| loaded.persisted.as_ref()) + .filter(|profile| profile.is_compatible(calibration, bus_layouts)) + { + self.apply_saved_operating_point(bus_layouts, profile) + .await?; + info!( + board = %self.config.device_id(), + asic_count = profile.saved_state.per_asic_pll_mhz.len(), + "BZM2 replayed saved operating point profile" + ); + return Ok(()); + } + + let telemetry = self.config.telemetry.snapshot(); + let site_temp_c = calibration + .site_temp_c + .or_else(|| snapshot_temperature(&telemetry, "board")) + .or_else(|| snapshot_temperature(&telemetry, "asic")) + .unwrap_or(DEFAULT_CALIBRATION_SITE_TEMP_C); + let saved_operating_point = loaded_profile + .as_ref() + .and_then(saved_operating_point_from_loaded_profile); + let engine_topology = self + .resolve_engine_topology_for_calibration(bus_layouts, saved_operating_point.as_ref()) + .await; + let (voltage_domains, domain_lookup) = build_voltage_domains( + total_asics as u16, + &calibration.asics_per_domain, + &calibration.domain_voltage_offsets_mv, + ); + let asics = build_topology(bus_layouts, &domain_lookup, &engine_topology); + let per_asic_throughput = saved_operating_point + .as_ref() + .map(|stored| distribute_saved_throughput(stored.board_throughput_ths, &asics)); + let shared_temp = snapshot_temperature(&telemetry, "asic") + .or_else(|| snapshot_temperature(&telemetry, "board")); + let asic_measurements = asics + .iter() + .map(|asic| Bzm2AsicMeasurement { + asic_id: asic.asic_id, + temperature_c: shared_temp, + throughput_ths: per_asic_throughput + .as_ref() + .and_then(|throughput| throughput.get(&asic.asic_id).copied()), + average_pass_rate: None, + pll_pass_rates: [None, None], + }) + .collect::>(); + let shared_domain_power = snapshot_input_power(&telemetry).map(|power| { + if voltage_domains.is_empty() { + power + } else { + power / voltage_domains.len() as f32 + } + }); + let domain_measurements = voltage_domains + .iter() + .map(|domain| Bzm2DomainMeasurement { + domain_id: domain.domain_id, + measured_voltage_mv: None, + measured_power_w: shared_domain_power, + }) + .collect::>(); + + let planner = Bzm2CalibrationPlanner; + let plan = planner.plan(&Bzm2BoardCalibrationInput { + operating_class: calibration.operating_class, + site_temp_c, + target_mode: calibration.performance_mode, + mode: calibration.mode, + per_stack_clocking: calibration.per_stack_clocking, + voltage_domains: voltage_domains.clone(), + asics: asics.clone(), + saved_operating_point, + domain_measurements, + asic_measurements, + constraints: Bzm2CalibrationConstraints::default(), + force_retune: calibration.force_retune, + }); + let per_domain_voltage_mv = plan + .domain_plans + .iter() + .map(|domain| (domain.domain_id, domain.voltage_mv)) + .collect::>(); + self.apply_domain_voltage_map(&per_domain_voltage_mv) + .await?; + + let per_asic_pll_mhz = plan + .asic_plans + .iter() + .map(|plan| (plan.asic_id, plan.pll_frequencies_mhz)) + .collect::>(); + self.apply_frequency_map( + bus_layouts, + [plan.initial_frequency_mhz; 2], + &per_asic_pll_mhz, + ) + .await?; + let current_saved_operating_point = Bzm2SavedOperatingPoint { + board_voltage_mv: average_u32(plan.domain_plans.iter().map(|domain| domain.voltage_mv)) + .unwrap_or(plan.desired_voltage_mv), + board_throughput_ths: estimate_planned_hashrate( + &plan, + self.config.nominal_hashrate_ths as f32, + &asics, + ), + per_domain_voltage_mv: per_domain_voltage_mv.clone(), + per_asic_engine_topology: engine_topology.clone(), + per_asic_pll_mhz: per_asic_pll_mhz.clone(), + }; + store_applied_operating_state( + &self.applied_operating_state, + &per_domain_voltage_mv, + &per_asic_pll_mhz, + Some(current_saved_operating_point.clone()), + Some(Bzm2StartupPath::LiveCalibration), + Some(Bzm2SavedOperatingPointStatus::Pending), + &[], + ); + + if let Some(profile_path) = calibration.profile_path.as_deref() { + let profile = Bzm2PersistedCalibrationProfile { + schema_version: Bzm2PersistedCalibrationProfile::SCHEMA_VERSION, + operating_class: operating_class_name(calibration.operating_class).into(), + performance_mode: performance_mode_name(calibration.performance_mode).into(), + asics_per_bus: bus_layouts.iter().map(|bus| bus.asic_count).collect(), + pll_post1_divider: calibration.pll_post1_divider, + saved_operating_point_status: Bzm2SavedOperatingPointStatus::Pending, + saved_operating_point_reasons: Vec::new(), + saved_state: current_saved_operating_point, + }; + store_calibration_profile(profile_path, &profile) + .map_err(BoardError::InitializationFailed)?; + } + + info!(board = %self.config.device_id(), reuse_saved_operating_point = plan.reuse_saved_operating_point, needs_retune = plan.needs_retune, initial_frequency_mhz = plan.initial_frequency_mhz, asic_count = plan.asic_plans.len(), "BZM2 live calibration completed"); + Ok(()) + } + + async fn resolve_engine_topology_for_calibration( + &self, + bus_layouts: &[Bzm2BusLayout], + saved_operating_point: Option<&Bzm2SavedOperatingPoint>, + ) -> BTreeMap { + let mut topology = saved_operating_point + .map(|saved| saved.per_asic_engine_topology.clone()) + .unwrap_or_default(); + + if self.config.calibration.discover_engine_topology { + for (asic_id, discovery) in self + .discover_engine_topology_for_calibration(bus_layouts) + .await + { + topology.insert(asic_id, saved_engine_topology_from_discovery(&discovery)); + } + } + + for (thread_index, bus) in bus_layouts.iter().enumerate() { + for asic_id in bus.asic_start..bus.asic_start + bus.asic_count { + let saved = topology + .entry(asic_id) + .or_insert_with(default_saved_engine_topology) + .clone(); + if let Some(local_asic) = bus.local_asic_id(asic_id) { + publish_saved_engine_topology( + &self.telemetry_tx, + thread_index, + &bus.serial_path, + local_asic, + &saved, + ); + } + } + } + + topology + } + + async fn discover_engine_topology_for_calibration( + &self, + bus_layouts: &[Bzm2BusLayout], + ) -> BTreeMap { + let mut topology = BTreeMap::new(); + + for (thread_index, bus) in bus_layouts.iter().enumerate() { + if bus.asic_count == 0 { + continue; + } + let stream = match SerialStream::new(&bus.serial_path, self.config.baud_rate) { + Ok(stream) => stream, + Err(err) => { + warn!( + board = %self.config.device_id(), + path = %bus.serial_path, + error = %err, + "Failed to open BZM2 calibration discovery transport" + ); + continue; + } + }; + let (reader, writer, _control) = stream.split(); + let mut uart = Bzm2UartController::new(reader, writer); + + for local_asic in 0..bus.asic_count { + let global_asic = bus.asic_start + local_asic; + match uart + .discover_engine_map( + local_asic as u8, + self.config.calibration.engine_discovery_tdm_prediv_raw, + self.config.calibration.engine_discovery_tdm_counter, + self.config.calibration.engine_discovery_timeout, + ) + .await + { + Ok(discovery) => { + publish_discovered_engine_map( + &self.telemetry_tx, + thread_index, + &bus.serial_path, + &discovery, + ); + topology.insert(global_asic, discovery); + } + Err(err) => { + warn!( + board = %self.config.device_id(), + path = %bus.serial_path, + asic = local_asic, + error = %err, + "BZM2 calibration engine discovery failed; falling back to saved or default topology" + ); + } + } + } + } + + topology + } +} + +pub(super) fn build_voltage_domains( + total_asics: u16, + asics_per_domain: &[u16], + domain_voltage_offsets_mv: &[i32], +) -> (Vec, BTreeMap) { + let mut domains = Vec::new(); + let mut lookup = BTreeMap::new(); + let mut domain_id = 0u16; + let mut asic_start = 0u16; + while asic_start < total_asics { + let requested = *asics_per_domain + .get(domain_id as usize) + .or_else(|| asics_per_domain.last()) + .unwrap_or(&total_asics) + .max(&1); + let asic_end = (asic_start.saturating_add(requested)).min(total_asics); + let asic_ids = (asic_start..asic_end).collect::>(); + for asic_id in &asic_ids { + lookup.insert(*asic_id, domain_id); + } + domains.push(Bzm2VoltageDomain { + domain_id, + asic_ids, + voltage_offset_mv: *domain_voltage_offsets_mv + .get(domain_id as usize) + .or_else(|| domain_voltage_offsets_mv.last()) + .unwrap_or(&0), + max_power_w: None, + }); + domain_id = domain_id.saturating_add(1); + asic_start = asic_end; + } + (domains, lookup) +} + +pub(super) fn build_topology( + bus_layouts: &[Bzm2BusLayout], + domain_lookup: &BTreeMap, + engine_topology: &BTreeMap, +) -> Vec { + let mut asics = Vec::new(); + for layout in bus_layouts { + for asic_id in layout.asic_start..layout.asic_start + layout.asic_count { + let saved_topology = engine_topology + .get(&asic_id) + .cloned() + .unwrap_or_else(default_saved_engine_topology); + asics.push(Bzm2AsicTopology { + asic_id, + domain_id: *domain_lookup.get(&asic_id).unwrap_or(&0), + pll_count: 2, + alive: true, + active_engine_count: saved_topology.active_engine_count, + missing_engines: saved_topology.missing_engines, + }); + } + } + asics +} + +pub(super) fn default_saved_engine_topology() -> Bzm2SavedEngineTopology { + Bzm2SavedEngineTopology { + active_engine_count: crate::asic::bzm2::protocol::default_engine_coordinates().len() as u16, + missing_engines: crate::asic::bzm2::protocol::default_excluded_engines() + .into_iter() + .map(|(row, col)| Bzm2SavedEngineCoordinate { row, col }) + .collect(), + } +} + +pub(super) fn store_applied_operating_state( + state: &Arc>, + per_domain_voltage_mv: &BTreeMap, + per_asic_pll_mhz: &BTreeMap, + saved_operating_point: Option, + startup_path: Option, + saved_operating_point_status: Option, + saved_operating_point_reasons: &[String], +) { + let mut guard = state.lock().unwrap_or_else(|e| e.into_inner()); + guard.per_domain_voltage_mv = per_domain_voltage_mv.clone(); + guard.per_asic_pll_mhz = per_asic_pll_mhz.clone(); + guard.saved_operating_point = saved_operating_point; + guard.startup_path = startup_path; + guard.saved_operating_point_status = saved_operating_point_status; + guard.saved_operating_point_reasons = saved_operating_point_reasons.to_vec(); +} + +pub(super) fn load_saved_operating_point_profile( + path: Option<&Path>, +) -> Result, String> { + let Some(path) = path else { + return Ok(None); + }; + if !path.exists() { + return Ok(None); + } + let raw = fs::read_to_string(path).map_err(|err| { + format!( + "Failed to read calibration profile {}: {}", + path.display(), + err + ) + })?; + + if let Ok(profile) = serde_json::from_str::(&raw) { + return Ok(Some(Bzm2LoadedCalibrationProfile { + saved_state: profile.saved_state.clone(), + persisted: Some(profile), + })); + } + + serde_json::from_str::(&raw) + .map(|saved_state| { + Some(Bzm2LoadedCalibrationProfile { + persisted: None, + saved_state, + }) + }) + .map_err(|err| { + format!( + "Failed to parse calibration profile {}: {}", + path.display(), + err + ) + }) +} + +pub(super) fn store_saved_operating_point_status( + path: &Path, + calibration: &Bzm2CalibrationConfig, + bus_layouts: &[Bzm2BusLayout], + saved_state: &Bzm2SavedOperatingPoint, + status: Bzm2SavedOperatingPointStatus, + reasons: &[String], +) -> Result<(), String> { + store_calibration_profile( + path, + &Bzm2PersistedCalibrationProfile { + schema_version: Bzm2PersistedCalibrationProfile::SCHEMA_VERSION, + operating_class: operating_class_name(calibration.operating_class).into(), + performance_mode: performance_mode_name(calibration.performance_mode).into(), + asics_per_bus: bus_layouts.iter().map(|bus| bus.asic_count).collect(), + pll_post1_divider: calibration.pll_post1_divider, + saved_operating_point_status: status, + saved_operating_point_reasons: reasons.to_vec(), + saved_state: saved_state.clone(), + }, + ) +} + +fn estimate_planned_hashrate( + plan: &crate::tuning::blockscale::Bzm2CalibrationPlan, + nominal_hashrate_ths: f32, + asics: &[Bzm2AsicTopology], +) -> f32 { + let nominal_board_hashrate = + nominal_hashrate_ths * asics.iter().filter(|asic| asic.alive).count().max(1) as f32; + let average_frequency_mhz = if plan.asic_plans.is_empty() { + plan.desired_clock_mhz + } else { + plan.asic_plans + .iter() + .map(|asic| (asic.pll_frequencies_mhz[0] + asic.pll_frequencies_mhz[1]) / 2.0) + .sum::() + / plan.asic_plans.len() as f32 + }; + let ratio = if plan.desired_clock_mhz > 0.0 { + average_frequency_mhz / plan.desired_clock_mhz + } else { + 1.0 + }; + let active_engine_ratio = { + let total_active = asics + .iter() + .filter(|asic| asic.alive) + .map(|asic| asic.active_engine_count.max(1) as f32) + .sum::() + .max(1.0); + let total_nominal = asics.iter().filter(|asic| asic.alive).count().max(1) as f32 + * default_saved_engine_topology().active_engine_count as f32; + (total_active / total_nominal).max(0.1) + }; + nominal_board_hashrate * ratio.max(0.1) * active_engine_ratio +} + +fn saved_operating_point_from_loaded_profile( + profile: &Bzm2LoadedCalibrationProfile, +) -> Option { + match profile.persisted.as_ref() { + Some(persisted) + if persisted.saved_operating_point_status + == Bzm2SavedOperatingPointStatus::Invalidated => + { + None + } + _ => Some(profile.saved_state.clone()), + } +} + +fn store_calibration_profile( + path: &Path, + profile: &Bzm2PersistedCalibrationProfile, +) -> Result<(), String> { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).map_err(|err| { + format!( + "Failed to create calibration profile directory {}: {}", + parent.display(), + err + ) + })?; + } + let raw = serde_json::to_string_pretty(profile) + .map_err(|err| format!("Failed to serialize calibration profile: {}", err))?; + fs::write(path, raw).map_err(|err| { + format!( + "Failed to write calibration profile {}: {}", + path.display(), + err + ) + }) +} + +fn saved_engine_topology_from_discovery( + discovery: &Bzm2DiscoveredEngineMap, +) -> Bzm2SavedEngineTopology { + Bzm2SavedEngineTopology { + active_engine_count: discovery.present_count() as u16, + missing_engines: discovery + .missing + .iter() + .map(|coord| Bzm2SavedEngineCoordinate { + row: coord.row, + col: coord.col, + }) + .collect(), + } +} + +fn distribute_saved_throughput( + total_throughput_ths: f32, + asics: &[Bzm2AsicTopology], +) -> BTreeMap { + let total_active = asics + .iter() + .filter(|asic| asic.alive) + .map(|asic| asic.active_engine_count.max(1) as f32) + .sum::() + .max(1.0); + + asics + .iter() + .filter(|asic| asic.alive) + .map(|asic| { + ( + asic.asic_id, + total_throughput_ths * (asic.active_engine_count.max(1) as f32 / total_active), + ) + }) + .collect() +} + +fn build_bus_layouts(serial_paths: &[String], asics_per_bus: &[u16]) -> Vec { + build_bus_layouts_with_minimum(serial_paths, asics_per_bus, 1) +} + +fn build_discovered_bus_layouts( + serial_paths: &[String], + asics_per_bus: &[u16], +) -> Vec { + build_bus_layouts_with_minimum(serial_paths, asics_per_bus, 0) +} + +fn build_bus_layouts_with_minimum( + serial_paths: &[String], + asics_per_bus: &[u16], + minimum_asic_count: u16, +) -> Vec { + let mut next_asic = 0u16; + serial_paths + .iter() + .enumerate() + .map(|(index, path)| { + let asic_count = *asics_per_bus + .get(index) + .or_else(|| asics_per_bus.last()) + .unwrap_or(&1) + .max(&minimum_asic_count); + let layout = Bzm2BusLayout { + serial_path: path.clone(), + asic_start: next_asic, + asic_count, + }; + next_asic = next_asic.saturating_add(asic_count); + layout + }) + .collect() +} + +fn should_fallback_to_configured_bus_layouts( + discovered: &[Bzm2BusLayout], + configured: &[Bzm2BusLayout], +) -> bool { + let discovered_total = discovered + .iter() + .map(|layout| layout.asic_count as usize) + .sum::(); + let configured_total = configured + .iter() + .map(|layout| layout.asic_count as usize) + .sum::(); + discovered_total == 0 && configured_total > 0 +} + +#[cfg(all(test, unix))] +mod tests { + use super::super::bringup::Bzm2BringupConfig; + use super::super::config::{ + Bzm2EnumerationConfig, Bzm2RuntimeConfig, DEFAULT_BAUD_RATE, + DEFAULT_CALIBRATION_POST1_DIVIDER, DEFAULT_NOMINAL_HASHRATE_THS, + }; + use super::super::telemetry::Bzm2TelemetryConfig; + use super::super::test_support::spawn_chain_emulator; + use super::*; + use crate::api_client::types::BoardTelemetry; + use crate::asic::bzm2::protocol::{OPCODE_UART_NOOP, encode_noop}; + use crate::tuning::blockscale::{Bzm2OperatingClass, Bzm2PerformanceMode}; + use nix::pty::openpty; + use std::io::{Read, Write}; + use std::os::fd::AsRawFd; + use std::time::{Duration, SystemTime, UNIX_EPOCH}; + use tokio::sync::{mpsc, watch}; + + #[tokio::test] + async fn live_calibration_persists_profile() { + let pty = openpty(None, None).unwrap(); + let serial_path = fs::read_link(format!("/proc/self/fd/{}", pty.slave.as_raw_fd())) + .unwrap() + .to_string_lossy() + .into_owned(); + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let profile_path = std::env::temp_dir().join(format!( + "bzm2-profile-{}-{}.json", + std::process::id(), + unique + )); + let rail0_path = std::env::temp_dir().join(format!("bzm2-domain-rail0-{unique}.txt")); + let rail1_path = std::env::temp_dir().join(format!("bzm2-domain-rail1-{unique}.txt")); + + let config = Bzm2RuntimeConfig { + serial_paths: vec![serial_path], + baud_rate: DEFAULT_BAUD_RATE, + timestamp_count: crate::asic::bzm2::protocol::DEFAULT_TIMESTAMP_COUNT, + nonce_gap: crate::asic::bzm2::protocol::DEFAULT_NONCE_GAP, + dispatch_interval: Duration::from_millis(50), + nominal_hashrate_ths: DEFAULT_NOMINAL_HASHRATE_THS, + dts_vs_generation: crate::asic::bzm2::protocol::DtsVsGeneration::Gen2, + telemetry: Bzm2TelemetryConfig::default(), + enumeration: Bzm2EnumerationConfig::default(), + bringup: Bzm2BringupConfig { + rail_set_paths: vec![ + rail0_path.to_string_lossy().into_owned(), + rail1_path.to_string_lossy().into_owned(), + ], + rail_write_scales: vec![1000.0, 1000.0], + ..Default::default() + }, + calibration: Bzm2CalibrationConfig { + enabled: true, + asics_per_bus: vec![2], + asics_per_domain: vec![1], + domain_voltage_offsets_mv: vec![0, 100], + profile_path: Some(profile_path.clone()), + skip_lock_check: true, + ..Default::default() + }, + }; + let (telemetry_tx, _telemetry_rx) = watch::channel(BoardTelemetry { + name: "bzm2-test".into(), + model: "BZM2".into(), + serial: Some("bzm2-test".into()), + ..Default::default() + }); + let board = Bzm2Board::new(config, telemetry_tx, mpsc::channel(1).1); + let bus_layouts = board.resolve_bus_layouts().await.unwrap(); + + board.execute_live_calibration(&bus_layouts).await.unwrap(); + + let profile = load_saved_operating_point_profile(Some(&profile_path)) + .unwrap() + .unwrap(); + assert_eq!(profile.saved_state.per_asic_pll_mhz.len(), 2); + assert_eq!(profile.saved_state.per_domain_voltage_mv.len(), 2); + assert_eq!(profile.saved_state.per_asic_engine_topology.len(), 2); + assert_eq!( + profile + .saved_state + .per_asic_engine_topology + .get(&0) + .unwrap() + .active_engine_count, + default_saved_engine_topology().active_engine_count + ); + assert_eq!( + fs::read_to_string(&rail0_path).unwrap().trim(), + profile + .saved_state + .per_domain_voltage_mv + .get(&0) + .unwrap() + .to_string() + ); + assert_eq!( + fs::read_to_string(&rail1_path).unwrap().trim(), + profile + .saved_state + .per_domain_voltage_mv + .get(&1) + .unwrap() + .to_string() + ); + assert!(profile.persisted.is_some()); + + let _ = fs::remove_file(profile_path); + let _ = fs::remove_file(rail0_path); + let _ = fs::remove_file(rail1_path); + drop(pty); + } + + #[tokio::test] + async fn stored_profile_replays_on_restart_without_rewrite() { + let pty = openpty(None, None).unwrap(); + let serial_path = fs::read_link(format!("/proc/self/fd/{}", pty.slave.as_raw_fd())) + .unwrap() + .to_string_lossy() + .into_owned(); + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let profile_path = std::env::temp_dir().join(format!( + "bzm2-replay-{}-{}.json", + std::process::id(), + unique + )); + let rail0_path = std::env::temp_dir().join(format!("bzm2-replay-rail0-{unique}.txt")); + let rail1_path = std::env::temp_dir().join(format!("bzm2-replay-rail1-{unique}.txt")); + let persisted = Bzm2PersistedCalibrationProfile { + schema_version: Bzm2PersistedCalibrationProfile::SCHEMA_VERSION, + operating_class: operating_class_name(Bzm2OperatingClass::Generic).into(), + performance_mode: performance_mode_name(Bzm2PerformanceMode::Standard).into(), + asics_per_bus: vec![2], + pll_post1_divider: DEFAULT_CALIBRATION_POST1_DIVIDER, + saved_operating_point_status: Bzm2SavedOperatingPointStatus::Validated, + saved_operating_point_reasons: Vec::new(), + saved_state: Bzm2SavedOperatingPoint { + board_voltage_mv: 17_500, + board_throughput_ths: 80.0, + per_domain_voltage_mv: BTreeMap::from([(0, 17_450), (1, 17_600)]), + per_asic_engine_topology: BTreeMap::new(), + per_asic_pll_mhz: BTreeMap::from([ + (0, [1_100.0, 1_125.0]), + (1, [1_150.0, 1_175.0]), + ]), + }, + }; + let original = serde_json::to_string_pretty(&persisted).unwrap(); + fs::write(&profile_path, &original).unwrap(); + + let config = Bzm2RuntimeConfig { + serial_paths: vec![serial_path], + baud_rate: DEFAULT_BAUD_RATE, + timestamp_count: crate::asic::bzm2::protocol::DEFAULT_TIMESTAMP_COUNT, + nonce_gap: crate::asic::bzm2::protocol::DEFAULT_NONCE_GAP, + dispatch_interval: Duration::from_millis(50), + nominal_hashrate_ths: DEFAULT_NOMINAL_HASHRATE_THS, + dts_vs_generation: crate::asic::bzm2::protocol::DtsVsGeneration::Gen2, + telemetry: Bzm2TelemetryConfig::default(), + enumeration: Bzm2EnumerationConfig::default(), + bringup: Bzm2BringupConfig { + rail_set_paths: vec![ + rail0_path.to_string_lossy().into_owned(), + rail1_path.to_string_lossy().into_owned(), + ], + rail_write_scales: vec![1000.0, 1000.0], + ..Default::default() + }, + calibration: Bzm2CalibrationConfig { + enabled: true, + apply_saved_operating_point: true, + asics_per_bus: vec![2], + profile_path: Some(profile_path.clone()), + skip_lock_check: true, + ..Default::default() + }, + }; + let (telemetry_tx, _telemetry_rx) = watch::channel(BoardTelemetry { + name: "bzm2-test".into(), + model: "BZM2".into(), + serial: Some("bzm2-test".into()), + ..Default::default() + }); + let board = Bzm2Board::new(config, telemetry_tx, mpsc::channel(1).1); + let bus_layouts = board.resolve_bus_layouts().await.unwrap(); + + board.execute_live_calibration(&bus_layouts).await.unwrap(); + + assert_eq!(fs::read_to_string(&profile_path).unwrap(), original); + assert_eq!(fs::read_to_string(&rail0_path).unwrap().trim(), "17450"); + assert_eq!(fs::read_to_string(&rail1_path).unwrap().trim(), "17600"); + + let _ = fs::remove_file(profile_path); + let _ = fs::remove_file(rail0_path); + let _ = fs::remove_file(rail1_path); + drop(pty); + } + + #[test] + fn build_bus_layouts_assigns_global_ranges() { + let layouts = build_bus_layouts(&["/dev/ttyUSB0".into(), "/dev/ttyUSB1".into()], &[4, 6]); + assert_eq!(layouts[0].asic_start, 0); + assert_eq!(layouts[0].asic_count, 4); + assert_eq!(layouts[1].asic_start, 4); + assert_eq!(layouts[1].asic_count, 6); + } + + #[tokio::test] + async fn resolve_bus_layouts_uses_startup_enumeration_counts() { + let pty = openpty(None, None).unwrap(); + let master = pty.master; + let slave = pty.slave; + let serial_path = fs::read_link(format!("/proc/self/fd/{}", slave.as_raw_fd())) + .unwrap() + .to_string_lossy() + .into_owned(); + let emulator = spawn_chain_emulator(master, 2, 0); + + let config = Bzm2RuntimeConfig { + serial_paths: vec![serial_path], + baud_rate: DEFAULT_BAUD_RATE, + timestamp_count: crate::asic::bzm2::protocol::DEFAULT_TIMESTAMP_COUNT, + nonce_gap: crate::asic::bzm2::protocol::DEFAULT_NONCE_GAP, + dispatch_interval: Duration::from_millis(50), + nominal_hashrate_ths: DEFAULT_NOMINAL_HASHRATE_THS, + dts_vs_generation: crate::asic::bzm2::protocol::DtsVsGeneration::Gen2, + telemetry: Bzm2TelemetryConfig::default(), + enumeration: Bzm2EnumerationConfig { + enabled: true, + start_id: 0, + max_asics_per_bus: vec![4], + }, + bringup: Bzm2BringupConfig::default(), + calibration: Bzm2CalibrationConfig::default(), + }; + let (telemetry_tx, _telemetry_rx) = watch::channel(BoardTelemetry { + name: "bzm2-test".into(), + model: "BZM2".into(), + serial: Some("bzm2-test".into()), + ..Default::default() + }); + let board = Bzm2Board::new(config, telemetry_tx, mpsc::channel(1).1); + + let layouts = board.resolve_bus_layouts().await.unwrap(); + assert_eq!(layouts.len(), 1); + assert_eq!(layouts[0].asic_count, 2); + + emulator.join().unwrap(); + } + + #[tokio::test] + async fn resolve_bus_layouts_falls_back_to_configured_counts_when_default_id_is_silent() { + let pty = openpty(None, None).unwrap(); + let master = pty.master; + let slave = pty.slave; + let serial_path = fs::read_link(format!("/proc/self/fd/{}", slave.as_raw_fd())) + .unwrap() + .to_string_lossy() + .into_owned(); + let emulator = std::thread::spawn(move || { + let mut file = fs::File::from(master); + let mut probe = vec![0u8; encode_noop(crate::asic::bzm2::DEFAULT_ASIC_ID).len()]; + file.read_exact(&mut probe).unwrap(); + assert_eq!(probe, encode_noop(crate::asic::bzm2::DEFAULT_ASIC_ID)); + file.write_all(&[ + crate::asic::bzm2::DEFAULT_ASIC_ID, + OPCODE_UART_NOOP, + b'N', + b'O', + b'P', + ]) + .unwrap(); + }); + + let config = Bzm2RuntimeConfig { + serial_paths: vec![serial_path], + baud_rate: DEFAULT_BAUD_RATE, + timestamp_count: crate::asic::bzm2::protocol::DEFAULT_TIMESTAMP_COUNT, + nonce_gap: crate::asic::bzm2::protocol::DEFAULT_NONCE_GAP, + dispatch_interval: Duration::from_millis(50), + nominal_hashrate_ths: DEFAULT_NOMINAL_HASHRATE_THS, + dts_vs_generation: crate::asic::bzm2::protocol::DtsVsGeneration::Gen2, + telemetry: Bzm2TelemetryConfig::default(), + enumeration: Bzm2EnumerationConfig { + enabled: true, + start_id: 0, + max_asics_per_bus: vec![4], + }, + bringup: Bzm2BringupConfig::default(), + calibration: Bzm2CalibrationConfig { + asics_per_bus: vec![3], + ..Default::default() + }, + }; + let (telemetry_tx, _telemetry_rx) = watch::channel(BoardTelemetry { + name: "bzm2-test".into(), + model: "BZM2".into(), + serial: Some("bzm2-test".into()), + ..Default::default() + }); + let board = Bzm2Board::new(config, telemetry_tx, mpsc::channel(1).1); + + let layouts = board.resolve_bus_layouts().await.unwrap(); + assert_eq!(layouts.len(), 1); + assert_eq!(layouts[0].asic_count, 3); + + emulator.join().unwrap(); + } +} diff --git a/mujina-miner/src/board/bzm2/commands.rs b/mujina-miner/src/board/bzm2/commands.rs new file mode 100644 index 00000000..8318a33a --- /dev/null +++ b/mujina-miner/src/board/bzm2/commands.rs @@ -0,0 +1,242 @@ +//! BoardCommand dispatch loop for the BZM2 board. + +use std::sync::Arc; +use std::time::Duration; + +use tokio::sync::watch; + +use crate::api::commands::BoardCommand; +use crate::api_client::types::{Bzm2BusSummary, Bzm2ChainSummaryResponse}; + +use super::config::DEFAULT_ENGINE_DISCOVERY_TIMEOUT_MS; +use super::telemetry::{map_clock_report, publish_discovered_engine_map, publish_thread_telemetry}; +use super::{BoardError, Bzm2Board}; + +impl Bzm2Board { + pub(super) fn spawn_command_loop(&mut self) { + if self.command_task.is_some() { + return; + } + let Some(mut command_rx) = self.command_rx.take() else { + return; + }; + + let telemetry_tx = self.telemetry_tx.clone(); + let shutdown_handles = self.shutdown_handles.clone(); + let serial_paths = self.config.serial_paths.clone(); + let bus_layouts = Arc::clone(&self.bus_layouts); + let applied_operating_state = Arc::clone(&self.applied_operating_state); + let board_name = self.config.device_id(); + let (shutdown_tx, mut shutdown_rx) = watch::channel(false); + self.command_shutdown = Some(shutdown_tx); + + self.command_task = Some(tokio::spawn(async move { + loop { + tokio::select! { + command = command_rx.recv() => { + let Some(command) = command else { + break; + }; + match command { + BoardCommand::QueryBzm2DtsVs { thread_index, asic, reply } => { + let result: Result<_, BoardError> = async { + let handle = shutdown_handles.get(thread_index).ok_or_else(|| { + BoardError::HardwareControl(format!( + "invalid BZM2 thread index {thread_index} for board {board_name}" + )) + })?; + let update = handle + .query_dts_vs(asic) + .await + .map_err(|err| BoardError::HardwareControl(err.to_string()))?; + publish_thread_telemetry(&telemetry_tx, &update); + Ok(()) + } + .await; + let _ = reply.send(result.map_err(anyhow::Error::from)); + } + BoardCommand::QueryBzm2Noop { thread_index, asic, reply } => { + let result: Result<_, BoardError> = async { + let handle = shutdown_handles.get(thread_index).ok_or_else(|| { + BoardError::HardwareControl(format!( + "invalid BZM2 thread index {thread_index} for board {board_name}" + )) + })?; + handle + .noop(asic) + .await + .map_err(|err| BoardError::HardwareControl(err.to_string())) + } + .await; + let _ = reply.send(result.map_err(anyhow::Error::from)); + } + BoardCommand::QueryBzm2ChainSummary { reply } => { + let bus_layouts = + bus_layouts.lock().unwrap_or_else(|e| e.into_inner()).clone(); + let applied = applied_operating_state + .lock() + .unwrap_or_else(|e| e.into_inner()) + .clone(); + let summary = Bzm2ChainSummaryResponse { + total_asics: bus_layouts + .iter() + .map(|bus| bus.asic_count) + .sum::(), + startup_path: applied.startup_path, + saved_operating_point_status: applied.saved_operating_point_status, + buses: bus_layouts + .iter() + .enumerate() + .map(|(thread_index, bus)| Bzm2BusSummary { + thread_index, + serial_path: bus.serial_path.clone(), + asic_start: bus.asic_start, + asic_count: bus.asic_count, + }) + .collect(), + }; + let _ = reply.send(Ok(summary)); + } + BoardCommand::QueryBzm2ClockReport { + thread_index, + asic, + reply, + } => { + let result: Result<_, BoardError> = async { + let handle = shutdown_handles.get(thread_index).ok_or_else(|| { + BoardError::HardwareControl(format!( + "invalid BZM2 thread index {thread_index} for board {board_name}" + )) + })?; + let report = handle + .clock_report(asic) + .await + .map_err(|err| BoardError::HardwareControl(err.to_string()))?; + Ok(map_clock_report(report)) + } + .await; + let _ = reply.send(result.map_err(anyhow::Error::from)); + } + BoardCommand::QueryBzm2Loopback { + thread_index, + asic, + payload, + reply, + } => { + let result: Result<_, BoardError> = async { + let handle = shutdown_handles.get(thread_index).ok_or_else(|| { + BoardError::HardwareControl(format!( + "invalid BZM2 thread index {thread_index} for board {board_name}" + )) + })?; + handle + .loopback(asic, payload) + .await + .map_err(|err| BoardError::HardwareControl(err.to_string())) + } + .await; + let _ = reply.send(result.map_err(anyhow::Error::from)); + } + BoardCommand::ReadBzm2Register { + thread_index, + asic, + engine_address, + offset, + count, + reply, + } => { + let result: Result<_, BoardError> = async { + let handle = shutdown_handles.get(thread_index).ok_or_else(|| { + BoardError::HardwareControl(format!( + "invalid BZM2 thread index {thread_index} for board {board_name}" + )) + })?; + handle + .read_register(asic, engine_address, offset, count) + .await + .map_err(|err| BoardError::HardwareControl(err.to_string())) + } + .await; + let _ = reply.send(result.map_err(anyhow::Error::from)); + } + BoardCommand::WriteBzm2Register { + thread_index, + asic, + engine_address, + offset, + value, + reply, + } => { + let result: Result<_, BoardError> = async { + let handle = shutdown_handles.get(thread_index).ok_or_else(|| { + BoardError::HardwareControl(format!( + "invalid BZM2 thread index {thread_index} for board {board_name}" + )) + })?; + handle + .write_register(asic, engine_address, offset, value) + .await + .map_err(|err| BoardError::HardwareControl(err.to_string())) + } + .await; + let _ = reply.send(result.map_err(anyhow::Error::from)); + } + BoardCommand::DiscoverBzm2Engines { + thread_index, + asic, + tdm_prediv_raw, + tdm_counter, + timeout_ms, + reply, + } => { + let result: Result<_, BoardError> = async { + let handle = shutdown_handles.get(thread_index).ok_or_else(|| { + BoardError::HardwareControl(format!( + "invalid BZM2 thread index {thread_index} for board {board_name}" + )) + })?; + let serial_path = serial_paths.get(thread_index).ok_or_else(|| { + BoardError::HardwareControl(format!( + "missing serial path for BZM2 thread index {thread_index} on board {board_name}" + )) + })?; + let discovery = handle + .discover_engine_map( + asic, + tdm_prediv_raw, + tdm_counter, + Duration::from_millis(u64::from( + timeout_ms.unwrap_or( + DEFAULT_ENGINE_DISCOVERY_TIMEOUT_MS as u32, + ), + )), + ) + .await + .map_err(|err| BoardError::HardwareControl(err.to_string()))?; + publish_discovered_engine_map( + &telemetry_tx, + thread_index, + serial_path, + &discovery, + ); + Ok(()) + } + .await; + let _ = reply.send(result.map_err(anyhow::Error::from)); + } + BoardCommand::SetFanTarget { reply, .. } => { + let _ = reply + .send(Err(anyhow::anyhow!("BZM2 board has no controllable fans"))); + } + } + } + changed = shutdown_rx.changed() => { + if changed.is_err() || *shutdown_rx.borrow() { + break; + } + } + } + } + })); + } +} diff --git a/mujina-miner/src/board/bzm2/config.rs b/mujina-miner/src/board/bzm2/config.rs new file mode 100644 index 00000000..25ccca87 --- /dev/null +++ b/mujina-miner/src/board/bzm2/config.rs @@ -0,0 +1,566 @@ +//! Environment-driven configuration for the BZM2 board driver. + +use std::env; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use crate::tuning::blockscale::{Bzm2CalibrationMode, Bzm2OperatingClass, Bzm2PerformanceMode}; + +use super::bringup::Bzm2BringupConfig; +use super::telemetry::Bzm2TelemetryConfig; + +pub(super) const DEFAULT_BAUD_RATE: u32 = 5_000_000; +const DEFAULT_DISPATCH_INTERVAL_MS: u64 = 500; +pub(super) const DEFAULT_NOMINAL_HASHRATE_THS: f64 = 40.0; +pub(super) const DEFAULT_TELEMETRY_INTERVAL_SECS: u64 = 5; +pub(super) const DEFAULT_ASIC_TEMP_SCALE: f32 = 0.001; +pub(super) const DEFAULT_BOARD_TEMP_SCALE: f32 = 0.001; +pub(super) const DEFAULT_FAN_RPM_SCALE: f32 = 1.0; +pub(super) const DEFAULT_FAN_PERCENT_SCALE: f32 = 1.0; +pub(super) const DEFAULT_VOLTAGE_SCALE: f32 = 0.001; +pub(super) const DEFAULT_CURRENT_SCALE: f32 = 0.001; +pub(super) const DEFAULT_POWER_SCALE: f32 = 0.000001; +pub(super) const DEFAULT_CALIBRATION_SITE_TEMP_C: f32 = 20.0; +pub(super) const DEFAULT_CALIBRATION_POST1_DIVIDER: u8 = 0; +const DEFAULT_CALIBRATION_LOCK_TIMEOUT_MS: u64 = 1_000; +const DEFAULT_CALIBRATION_LOCK_POLL_MS: u64 = 100; +pub(super) const DEFAULT_CALIBRATION_REPLAY_FREQ_MHZ: f32 = 800.0; +const DEFAULT_CALIBRATION_ENGINE_DISCOVERY_TDM_PREDIV_RAW: u32 = 0x0f; +const DEFAULT_CALIBRATION_ENGINE_DISCOVERY_TDM_COUNTER: u8 = 16; +const DEFAULT_CALIBRATION_ENGINE_DISCOVERY_TIMEOUT_MS: u64 = 100; +const DEFAULT_RUNTIME_RETUNE_PERSISTENCE_POLLS: u8 = 3; +const DEFAULT_RUNTIME_RETUNE_THERMAL_C: f32 = 85.0; +const DEFAULT_RUNTIME_RETUNE_VOLTAGE_IMBALANCE_MV: u32 = 150; +pub(super) const DEFAULT_ENUMERATION_MAX_ASICS_PER_BUS: u16 = 100; +pub(super) const DEFAULT_BRINGUP_PRE_POWER_MS: u64 = 10; +pub(super) const DEFAULT_BRINGUP_POST_POWER_MS: u64 = 25; +pub(super) const DEFAULT_BRINGUP_RELEASE_RESET_MS: u64 = 25; +pub(super) const DEFAULT_ENGINE_DISCOVERY_TIMEOUT_MS: u64 = 100; + +#[derive(Debug, Clone)] +pub struct Bzm2RuntimeConfig { + pub serial_paths: Vec, + pub baud_rate: u32, + pub timestamp_count: u8, + pub nonce_gap: u32, + pub dispatch_interval: Duration, + pub nominal_hashrate_ths: f64, + pub dts_vs_generation: crate::asic::bzm2::protocol::DtsVsGeneration, + pub telemetry: Bzm2TelemetryConfig, + pub calibration: Bzm2CalibrationConfig, + pub enumeration: Bzm2EnumerationConfig, + pub bringup: Bzm2BringupConfig, +} + +impl Bzm2RuntimeConfig { + pub fn from_env() -> Option { + let raw_paths = env::var("MUJINA_BZM2_SERIAL") + .ok() + .or_else(|| env::var("MUJINA_BZM2_SERIAL_PATHS").ok())?; + + let serial_paths: Vec = raw_paths + .split(',') + .map(str::trim) + .filter(|path| !path.is_empty()) + .map(ToOwned::to_owned) + .collect(); + if serial_paths.is_empty() { + return None; + } + + let baud_rate = env::var("MUJINA_BZM2_BAUD") + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or(DEFAULT_BAUD_RATE); + let timestamp_count = env::var("MUJINA_BZM2_TIMESTAMP_COUNT") + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or(crate::asic::bzm2::protocol::DEFAULT_TIMESTAMP_COUNT); + let nonce_gap = env::var("MUJINA_BZM2_NONCE_GAP") + .ok() + .and_then(|value| parse_u32(&value)) + .unwrap_or(crate::asic::bzm2::protocol::DEFAULT_NONCE_GAP); + let dispatch_interval = Duration::from_millis( + env::var("MUJINA_BZM2_DISPATCH_MS") + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or(DEFAULT_DISPATCH_INTERVAL_MS), + ); + let nominal_hashrate_ths = env::var("MUJINA_BZM2_HASHRATE_THS") + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or(DEFAULT_NOMINAL_HASHRATE_THS); + let dts_vs_generation = env::var("MUJINA_BZM2_DTS_VS_GEN") + .ok() + .as_deref() + .and_then(crate::asic::bzm2::protocol::DtsVsGeneration::from_env_value) + .unwrap_or(crate::asic::bzm2::protocol::DtsVsGeneration::Gen2); + let calibration = Bzm2CalibrationConfig::from_env(serial_paths.len()); + let bringup = Bzm2BringupConfig::from_env(); + + Some(Self { + serial_paths: serial_paths.clone(), + baud_rate, + timestamp_count, + nonce_gap, + dispatch_interval, + nominal_hashrate_ths, + dts_vs_generation, + telemetry: Bzm2TelemetryConfig::from_env(), + enumeration: Bzm2EnumerationConfig::from_env(serial_paths.len(), &calibration), + bringup, + calibration, + }) + } + + pub fn device_id(&self) -> String { + let suffix = self + .serial_paths + .iter() + .map(|path| { + Path::new(path) + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or(path) + .chars() + .map(|ch| if ch.is_ascii_alphanumeric() { ch } else { '_' }) + .collect::() + }) + .collect::>() + .join("-"); + format!("bzm2-{}", suffix) + } +} + +#[derive(Debug, Clone)] +pub struct Bzm2EnumerationConfig { + pub enabled: bool, + pub start_id: u8, + pub max_asics_per_bus: Vec, +} + +impl Default for Bzm2EnumerationConfig { + fn default() -> Self { + Self { + enabled: false, + start_id: 0, + max_asics_per_bus: vec![DEFAULT_ENUMERATION_MAX_ASICS_PER_BUS], + } + } +} + +impl Bzm2EnumerationConfig { + fn from_env(serial_count: usize, calibration: &Bzm2CalibrationConfig) -> Self { + let mut max_asics_per_bus = parse_csv_numbers::("MUJINA_BZM2_ENUM_MAX_ASICS_PER_BUS") + .unwrap_or_else(|| { + if calibration.asics_per_bus.iter().any(|count| *count > 1) { + calibration.asics_per_bus.clone() + } else if serial_count == 0 { + Vec::new() + } else { + vec![DEFAULT_ENUMERATION_MAX_ASICS_PER_BUS; serial_count] + } + }); + if max_asics_per_bus.is_empty() && serial_count > 0 { + max_asics_per_bus = vec![DEFAULT_ENUMERATION_MAX_ASICS_PER_BUS; serial_count]; + } + + Self { + enabled: env_flag_any(&["MUJINA_BZM2_ENUMERATE_CHAIN", "MUJINA_BZM2_AUTO_ENUMERATE"]), + start_id: env::var("MUJINA_BZM2_ENUM_START_ID") + .ok() + .and_then(|value| value.parse::().ok()) + .unwrap_or(0), + max_asics_per_bus, + } + } +} + +#[derive(Debug, Clone)] +pub struct Bzm2CalibrationConfig { + pub enabled: bool, + pub apply_saved_operating_point: bool, + pub discover_engine_topology: bool, + pub operating_class: Bzm2OperatingClass, + pub performance_mode: Bzm2PerformanceMode, + pub mode: Bzm2CalibrationMode, + pub per_stack_clocking: bool, + pub force_retune: bool, + pub asics_per_bus: Vec, + pub asics_per_domain: Vec, + pub domain_voltage_offsets_mv: Vec, + pub profile_path: Option, + pub site_temp_c: Option, + pub pll_post1_divider: u8, + pub skip_lock_check: bool, + pub lock_timeout: Duration, + pub lock_poll_interval: Duration, + pub engine_discovery_tdm_prediv_raw: u32, + pub engine_discovery_tdm_counter: u8, + pub engine_discovery_timeout: Duration, + pub runtime_retune_enabled: bool, + pub runtime_retune_persistence_polls: u8, + pub runtime_retune_thermal_c: f32, + pub runtime_retune_voltage_imbalance_mv: u32, +} + +impl Default for Bzm2CalibrationConfig { + fn default() -> Self { + Self { + enabled: false, + apply_saved_operating_point: true, + discover_engine_topology: true, + operating_class: Bzm2OperatingClass::Generic, + performance_mode: Bzm2PerformanceMode::Standard, + mode: Bzm2CalibrationMode::default(), + per_stack_clocking: false, + force_retune: false, + asics_per_bus: vec![1], + asics_per_domain: vec![1], + domain_voltage_offsets_mv: Vec::new(), + profile_path: None, + site_temp_c: None, + pll_post1_divider: DEFAULT_CALIBRATION_POST1_DIVIDER, + skip_lock_check: false, + lock_timeout: Duration::from_millis(DEFAULT_CALIBRATION_LOCK_TIMEOUT_MS), + lock_poll_interval: Duration::from_millis(DEFAULT_CALIBRATION_LOCK_POLL_MS), + engine_discovery_tdm_prediv_raw: DEFAULT_CALIBRATION_ENGINE_DISCOVERY_TDM_PREDIV_RAW, + engine_discovery_tdm_counter: DEFAULT_CALIBRATION_ENGINE_DISCOVERY_TDM_COUNTER, + engine_discovery_timeout: Duration::from_millis( + DEFAULT_CALIBRATION_ENGINE_DISCOVERY_TIMEOUT_MS, + ), + runtime_retune_enabled: true, + runtime_retune_persistence_polls: DEFAULT_RUNTIME_RETUNE_PERSISTENCE_POLLS, + runtime_retune_thermal_c: DEFAULT_RUNTIME_RETUNE_THERMAL_C, + runtime_retune_voltage_imbalance_mv: DEFAULT_RUNTIME_RETUNE_VOLTAGE_IMBALANCE_MV, + } + } +} + +impl Bzm2CalibrationConfig { + fn from_env(serial_count: usize) -> Self { + let mut config = Self { + enabled: env_flag("MUJINA_BZM2_CALIBRATE") || env_flag("MUJINA_BZM2_ENABLE_PNP"), + apply_saved_operating_point: env_flag_default_any( + &[ + "MUJINA_BZM2_APPLY_SAVED_OPERATING_POINT", + "MUJINA_BZM2_REPLAY_STORED_CALIBRATION", + ], + true, + ), + discover_engine_topology: env_flag_default_any( + &[ + "MUJINA_BZM2_CALIBRATION_DISCOVER_ENGINES", + "MUJINA_BZM2_DISCOVER_ENGINES_FOR_CALIBRATION", + ], + true, + ), + operating_class: env_var_any(&["MUJINA_BZM2_OPERATING_CLASS", "MUJINA_BZM2_BOARD_BIN"]) + .as_deref() + .and_then(parse_operating_class) + .unwrap_or(Bzm2OperatingClass::Generic), + performance_mode: env_var_any(&[ + "MUJINA_BZM2_PERFORMANCE_MODE", + "MUJINA_BZM2_MINING_STRATEGY", + ]) + .as_deref() + .and_then(parse_performance_mode) + .unwrap_or(Bzm2PerformanceMode::Standard), + mode: Bzm2CalibrationMode { + sweep_strategy: env_flag_any(&[ + "MUJINA_BZM2_SWEEP_MODE", + "MUJINA_BZM2_SWEEP_STRATEGY", + ]), + sweep_voltage: env_flag("MUJINA_BZM2_SWEEP_VOLTAGE"), + sweep_frequency: env_flag("MUJINA_BZM2_SWEEP_FREQUENCY"), + sweep_pass_rate: env_flag("MUJINA_BZM2_SWEEP_PASS_RATE"), + }, + per_stack_clocking: env_flag_any(&[ + "MUJINA_BZM2_PER_STACK_CLOCKING", + "MUJINA_BZM2_SPLIT_STACK_FREQUENCY", + ]), + force_retune: env_flag_any(&[ + "MUJINA_BZM2_FORCE_RETUNE", + "MUJINA_BZM2_FORCE_RECALIBRATION", + ]), + asics_per_bus: parse_csv_numbers::("MUJINA_BZM2_ASICS_PER_BUS").unwrap_or_else( + || { + if serial_count == 0 { + Vec::new() + } else { + vec![1; serial_count] + } + }, + ), + asics_per_domain: parse_csv_numbers::("MUJINA_BZM2_ASICS_PER_DOMAIN") + .unwrap_or_else(|| vec![1]), + domain_voltage_offsets_mv: parse_csv_numbers::( + "MUJINA_BZM2_DOMAIN_VOLTAGE_OFFSETS_MV", + ) + .unwrap_or_default(), + profile_path: env_var_any(&[ + "MUJINA_BZM2_SAVED_OPERATING_POINT_PATH", + "MUJINA_BZM2_CALIBRATION_PROFILE", + ]) + .map(PathBuf::from), + site_temp_c: env_f32_any(&["MUJINA_BZM2_SITE_TEMP_C", "MUJINA_BZM2_AMBIENT_TEMP_C"]), + pll_post1_divider: env::var("MUJINA_BZM2_CALIBRATION_POST1_DIVIDER") + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or(DEFAULT_CALIBRATION_POST1_DIVIDER), + skip_lock_check: env_flag("MUJINA_BZM2_CALIBRATION_SKIP_LOCK_CHECK"), + lock_timeout: Duration::from_millis( + env::var("MUJINA_BZM2_CALIBRATION_LOCK_TIMEOUT_MS") + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or(DEFAULT_CALIBRATION_LOCK_TIMEOUT_MS), + ), + lock_poll_interval: Duration::from_millis( + env::var("MUJINA_BZM2_CALIBRATION_LOCK_POLL_MS") + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or(DEFAULT_CALIBRATION_LOCK_POLL_MS), + ), + engine_discovery_tdm_prediv_raw: env_var_any(&[ + "MUJINA_BZM2_ENGINE_DISCOVERY_TDM_PREDIV_RAW", + "MUJINA_BZM2_CALIBRATION_ENGINE_DISCOVERY_TDM_PREDIV_RAW", + ]) + .as_deref() + .and_then(parse_u32_any_radix) + .unwrap_or(DEFAULT_CALIBRATION_ENGINE_DISCOVERY_TDM_PREDIV_RAW), + engine_discovery_tdm_counter: env_var_any(&[ + "MUJINA_BZM2_ENGINE_DISCOVERY_TDM_COUNTER", + "MUJINA_BZM2_CALIBRATION_ENGINE_DISCOVERY_TDM_COUNTER", + ]) + .as_deref() + .and_then(parse_u8_any_radix) + .unwrap_or(DEFAULT_CALIBRATION_ENGINE_DISCOVERY_TDM_COUNTER), + engine_discovery_timeout: Duration::from_millis( + env_var_any(&[ + "MUJINA_BZM2_ENGINE_DISCOVERY_TIMEOUT_MS", + "MUJINA_BZM2_CALIBRATION_ENGINE_DISCOVERY_TIMEOUT_MS", + ]) + .as_deref() + .and_then(parse_u64_any_radix) + .unwrap_or(DEFAULT_CALIBRATION_ENGINE_DISCOVERY_TIMEOUT_MS), + ), + runtime_retune_enabled: env_flag_default_any( + &[ + "MUJINA_BZM2_RUNTIME_RETUNE", + "MUJINA_BZM2_ENABLE_RUNTIME_RETUNE", + ], + true, + ), + runtime_retune_persistence_polls: env_var_any(&[ + "MUJINA_BZM2_RUNTIME_RETUNE_PERSISTENCE_POLLS", + "MUJINA_BZM2_RETUNE_PERSISTENCE_POLLS", + ]) + .as_deref() + .and_then(parse_u8_any_radix) + .unwrap_or(DEFAULT_RUNTIME_RETUNE_PERSISTENCE_POLLS), + runtime_retune_thermal_c: env_f32_any(&[ + "MUJINA_BZM2_RUNTIME_RETUNE_THERMAL_C", + "MUJINA_BZM2_RETUNE_THERMAL_C", + ]) + .unwrap_or(DEFAULT_RUNTIME_RETUNE_THERMAL_C), + runtime_retune_voltage_imbalance_mv: env_var_any(&[ + "MUJINA_BZM2_RUNTIME_RETUNE_VOLTAGE_IMBALANCE_MV", + "MUJINA_BZM2_RETUNE_VOLTAGE_IMBALANCE_MV", + ]) + .as_deref() + .and_then(parse_u32_any_radix) + .unwrap_or(DEFAULT_RUNTIME_RETUNE_VOLTAGE_IMBALANCE_MV), + }; + + if config.asics_per_bus.is_empty() && serial_count > 0 { + config.asics_per_bus = vec![1; serial_count]; + } + if config.asics_per_domain.is_empty() { + config.asics_per_domain = vec![1]; + } + + config + } +} + +pub(super) fn env_var_any(keys: &[&str]) -> Option { + keys.iter().find_map(|key| env::var(key).ok()) +} + +pub(super) fn parse_u8_any_radix(raw: &str) -> Option { + parse_u64_any_radix(raw).and_then(|value| u8::try_from(value).ok()) +} + +pub(super) fn parse_u32_any_radix(raw: &str) -> Option { + parse_u64_any_radix(raw).and_then(|value| u32::try_from(value).ok()) +} + +pub(super) fn parse_u64_any_radix(raw: &str) -> Option { + let trimmed = raw.trim(); + if let Some(hex) = trimmed + .strip_prefix("0x") + .or_else(|| trimmed.strip_prefix("0X")) + { + u64::from_str_radix(hex, 16).ok() + } else { + trimmed.parse::().ok() + } +} + +pub(super) fn env_csv_strings_any(keys: &[&str]) -> Vec { + env_var_any(keys) + .map(|value| { + value + .split(',') + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(ToOwned::to_owned) + .collect::>() + }) + .unwrap_or_default() +} + +pub(super) fn env_flag(key: &str) -> bool { + env_var_any(&[key]).as_deref().is_some_and(|value| { + matches!( + value.trim().to_ascii_lowercase().as_str(), + "1" | "true" | "yes" | "on" + ) + }) +} + +pub(super) fn env_flag_any(keys: &[&str]) -> bool { + env_var_any(keys).as_deref().is_some_and(|value| { + matches!( + value.trim().to_ascii_lowercase().as_str(), + "1" | "true" | "yes" | "on" + ) + }) +} + +pub(super) fn env_flag_default_any(keys: &[&str], default: bool) -> bool { + env_var_any(keys) + .map(|value| { + matches!( + value.trim().to_ascii_lowercase().as_str(), + "1" | "true" | "yes" | "on" + ) + }) + .unwrap_or(default) +} + +pub(super) fn env_f32(key: &str) -> Option { + env_f32_any(&[key]) +} + +pub(super) fn env_f32_any(keys: &[&str]) -> Option { + env_var_any(keys).and_then(|value| value.parse().ok()) +} + +pub(super) fn parse_u32(value: &str) -> Option { + let trimmed = value.trim(); + if let Some(hex) = trimmed.strip_prefix("0x") { + u32::from_str_radix(hex, 16).ok() + } else { + trimmed.parse().ok() + } +} + +pub(super) fn parse_csv_numbers(key: &str) -> Option> +where + T: std::str::FromStr, +{ + let value = env::var(key).ok()?; + let parsed = value + .split(',') + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(|value| value.parse().ok()) + .collect::>>()?; + Some(parsed) +} + +pub(super) fn parse_csv_numbers_any(keys: &[&str]) -> Option> +where + T: std::str::FromStr, +{ + keys.iter().find_map(|key| parse_csv_numbers::(key)) +} + +pub(super) fn parse_operating_class(value: &str) -> Option { + match value.trim().to_ascii_lowercase().as_str() { + "generic" => Some(Bzm2OperatingClass::Generic), + "early-validation" | "early_validation" | "dvt1" => { + Some(Bzm2OperatingClass::EarlyValidation) + } + "production-validation" | "production_validation" | "pvt" => { + Some(Bzm2OperatingClass::ProductionValidation) + } + "stack-tuned-a" | "stack_tuned_a" | "dvt2-bin1" | "dvt2_bin1" | "dvt2bin1" | "bin1" => { + Some(Bzm2OperatingClass::StackTunedA) + } + "stack-tuned-b" | "stack_tuned_b" | "dvt2-bin2" | "dvt2_bin2" | "dvt2bin2" | "bin2" => { + Some(Bzm2OperatingClass::StackTunedB) + } + "extended-headroom" | "extended_headroom" | "plus" => { + Some(Bzm2OperatingClass::ExtendedHeadroom) + } + "extended-headroom-b" + | "extended_headroom_b" + | "plus-ebin2" + | "plus_ebin2" + | "plusebin2" + | "ebin2" => Some(Bzm2OperatingClass::ExtendedHeadroomB), + _ => None, + } +} + +pub(super) fn parse_performance_mode(value: &str) -> Option { + match value.trim().to_ascii_lowercase().as_str() { + "max-throughput" | "max_throughput" | "high" | "high-performance" | "high_performance" + | "performance" => Some(Bzm2PerformanceMode::MaxThroughput), + "standard" | "balanced" => Some(Bzm2PerformanceMode::Standard), + "efficiency" | "low" | "low-power" | "low_power" => Some(Bzm2PerformanceMode::Efficiency), + _ => None, + } +} + +pub(super) fn average_u32(values: impl Iterator) -> Option { + let mut total = 0u64; + let mut count = 0u64; + for value in values { + total += value as u64; + count += 1; + } + (count > 0).then_some((total / count) as u32) +} + +pub(super) fn average_f32(values: impl Iterator) -> Option { + let mut total = 0.0f32; + let mut count = 0usize; + for value in values { + total += value; + count += 1; + } + (count > 0).then_some(total / count as f32) +} + +pub(super) fn operating_class_name(operating_class: Bzm2OperatingClass) -> &'static str { + match operating_class { + Bzm2OperatingClass::Generic => "generic", + Bzm2OperatingClass::EarlyValidation => "early-validation", + Bzm2OperatingClass::ProductionValidation => "production-validation", + Bzm2OperatingClass::StackTunedA => "stack-tuned-a", + Bzm2OperatingClass::StackTunedB => "stack-tuned-b", + Bzm2OperatingClass::ExtendedHeadroom => "extended-headroom", + Bzm2OperatingClass::ExtendedHeadroomB => "extended-headroom-b", + } +} + +pub(super) fn performance_mode_name(performance_mode: Bzm2PerformanceMode) -> &'static str { + match performance_mode { + Bzm2PerformanceMode::MaxThroughput => "max-throughput", + Bzm2PerformanceMode::Standard => "standard", + Bzm2PerformanceMode::Efficiency => "efficiency", + } +} diff --git a/mujina-miner/src/board/bzm2/mod.rs b/mujina-miner/src/board/bzm2/mod.rs new file mode 100644 index 00000000..cabb89d4 --- /dev/null +++ b/mujina-miner/src/board/bzm2/mod.rs @@ -0,0 +1,562 @@ +use std::sync::{Arc, Mutex}; + +use anyhow::Result as AnyhowResult; +use async_trait::async_trait; +use tokio::sync::{mpsc, watch}; +use tokio::task::JoinHandle; + +use super::{BackplaneConnector, BoardInfo, VirtualBoardDescriptor}; +use crate::api::commands::BoardCommand; +use crate::{ + api_client::types::{BoardTelemetry, ThreadTelemetry}, + asic::{ + bzm2::{Bzm2Thread, Bzm2ThreadConfig, Bzm2ThreadHandle}, + hash_thread::{ + HashTask, HashThread, HashThreadCapabilities, HashThreadEvent, HashThreadStatus, + }, + }, + tracing::prelude::*, + transport::{SerialControl, SerialStream}, +}; + +mod bringup; +mod calibration; +mod commands; +mod config; +mod monitor; +mod telemetry; +#[cfg(all(test, unix))] +mod test_support; + +use calibration::{Bzm2AppliedOperatingState, Bzm2BusLayout}; +pub use config::Bzm2RuntimeConfig; +use monitor::Bzm2RuntimeMeasurementCache; +use telemetry::{ + merge_power_readings, merge_temperature_readings, publish_thread_status, + publish_thread_telemetry, +}; + +// Register this board type with the inventory system +inventory::submit! { + VirtualBoardDescriptor { + device_type: "bzm2", + name: "BZM2", + create_fn: || Box::pin(create_bzm2_board()), + } +} + +/// Errors raised by BZM2 board bring-up and hardware control. +#[derive(Debug)] +pub enum BoardError { + /// Board initialization failed (bring-up, serial open, calibration). + InitializationFailed(String), + /// Serial or file I/O failure while talking to the board. + Communication(std::io::Error), + /// A hardware control operation (rails, reset, clocks) failed. + HardwareControl(String), +} + +impl std::fmt::Display for BoardError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + BoardError::InitializationFailed(msg) => { + write!(f, "board initialization failed: {msg}") + } + BoardError::Communication(err) => write!(f, "board communication error: {err}"), + BoardError::HardwareControl(msg) => write!(f, "hardware control error: {msg}"), + } + } +} + +impl std::error::Error for BoardError {} + +impl From for BoardError { + fn from(err: std::io::Error) -> Self { + BoardError::Communication(err) + } +} + +pub struct Bzm2Board { + config: Bzm2RuntimeConfig, + bringup_applied: bool, + shutdown_handles: Vec, + serial_controls: Vec, + bus_layouts: Arc>>, + applied_operating_state: Arc>, + runtime_measurements: Arc>, + telemetry_tx: watch::Sender, + command_rx: Option>, + monitor_shutdown: Option>, + monitor_task: Option>, + command_shutdown: Option>, + command_task: Option>, +} + +impl Bzm2Board { + pub fn new( + config: Bzm2RuntimeConfig, + telemetry_tx: watch::Sender, + command_rx: mpsc::Receiver, + ) -> Self { + Self { + config, + bringup_applied: false, + shutdown_handles: Vec::new(), + serial_controls: Vec::new(), + bus_layouts: Arc::new(Mutex::new(Vec::new())), + applied_operating_state: Arc::new(Mutex::new(Bzm2AppliedOperatingState::default())), + runtime_measurements: Arc::new(Mutex::new(Bzm2RuntimeMeasurementCache::default())), + telemetry_tx, + command_rx: Some(command_rx), + monitor_shutdown: None, + monitor_task: None, + command_shutdown: None, + command_task: None, + } + } +} + +impl Bzm2Board { + fn board_info(&self) -> BoardInfo { + BoardInfo { + model: "BZM2".into(), + firmware_version: None, + serial_number: Some(self.config.device_id()), + } + } + + async fn shutdown(&mut self) -> AnyhowResult<()> { + if let Some(tx) = self.monitor_shutdown.take() { + let _ = tx.send(true); + } + if let Some(tx) = self.command_shutdown.take() { + let _ = tx.send(true); + } + if let Some(handle) = self.monitor_task.take() { + let _ = handle.await; + } + if let Some(handle) = self.command_task.take() { + let _ = handle.await; + } + for handle in &self.shutdown_handles { + handle.shutdown(); + } + self.shutdown_handles.clear(); + self.serial_controls.clear(); + self.command_rx = None; + self.telemetry_tx.send_modify(|state| { + for thread in &mut state.threads { + thread.is_active = false; + thread.hashrate = 0; + } + }); + self.apply_shutdown_sequence().await?; + Ok(()) + } + + async fn create_hash_threads(&mut self) -> AnyhowResult>> { + let mut threads: Vec> = Vec::new(); + let mut thread_states = Vec::new(); + self.apply_bringup_sequence().await?; + let bus_layouts = self.resolve_bus_layouts().await?; + *self.bus_layouts.lock().unwrap_or_else(|e| e.into_inner()) = bus_layouts.clone(); + let initial_snapshot = self.config.telemetry.snapshot(); + let initial_rail_snapshot = self.config.bringup.snapshot_telemetry(); + self.telemetry_tx.send_modify(|state| { + state.fans = initial_snapshot.fans.clone(); + merge_temperature_readings(&mut state.temperatures, &initial_snapshot.temperatures); + merge_power_readings(&mut state.powers, &initial_snapshot.powers); + merge_temperature_readings( + &mut state.temperatures, + &initial_rail_snapshot.temperatures, + ); + merge_power_readings(&mut state.powers, &initial_rail_snapshot.powers); + }); + + self.execute_live_calibration(&bus_layouts).await?; + let post_calibration_rail_snapshot = self.config.bringup.snapshot_telemetry(); + self.telemetry_tx.send_modify(|state| { + merge_temperature_readings( + &mut state.temperatures, + &post_calibration_rail_snapshot.temperatures, + ); + merge_power_readings(&mut state.powers, &post_calibration_rail_snapshot.powers); + }); + + for (index, serial_path) in self.config.serial_paths.iter().enumerate() { + let stream = SerialStream::new(serial_path, self.config.baud_rate).map_err(|err| { + BoardError::InitializationFailed(format!( + "Failed to open BZM2 serial transport {}: {}", + serial_path, err + )) + })?; + let (reader, writer, control) = stream.split(); + let thread_name = format!("BZM2 UART {}", index); + let mut config = Bzm2ThreadConfig::new(serial_path.clone(), self.config.baud_rate); + config.timestamp_count = self.config.timestamp_count; + config.nonce_gap = self.config.nonce_gap; + config.dispatch_interval = self.config.dispatch_interval; + config.nominal_hashrate_ths = self.config.nominal_hashrate_ths; + config.dts_vs_generation = self.config.dts_vs_generation; + + self.serial_controls.push(control.clone()); + let thread = Bzm2Thread::new(thread_name.clone(), reader, writer, control, config); + self.shutdown_handles.push(thread.shutdown_handle()); + thread_states.push(ThreadTelemetry { + name: thread_name, + hashrate: 0, + is_active: false, + }); + threads.push(Box::new(Bzm2ManagedThread::new( + Box::new(thread), + self.telemetry_tx.clone(), + index, + ))); + } + + self.telemetry_tx.send_modify(|state| { + state.threads = thread_states.clone(); + }); + + self.spawn_monitor(); + self.spawn_command_loop(); + Ok(threads) + } +} + +struct Bzm2ManagedThread { + inner: Box, + telemetry_tx: watch::Sender, + thread_index: usize, +} + +impl Bzm2ManagedThread { + fn new( + inner: Box, + telemetry_tx: watch::Sender, + thread_index: usize, + ) -> Self { + Self { + inner, + telemetry_tx, + thread_index, + } + } + + fn publish_status(&self, status: &HashThreadStatus) { + publish_thread_status(&self.telemetry_tx, self.thread_index, status); + } +} + +#[async_trait] +impl HashThread for Bzm2ManagedThread { + fn name(&self) -> &str { + self.inner.name() + } + fn capabilities(&self) -> &HashThreadCapabilities { + self.inner.capabilities() + } + + async fn configure(&mut self) -> AnyhowResult<()> { + self.inner.configure().await + } + + async fn update_task(&mut self, new_task: HashTask) -> AnyhowResult> { + let result = self.inner.update_task(new_task).await; + self.publish_status(&self.inner.status()); + result + } + + async fn replace_task(&mut self, new_task: HashTask) -> AnyhowResult> { + let result = self.inner.replace_task(new_task).await; + self.publish_status(&self.inner.status()); + result + } + + async fn go_idle(&mut self) -> AnyhowResult> { + let result = self.inner.go_idle().await; + self.publish_status(&self.inner.status()); + result + } + + fn take_event_receiver(&mut self) -> Option> { + let mut inner_rx = self.inner.take_event_receiver()?; + let (event_tx, event_rx) = mpsc::channel(64); + let telemetry_tx = self.telemetry_tx.clone(); + let thread_index = self.thread_index; + tokio::spawn(async move { + while let Some(event) = inner_rx.recv().await { + match &event { + HashThreadEvent::StatusUpdate(status) => { + publish_thread_status(&telemetry_tx, thread_index, status); + } + HashThreadEvent::TelemetryUpdate(update) => { + publish_thread_telemetry(&telemetry_tx, update); + } + _ => {} + } + if event_tx.send(event).await.is_err() { + break; + } + } + }); + Some(event_rx) + } + + fn status(&self) -> HashThreadStatus { + self.inner.status() + } +} +async fn create_bzm2_board() -> AnyhowResult { + let config = Bzm2RuntimeConfig::from_env() + .ok_or_else(|| anyhow::anyhow!("BZM2 not configured (MUJINA_BZM2_SERIAL not set)"))?; + + let serial = config.device_id(); + let initial_state = BoardTelemetry { + name: serial.clone(), + model: "BZM2".into(), + serial: Some(serial), + ..Default::default() + }; + let (telemetry_tx, telemetry_rx) = watch::channel(initial_state); + let (command_tx, command_rx) = mpsc::channel(16); + + let mut board = Bzm2Board::new(config, telemetry_tx, command_rx); + let info = board.board_info(); + + // Bring-up, enumeration, calibration, and the monitor/command loops + // all happen here; the returned threads are ready for the scheduler. + let threads = board.create_hash_threads().await?; + + let shutdown = Box::pin(async move { + if let Err(err) = board.shutdown().await { + warn!(error = %err, "BZM2 board shutdown reported an error"); + } + }); + + Ok(BackplaneConnector { + info, + threads, + telemetry_rx, + command_tx: Some(command_tx), + shutdown: Some(shutdown), + }) +} + +#[cfg(all(test, unix))] +mod tests { + use super::bringup::Bzm2BringupConfig; + use super::config::{ + Bzm2CalibrationConfig, Bzm2EnumerationConfig, DEFAULT_BAUD_RATE, + DEFAULT_NOMINAL_HASHRATE_THS, + }; + use super::telemetry::{Bzm2TelemetryConfig, SensorSpec}; + use super::*; + use crate::board::power::{VoltageStackBringupPlan, VoltageStackStep}; + use crate::types::Temperature; + use nix::pty::openpty; + use std::fs; + use std::os::fd::AsRawFd; + use std::time::{Duration, SystemTime, UNIX_EPOCH}; + + #[tokio::test] + async fn create_hash_threads_applies_bringup_and_shutdown_sequences() { + let pty = openpty(None, None).unwrap(); + let serial_path = fs::read_link(format!("/proc/self/fd/{}", pty.slave.as_raw_fd())) + .unwrap() + .to_string_lossy() + .into_owned(); + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let rail0_path = std::env::temp_dir().join(format!("bzm2-rail0-{unique}.txt")); + let rail1_path = std::env::temp_dir().join(format!("bzm2-rail1-{unique}.txt")); + let enable0_path = std::env::temp_dir().join(format!("bzm2-enable0-{unique}.txt")); + let enable1_path = std::env::temp_dir().join(format!("bzm2-enable1-{unique}.txt")); + let reset_path = std::env::temp_dir().join(format!("bzm2-reset-{unique}.txt")); + + let config = Bzm2RuntimeConfig { + serial_paths: vec![serial_path], + baud_rate: DEFAULT_BAUD_RATE, + timestamp_count: crate::asic::bzm2::protocol::DEFAULT_TIMESTAMP_COUNT, + nonce_gap: crate::asic::bzm2::protocol::DEFAULT_NONCE_GAP, + dispatch_interval: Duration::from_millis(50), + nominal_hashrate_ths: DEFAULT_NOMINAL_HASHRATE_THS, + dts_vs_generation: crate::asic::bzm2::protocol::DtsVsGeneration::Gen2, + telemetry: Bzm2TelemetryConfig::default(), + enumeration: Bzm2EnumerationConfig::default(), + bringup: Bzm2BringupConfig { + enabled: true, + rail_set_paths: vec![ + rail0_path.to_string_lossy().into_owned(), + rail1_path.to_string_lossy().into_owned(), + ], + rail_write_scales: vec![1000.0, 1000.0], + domain_rail_indices: Vec::new(), + rail_enable_paths: vec![ + enable0_path.to_string_lossy().into_owned(), + enable1_path.to_string_lossy().into_owned(), + ], + rail_enable_values: vec!["EN".into(), "ON".into()], + rail_vin: Vec::new(), + rail_vout: Vec::new(), + rail_current: Vec::new(), + rail_power: Vec::new(), + rail_temperature: Vec::new(), + reset_path: Some(reset_path.to_string_lossy().into_owned()), + reset_active_low: true, + plan: VoltageStackBringupPlan { + pre_power_delay: Duration::ZERO, + post_power_delay: Duration::ZERO, + release_reset_delay: Duration::ZERO, + steps: vec![ + VoltageStackStep { + rail_index: 0, + voltage: 1.1, + settle_for: Duration::ZERO, + }, + VoltageStackStep { + rail_index: 1, + voltage: 1.25, + settle_for: Duration::ZERO, + }, + ], + ..Default::default() + }, + }, + calibration: Bzm2CalibrationConfig::default(), + }; + let (telemetry_tx, _telemetry_rx) = watch::channel(BoardTelemetry { + name: "bzm2-test".into(), + model: "BZM2".into(), + serial: Some("bzm2-test".into()), + ..Default::default() + }); + let mut board = Bzm2Board::new(config, telemetry_tx, mpsc::channel(1).1); + + let _threads = board.create_hash_threads().await.unwrap(); + + assert_eq!(fs::read_to_string(&rail0_path).unwrap(), "1100"); + assert_eq!(fs::read_to_string(&rail1_path).unwrap(), "1250"); + assert_eq!(fs::read_to_string(&enable0_path).unwrap(), "EN"); + assert_eq!(fs::read_to_string(&enable1_path).unwrap(), "ON"); + assert_eq!(fs::read_to_string(&reset_path).unwrap(), "1"); + + board.shutdown().await.unwrap(); + + assert_eq!(fs::read_to_string(&rail0_path).unwrap(), "0"); + assert_eq!(fs::read_to_string(&rail1_path).unwrap(), "0"); + assert_eq!(fs::read_to_string(&reset_path).unwrap(), "0"); + + let _ = fs::remove_file(rail0_path); + let _ = fs::remove_file(rail1_path); + let _ = fs::remove_file(enable0_path); + let _ = fs::remove_file(enable1_path); + let _ = fs::remove_file(reset_path); + drop(pty); + } + + #[tokio::test] + async fn create_hash_threads_publishes_rail_telemetry() { + let pty = openpty(None, None).unwrap(); + let serial_path = fs::read_link(format!("/proc/self/fd/{}", pty.slave.as_raw_fd())) + .unwrap() + .to_string_lossy() + .into_owned(); + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let vin_path = std::env::temp_dir().join(format!("bzm2-vin-{unique}.txt")); + let vout_path = std::env::temp_dir().join(format!("bzm2-vout-{unique}.txt")); + let current_path = std::env::temp_dir().join(format!("bzm2-current-{unique}.txt")); + let power_path = std::env::temp_dir().join(format!("bzm2-power-{unique}.txt")); + let temp_path = std::env::temp_dir().join(format!("bzm2-temp-{unique}.txt")); + fs::write(&vin_path, "12000\n").unwrap(); + fs::write(&vout_path, "850\n").unwrap(); + fs::write(¤t_path, "1500\n").unwrap(); + fs::write(&power_path, "1275\n").unwrap(); + fs::write(&temp_path, "47000\n").unwrap(); + + let config = Bzm2RuntimeConfig { + serial_paths: vec![serial_path], + baud_rate: DEFAULT_BAUD_RATE, + timestamp_count: crate::asic::bzm2::protocol::DEFAULT_TIMESTAMP_COUNT, + nonce_gap: crate::asic::bzm2::protocol::DEFAULT_NONCE_GAP, + dispatch_interval: Duration::from_millis(50), + nominal_hashrate_ths: DEFAULT_NOMINAL_HASHRATE_THS, + dts_vs_generation: crate::asic::bzm2::protocol::DtsVsGeneration::Gen2, + telemetry: Bzm2TelemetryConfig::default(), + enumeration: Bzm2EnumerationConfig::default(), + bringup: Bzm2BringupConfig { + rail_vin: vec![SensorSpec { + path: vin_path.to_string_lossy().into_owned(), + scale: 0.001, + }], + rail_vout: vec![SensorSpec { + path: vout_path.to_string_lossy().into_owned(), + scale: 0.001, + }], + rail_current: vec![SensorSpec { + path: current_path.to_string_lossy().into_owned(), + scale: 0.001, + }], + rail_power: vec![SensorSpec { + path: power_path.to_string_lossy().into_owned(), + scale: 0.001, + }], + rail_temperature: vec![SensorSpec { + path: temp_path.to_string_lossy().into_owned(), + scale: 0.001, + }], + ..Default::default() + }, + calibration: Bzm2CalibrationConfig::default(), + }; + let (telemetry_tx, telemetry_rx) = watch::channel(BoardTelemetry { + name: "bzm2-test".into(), + model: "BZM2".into(), + serial: Some("bzm2-test".into()), + ..Default::default() + }); + let mut board = Bzm2Board::new(config, telemetry_tx, mpsc::channel(1).1); + + let _threads = board.create_hash_threads().await.unwrap(); + let state = telemetry_rx.borrow().clone(); + assert!(state.temperatures.iter().any(|sensor| { + sensor.name == "rail0-regulator" + && sensor + .temperature + .map(Temperature::as_degrees_c) + .is_some_and(|value| (value - 47.0).abs() < 0.001) + })); + assert!(state.powers.iter().any(|power| { + power.name == "rail0-input" + && power + .voltage_v + .is_some_and(|value| (value - 12.0).abs() < 0.001) + })); + assert!(state.powers.iter().any(|power| { + power.name == "rail0-output" + && power + .voltage_v + .is_some_and(|value| (value - 0.85).abs() < 0.001) + && power + .current_a + .is_some_and(|value| (value - 1.5).abs() < 0.001) + && power + .power_w + .is_some_and(|value| (value - 1.275).abs() < 0.001) + })); + + board.shutdown().await.unwrap(); + + let _ = fs::remove_file(vin_path); + let _ = fs::remove_file(vout_path); + let _ = fs::remove_file(current_path); + let _ = fs::remove_file(power_path); + let _ = fs::remove_file(temp_path); + drop(pty); + } +} diff --git a/mujina-miner/src/board/bzm2/monitor.rs b/mujina-miner/src/board/bzm2/monitor.rs new file mode 100644 index 00000000..c5eab0e5 --- /dev/null +++ b/mujina-miner/src/board/bzm2/monitor.rs @@ -0,0 +1,1311 @@ +//! Runtime monitor loop and tuning evaluation for the BZM2 board. + +use std::collections::BTreeMap; +use std::path::Path; +use std::sync::{Arc, Mutex}; + +use tokio::sync::watch; + +use crate::api_client::types::{ + AsicState, Bzm2AsicTuningState, Bzm2DomainTuningState, Bzm2PllTuningState, + Bzm2SavedOperatingPointStatus, Bzm2TuningState, EngineCoordinate, TemperatureSensor, +}; +use crate::asic::bzm2::{Bzm2ThreadHandle, Bzm2ThreadRuntimeMetrics}; +use crate::tracing::prelude::*; +use crate::tuning::blockscale::{ + Bzm2AsicMeasurement, Bzm2BoardCalibrationInput, Bzm2CalibrationConstraints, + Bzm2CalibrationPlanner, Bzm2DomainMeasurement, Bzm2SavedEngineCoordinate, + Bzm2SavedEngineTopology, +}; +use crate::types::Temperature; + +use super::Bzm2Board; +use super::bringup::Bzm2BringupConfig; +use super::calibration::{ + Bzm2AppliedOperatingState, Bzm2BusLayout, build_topology, build_voltage_domains, + default_saved_engine_topology, store_saved_operating_point_status, +}; +use super::config::{Bzm2CalibrationConfig, DEFAULT_CALIBRATION_SITE_TEMP_C}; +use super::telemetry::{Bzm2TelemetrySnapshot, merge_power_readings, merge_temperature_readings}; + +#[derive(Debug, Clone, Default)] +pub(super) struct Bzm2RuntimeMeasurementCache { + domain_measurements: BTreeMap, + asic_measurements: BTreeMap, +} + +#[derive(Debug, Clone, Default)] +struct Bzm2RetuneTriggerTracker { + throughput_regression_polls: u8, + thermal_drift_polls: u8, + voltage_imbalance_polls: u8, +} + +impl Bzm2Board { + pub(super) fn spawn_monitor(&mut self) { + if (!self.config.telemetry.is_enabled() && !self.config.bringup.has_telemetry()) + || self.monitor_task.is_some() + { + return; + } + + let telemetry = self.config.telemetry.clone(); + let rail_telemetry = self.config.bringup.clone(); + let calibration = self.config.calibration.clone(); + let telemetry_tx = self.telemetry_tx.clone(); + let shutdown_handles = self.shutdown_handles.clone(); + let serial_controls = self.serial_controls.clone(); + let bus_layouts = Arc::clone(&self.bus_layouts); + let applied_operating_state = Arc::clone(&self.applied_operating_state); + let runtime_measurements = Arc::clone(&self.runtime_measurements); + let board_name = self.config.device_id(); + let (shutdown_tx, mut shutdown_rx) = watch::channel(false); + self.monitor_shutdown = Some(shutdown_tx); + + self.monitor_task = Some(tokio::spawn(async move { + let mut interval = tokio::time::interval(telemetry.poll_interval); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + let mut retune_tracker = Bzm2RetuneTriggerTracker::default(); + loop { + tokio::select! { + _ = interval.tick() => { + let snapshot = telemetry.snapshot(); + let rail_snapshot = rail_telemetry.snapshot_telemetry(); + let thread_metrics = collect_thread_runtime_metrics(&shutdown_handles).await; + let bus_layouts = bus_layouts.lock().unwrap_or_else(|e| e.into_inner()).clone(); + let applied_operating_snapshot = applied_operating_state.lock().unwrap_or_else(|e| e.into_inner()).clone(); + let current_state = telemetry_tx.borrow().clone(); + let (tuning_state, measurement_cache) = build_runtime_tuning_state( + ¤t_state.asics, + ¤t_state.temperatures, + &bus_layouts, + &calibration, + &rail_telemetry, + &rail_snapshot, + &applied_operating_snapshot, + &thread_metrics, + ); + let tuning_state = apply_runtime_tuning_plan( + tuning_state, + evaluate_runtime_tuning_plan( + ¤t_state.asics, + ¤t_state.temperatures, + &bus_layouts, + &calibration, + &applied_operating_snapshot, + &measurement_cache, + ), + ); + let tuning_state = apply_runtime_retune_triggers( + tuning_state, + &calibration, + &measurement_cache, + &mut retune_tracker, + ); + let tuning_state = reconcile_saved_operating_point_status( + tuning_state, + &calibration, + &bus_layouts, + &applied_operating_state, + ); + let runtime_domain_count = measurement_cache.domain_measurements.len(); + let runtime_asic_count = measurement_cache.asic_measurements.len(); + *runtime_measurements.lock().unwrap_or_else(|e| e.into_inner()) = measurement_cache; + let total_stats = serial_controls.iter().fold((0u64, 0u64), |acc, control| { + let stats = control.stats(); + (acc.0 + stats.bytes_read, acc.1 + stats.bytes_written) + }); + telemetry_tx.send_modify(|state| { + state.fans = snapshot.fans.clone(); + merge_temperature_readings(&mut state.temperatures, &snapshot.temperatures); + merge_power_readings(&mut state.powers, &snapshot.powers); + merge_temperature_readings(&mut state.temperatures, &rail_snapshot.temperatures); + merge_power_readings(&mut state.powers, &rail_snapshot.powers); + state.bzm2_tuning = + (!tuning_state.asics.is_empty() || !tuning_state.domains.is_empty() + || tuning_state.board_throughput_hs.is_some()) + .then_some(tuning_state.clone()); + }); + trace!( + board = %board_name, + bytes_read = total_stats.0, + bytes_written = total_stats.1, + runtime_domain_count, + runtime_asic_count, + "BZM2 board telemetry updated" + ); + if let Some(reason) = snapshot.trip_reason.clone() { + warn!(board = %board_name, reason = %reason, "BZM2 safety trip triggered"); + for handle in &shutdown_handles { + handle.shutdown(); + } + telemetry_tx.send_modify(|state| { + for thread in &mut state.threads { + thread.is_active = false; + thread.hashrate = 0; + } + }); + break; + } + } + changed = shutdown_rx.changed() => { + if changed.is_err() || *shutdown_rx.borrow() { + break; + } + } + } + } + })); + } +} + +async fn collect_thread_runtime_metrics( + handles: &[Bzm2ThreadHandle], +) -> BTreeMap { + let mut metrics = BTreeMap::new(); + for (thread_index, handle) in handles.iter().enumerate() { + match handle.runtime_metrics().await { + Ok(snapshot) => { + metrics.insert(thread_index, snapshot); + } + Err(err) => { + warn!(thread_index, error = %err, "Failed to query BZM2 runtime metrics"); + } + } + } + metrics +} + +// Aggregates the monitor's per-poll working set; a parameter struct would be +// built and torn down at the single call site for no clarity gain. +#[allow(clippy::too_many_arguments)] +fn build_runtime_tuning_state( + asics: &[AsicState], + temperatures: &[TemperatureSensor], + bus_layouts: &[Bzm2BusLayout], + calibration: &Bzm2CalibrationConfig, + bringup: &Bzm2BringupConfig, + rail_snapshot: &Bzm2TelemetrySnapshot, + applied_operating_state: &Bzm2AppliedOperatingState, + thread_metrics: &BTreeMap, +) -> (Bzm2TuningState, Bzm2RuntimeMeasurementCache) { + let total_asics = bus_layouts.iter().map(|bus| bus.asic_count).sum::(); + let (domains, _domain_lookup) = build_voltage_domains( + total_asics, + &calibration.asics_per_domain, + &calibration.domain_voltage_offsets_mv, + ); + + let mut tuning_domains = Vec::new(); + let mut cache_domains = BTreeMap::new(); + for domain in domains { + let rail_index = bringup.rail_index_for_domain(domain.domain_id); + let rail_output_name = rail_index.map(|index| format!("rail{index}-output")); + let measured_voltage_mv = rail_output_name + .as_ref() + .and_then(|name| { + rail_snapshot + .powers + .iter() + .find(|power| power.name == *name) + }) + .and_then(|power| power.voltage_v) + .map(|voltage| (voltage * 1000.0).round() as u32); + let measured_power_w = rail_output_name + .as_ref() + .and_then(|name| { + rail_snapshot + .powers + .iter() + .find(|power| power.name == *name) + }) + .and_then(|power| power.power_w); + tuning_domains.push(Bzm2DomainTuningState { + domain_id: domain.domain_id, + rail_index, + target_voltage_mv: applied_operating_state + .per_domain_voltage_mv + .get(&domain.domain_id) + .copied(), + measured_voltage_mv, + measured_power_w, + }); + cache_domains.insert( + domain.domain_id, + Bzm2DomainMeasurement { + domain_id: domain.domain_id, + measured_voltage_mv, + measured_power_w, + }, + ); + } + tuning_domains.sort_by_key(|domain| domain.domain_id); + + let mut board_throughput_hs = 0u64; + let mut board_has_throughput = false; + let mut tuning_asics = Vec::new(); + let mut cache_asics = BTreeMap::new(); + + for asic in asics { + let Some(thread_index) = asic.thread_index else { + continue; + }; + let Some(bus) = bus_layouts.get(thread_index) else { + continue; + }; + let Some(global_asic_id) = bus.global_asic_id(asic.id) else { + continue; + }; + let runtime_asic = thread_metrics + .get(&thread_index) + .and_then(|metrics| metrics.asics.iter().find(|metrics| metrics.asic == asic.id)); + let missing_engines = + if asic.missing_engines.is_empty() && asic.discovered_engine_count.is_none() { + default_saved_engine_topology() + .missing_engines + .into_iter() + .map(|engine| EngineCoordinate { + row: engine.row, + col: engine.col, + }) + .collect::>() + } else { + asic.missing_engines.clone() + }; + let (stack0_active, stack1_active) = + split_active_engine_counts(asic.discovered_engine_count, &missing_engines); + let frequencies = applied_operating_state + .per_asic_pll_mhz + .get(&global_asic_id) + .copied(); + let mut pll_states = Vec::with_capacity(2); + let mut pll_pass_rates = [None, None]; + + for pll_index in 0..2usize { + let throughput_hs = runtime_asic.and_then(|asic| asic.plls[pll_index].throughput_hs); + let frequency_mhz = frequencies.map(|freq| freq[pll_index]); + let active_engines = if pll_index == 0 { + stack0_active + } else { + stack1_active + }; + let pass_rate = + throughput_hs + .zip(frequency_mhz) + .and_then(|(throughput_hs, frequency_mhz)| { + expected_stack_throughput_hs(active_engines, frequency_mhz) + .map(|expected| throughput_hs as f32 / expected.max(1) as f32) + }); + pll_pass_rates[pll_index] = pass_rate; + pll_states.push(Bzm2PllTuningState { + pll_index: pll_index as u8, + frequency_mhz, + throughput_hs, + pass_rate, + }); + } + + let average_pass_rate = weighted_average_pass_rate(&[ + (pll_pass_rates[0], stack0_active), + (pll_pass_rates[1], stack1_active), + ]); + let throughput_hs = runtime_asic.and_then(|asic| asic.throughput_hs); + if let Some(throughput_hs) = throughput_hs { + board_throughput_hs = board_throughput_hs.saturating_add(throughput_hs); + board_has_throughput = true; + } + + tuning_asics.push(Bzm2AsicTuningState { + id: asic.id, + thread_index: asic.thread_index, + active_engine_count: asic.discovered_engine_count, + throughput_hs, + average_pass_rate, + scheduler_share_count: runtime_asic.map(|asic| asic.scheduler_share_count), + plls: pll_states, + }); + + cache_asics.insert( + global_asic_id, + Bzm2AsicMeasurement { + asic_id: global_asic_id, + temperature_c: asic_temperature_for_sensor( + temperatures, + bus.serial_path.as_str(), + asic.id, + ), + throughput_ths: throughput_hs + .map(|throughput| throughput as f32 / 1_000_000_000_000.0), + average_pass_rate, + pll_pass_rates, + }, + ); + } + tuning_asics.sort_by_key(|asic| (asic.thread_index.unwrap_or(usize::MAX), asic.id)); + + ( + Bzm2TuningState { + board_throughput_hs: board_has_throughput.then_some(board_throughput_hs), + reuse_saved_operating_point: None, + needs_retune: None, + desired_voltage_mv: None, + desired_clock_mhz: None, + desired_accept_ratio: None, + retune_pending: None, + retune_reasons: Vec::new(), + saved_operating_point_status: applied_operating_state.saved_operating_point_status, + saved_operating_point_reasons: applied_operating_state + .saved_operating_point_reasons + .clone(), + planner_notes: Vec::new(), + domains: tuning_domains, + asics: tuning_asics, + }, + Bzm2RuntimeMeasurementCache { + domain_measurements: cache_domains, + asic_measurements: cache_asics, + }, + ) +} + +fn apply_runtime_tuning_plan( + mut tuning_state: Bzm2TuningState, + plan: Option, +) -> Bzm2TuningState { + if let Some(plan) = plan { + tuning_state.reuse_saved_operating_point = Some(plan.reuse_saved_operating_point); + tuning_state.needs_retune = Some(plan.needs_retune); + tuning_state.desired_voltage_mv = Some(plan.desired_voltage_mv); + tuning_state.desired_clock_mhz = Some(plan.desired_clock_mhz); + tuning_state.desired_accept_ratio = Some(plan.desired_accept_ratio); + tuning_state.planner_notes = plan.notes; + } + tuning_state +} + +fn apply_runtime_retune_triggers( + mut tuning_state: Bzm2TuningState, + calibration: &Bzm2CalibrationConfig, + measurement_cache: &Bzm2RuntimeMeasurementCache, + tracker: &mut Bzm2RetuneTriggerTracker, +) -> Bzm2TuningState { + if !calibration.runtime_retune_enabled { + tuning_state.retune_pending = Some(false); + tuning_state.retune_reasons.clear(); + return tuning_state; + } + + let persistence = calibration.runtime_retune_persistence_polls.max(1); + let throughput_regression = tuning_state.needs_retune.unwrap_or(false); + let thermal_drift = measurement_cache + .asic_measurements + .values() + .filter_map(|asic| asic.temperature_c) + .any(|temp| temp >= calibration.runtime_retune_thermal_c); + let voltage_imbalance = tuning_state.domains.iter().any(|domain| { + domain + .target_voltage_mv + .zip(domain.measured_voltage_mv) + .is_some_and(|(target, measured)| { + target.abs_diff(measured) >= calibration.runtime_retune_voltage_imbalance_mv + }) + }); + + let mut retune_reasons = Vec::new(); + if update_trigger_counter( + &mut tracker.throughput_regression_polls, + throughput_regression, + ) >= persistence + { + retune_reasons.push("throughput regression".into()); + } + if update_trigger_counter(&mut tracker.thermal_drift_polls, thermal_drift) >= persistence { + retune_reasons.push("thermal drift".into()); + } + if update_trigger_counter(&mut tracker.voltage_imbalance_polls, voltage_imbalance) + >= persistence + { + retune_reasons.push("persistent voltage imbalance".into()); + } + + tuning_state.retune_pending = Some(!retune_reasons.is_empty()); + tuning_state.retune_reasons = retune_reasons; + tuning_state +} + +fn reconcile_saved_operating_point_status( + mut tuning_state: Bzm2TuningState, + calibration: &Bzm2CalibrationConfig, + bus_layouts: &[Bzm2BusLayout], + applied_operating_state: &Arc>, +) -> Bzm2TuningState { + let mut guard = applied_operating_state + .lock() + .unwrap_or_else(|e| e.into_inner()); + + let desired = if tuning_state.retune_pending == Some(true) { + // Retune triggers only set retune_pending once they persist past the + // trigger tracker's threshold, so the operating point is known bad: + // invalidate it so startup replay refuses it and falls back to live + // calibration. + guard.saved_operating_point.as_ref().map(|_| { + ( + Bzm2SavedOperatingPointStatus::Invalidated, + tuning_state.retune_reasons.clone(), + ) + }) + } else if guard.saved_operating_point.is_some() { + Some((Bzm2SavedOperatingPointStatus::Validated, Vec::new())) + } else { + guard + .saved_operating_point_status + .map(|status| (status, guard.saved_operating_point_reasons.clone())) + }; + + if let Some((status, reasons)) = desired.clone() { + let status_changed = guard.saved_operating_point_status != Some(status) + || guard.saved_operating_point_reasons != reasons; + if status_changed { + if let (Some(profile_path), Some(saved_state)) = ( + calibration.profile_path.as_deref(), + guard.saved_operating_point.as_ref(), + ) && let Err(err) = store_saved_operating_point_status( + profile_path, + calibration, + bus_layouts, + saved_state, + status, + &reasons, + ) { + warn!( + path = %profile_path.display(), + error = %err, + "Failed to persist BZM2 saved operating point status" + ); + } + guard.saved_operating_point_status = Some(status); + guard.saved_operating_point_reasons = reasons.clone(); + } + + tuning_state.saved_operating_point_status = Some(status); + tuning_state.saved_operating_point_reasons = reasons; + if tuning_state.retune_pending == Some(true) { + tuning_state.reuse_saved_operating_point = Some(false); + } + } else { + tuning_state.saved_operating_point_status = None; + tuning_state.saved_operating_point_reasons.clear(); + } + + tuning_state +} + +fn update_trigger_counter(counter: &mut u8, active: bool) -> u8 { + if active { + *counter = counter.saturating_add(1); + } else { + *counter = 0; + } + *counter +} + +fn evaluate_runtime_tuning_plan( + asics: &[AsicState], + temperatures: &[TemperatureSensor], + bus_layouts: &[Bzm2BusLayout], + calibration: &Bzm2CalibrationConfig, + applied_operating_state: &Bzm2AppliedOperatingState, + measurement_cache: &Bzm2RuntimeMeasurementCache, +) -> Option { + if bus_layouts.is_empty() { + return None; + } + + let total_asics = bus_layouts.iter().map(|bus| bus.asic_count).sum::(); + if total_asics == 0 { + return None; + } + + let (_voltage_domains, domain_lookup) = build_voltage_domains( + total_asics, + &calibration.asics_per_domain, + &calibration.domain_voltage_offsets_mv, + ); + let engine_topology = saved_engine_topology_from_state(asics, bus_layouts); + let voltage_domains = build_voltage_domains( + total_asics, + &calibration.asics_per_domain, + &calibration.domain_voltage_offsets_mv, + ) + .0; + let asic_topology = build_topology(bus_layouts, &domain_lookup, &engine_topology); + let domain_measurements = voltage_domains + .iter() + .map(|domain| { + measurement_cache + .domain_measurements + .get(&domain.domain_id) + .cloned() + .unwrap_or(Bzm2DomainMeasurement { + domain_id: domain.domain_id, + measured_voltage_mv: None, + measured_power_w: None, + }) + }) + .collect::>(); + let asic_measurements = asic_topology + .iter() + .map(|asic| { + measurement_cache + .asic_measurements + .get(&asic.asic_id) + .cloned() + .unwrap_or(Bzm2AsicMeasurement { + asic_id: asic.asic_id, + temperature_c: temperatures + .iter() + .find(|sensor| { + bus_layouts + .iter() + .find(|layout| layout.contains(asic.asic_id)) + .and_then(|layout| layout.local_asic_id(asic.asic_id)) + .map(|local_asic| { + sensor.name + == format!( + "{}-asic-{local_asic}-dts", + sensor_prefix_from_serial( + bus_layouts + .iter() + .find(|layout| layout.contains(asic.asic_id)) + .map(|layout| layout.serial_path.as_str()) + .unwrap_or(""), + ) + ) + }) + .unwrap_or(false) + }) + .and_then(|sensor| sensor.temperature.map(Temperature::as_degrees_c)), + throughput_ths: None, + average_pass_rate: None, + pll_pass_rates: [None, None], + }) + }) + .collect::>(); + let site_temp_c = temperatures + .iter() + .find(|sensor| sensor.name == "board") + .and_then(|sensor| sensor.temperature.map(Temperature::as_degrees_c)) + .or_else(|| { + temperatures + .iter() + .find(|sensor| sensor.name == "asic") + .and_then(|sensor| sensor.temperature.map(Temperature::as_degrees_c)) + }) + .or(calibration.site_temp_c) + .unwrap_or(DEFAULT_CALIBRATION_SITE_TEMP_C); + + Some(Bzm2CalibrationPlanner.plan(&Bzm2BoardCalibrationInput { + operating_class: calibration.operating_class, + site_temp_c, + target_mode: calibration.performance_mode, + mode: calibration.mode, + per_stack_clocking: calibration.per_stack_clocking, + voltage_domains, + asics: asic_topology, + saved_operating_point: applied_operating_state.saved_operating_point.clone(), + domain_measurements, + asic_measurements, + constraints: Bzm2CalibrationConstraints::default(), + force_retune: calibration.force_retune, + })) +} + +fn saved_engine_topology_from_state( + asics: &[AsicState], + bus_layouts: &[Bzm2BusLayout], +) -> BTreeMap { + let mut topology = BTreeMap::new(); + for asic in asics { + let Some(thread_index) = asic.thread_index else { + continue; + }; + let Some(bus) = bus_layouts.get(thread_index) else { + continue; + }; + let Some(global_asic_id) = bus.global_asic_id(asic.id) else { + continue; + }; + topology.insert( + global_asic_id, + Bzm2SavedEngineTopology { + active_engine_count: asic + .discovered_engine_count + .unwrap_or_else(|| default_saved_engine_topology().active_engine_count), + missing_engines: if asic.missing_engines.is_empty() + && asic.discovered_engine_count.is_none() + { + default_saved_engine_topology().missing_engines + } else { + asic.missing_engines + .iter() + .map(|engine| Bzm2SavedEngineCoordinate { + row: engine.row, + col: engine.col, + }) + .collect() + }, + }, + ); + } + topology +} + +fn sensor_prefix_from_serial(serial_path: &str) -> String { + Path::new(serial_path) + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or(serial_path) + .chars() + .map(|ch| if ch.is_ascii_alphanumeric() { ch } else { '_' }) + .collect() +} + +fn split_active_engine_counts( + active_engine_count: Option, + missing_engines: &[EngineCoordinate], +) -> (u16, u16) { + if missing_engines.is_empty() + && let Some(active_engine_count) = active_engine_count + { + let lower = active_engine_count / 2; + return (lower, active_engine_count.saturating_sub(lower)); + } + let mut bottom_missing = 0u16; + let mut top_missing = 0u16; + for engine in missing_engines { + if engine.row < 10 { + bottom_missing = bottom_missing.saturating_add(1); + } else { + top_missing = top_missing.saturating_add(1); + } + } + let engines_per_stack = 10u16 * 12u16; + ( + engines_per_stack.saturating_sub(bottom_missing), + engines_per_stack.saturating_sub(top_missing), + ) +} + +fn expected_stack_throughput_hs(active_engines: u16, frequency_mhz: f32) -> Option { + (frequency_mhz > 0.0).then(|| { + let ghs = active_engines as f32 * 4.0 * (frequency_mhz / 1000.0) / 3.0; + (ghs * 1_000_000_000.0).round() as u64 + }) +} + +fn weighted_average_pass_rate(samples: &[(Option, u16)]) -> Option { + let mut weighted = 0.0f32; + let mut total_weight = 0u32; + for (pass_rate, weight) in samples { + if let Some(pass_rate) = pass_rate { + weighted += pass_rate * *weight as f32; + total_weight += u32::from(*weight); + } + } + (total_weight > 0).then_some(weighted / total_weight as f32) +} + +fn asic_temperature_for_sensor( + temperatures: &[TemperatureSensor], + serial_path: &str, + asic: u8, +) -> Option { + let prefix = Path::new(serial_path) + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or(serial_path) + .chars() + .map(|ch| if ch.is_ascii_alphanumeric() { ch } else { '_' }) + .collect::(); + let name = format!("{prefix}-asic-{asic}-dts"); + temperatures + .iter() + .find(|sensor| sensor.name == name) + .and_then(|sensor| sensor.temperature.map(Temperature::as_degrees_c)) +} + +#[cfg(test)] +mod tests { + use super::super::calibration::load_saved_operating_point_profile; + #[cfg(unix)] + use super::super::config::{ + Bzm2EnumerationConfig, Bzm2RuntimeConfig, DEFAULT_BAUD_RATE, DEFAULT_NOMINAL_HASHRATE_THS, + }; + #[cfg(unix)] + use super::super::telemetry::{Bzm2TelemetryConfig, SensorSpec}; + use super::*; + #[cfg(unix)] + use crate::api_client::types::BoardTelemetry; + use crate::api_client::types::{Bzm2StartupPath, PowerMeasurement}; + use crate::tuning::blockscale::Bzm2SavedOperatingPoint; + #[cfg(unix)] + use nix::pty::openpty; + use std::fs; + #[cfg(unix)] + use std::os::fd::AsRawFd; + #[cfg(unix)] + use std::time::Duration; + use std::time::{SystemTime, UNIX_EPOCH}; + #[cfg(unix)] + use tokio::sync::{mpsc, watch}; + + #[cfg(unix)] + #[tokio::test] + async fn board_safety_trip_closes_scheduler_event_stream() { + let pty = openpty(None, None).unwrap(); + let serial_path = fs::read_link(format!("/proc/self/fd/{}", pty.slave.as_raw_fd())) + .unwrap() + .to_string_lossy() + .into_owned(); + + let sensor_path = std::env::temp_dir().join(format!( + "bzm2-trip-{}-{}.txt", + std::process::id(), + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + fs::write(&sensor_path, "90\n").unwrap(); + + let config = Bzm2RuntimeConfig { + serial_paths: vec![serial_path], + baud_rate: DEFAULT_BAUD_RATE, + timestamp_count: crate::asic::bzm2::protocol::DEFAULT_TIMESTAMP_COUNT, + nonce_gap: crate::asic::bzm2::protocol::DEFAULT_NONCE_GAP, + dispatch_interval: Duration::from_millis(50), + nominal_hashrate_ths: DEFAULT_NOMINAL_HASHRATE_THS, + dts_vs_generation: crate::asic::bzm2::protocol::DtsVsGeneration::Gen2, + telemetry: Bzm2TelemetryConfig { + poll_interval: Duration::from_millis(20), + asic_temp: Some(SensorSpec { + path: sensor_path.to_string_lossy().into_owned(), + scale: 1.0, + }), + max_asic_temp_c: Some(80.0), + ..Default::default() + }, + enumeration: Bzm2EnumerationConfig::default(), + bringup: Bzm2BringupConfig::default(), + calibration: Bzm2CalibrationConfig::default(), + }; + let (telemetry_tx, mut telemetry_rx) = watch::channel(BoardTelemetry { + name: "bzm2-test".into(), + model: "BZM2".into(), + serial: Some("bzm2-test".into()), + ..Default::default() + }); + let mut board = Bzm2Board::new(config, telemetry_tx, mpsc::channel(1).1); + + let mut threads = board.create_hash_threads().await.unwrap(); + let mut event_rx = threads[0].take_event_receiver().unwrap(); + + let closed = tokio::time::timeout(Duration::from_secs(1), async { + while event_rx.recv().await.is_some() {} + }) + .await; + assert!( + closed.is_ok(), + "event stream should close after safety trip" + ); + + let state = tokio::time::timeout(Duration::from_secs(1), async { + loop { + let snapshot = telemetry_rx.borrow().clone(); + if snapshot.temperatures.iter().any(|sensor| { + sensor.name == "asic" + && sensor.temperature.map(Temperature::as_degrees_c) == Some(90.0) + }) { + break snapshot; + } + telemetry_rx.changed().await.unwrap(); + } + }) + .await + .unwrap(); + assert_eq!(state.threads[0].hashrate, 0); + + board.shutdown().await.unwrap(); + let _ = fs::remove_file(sensor_path); + drop(pty); + } + + #[test] + fn build_runtime_tuning_state_maps_live_measurements() { + let asics = vec![AsicState { + id: 0, + thread_index: Some(0), + serial_path: Some("/dev/ttyUSB0".into()), + discovered_engine_count: Some(236), + missing_engines: Vec::new(), + }]; + let temperatures = vec![TemperatureSensor { + name: "ttyUSB0-asic-0-dts".into(), + temperature: Some(Temperature::from_celsius(67.0)), + }]; + let bus_layouts = vec![Bzm2BusLayout { + serial_path: "/dev/ttyUSB0".into(), + asic_start: 0, + asic_count: 1, + }]; + let calibration = Bzm2CalibrationConfig::default(); + let bringup = Bzm2BringupConfig { + rail_set_paths: vec!["/tmp/rail0".into()], + ..Default::default() + }; + let rail_snapshot = Bzm2TelemetrySnapshot { + powers: vec![PowerMeasurement { + name: "rail0-output".into(), + voltage_v: Some(0.9), + current_a: Some(44.0), + power_w: Some(40.0), + }], + ..Default::default() + }; + let applied = Bzm2AppliedOperatingState { + per_domain_voltage_mv: BTreeMap::from([(0, 18_500)]), + per_asic_pll_mhz: BTreeMap::from([(0, [1_200.0, 1_200.0])]), + saved_operating_point: None, + startup_path: None, + saved_operating_point_status: None, + saved_operating_point_reasons: Vec::new(), + }; + let thread_metrics = BTreeMap::from([( + 0usize, + Bzm2ThreadRuntimeMetrics { + throughput_hs: Some(358_720_000_000), + asics: vec![crate::asic::bzm2::Bzm2AsicRuntimeMetrics { + asic: 0, + throughput_hs: Some(358_720_000_000), + scheduler_share_count: 12, + plls: [ + crate::asic::bzm2::Bzm2PllRuntimeMetrics { + throughput_hs: Some(179_360_000_000), + scheduler_share_count: 6, + }, + crate::asic::bzm2::Bzm2PllRuntimeMetrics { + throughput_hs: Some(179_360_000_000), + scheduler_share_count: 6, + }, + ], + }], + }, + )]); + + let (tuning, cache) = build_runtime_tuning_state( + &asics, + &temperatures, + &bus_layouts, + &calibration, + &bringup, + &rail_snapshot, + &applied, + &thread_metrics, + ); + + assert_eq!(tuning.board_throughput_hs, Some(358_720_000_000)); + assert_eq!(tuning.domains.len(), 1); + assert_eq!(tuning.domains[0].target_voltage_mv, Some(18_500)); + assert_eq!(tuning.domains[0].measured_voltage_mv, Some(900)); + assert_eq!(tuning.domains[0].measured_power_w, Some(40.0)); + assert_eq!(tuning.asics.len(), 1); + assert_eq!(tuning.asics[0].throughput_hs, Some(358_720_000_000)); + assert_eq!(tuning.asics[0].scheduler_share_count, Some(12)); + assert!( + tuning.asics[0] + .average_pass_rate + .is_some_and(|pass_rate| (pass_rate - 0.95).abs() < 0.0001) + ); + assert!( + tuning.asics[0].plls[0] + .pass_rate + .is_some_and(|pass_rate| (pass_rate - 0.95).abs() < 0.0001) + ); + assert!( + cache.asic_measurements[&0] + .temperature_c + .is_some_and(|temp| (temp - 67.0).abs() < 0.0001) + ); + assert!( + cache.asic_measurements[&0] + .throughput_ths + .is_some_and(|throughput| (throughput - 0.35872).abs() < 0.0001) + ); + assert_eq!(cache.domain_measurements[&0].measured_voltage_mv, Some(900)); + assert_eq!(cache.domain_measurements[&0].measured_power_w, Some(40.0)); + } + + #[test] + fn evaluate_runtime_tuning_plan_flags_underperforming_saved_point() { + let asics = vec![AsicState { + id: 0, + thread_index: Some(0), + serial_path: Some("/dev/ttyUSB0".into()), + discovered_engine_count: Some(236), + missing_engines: Vec::new(), + }]; + let temperatures = vec![TemperatureSensor { + name: "ttyUSB0-asic-0-dts".into(), + temperature: Some(Temperature::from_celsius(72.0)), + }]; + let bus_layouts = vec![Bzm2BusLayout { + serial_path: "/dev/ttyUSB0".into(), + asic_start: 0, + asic_count: 1, + }]; + let calibration = Bzm2CalibrationConfig::default(); + let applied = Bzm2AppliedOperatingState { + per_domain_voltage_mv: BTreeMap::from([(0, 18_500)]), + per_asic_pll_mhz: BTreeMap::from([(0, [1_200.0, 1_200.0])]), + saved_operating_point: Some(Bzm2SavedOperatingPoint { + board_voltage_mv: 18_500, + board_throughput_ths: 0.40, + per_domain_voltage_mv: BTreeMap::from([(0, 18_500)]), + per_asic_engine_topology: BTreeMap::new(), + per_asic_pll_mhz: BTreeMap::from([(0, [1_200.0, 1_200.0])]), + }), + startup_path: Some(Bzm2StartupPath::SavedReplay), + saved_operating_point_status: Some(Bzm2SavedOperatingPointStatus::Validated), + saved_operating_point_reasons: Vec::new(), + }; + let measurement_cache = Bzm2RuntimeMeasurementCache { + domain_measurements: BTreeMap::from([( + 0, + Bzm2DomainMeasurement { + domain_id: 0, + measured_voltage_mv: Some(18_300), + measured_power_w: Some(55.0), + }, + )]), + asic_measurements: BTreeMap::from([( + 0, + Bzm2AsicMeasurement { + asic_id: 0, + temperature_c: Some(72.0), + throughput_ths: Some(0.20), + average_pass_rate: Some(0.94), + pll_pass_rates: [Some(0.94), Some(0.94)], + }, + )]), + }; + + let plan = evaluate_runtime_tuning_plan( + &asics, + &temperatures, + &bus_layouts, + &calibration, + &applied, + &measurement_cache, + ) + .unwrap(); + + assert!(plan.needs_retune); + assert!(!plan.reuse_saved_operating_point); + } + + #[test] + fn runtime_retune_triggers_require_persistence() { + let mut calibration = Bzm2CalibrationConfig::default(); + calibration.runtime_retune_persistence_polls = 2; + calibration.runtime_retune_thermal_c = 80.0; + let measurement_cache = Bzm2RuntimeMeasurementCache { + domain_measurements: BTreeMap::new(), + asic_measurements: BTreeMap::from([( + 0, + Bzm2AsicMeasurement { + asic_id: 0, + temperature_c: Some(82.0), + throughput_ths: Some(0.30), + average_pass_rate: Some(0.97), + pll_pass_rates: [Some(0.97), Some(0.97)], + }, + )]), + }; + let mut tracker = Bzm2RetuneTriggerTracker::default(); + let tuning = Bzm2TuningState { + needs_retune: Some(true), + domains: vec![Bzm2DomainTuningState { + domain_id: 0, + rail_index: Some(0), + target_voltage_mv: Some(18_500), + measured_voltage_mv: Some(18_650), + measured_power_w: Some(40.0), + }], + ..Default::default() + }; + + let first = apply_runtime_retune_triggers( + tuning.clone(), + &calibration, + &measurement_cache, + &mut tracker, + ); + assert_eq!(first.retune_pending, Some(false)); + assert!(first.retune_reasons.is_empty()); + + let second = + apply_runtime_retune_triggers(tuning, &calibration, &measurement_cache, &mut tracker); + assert_eq!(second.retune_pending, Some(true)); + assert!( + second + .retune_reasons + .iter() + .any(|reason| reason == "throughput regression") + ); + assert!( + second + .retune_reasons + .iter() + .any(|reason| reason == "thermal drift") + ); + assert!( + second + .retune_reasons + .iter() + .any(|reason| reason == "persistent voltage imbalance") + ); + } + + #[test] + fn reconcile_saved_operating_point_status_validates_profile() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let profile_path = std::env::temp_dir().join(format!( + "bzm2-validate-profile-{}-{}.json", + std::process::id(), + unique + )); + let mut calibration = Bzm2CalibrationConfig::default(); + calibration.profile_path = Some(profile_path.clone()); + let bus_layouts = vec![Bzm2BusLayout { + serial_path: "/dev/ttyUSB0".into(), + asic_start: 0, + asic_count: 1, + }]; + let saved_state = Bzm2SavedOperatingPoint { + board_voltage_mv: 17_500, + board_throughput_ths: 40.0, + per_domain_voltage_mv: BTreeMap::from([(0, 17_500)]), + per_asic_engine_topology: BTreeMap::new(), + per_asic_pll_mhz: BTreeMap::from([(0, [1_100.0, 1_100.0])]), + }; + let applied_state = Arc::new(Mutex::new(Bzm2AppliedOperatingState { + per_domain_voltage_mv: saved_state.per_domain_voltage_mv.clone(), + per_asic_pll_mhz: saved_state.per_asic_pll_mhz.clone(), + saved_operating_point: Some(saved_state), + startup_path: Some(Bzm2StartupPath::LiveCalibration), + saved_operating_point_status: Some(Bzm2SavedOperatingPointStatus::Pending), + saved_operating_point_reasons: vec!["awaiting runtime validation".into()], + })); + + let tuning = reconcile_saved_operating_point_status( + Bzm2TuningState::default(), + &calibration, + &bus_layouts, + &applied_state, + ); + assert_eq!( + tuning.saved_operating_point_status, + Some(Bzm2SavedOperatingPointStatus::Validated) + ); + assert!(tuning.saved_operating_point_reasons.is_empty()); + + let stored = load_saved_operating_point_profile(Some(&profile_path)) + .unwrap() + .unwrap(); + assert_eq!( + stored.persisted.unwrap().saved_operating_point_status, + Bzm2SavedOperatingPointStatus::Validated + ); + + let _ = fs::remove_file(profile_path); + } + + #[test] + fn reconcile_saved_operating_point_status_invalidates_profile() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let profile_path = std::env::temp_dir().join(format!( + "bzm2-invalidate-profile-{}-{}.json", + std::process::id(), + unique + )); + let mut calibration = Bzm2CalibrationConfig::default(); + calibration.profile_path = Some(profile_path.clone()); + let bus_layouts = vec![Bzm2BusLayout { + serial_path: "/dev/ttyUSB0".into(), + asic_start: 0, + asic_count: 1, + }]; + let saved_state = Bzm2SavedOperatingPoint { + board_voltage_mv: 17_500, + board_throughput_ths: 40.0, + per_domain_voltage_mv: BTreeMap::from([(0, 17_500)]), + per_asic_engine_topology: BTreeMap::new(), + per_asic_pll_mhz: BTreeMap::from([(0, [1_100.0, 1_100.0])]), + }; + let applied_state = Arc::new(Mutex::new(Bzm2AppliedOperatingState { + per_domain_voltage_mv: saved_state.per_domain_voltage_mv.clone(), + per_asic_pll_mhz: saved_state.per_asic_pll_mhz.clone(), + saved_operating_point: Some(saved_state), + startup_path: Some(Bzm2StartupPath::SavedReplay), + saved_operating_point_status: Some(Bzm2SavedOperatingPointStatus::Validated), + saved_operating_point_reasons: Vec::new(), + })); + + let tuning = reconcile_saved_operating_point_status( + Bzm2TuningState { + retune_pending: Some(true), + retune_reasons: vec!["throughput regression".into()], + ..Default::default() + }, + &calibration, + &bus_layouts, + &applied_state, + ); + assert_eq!( + tuning.saved_operating_point_status, + Some(Bzm2SavedOperatingPointStatus::Invalidated) + ); + assert_eq!( + tuning.saved_operating_point_reasons, + vec!["throughput regression"] + ); + assert_eq!(tuning.reuse_saved_operating_point, Some(false)); + + let applied = applied_state + .lock() + .unwrap_or_else(|e| e.into_inner()) + .clone(); + assert!(applied.saved_operating_point.is_some()); + assert_eq!( + applied.saved_operating_point_status, + Some(Bzm2SavedOperatingPointStatus::Invalidated) + ); + + let stored = load_saved_operating_point_profile(Some(&profile_path)) + .unwrap() + .unwrap(); + assert_eq!( + stored.persisted.unwrap().saved_operating_point_status, + Bzm2SavedOperatingPointStatus::Invalidated + ); + + let _ = fs::remove_file(profile_path); + } + + #[test] + fn persistent_retune_triggers_invalidate_saved_operating_point() { + let mut calibration = Bzm2CalibrationConfig::default(); + calibration.runtime_retune_persistence_polls = 2; + calibration.runtime_retune_thermal_c = 80.0; + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let profile_path = std::env::temp_dir().join(format!( + "bzm2-persistent-invalidate-{}-{}.json", + std::process::id(), + unique + )); + calibration.profile_path = Some(profile_path.clone()); + let bus_layouts = vec![Bzm2BusLayout { + serial_path: "/dev/ttyUSB0".into(), + asic_start: 0, + asic_count: 1, + }]; + let saved_state = Bzm2SavedOperatingPoint { + board_voltage_mv: 17_500, + board_throughput_ths: 40.0, + per_domain_voltage_mv: BTreeMap::from([(0, 17_500)]), + per_asic_engine_topology: BTreeMap::new(), + per_asic_pll_mhz: BTreeMap::from([(0, [1_100.0, 1_100.0])]), + }; + let applied_state = Arc::new(Mutex::new(Bzm2AppliedOperatingState { + per_domain_voltage_mv: saved_state.per_domain_voltage_mv.clone(), + per_asic_pll_mhz: saved_state.per_asic_pll_mhz.clone(), + saved_operating_point: Some(saved_state), + startup_path: Some(Bzm2StartupPath::SavedReplay), + saved_operating_point_status: Some(Bzm2SavedOperatingPointStatus::Validated), + saved_operating_point_reasons: Vec::new(), + })); + let measurement_cache = Bzm2RuntimeMeasurementCache { + domain_measurements: BTreeMap::new(), + asic_measurements: BTreeMap::from([( + 0, + Bzm2AsicMeasurement { + asic_id: 0, + temperature_c: Some(82.0), + throughput_ths: Some(0.30), + average_pass_rate: Some(0.97), + pll_pass_rates: [Some(0.97), Some(0.97)], + }, + )]), + }; + let mut tracker = Bzm2RetuneTriggerTracker::default(); + let tuning = Bzm2TuningState { + needs_retune: Some(true), + ..Default::default() + }; + + // First poll: the trigger fires but has not persisted; the saved + // point keeps its validated status. + let first = apply_runtime_retune_triggers( + tuning.clone(), + &calibration, + &measurement_cache, + &mut tracker, + ); + assert_eq!(first.retune_pending, Some(false)); + let first = reconcile_saved_operating_point_status( + first, + &calibration, + &bus_layouts, + &applied_state, + ); + assert_eq!( + first.saved_operating_point_status, + Some(Bzm2SavedOperatingPointStatus::Validated) + ); + + // Second poll: the trigger passes the persistence threshold; the + // saved point is invalidated and the invalidation is persisted. + let second = + apply_runtime_retune_triggers(tuning, &calibration, &measurement_cache, &mut tracker); + assert_eq!(second.retune_pending, Some(true)); + let second = reconcile_saved_operating_point_status( + second, + &calibration, + &bus_layouts, + &applied_state, + ); + assert_eq!( + second.saved_operating_point_status, + Some(Bzm2SavedOperatingPointStatus::Invalidated) + ); + + let stored = load_saved_operating_point_profile(Some(&profile_path)) + .unwrap() + .unwrap(); + assert_eq!( + stored.persisted.unwrap().saved_operating_point_status, + Bzm2SavedOperatingPointStatus::Invalidated + ); + + let _ = fs::remove_file(profile_path); + } +} diff --git a/mujina-miner/src/board/bzm2/telemetry.rs b/mujina-miner/src/board/bzm2/telemetry.rs new file mode 100644 index 00000000..79a4d1b4 --- /dev/null +++ b/mujina-miner/src/board/bzm2/telemetry.rs @@ -0,0 +1,647 @@ +//! Sensor polling and board telemetry publishing for the BZM2 board. + +use std::env; +use std::fs; +use std::time::Duration; + +use tokio::sync::watch; + +use crate::api_client::types::{ + AsicState, BoardTelemetry, Bzm2ClockReportResponse, Bzm2DllClockStatus, Bzm2PllClockStatus, + EngineCoordinate, Fan, PowerMeasurement, TemperatureSensor, +}; +use crate::asic::bzm2::Bzm2DiscoveredEngineMap; +use crate::asic::hash_thread::{HashThreadStatus, HashThreadTelemetryUpdate}; +use crate::tuning::blockscale::Bzm2SavedEngineTopology; +use crate::types::Temperature; + +use super::config::{ + DEFAULT_ASIC_TEMP_SCALE, DEFAULT_BOARD_TEMP_SCALE, DEFAULT_CURRENT_SCALE, + DEFAULT_FAN_PERCENT_SCALE, DEFAULT_FAN_RPM_SCALE, DEFAULT_POWER_SCALE, + DEFAULT_TELEMETRY_INTERVAL_SECS, DEFAULT_VOLTAGE_SCALE, env_csv_strings_any, env_f32, + parse_csv_numbers_any, +}; + +#[derive(Debug, Clone, Default)] +pub struct Bzm2TelemetryConfig { + pub poll_interval: Duration, + pub asic_temp: Option, + pub board_temp: Option, + pub fan_rpm: Option, + pub fan_percent: Option, + pub input_voltage: Option, + pub input_current: Option, + pub input_power: Option, + pub max_asic_temp_c: Option, + pub max_board_temp_c: Option, + pub max_input_power_w: Option, +} +impl Bzm2TelemetryConfig { + pub(super) fn from_env() -> Self { + Self { + poll_interval: Duration::from_secs( + env::var("MUJINA_BZM2_TELEMETRY_INTERVAL_SECS") + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or(DEFAULT_TELEMETRY_INTERVAL_SECS), + ), + asic_temp: SensorSpec::from_env( + "MUJINA_BZM2_ASIC_TEMP_PATH", + "MUJINA_BZM2_ASIC_TEMP_SCALE", + DEFAULT_ASIC_TEMP_SCALE, + ), + board_temp: SensorSpec::from_env( + "MUJINA_BZM2_BOARD_TEMP_PATH", + "MUJINA_BZM2_BOARD_TEMP_SCALE", + DEFAULT_BOARD_TEMP_SCALE, + ), + fan_rpm: SensorSpec::from_env( + "MUJINA_BZM2_FAN_RPM_PATH", + "MUJINA_BZM2_FAN_RPM_SCALE", + DEFAULT_FAN_RPM_SCALE, + ), + fan_percent: SensorSpec::from_env( + "MUJINA_BZM2_FAN_PERCENT_PATH", + "MUJINA_BZM2_FAN_PERCENT_SCALE", + DEFAULT_FAN_PERCENT_SCALE, + ), + input_voltage: SensorSpec::from_env( + "MUJINA_BZM2_INPUT_VOLTAGE_PATH", + "MUJINA_BZM2_INPUT_VOLTAGE_SCALE", + DEFAULT_VOLTAGE_SCALE, + ), + input_current: SensorSpec::from_env( + "MUJINA_BZM2_INPUT_CURRENT_PATH", + "MUJINA_BZM2_INPUT_CURRENT_SCALE", + DEFAULT_CURRENT_SCALE, + ), + input_power: SensorSpec::from_env( + "MUJINA_BZM2_INPUT_POWER_PATH", + "MUJINA_BZM2_INPUT_POWER_SCALE", + DEFAULT_POWER_SCALE, + ), + max_asic_temp_c: env_f32("MUJINA_BZM2_MAX_ASIC_TEMP_C"), + max_board_temp_c: env_f32("MUJINA_BZM2_MAX_BOARD_TEMP_C"), + max_input_power_w: env_f32("MUJINA_BZM2_MAX_INPUT_POWER_W"), + } + } + + pub(super) fn is_enabled(&self) -> bool { + self.asic_temp.is_some() + || self.board_temp.is_some() + || self.fan_rpm.is_some() + || self.fan_percent.is_some() + || self.input_voltage.is_some() + || self.input_current.is_some() + || self.input_power.is_some() + || self.max_asic_temp_c.is_some() + || self.max_board_temp_c.is_some() + || self.max_input_power_w.is_some() + } + + pub(super) fn snapshot(&self) -> Bzm2TelemetrySnapshot { + let asic_temp = self.asic_temp.as_ref().and_then(SensorSpec::read); + let board_temp = self.board_temp.as_ref().and_then(SensorSpec::read); + let fan_rpm = self + .fan_rpm + .as_ref() + .and_then(SensorSpec::read) + .map(|v| v.round() as u32); + let fan_percent = self + .fan_percent + .as_ref() + .and_then(SensorSpec::read) + .map(|v| v.round().clamp(0.0, 100.0) as u8); + let voltage_v = self.input_voltage.as_ref().and_then(SensorSpec::read); + let current_a = self.input_current.as_ref().and_then(SensorSpec::read); + let power_w = self + .input_power + .as_ref() + .and_then(SensorSpec::read) + .or_else(|| voltage_v.zip(current_a).map(|(v, c)| v * c)); + + let fans = if fan_rpm.is_some() || fan_percent.is_some() { + vec![Fan { + name: "fan".into(), + rpm: fan_rpm, + percent: fan_percent, + target_percent: None, + }] + } else { + Vec::new() + }; + + let mut temperatures = Vec::new(); + if self.asic_temp.is_some() || asic_temp.is_some() { + temperatures.push(TemperatureSensor { + name: "asic".into(), + temperature: asic_temp.map(Temperature::from_celsius), + }); + } + if self.board_temp.is_some() || board_temp.is_some() { + temperatures.push(TemperatureSensor { + name: "board".into(), + temperature: board_temp.map(Temperature::from_celsius), + }); + } + + let powers = if self.input_voltage.is_some() + || self.input_current.is_some() + || self.input_power.is_some() + || power_w.is_some() + { + vec![PowerMeasurement { + name: "input".into(), + voltage_v, + current_a, + power_w, + }] + } else { + Vec::new() + }; + + let trip_reason = self.trip_reason(asic_temp, board_temp, power_w); + Bzm2TelemetrySnapshot { + fans, + temperatures, + powers, + trip_reason, + } + } + + fn trip_reason( + &self, + asic_temp: Option, + board_temp: Option, + input_power_w: Option, + ) -> Option { + if let (Some(limit), Some(value)) = (self.max_asic_temp_c, asic_temp) + && value > limit + { + return Some(format!( + "ASIC temperature {:.1}C exceeded limit {:.1}C", + value, limit + )); + } + if let (Some(limit), Some(value)) = (self.max_board_temp_c, board_temp) + && value > limit + { + return Some(format!( + "Board temperature {:.1}C exceeded limit {:.1}C", + value, limit + )); + } + if let (Some(limit), Some(value)) = (self.max_input_power_w, input_power_w) + && value > limit + { + return Some(format!( + "Input power {:.1}W exceeded limit {:.1}W", + value, limit + )); + } + None + } +} + +#[derive(Debug, Clone)] +pub struct SensorSpec { + pub path: String, + pub scale: f32, +} + +impl SensorSpec { + fn from_env(path_var: &str, scale_var: &str, default_scale: f32) -> Option { + let path = env::var(path_var).ok()?; + let scale = env::var(scale_var) + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or(default_scale); + Some(Self { path, scale }) + } + + pub(super) fn read(&self) -> Option { + let raw = fs::read_to_string(&self.path).ok()?; + parse_scaled_sensor_value(&raw, self.scale) + } +} + +#[derive(Debug, Clone, Default)] +pub(super) struct Bzm2TelemetrySnapshot { + pub(super) fans: Vec, + pub(super) temperatures: Vec, + pub(super) powers: Vec, + pub(super) trip_reason: Option, +} + +pub(super) fn publish_thread_status( + telemetry_tx: &watch::Sender, + thread_index: usize, + status: &HashThreadStatus, +) { + telemetry_tx.send_modify(|state| { + if let Some(thread) = state.threads.get_mut(thread_index) { + thread.hashrate = status.hashrate.0; + thread.is_active = status.is_active; + } + }); +} + +pub(super) fn publish_thread_telemetry( + telemetry_tx: &watch::Sender, + update: &HashThreadTelemetryUpdate, +) { + telemetry_tx.send_modify(|state| { + merge_temperature_readings( + &mut state.temperatures, + &update + .temperatures + .iter() + .map(|reading| TemperatureSensor { + name: reading.name.clone(), + temperature: reading.temperature_c.map(Temperature::from_celsius), + }) + .collect::>(), + ); + merge_power_readings( + &mut state.powers, + &update + .powers + .iter() + .map(|reading| PowerMeasurement { + name: reading.name.clone(), + voltage_v: reading.voltage_v, + current_a: reading.current_a, + power_w: reading.power_w, + }) + .collect::>(), + ); + }); +} + +pub(super) fn publish_discovered_engine_map( + telemetry_tx: &watch::Sender, + thread_index: usize, + serial_path: &str, + discovery: &Bzm2DiscoveredEngineMap, +) { + upsert_asic_state( + telemetry_tx, + thread_index, + serial_path, + discovery.asic, + discovery.present_count() as u16, + discovery + .missing + .iter() + .map(|engine| EngineCoordinate { + row: engine.row, + col: engine.col, + }) + .collect(), + ); +} + +pub(super) fn publish_saved_engine_topology( + telemetry_tx: &watch::Sender, + thread_index: usize, + serial_path: &str, + asic_id: u8, + topology: &Bzm2SavedEngineTopology, +) { + upsert_asic_state( + telemetry_tx, + thread_index, + serial_path, + asic_id, + topology.active_engine_count, + topology + .missing_engines + .iter() + .map(|engine| EngineCoordinate { + row: engine.row, + col: engine.col, + }) + .collect(), + ); +} + +pub(super) fn merge_temperature_readings( + existing: &mut Vec, + updates: &[TemperatureSensor], +) { + for update in updates { + if let Some(sensor) = existing + .iter_mut() + .find(|sensor| sensor.name == update.name) + { + sensor.temperature = update.temperature; + } else { + existing.push(update.clone()); + } + } +} + +pub(super) fn merge_power_readings( + existing: &mut Vec, + updates: &[PowerMeasurement], +) { + for update in updates { + if let Some(sensor) = existing + .iter_mut() + .find(|sensor| sensor.name == update.name) + { + sensor.voltage_v = update.voltage_v; + sensor.current_a = update.current_a; + sensor.power_w = update.power_w; + } else { + existing.push(update.clone()); + } + } +} + +pub(super) fn map_clock_report( + report: crate::asic::bzm2::Bzm2ClockDebugReport, +) -> Bzm2ClockReportResponse { + Bzm2ClockReportResponse { + asic: report.asic, + pll0: Bzm2PllClockStatus { + enable_register: report.pll0.enable_register, + misc_register: report.pll0.misc_register, + enabled: report.pll0.enabled, + locked: report.pll0.locked, + }, + pll1: Bzm2PllClockStatus { + enable_register: report.pll1.enable_register, + misc_register: report.pll1.misc_register, + enabled: report.pll1.enabled, + locked: report.pll1.locked, + }, + dll0: Bzm2DllClockStatus { + control2: report.dll0.control2, + control5: report.dll0.control5, + coarsecon: report.dll0.coarsecon, + fincon: report.dll0.fincon, + freeze_valid: report.dll0.freeze_valid, + locked: report.dll0.locked, + fincon_valid: report.dll0.fincon_valid, + }, + dll1: Bzm2DllClockStatus { + control2: report.dll1.control2, + control5: report.dll1.control5, + coarsecon: report.dll1.coarsecon, + fincon: report.dll1.fincon, + freeze_valid: report.dll1.freeze_valid, + locked: report.dll1.locked, + fincon_valid: report.dll1.fincon_valid, + }, + } +} + +pub(super) fn snapshot_temperature(snapshot: &Bzm2TelemetrySnapshot, name: &str) -> Option { + snapshot + .temperatures + .iter() + .find(|sensor| sensor.name == name) + .and_then(|sensor| sensor.temperature.map(Temperature::as_degrees_c)) +} + +pub(super) fn snapshot_input_power(snapshot: &Bzm2TelemetrySnapshot) -> Option { + snapshot + .powers + .iter() + .find(|power| power.name == "input") + .and_then(|power| power.power_w) +} + +pub(super) fn sensor_specs_from_env( + paths_keys: &[&str], + scales_keys: &[&str], + default_scale: f32, +) -> Vec { + let paths = env_csv_strings_any(paths_keys); + let scales = parse_csv_numbers_any::(scales_keys).unwrap_or_default(); + paths + .into_iter() + .enumerate() + .map(|(index, path)| SensorSpec { + path, + scale: *scales + .get(index) + .or_else(|| scales.last()) + .unwrap_or(&default_scale), + }) + .collect() +} + +fn parse_scaled_sensor_value(raw: &str, scale: f32) -> Option { + let trimmed = raw.trim(); + if trimmed.is_empty() { + return None; + } + trimmed.parse::().ok().map(|value| value * scale) +} + +fn upsert_asic_state( + telemetry_tx: &watch::Sender, + thread_index: usize, + serial_path: &str, + asic_id: u8, + active_engine_count: u16, + missing_engines: Vec, +) { + telemetry_tx.send_modify(|state| { + if let Some(asic) = state + .asics + .iter_mut() + .find(|asic| asic.thread_index == Some(thread_index) && asic.id == asic_id) + { + asic.serial_path = Some(serial_path.to_owned()); + asic.discovered_engine_count = Some(active_engine_count); + asic.missing_engines = missing_engines.clone(); + } else { + state.asics.push(AsicState { + id: asic_id, + thread_index: Some(thread_index), + serial_path: Some(serial_path.to_owned()), + discovered_engine_count: Some(active_engine_count), + missing_engines: missing_engines.clone(), + }); + } + state + .asics + .sort_by_key(|asic| (asic.thread_index.unwrap_or(usize::MAX), asic.id)); + }); +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::api_client::types::ThreadTelemetry; + + #[test] + fn parse_scaled_sensor_value_applies_scale() { + let parsed = parse_scaled_sensor_value("42500\n", 0.001).unwrap(); + assert!((parsed - 42.5).abs() < 0.001); + assert_eq!(parse_scaled_sensor_value("", 0.001), None); + assert_eq!(parse_scaled_sensor_value("nope", 1.0), None); + } + + #[test] + fn telemetry_trip_detects_thresholds() { + let telemetry = Bzm2TelemetryConfig { + max_asic_temp_c: Some(80.0), + max_input_power_w: Some(1200.0), + ..Default::default() + }; + assert!( + telemetry + .trip_reason(Some(81.0), None, None) + .unwrap() + .contains("ASIC temperature") + ); + assert!(telemetry.trip_reason(None, None, Some(1250.0)).is_some()); + assert!( + telemetry + .trip_reason(Some(75.0), None, Some(1100.0)) + .is_none() + ); + } + + #[test] + fn publish_thread_telemetry_updates_board_state() { + let (telemetry_tx, telemetry_rx) = watch::channel(BoardTelemetry { + name: "bzm2-test".into(), + model: "BZM2".into(), + serial: Some("bzm2-test".into()), + temperatures: vec![TemperatureSensor { + name: "host-board-temp".into(), + temperature: Some(Temperature::from_celsius(52.0)), + }], + powers: vec![PowerMeasurement { + name: "host-input".into(), + voltage_v: Some(12.0), + current_a: Some(10.0), + power_w: Some(120.0), + }], + ..Default::default() + }); + + publish_thread_telemetry( + &telemetry_tx, + &HashThreadTelemetryUpdate { + temperatures: vec![crate::asic::hash_thread::HashThreadTemperatureReading { + name: "ttyUSB0-asic-2-dts".into(), + temperature_c: Some(64.5), + }], + powers: vec![crate::asic::hash_thread::HashThreadPowerReading { + name: "ttyUSB0-asic-2-vs-ch0".into(), + voltage_v: Some(0.78), + current_a: None, + power_w: None, + }], + }, + ); + + let state = telemetry_rx.borrow().clone(); + assert_eq!(state.temperatures.len(), 2); + assert!( + state + .temperatures + .iter() + .any(|sensor| sensor.name == "host-board-temp" + && sensor.temperature.map(Temperature::as_degrees_c) == Some(52.0)) + ); + assert!( + state + .temperatures + .iter() + .any(|sensor| sensor.name == "ttyUSB0-asic-2-dts" + && sensor.temperature.map(Temperature::as_degrees_c) == Some(64.5)) + ); + assert_eq!(state.powers.len(), 2); + assert!( + state + .powers + .iter() + .any(|sensor| sensor.name == "host-input" && sensor.voltage_v == Some(12.0)) + ); + assert!( + state + .powers + .iter() + .any(|sensor| sensor.name == "ttyUSB0-asic-2-vs-ch0" + && sensor.voltage_v == Some(0.78)) + ); + } + + #[test] + fn publish_thread_status_updates_state_slot() { + let (telemetry_tx, telemetry_rx) = watch::channel(BoardTelemetry { + name: "bzm2-test".into(), + model: "BZM2".into(), + serial: Some("bzm2-test".into()), + threads: vec![ThreadTelemetry { + name: "BZM2 UART 0".into(), + hashrate: 0, + is_active: false, + }], + ..Default::default() + }); + + let status = HashThreadStatus { + hashrate: crate::types::HashRate::from_terahashes(42.0), + is_active: true, + ..Default::default() + }; + + publish_thread_status(&telemetry_tx, 0, &status); + + let state = telemetry_rx.borrow().clone(); + assert_eq!( + state.threads[0].hashrate, + crate::types::HashRate::from_terahashes(42.0).0 + ); + assert!(state.threads[0].is_active); + } + + #[test] + fn publish_discovered_engine_map_updates_board_state() { + let (telemetry_tx, telemetry_rx) = watch::channel(BoardTelemetry { + name: "bzm2-test".into(), + model: "BZM2".into(), + serial: Some("bzm2-test".into()), + ..Default::default() + }); + + publish_discovered_engine_map( + &telemetry_tx, + 1, + "/dev/ttyUSB1", + &Bzm2DiscoveredEngineMap { + asic: 2, + present: vec![ + crate::asic::bzm2::Bzm2EngineCoordinate::new(0, 0), + crate::asic::bzm2::Bzm2EngineCoordinate::new(0, 1), + ], + missing: vec![ + crate::asic::bzm2::Bzm2EngineCoordinate::new(3, 7), + crate::asic::bzm2::Bzm2EngineCoordinate::new(5, 11), + ], + }, + ); + + let state = telemetry_rx.borrow().clone(); + assert_eq!(state.asics.len(), 1); + assert_eq!(state.asics[0].id, 2); + assert_eq!(state.asics[0].thread_index, Some(1)); + assert_eq!(state.asics[0].serial_path.as_deref(), Some("/dev/ttyUSB1")); + assert_eq!(state.asics[0].discovered_engine_count, Some(2)); + assert_eq!( + state.asics[0].missing_engines, + vec![ + EngineCoordinate { row: 3, col: 7 }, + EngineCoordinate { row: 5, col: 11 }, + ] + ); + } +} diff --git a/mujina-miner/src/board/bzm2/test_support.rs b/mujina-miner/src/board/bzm2/test_support.rs new file mode 100644 index 00000000..f14a17b2 --- /dev/null +++ b/mujina-miner/src/board/bzm2/test_support.rs @@ -0,0 +1,53 @@ +//! Shared PTY-backed BZM2 chain emulator for board tests. + +use std::fs; +use std::io::{Read, Write}; + +use crate::asic::bzm2::protocol::{OPCODE_UART_NOOP, encode_noop, encode_write_register}; + +pub(super) fn spawn_chain_emulator( + master: std::os::fd::OwnedFd, + chain_len: u8, + start_id: u8, +) -> std::thread::JoinHandle<()> { + std::thread::spawn(move || { + let mut file = fs::File::from(master); + for offset in 0..chain_len { + let mut noop_request = vec![0u8; encode_noop(crate::asic::bzm2::DEFAULT_ASIC_ID).len()]; + file.read_exact(&mut noop_request).unwrap(); + assert_eq!( + noop_request, + encode_noop(crate::asic::bzm2::DEFAULT_ASIC_ID) + ); + file.write_all(&[ + crate::asic::bzm2::DEFAULT_ASIC_ID, + OPCODE_UART_NOOP, + b'B', + b'Z', + b'2', + ]) + .unwrap(); + + let assigned = start_id.saturating_add(offset); + let expected_write = encode_write_register( + crate::asic::bzm2::DEFAULT_ASIC_ID, + crate::asic::bzm2::NOTCH_REG, + 0x0b, + &(assigned as u32).to_le_bytes(), + ); + let mut write_request = vec![0u8; expected_write.len()]; + file.read_exact(&mut write_request).unwrap(); + assert_eq!(write_request, expected_write); + + let mut assigned_noop = vec![0u8; encode_noop(assigned).len()]; + file.read_exact(&mut assigned_noop).unwrap(); + assert_eq!(assigned_noop, encode_noop(assigned)); + file.write_all(&[assigned, OPCODE_UART_NOOP, b'B', b'Z', b'2']) + .unwrap(); + } + + let mut final_probe = vec![0u8; encode_noop(crate::asic::bzm2::DEFAULT_ASIC_ID).len()]; + file.read_exact(&mut final_probe).unwrap(); + assert_eq!(final_probe, encode_noop(crate::asic::bzm2::DEFAULT_ASIC_ID)); + }) +} diff --git a/mujina-miner/src/board/cpu.rs b/mujina-miner/src/board/cpu.rs index 41e30a1f..850a9255 100644 --- a/mujina-miner/src/board/cpu.rs +++ b/mujina-miner/src/board/cpu.rs @@ -55,6 +55,7 @@ async fn create_cpu_board() -> Result { info, threads, telemetry_rx, + command_tx: None, shutdown: None, }) } diff --git a/mujina-miner/src/board/emberone00.rs b/mujina-miner/src/board/emberone00.rs index 60c8e56b..c17cb67c 100644 --- a/mujina-miner/src/board/emberone00.rs +++ b/mujina-miner/src/board/emberone00.rs @@ -171,6 +171,7 @@ async fn create_from_usb(device: UsbDeviceInfo) -> Result { info, threads: Vec::new(), telemetry_rx, + command_tx: None, shutdown: Some(shutdown), }) } diff --git a/mujina-miner/src/board/mod.rs b/mujina-miner/src/board/mod.rs index 3b9e161b..067f8a39 100644 --- a/mujina-miner/src/board/mod.rs +++ b/mujina-miner/src/board/mod.rs @@ -1,14 +1,17 @@ pub(crate) mod bitaxe; +pub(crate) mod bzm2; pub(crate) mod cpu; pub(crate) mod emberone00; pub mod pattern; +pub mod power; use anyhow::Result; use futures::future::BoxFuture; -use tokio::sync::watch; +use tokio::sync::{mpsc, watch}; use crate::{ - api_client::types::BoardTelemetry, asic::hash_thread::HashThread, transport::UsbDeviceInfo, + api::commands::BoardCommand, api_client::types::BoardTelemetry, asic::hash_thread::HashThread, + transport::UsbDeviceInfo, }; /// Returned by board factory functions with everything the backplane @@ -23,6 +26,10 @@ pub struct BackplaneConnector { /// Watch receiver for the board's telemetry stream. pub telemetry_rx: watch::Receiver, + /// Sender for board commands (diagnostics, fan control). `None` if + /// the board accepts no commands. + pub command_tx: Option>, + /// Shuts down the board when awaited. `None` if the board has /// no shutdown work to do. pub shutdown: Option>, diff --git a/mujina-miner/src/board/power.rs b/mujina-miner/src/board/power.rs new file mode 100644 index 00000000..7f5e50ba --- /dev/null +++ b/mujina-miner/src/board/power.rs @@ -0,0 +1,423 @@ +use std::time::Duration; + +use anyhow::Result; +use async_trait::async_trait; +use tokio::fs; +use tokio::time::sleep; + +use crate::{ + asic::hash_thread::{AsicEnable, VoltageRegulator}, + hw_trait::{ + gpio::{GpioPin, PinValue}, + i2c::I2c, + }, + peripheral::tps546::Tps546, +}; + +#[async_trait] +pub trait PowerRail: Send + Sync { + async fn initialize(&mut self) -> Result<()>; + async fn set_voltage(&mut self, volts: f32) -> Result<()>; + async fn telemetry(&mut self) -> Result; +} +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct PowerRailTelemetry { + pub vin_volts: f32, + pub vout_volts: f32, + pub current_amps: f32, + pub temperature_c: f32, + pub power_watts: f32, +} + +pub struct Tps546PowerRail { + inner: Tps546, +} + +impl Tps546PowerRail { + pub fn new(inner: Tps546) -> Self { + Self { inner } + } + + pub fn into_inner(self) -> Tps546 { + self.inner + } +} + +#[async_trait] +impl PowerRail for Tps546PowerRail { + async fn initialize(&mut self) -> Result<()> { + self.inner.init().await + } + + async fn set_voltage(&mut self, volts: f32) -> Result<()> { + self.inner.set_vout(volts).await + } + + async fn telemetry(&mut self) -> Result { + Ok(PowerRailTelemetry { + vin_volts: self.inner.get_vin().await? as f32 / 1000.0, + vout_volts: self.inner.get_vout().await? as f32 / 1000.0, + current_amps: self.inner.get_iout().await? as f32 / 1000.0, + temperature_c: self.inner.get_temperature().await? as f32, + power_watts: self.inner.get_power().await? as f32 / 1000.0, + }) + } +} + +#[async_trait] +impl VoltageRegulator for Tps546PowerRail { + async fn set_voltage(&mut self, volts: f32) -> Result<()> { + PowerRail::set_voltage(self, volts).await + } +} + +pub struct GpioResetLine { + pin: PIN, + active_low: bool, +} + +impl GpioResetLine { + pub fn new(pin: PIN, active_low: bool) -> Self { + Self { pin, active_low } + } + + pub fn into_inner(self) -> PIN { + self.pin + } + + pub async fn pulse(&mut self, assert_for: Duration, settle_for: Duration) -> Result<()> + where + PIN: GpioPin, + { + self.drive(true).await?; + sleep(assert_for).await; + self.drive(false).await?; + sleep(settle_for).await; + Ok(()) + } + async fn drive(&mut self, asserted: bool) -> Result<()> + where + PIN: GpioPin, + { + let value = if asserted == self.active_low { + PinValue::Low + } else { + PinValue::High + }; + self.pin.write(value).await?; + Ok(()) + } +} + +#[async_trait] +impl AsicEnable for GpioResetLine { + async fn enable(&mut self) -> Result<()> { + self.drive(false).await + } + + async fn disable(&mut self) -> Result<()> { + self.drive(true).await + } +} + +#[derive(Debug, Clone)] +pub struct FileGpioPin { + path: String, + high_value: String, + low_value: String, +} + +impl FileGpioPin { + pub fn new( + path: impl Into, + high_value: impl Into, + low_value: impl Into, + ) -> Self { + Self { + path: path.into(), + high_value: high_value.into(), + low_value: low_value.into(), + } + } +} + +#[async_trait] +impl GpioPin for FileGpioPin { + async fn set_mode( + &mut self, + _mode: crate::hw_trait::gpio::PinMode, + ) -> crate::hw_trait::Result<()> { + Ok(()) + } + + async fn write(&mut self, value: PinValue) -> crate::hw_trait::Result<()> { + let raw = match value { + PinValue::Low => &self.low_value, + PinValue::High => &self.high_value, + }; + fs::write(&self.path, raw).await?; + Ok(()) + } + + async fn read(&mut self) -> crate::hw_trait::Result { + let raw = fs::read_to_string(&self.path).await?; + if raw.trim() == self.high_value.trim() { + Ok(PinValue::High) + } else { + Ok(PinValue::Low) + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct VoltageStackStep { + pub rail_index: usize, + pub voltage: f32, + pub settle_for: Duration, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct VoltageStackBringupPlan { + pub assert_reset_before_power: bool, + pub pre_power_delay: Duration, + pub post_power_delay: Duration, + pub release_reset_delay: Duration, + pub steps: Vec, +} + +impl Default for VoltageStackBringupPlan { + fn default() -> Self { + Self { + assert_reset_before_power: true, + pre_power_delay: Duration::from_millis(10), + post_power_delay: Duration::from_millis(25), + release_reset_delay: Duration::from_millis(25), + steps: Vec::new(), + } + } +} + +impl VoltageStackBringupPlan { + pub async fn apply( + &self, + rails: &mut [R], + mut reset_line: Option<&mut GpioResetLine>, + ) -> Result<()> + where + R: PowerRail, + PIN: GpioPin, + { + if self.assert_reset_before_power { + if let Some(reset_line) = reset_line.as_deref_mut() { + reset_line.disable().await?; + } + sleep(self.pre_power_delay).await; + } + + for rail in rails.iter_mut() { + rail.initialize().await?; + } + + for step in &self.steps { + let rail = rails + .get_mut(step.rail_index) + .ok_or_else(|| anyhow::anyhow!("rail index {} out of range", step.rail_index))?; + rail.set_voltage(step.voltage).await?; + sleep(step.settle_for).await; + } + + sleep(self.post_power_delay).await; + + if let Some(reset_line) = reset_line { + reset_line.enable().await?; + sleep(self.release_reset_delay).await; + } + + Ok(()) + } + + pub async fn shutdown( + &self, + rails: &mut [R], + reset_line: Option<&mut GpioResetLine>, + ) -> Result<()> + where + R: PowerRail, + PIN: GpioPin, + { + if let Some(reset_line) = reset_line { + reset_line.disable().await?; + } + + for rail in rails.iter_mut().rev() { + rail.set_voltage(0.0).await?; + } + + Ok(()) + } +} + +#[derive(Debug, Clone)] +pub struct FilePowerRail { + set_path: String, + write_scale: f32, + enable_path: Option, + enable_value: Option, +} + +impl FilePowerRail { + pub fn new(path: impl Into, write_scale: f32) -> Self { + Self { + set_path: path.into(), + write_scale, + enable_path: None, + enable_value: None, + } + } + + pub fn with_enable( + mut self, + enable_path: impl Into, + enable_value: impl Into, + ) -> Self { + self.enable_path = Some(enable_path.into()); + self.enable_value = Some(enable_value.into()); + self + } + + fn encode_voltage(&self, volts: f32) -> String { + if (self.write_scale - 1.0).abs() < f32::EPSILON { + format!("{volts:.6}") + } else { + format!("{}", (volts * self.write_scale).round() as i64) + } + } +} + +#[async_trait] +impl PowerRail for FilePowerRail { + async fn initialize(&mut self) -> Result<()> { + if let (Some(path), Some(value)) = (&self.enable_path, &self.enable_value) { + fs::write(path, value).await?; + } + Ok(()) + } + + async fn set_voltage(&mut self, volts: f32) -> Result<()> { + fs::write(&self.set_path, self.encode_voltage(volts)).await?; + Ok(()) + } + + async fn telemetry(&mut self) -> Result { + Ok(PowerRailTelemetry { + vin_volts: 0.0, + vout_volts: 0.0, + current_amps: 0.0, + temperature_c: 0.0, + power_watts: 0.0, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::hw_trait::{Result as HwResult, gpio::PinMode}; + + #[derive(Default)] + struct MockPin { + writes: Vec, + } + + #[async_trait] + impl GpioPin for MockPin { + async fn set_mode(&mut self, _mode: PinMode) -> HwResult<()> { + Ok(()) + } + + async fn write(&mut self, value: PinValue) -> HwResult<()> { + self.writes.push(value); + Ok(()) + } + + async fn read(&mut self) -> HwResult { + Ok(self.writes.last().copied().unwrap_or(PinValue::Low)) + } + } + + #[derive(Default)] + struct MockRail { + initialized: bool, + voltages: Vec, + } + + #[async_trait] + impl PowerRail for MockRail { + async fn initialize(&mut self) -> Result<()> { + self.initialized = true; + Ok(()) + } + + async fn set_voltage(&mut self, volts: f32) -> Result<()> { + self.voltages.push(volts); + Ok(()) + } + + async fn telemetry(&mut self) -> Result { + Ok(PowerRailTelemetry { + vin_volts: 12.0, + vout_volts: self.voltages.last().copied().unwrap_or_default(), + current_amps: 1.0, + temperature_c: 42.0, + power_watts: 12.0, + }) + } + } + + #[tokio::test] + async fn bringup_plan_sequences_rails_then_releases_reset() { + let mut rails = vec![MockRail::default(), MockRail::default()]; + let mut reset = GpioResetLine::new(MockPin::default(), true); + let plan = VoltageStackBringupPlan { + pre_power_delay: Duration::from_millis(0), + post_power_delay: Duration::from_millis(0), + release_reset_delay: Duration::from_millis(0), + steps: vec![ + VoltageStackStep { + rail_index: 0, + voltage: 0.82, + settle_for: Duration::from_millis(0), + }, + VoltageStackStep { + rail_index: 1, + voltage: 0.79, + settle_for: Duration::from_millis(0), + }, + ], + ..Default::default() + }; + + plan.apply(&mut rails, Some(&mut reset)).await.unwrap(); + + assert!(rails[0].initialized); + assert!(rails[1].initialized); + assert_eq!(rails[0].voltages, vec![0.82]); + assert_eq!(rails[1].voltages, vec![0.79]); + let pin = reset.into_inner(); + assert_eq!(pin.writes, vec![PinValue::Low, PinValue::High]); + } + + #[tokio::test] + async fn shutdown_plan_asserts_reset_then_powers_off_rails() { + let mut rails = vec![MockRail::default(), MockRail::default()]; + let mut reset = GpioResetLine::new(MockPin::default(), true); + let plan = VoltageStackBringupPlan::default(); + + plan.shutdown(&mut rails, Some(&mut reset)).await.unwrap(); + + assert_eq!(rails[0].voltages, vec![0.0]); + assert_eq!(rails[1].voltages, vec![0.0]); + let pin = reset.into_inner(); + assert_eq!(pin.writes, vec![PinValue::Low]); + } +} diff --git a/mujina-miner/src/daemon.rs b/mujina-miner/src/daemon.rs index f14ad225..081f70bd 100644 --- a/mujina-miner/src/daemon.rs +++ b/mujina-miner/src/daemon.rs @@ -14,6 +14,7 @@ use crate::tracing::prelude::*; use crate::{ api::{self, ApiConfig, commands::SchedulerCommand}, backplane::Backplane, + board::bzm2::Bzm2RuntimeConfig, cpu_miner::CpuMinerConfig, job_source::{ SourceCommand, SourceEvent, @@ -94,6 +95,22 @@ impl Daemon { // Create and start backplane let mut backplane = Backplane::new(transport_rxs, thread_tx, board_reg_tx); + + // Attach a configured BZM2 board before the backplane starts draining + // transport events, so its threads register ahead of the + // initial-enumeration-complete signal and count toward the startup + // hold. + if let Some(config) = Bzm2RuntimeConfig::from_env() { + info!( + serials = config.serial_paths.len(), + baud = config.baud_rate, + "BZM2 board enabled from configured serial paths" + ); + backplane + .attach_configured_board("bzm2", config.device_id()) + .await?; + } + self.tracker.spawn({ let shutdown = self.shutdown.clone(); async move { diff --git a/mujina-miner/src/lib.rs b/mujina-miner/src/lib.rs index 18748bca..3520712e 100644 --- a/mujina-miner/src/lib.rs +++ b/mujina-miner/src/lib.rs @@ -17,5 +17,6 @@ pub mod stratum_v1; mod testing; pub mod tracing; pub mod transport; +pub mod tuning; pub mod types; mod u256; diff --git a/mujina-miner/src/scheduler.rs b/mujina-miner/src/scheduler.rs index e9f44bfa..6a9fa90f 100644 --- a/mujina-miner/src/scheduler.rs +++ b/mujina-miner/src/scheduler.rs @@ -634,6 +634,10 @@ impl Scheduler { ); } + HashThreadEvent::TelemetryUpdate(_) => { + trace!(thread = %thread_name, "Thread telemetry update"); + } + HashThreadEvent::ExpectedHashRate(rate) => { let Some(entry) = self.threads.get_mut(thread_id) else { return; diff --git a/mujina-miner/src/transport/serial.rs b/mujina-miner/src/transport/serial.rs index e04ad65a..44da0f6d 100644 --- a/mujina-miner/src/transport/serial.rs +++ b/mujina-miner/src/transport/serial.rs @@ -120,16 +120,19 @@ struct SerialInner { } /// Reader half of a split serial stream. +#[derive(Clone)] pub struct SerialReader { inner: Arc, } /// Writer half of a split serial stream. +#[derive(Clone)] pub struct SerialWriter { inner: Arc, } /// Control handle for a split serial stream. +#[derive(Clone)] pub struct SerialControl { inner: Arc, } diff --git a/mujina-miner/src/tuning/blockscale.rs b/mujina-miner/src/tuning/blockscale.rs new file mode 100644 index 00000000..06f9a3be --- /dev/null +++ b/mujina-miner/src/tuning/blockscale.rs @@ -0,0 +1,996 @@ +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +const CALI_VOLTAGE_MV: u32 = 50; +const CALI_FREQ_MHZ: f32 = 25.0; +const CALI_PASS_RATE_STEP: f32 = 0.0025; +const TARGET_VOLTAGE_MAX_MV: u32 = 21_000; +const TARGET_VOLTAGE_MIN_MV: u32 = 16_950; +const TARGET_FREQ_MAX_MHZ: f32 = 2_000.0; +const TARGET_FREQ_MIN_MHZ: f32 = 800.0; +const TARGET_FREQ_HIGH_PLUS_MHZ: f32 = 1_312.5; +const TARGET_FREQ_HIGH_MHZ: f32 = 1_200.0; +const TARGET_FREQ_BALANCED_MHZ: f32 = 1_150.0; +const TARGET_FREQ_LOW_MHZ: f32 = 1_000.0; +const FREQ_RANGE_MHZ: f32 = 100.0; +const MAX_FREQ_RANGE_MHZ: f32 = 150.0; +const ACCEPT_RATIO_BAND_MAX_THROUGHPUT: f32 = 0.02; +const ACCEPT_RATIO_BAND_STANDARD: f32 = 0.02; +const ACCEPT_RATIO_BAND_EFFICIENCY: f32 = 0.02; +const MIN_ACCEPT_RATIO: f32 = 0.90; +const DESIRED_ACCEPT_RATIO_MAX_THROUGHPUT: f32 = 0.975; +const DESIRED_ACCEPT_RATIO_STANDARD: f32 = 0.975; +const DESIRED_ACCEPT_RATIO_EFFICIENCY: f32 = 0.975; +const STARTUP_VOLTAGE_BIAS_MV: i32 = 50; +const SITE_TEMP_COLD_SOAK_C: f32 = -2.5; +const SITE_TEMP_COOL_C: f32 = 7.5; +const SITE_TEMP_NOMINAL_C: f32 = 17.5; +const SITE_TEMP_WARM_C: f32 = 27.5; +const DEFAULT_THERMAL_THRESHOLD_C: f32 = 100.0; +const DEFAULT_AVG_THERMAL_THRESHOLD_C: f32 = 85.0; +const DEFAULT_CURRENT_THRESHOLD_A: f32 = 260.0; +const DEFAULT_POWER_THRESHOLD_W: f32 = 4_900.0; +const DEFAULT_FREQ_INCREASE_RATIO_HIGH: f32 = 0.28; +const DEFAULT_FREQ_INCREASE_RATIO_LOW: f32 = 0.24; +const DEFAULT_RECALIBRATE_THROUGHPUT_RATIO: f32 = 0.80; +const NOMINAL_ACTIVE_ENGINE_COUNT: u16 = 236; + +#[derive(Debug, Default)] +pub struct Bzm2CalibrationPlanner; + +impl Bzm2CalibrationPlanner { + pub fn build_search_space( + &self, + request: &Bzm2CalibrationSweepRequest, + ) -> Vec { + let modes = if request.mode.sweep_strategy { + vec![ + Bzm2PerformanceMode::MaxThroughput, + Bzm2PerformanceMode::Standard, + Bzm2PerformanceMode::Efficiency, + ] + } else { + vec![request.target_mode] + }; + + let mut parameters = Vec::new(); + for mode in modes { + let target = operating_targets(request.operating_class, mode); + let voltage_offsets = build_offsets(request.mode.sweep_voltage, request.voltage_steps); + let frequency_offsets = + build_frequency_offsets(request.mode.sweep_frequency, request.frequency_steps); + let pass_rate_offsets = + build_pass_rate_offsets(request.mode.sweep_pass_rate, request.pass_rate_steps); + + for voltage_offset in &voltage_offsets { + for frequency_offset in &frequency_offsets { + for pass_rate_offset in &pass_rate_offsets { + parameters.push(Bzm2ParameterSet { + mode, + desired_voltage_mv: clamp_voltage(apply_i32( + target.voltage_mv, + *voltage_offset, + )), + desired_clock_mhz: clamp_frequency( + target.frequency_mhz + *frequency_offset, + ), + desired_accept_ratio: clamp_pass_rate( + target.pass_rate + *pass_rate_offset, + ), + }); + } + } + } + } + + parameters.sort_by(|a, b| { + a.mode + .cmp(&b.mode) + .then(a.desired_voltage_mv.cmp(&b.desired_voltage_mv)) + .then_with(|| a.desired_clock_mhz.total_cmp(&b.desired_clock_mhz)) + .then_with(|| a.desired_accept_ratio.total_cmp(&b.desired_accept_ratio)) + }); + parameters.dedup_by(|a, b| { + a.mode == b.mode + && a.desired_voltage_mv == b.desired_voltage_mv + && (a.desired_clock_mhz - b.desired_clock_mhz).abs() < f32::EPSILON + && (a.desired_accept_ratio - b.desired_accept_ratio).abs() < f32::EPSILON + }); + parameters + } + + pub fn plan(&self, input: &Bzm2BoardCalibrationInput) -> Bzm2CalibrationPlan { + let target = operating_targets(input.operating_class, input.target_mode); + let search_space = self.build_search_space(&Bzm2CalibrationSweepRequest { + operating_class: input.operating_class, + target_mode: input.target_mode, + mode: input.mode, + voltage_steps: 4, + frequency_steps: 4, + pass_rate_steps: 2, + }); + + let current_throughput = input + .asic_measurements + .iter() + .filter_map(|asic| asic.throughput_ths) + .sum::(); + let has_live_throughput = input + .asic_measurements + .iter() + .any(|asic| asic.throughput_ths.is_some()); + let live_active_engines = total_active_engine_count(&input.asics); + let reuse_saved_operating_point = + input.saved_operating_point.as_ref().is_some_and(|stored| { + let current_normalized_throughput = + normalize_throughput(current_throughput, live_active_engines); + let stored_normalized_throughput = normalize_throughput( + stored.board_throughput_ths, + stored_total_active_engine_count( + stored, + input.asics.iter().filter(|asic| asic.alive).count(), + ), + ); + !input.force_retune + && stored.per_asic_pll_mhz.len() + == input.asics.iter().filter(|asic| asic.alive).count() + && (!has_live_throughput + || current_normalized_throughput + >= stored_normalized_throughput + * input.constraints.recalibrate_throughput_ratio) + }); + let needs_retune = input.force_retune + || (has_live_throughput + && input.saved_operating_point.as_ref().is_some_and(|stored| { + normalize_throughput(current_throughput, live_active_engines) + < normalize_throughput( + stored.board_throughput_ths, + stored_total_active_engine_count( + stored, + input.asics.iter().filter(|asic| asic.alive).count(), + ), + ) * input.constraints.recalibrate_throughput_ratio + })); + + let (initial_voltage_mv, freq_increase_threshold_mhz) = initial_voltage_and_threshold( + target.voltage_mv, + input.site_temp_c, + input.mode.sweep_frequency, + ); + let initial_frequency_mhz = clamp_frequency( + (target.frequency_mhz - input.constraints.freq_range_mhz).max(TARGET_FREQ_MIN_MHZ), + ); + + let domain_measurements: BTreeMap = input + .domain_measurements + .iter() + .map(|measurement| (measurement.domain_id, measurement)) + .collect(); + let asic_measurements: BTreeMap = input + .asic_measurements + .iter() + .map(|measurement| (measurement.asic_id, measurement)) + .collect(); + + let mut domain_plans = Vec::new(); + let mut asic_plans = Vec::new(); + let mut notes = Vec::new(); + + for domain in &input.voltage_domains { + let domain_target_voltage = + clamp_voltage(apply_i32(initial_voltage_mv, domain.voltage_offset_mv)); + let domain_power = domain_measurements + .get(&domain.domain_id) + .and_then(|measurement| measurement.measured_power_w) + .unwrap_or_default(); + let domain_guarded = domain.max_power_w.is_some_and(|limit| domain_power > limit) + || domain_power > input.constraints.max_power_w; + + let domain_asics: Vec<&Bzm2AsicTopology> = input + .asics + .iter() + .filter(|asic| asic.alive && asic.domain_id == domain.domain_id) + .collect(); + let domain_avg_temp = average( + domain_asics + .iter() + .filter_map(|asic| asic_measurements.get(&asic.asic_id)) + .filter_map(|measurement| measurement.temperature_c), + ); + let domain_avg_pass_rate = average( + domain_asics + .iter() + .filter_map(|asic| asic_measurements.get(&asic.asic_id)) + .filter_map(|measurement| measurement.average_pass_rate), + ); + + let mut domain_frequency = initial_frequency_mhz; + let mut domain_notes = Vec::new(); + if let Some(pass_rate) = domain_avg_pass_rate { + if pass_rate >= target.pass_rate && !domain_guarded { + domain_frequency = clamp_frequency( + target + .frequency_mhz + .min(initial_frequency_mhz + input.constraints.max_freq_range_mhz), + ); + domain_notes.push(format!( + "domain average pass rate {:.2}% supports target frequency", + pass_rate * 100.0 + )); + } else { + domain_notes.push(format!( + "domain average pass rate {:.2}% below target {:.2}%", + pass_rate * 100.0, + target.pass_rate * 100.0 + )); + } + } + if let Some(temp) = domain_avg_temp + && temp >= input.constraints.max_avg_thermal_c + { + domain_frequency = clamp_frequency(domain_frequency - CALI_FREQ_MHZ); + domain_notes.push(format!( + "domain average temperature {:.1}C triggered thermal guard", + temp + )); + } + if domain_guarded { + domain_frequency = clamp_frequency(domain_frequency - CALI_FREQ_MHZ); + domain_notes.push("domain power guard active".into()); + } + + domain_plans.push(Bzm2DomainPlan { + domain_id: domain.domain_id, + voltage_mv: domain_target_voltage, + average_frequency_mhz: domain_frequency, + guarded: domain_guarded, + notes: domain_notes.clone(), + }); + + for asic in domain_asics { + let measurement = asic_measurements.get(&asic.asic_id).copied(); + let mut pll_frequencies = [domain_frequency; 2]; + let mut asic_notes = Vec::new(); + + if reuse_saved_operating_point { + if let Some(stored) = input + .saved_operating_point + .as_ref() + .and_then(|stored| stored.per_asic_pll_mhz.get(&asic.asic_id)) + { + pll_frequencies = *stored; + asic_notes.push("reusing stored per-ASIC calibration".into()); + } + } else if let Some(measurement) = measurement { + if let Some(temp) = measurement.temperature_c + && temp >= input.constraints.max_thermal_c + { + pll_frequencies = [clamp_frequency(domain_frequency - CALI_FREQ_MHZ); 2]; + asic_notes.push(format!( + "ASIC temperature {:.1}C exceeded thermal threshold", + temp + )); + } + + if input.per_stack_clocking { + for (pll_index, pass_rate) in measurement.pll_pass_rates.iter().enumerate() + { + if let Some(pass_rate) = pass_rate { + let low = target.pass_rate - input.target_mode.pass_rate_range(); + let high = target.pass_rate + input.target_mode.pass_rate_range(); + if *pass_rate < low { + pll_frequencies[pll_index] = + clamp_frequency(pll_frequencies[pll_index] - CALI_FREQ_MHZ); + asic_notes.push(format!( + "PLL {} pass rate {:.2}% below window", + pll_index, + pass_rate * 100.0 + )); + } else if *pass_rate > high && !domain_guarded { + pll_frequencies[pll_index] = clamp_frequency( + pll_frequencies[pll_index] + CALI_FREQ_MHZ / 2.0, + ); + asic_notes.push(format!( + "PLL {} pass rate {:.2}% above window", + pll_index, + pass_rate * 100.0 + )); + } + } + } + } else if let Some(pass_rate) = measurement.average_pass_rate + && pass_rate < target.pass_rate - input.target_mode.pass_rate_range() + { + pll_frequencies = [clamp_frequency(domain_frequency - CALI_FREQ_MHZ); 2]; + asic_notes.push(format!( + "ASIC pass rate {:.2}% below target window", + pass_rate * 100.0 + )); + } + } + + if domain_guarded { + asic_notes.push("bounded by domain power guard".into()); + } + if asic.active_engine_count < NOMINAL_ACTIVE_ENGINE_COUNT { + asic_notes.push(format!( + "ASIC has {} active engines and {} missing coordinates", + asic.active_engine_count, + asic.missing_engines.len() + )); + } + + asic_plans.push(Bzm2AsicPlan { + asic_id: asic.asic_id, + domain_id: asic.domain_id, + pll_frequencies_mhz: pll_frequencies, + notes: asic_notes, + }); + } + } + + if reuse_saved_operating_point { + notes.push("saved operating point is consistent with current throughput".into()); + } else if needs_retune { + notes.push( + "saved operating point is missing or underperforming; full retune required".into(), + ); + } else { + notes.push("building fresh domain-aware calibration plan".into()); + } + + if live_active_engines < total_nominal_engine_capacity(&input.asics) { + notes.push(format!( + "tuning normalized throughput against {:.1}% active engine capacity", + (live_active_engines as f32 / total_nominal_engine_capacity(&input.asics) as f32) + * 100.0 + )); + } + + if input.voltage_domains.len() > 1 { + notes.push("domain-first planning enabled for multi-domain hardware".into()); + } + if input.asics.len() >= 100 { + notes.push("planner uses one domain aggregation pass and one ASIC tuning pass".into()); + } + + Bzm2CalibrationPlan { + reuse_saved_operating_point, + needs_retune, + desired_voltage_mv: target.voltage_mv, + desired_clock_mhz: target.frequency_mhz, + desired_accept_ratio: target.pass_rate, + initial_voltage_mv, + initial_frequency_mhz, + freq_increase_threshold_mhz, + search_space, + domain_plans, + asic_plans, + notes, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum Bzm2PerformanceMode { + MaxThroughput, + Standard, + Efficiency, +} + +impl Bzm2PerformanceMode { + fn pass_rate_range(self) -> f32 { + match self { + Self::MaxThroughput => ACCEPT_RATIO_BAND_MAX_THROUGHPUT, + Self::Standard => ACCEPT_RATIO_BAND_STANDARD, + Self::Efficiency => ACCEPT_RATIO_BAND_EFFICIENCY, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Bzm2OperatingClass { + Generic, + EarlyValidation, + ProductionValidation, + StackTunedA, + StackTunedB, + ExtendedHeadroom, + ExtendedHeadroomB, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct Bzm2CalibrationMode { + pub sweep_strategy: bool, + pub sweep_voltage: bool, + pub sweep_frequency: bool, + pub sweep_pass_rate: bool, +} + +#[derive(Debug, Clone)] +pub struct Bzm2CalibrationConstraints { + pub max_power_w: f32, + pub max_current_a: f32, + pub max_thermal_c: f32, + pub max_avg_thermal_c: f32, + pub freq_range_mhz: f32, + pub max_freq_range_mhz: f32, + pub recalibrate_throughput_ratio: f32, +} + +impl Default for Bzm2CalibrationConstraints { + fn default() -> Self { + Self { + max_power_w: DEFAULT_POWER_THRESHOLD_W, + max_current_a: DEFAULT_CURRENT_THRESHOLD_A, + max_thermal_c: DEFAULT_THERMAL_THRESHOLD_C, + max_avg_thermal_c: DEFAULT_AVG_THERMAL_THRESHOLD_C, + freq_range_mhz: FREQ_RANGE_MHZ, + max_freq_range_mhz: MAX_FREQ_RANGE_MHZ, + recalibrate_throughput_ratio: DEFAULT_RECALIBRATE_THROUGHPUT_RATIO, + } + } +} + +#[derive(Debug, Clone)] +pub struct Bzm2CalibrationSweepRequest { + pub operating_class: Bzm2OperatingClass, + pub target_mode: Bzm2PerformanceMode, + pub mode: Bzm2CalibrationMode, + pub voltage_steps: u8, + pub frequency_steps: u8, + pub pass_rate_steps: u8, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct Bzm2SavedOperatingPoint { + pub board_voltage_mv: u32, + pub board_throughput_ths: f32, + #[serde(default)] + pub per_domain_voltage_mv: BTreeMap, + #[serde(default)] + pub per_asic_engine_topology: BTreeMap, + pub per_asic_pll_mhz: BTreeMap, +} + +#[derive(Debug, Clone, Default, Deserialize, Serialize, PartialEq, Eq)] +pub struct Bzm2SavedEngineCoordinate { + pub row: u8, + pub col: u8, +} + +#[derive(Debug, Clone, Default, Deserialize, Serialize, PartialEq, Eq)] +pub struct Bzm2SavedEngineTopology { + #[serde(default)] + pub active_engine_count: u16, + #[serde(default)] + pub missing_engines: Vec, +} + +#[derive(Debug, Clone)] +pub struct Bzm2AsicTopology { + pub asic_id: u16, + pub domain_id: u16, + pub pll_count: usize, + pub alive: bool, + pub active_engine_count: u16, + pub missing_engines: Vec, +} + +#[derive(Debug, Clone)] +pub struct Bzm2VoltageDomain { + pub domain_id: u16, + pub asic_ids: Vec, + pub voltage_offset_mv: i32, + pub max_power_w: Option, +} + +#[derive(Debug, Clone, Default)] +pub struct Bzm2DomainMeasurement { + pub domain_id: u16, + pub measured_voltage_mv: Option, + pub measured_power_w: Option, +} + +#[derive(Debug, Clone, Default)] +pub struct Bzm2AsicMeasurement { + pub asic_id: u16, + pub temperature_c: Option, + pub throughput_ths: Option, + pub average_pass_rate: Option, + pub pll_pass_rates: [Option; 2], +} + +#[derive(Debug, Clone)] +pub struct Bzm2BoardCalibrationInput { + pub operating_class: Bzm2OperatingClass, + pub site_temp_c: f32, + pub target_mode: Bzm2PerformanceMode, + pub mode: Bzm2CalibrationMode, + pub per_stack_clocking: bool, + pub voltage_domains: Vec, + pub asics: Vec, + pub saved_operating_point: Option, + pub domain_measurements: Vec, + pub asic_measurements: Vec, + pub constraints: Bzm2CalibrationConstraints, + pub force_retune: bool, +} + +#[derive(Debug, Clone)] +pub struct Bzm2DomainPlan { + pub domain_id: u16, + pub voltage_mv: u32, + pub average_frequency_mhz: f32, + pub guarded: bool, + pub notes: Vec, +} + +#[derive(Debug, Clone)] +pub struct Bzm2AsicPlan { + pub asic_id: u16, + pub domain_id: u16, + pub pll_frequencies_mhz: [f32; 2], + pub notes: Vec, +} + +#[derive(Debug, Clone)] +pub struct Bzm2CalibrationPlan { + pub reuse_saved_operating_point: bool, + pub needs_retune: bool, + pub desired_voltage_mv: u32, + pub desired_clock_mhz: f32, + pub desired_accept_ratio: f32, + pub initial_voltage_mv: u32, + pub initial_frequency_mhz: f32, + pub freq_increase_threshold_mhz: f32, + pub search_space: Vec, + pub domain_plans: Vec, + pub asic_plans: Vec, + pub notes: Vec, +} + +#[derive(Debug, Clone)] +pub struct Bzm2ParameterSet { + pub mode: Bzm2PerformanceMode, + pub desired_voltage_mv: u32, + pub desired_clock_mhz: f32, + pub desired_accept_ratio: f32, +} + +#[derive(Debug, Clone, Copy)] +struct OperatingTarget { + voltage_mv: u32, + frequency_mhz: f32, + pass_rate: f32, +} + +fn total_active_engine_count(asics: &[Bzm2AsicTopology]) -> u32 { + asics + .iter() + .filter(|asic| asic.alive) + .map(|asic| u32::from(asic.active_engine_count.max(1))) + .sum::() + .max(1) +} + +fn total_nominal_engine_capacity(asics: &[Bzm2AsicTopology]) -> u32 { + (asics.iter().filter(|asic| asic.alive).count() as u32 * u32::from(NOMINAL_ACTIVE_ENGINE_COUNT)) + .max(1) +} + +fn stored_total_active_engine_count(stored: &Bzm2SavedOperatingPoint, alive_asics: usize) -> u32 { + if stored.per_asic_engine_topology.is_empty() { + return (alive_asics as u32 * u32::from(NOMINAL_ACTIVE_ENGINE_COUNT)).max(1); + } + + stored + .per_asic_engine_topology + .values() + .map(|topology| u32::from(topology.active_engine_count.max(1))) + .sum::() + .max(1) +} + +fn normalize_throughput(throughput_ths: f32, active_engine_count: u32) -> f32 { + throughput_ths / active_engine_count.max(1) as f32 +} + +fn operating_targets( + operating_class: Bzm2OperatingClass, + mode: Bzm2PerformanceMode, +) -> OperatingTarget { + let (high_voltage, balanced_voltage, low_voltage, high_freq) = match operating_class { + Bzm2OperatingClass::Generic => (17_600, 17_500, 17_150, TARGET_FREQ_HIGH_MHZ), + Bzm2OperatingClass::EarlyValidation => (17_800, 17_700, 17_350, TARGET_FREQ_HIGH_MHZ), + Bzm2OperatingClass::ProductionValidation => (17_550, 17_450, 17_100, TARGET_FREQ_HIGH_MHZ), + Bzm2OperatingClass::StackTunedA => (17_300, 17_200, 16_850, TARGET_FREQ_HIGH_MHZ), + Bzm2OperatingClass::StackTunedB => (17_600, 17_500, 17_150, TARGET_FREQ_HIGH_MHZ), + Bzm2OperatingClass::ExtendedHeadroom => (17_900, 17_450, 17_100, TARGET_FREQ_HIGH_PLUS_MHZ), + Bzm2OperatingClass::ExtendedHeadroomB => { + (18_050, 17_550, 17_150, TARGET_FREQ_HIGH_PLUS_MHZ) + } + }; + + match mode { + Bzm2PerformanceMode::MaxThroughput => OperatingTarget { + voltage_mv: high_voltage, + frequency_mhz: high_freq, + pass_rate: DESIRED_ACCEPT_RATIO_MAX_THROUGHPUT, + }, + Bzm2PerformanceMode::Standard => OperatingTarget { + voltage_mv: balanced_voltage, + frequency_mhz: TARGET_FREQ_BALANCED_MHZ, + pass_rate: DESIRED_ACCEPT_RATIO_STANDARD, + }, + Bzm2PerformanceMode::Efficiency => OperatingTarget { + voltage_mv: low_voltage, + frequency_mhz: TARGET_FREQ_LOW_MHZ, + pass_rate: DESIRED_ACCEPT_RATIO_EFFICIENCY, + }, + } +} + +fn build_offsets(enabled: bool, steps: u8) -> Vec { + if !enabled { + return vec![0]; + } + + let steps = steps.min(20) as i32; + (-steps..=steps) + .map(|step| step * CALI_VOLTAGE_MV as i32) + .collect() +} + +fn build_frequency_offsets(enabled: bool, steps: u8) -> Vec { + if !enabled { + return vec![0.0]; + } + + let steps = steps.min(16) as i32; + (-steps..=steps) + .map(|step| step as f32 * CALI_FREQ_MHZ) + .collect() +} + +fn build_pass_rate_offsets(enabled: bool, steps: u8) -> Vec { + if !enabled { + return vec![0.0]; + } + + let steps = steps.min(4) as i32; + (-steps..=steps) + .map(|step| step as f32 * CALI_PASS_RATE_STEP) + .collect() +} + +fn initial_voltage_and_threshold( + desired_voltage_mv: u32, + site_temp_c: f32, + frequency_mode: bool, +) -> (u32, f32) { + let (offset, ratio) = if site_temp_c < SITE_TEMP_COLD_SOAK_C { + (STARTUP_VOLTAGE_BIAS_MV * 2, DEFAULT_FREQ_INCREASE_RATIO_LOW) + } else if site_temp_c < SITE_TEMP_COOL_C { + (STARTUP_VOLTAGE_BIAS_MV, DEFAULT_FREQ_INCREASE_RATIO_HIGH) + } else if site_temp_c < SITE_TEMP_NOMINAL_C { + (0, DEFAULT_FREQ_INCREASE_RATIO_HIGH) + } else if site_temp_c < SITE_TEMP_WARM_C { + (-STARTUP_VOLTAGE_BIAS_MV, DEFAULT_FREQ_INCREASE_RATIO_HIGH) + } else { + ( + -STARTUP_VOLTAGE_BIAS_MV * 2, + DEFAULT_FREQ_INCREASE_RATIO_LOW, + ) + }; + + let threshold = if frequency_mode { + CALI_FREQ_MHZ * DEFAULT_FREQ_INCREASE_RATIO_LOW + } else { + CALI_FREQ_MHZ * ratio + }; + + ( + clamp_voltage(apply_i32(desired_voltage_mv, offset)), + threshold, + ) +} + +fn clamp_voltage(voltage_mv: u32) -> u32 { + voltage_mv.clamp(TARGET_VOLTAGE_MIN_MV, TARGET_VOLTAGE_MAX_MV) +} + +fn clamp_frequency(frequency_mhz: f32) -> f32 { + frequency_mhz.clamp(TARGET_FREQ_MIN_MHZ, TARGET_FREQ_MAX_MHZ) +} + +fn clamp_pass_rate(pass_rate: f32) -> f32 { + pass_rate.clamp(MIN_ACCEPT_RATIO, DESIRED_ACCEPT_RATIO_MAX_THROUGHPUT) +} + +fn apply_i32(value: u32, offset: i32) -> u32 { + if offset >= 0 { + value.saturating_add(offset as u32) + } else { + value.saturating_sub(offset.unsigned_abs()) + } +} + +fn average(values: impl Iterator) -> Option { + let mut total = 0.0; + let mut count = 0usize; + for value in values { + total += value; + count += 1; + } + (count > 0).then_some(total / count as f32) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn search_space_expands_requested_axes() { + let planner = Bzm2CalibrationPlanner; + let parameters = planner.build_search_space(&Bzm2CalibrationSweepRequest { + operating_class: Bzm2OperatingClass::Generic, + target_mode: Bzm2PerformanceMode::Standard, + mode: Bzm2CalibrationMode { + sweep_strategy: true, + sweep_voltage: true, + sweep_frequency: true, + sweep_pass_rate: true, + }, + voltage_steps: 1, + frequency_steps: 1, + pass_rate_steps: 1, + }); + + assert!(parameters.len() > 20); + assert!( + parameters + .iter() + .any(|p| p.mode == Bzm2PerformanceMode::MaxThroughput) + ); + assert!(parameters.iter().any(|p| p.desired_voltage_mv < 17_500)); + assert!( + parameters + .iter() + .any(|p| p.desired_clock_mhz > TARGET_FREQ_BALANCED_MHZ) + ); + } + + #[test] + fn single_asic_plan_prefers_saved_operating_point_when_consistent() { + let planner = Bzm2CalibrationPlanner; + let mut stored = BTreeMap::new(); + stored.insert(0, [1_075.0, 1_075.0]); + let plan = planner.plan(&Bzm2BoardCalibrationInput { + operating_class: Bzm2OperatingClass::Generic, + site_temp_c: 15.0, + target_mode: Bzm2PerformanceMode::Standard, + mode: Bzm2CalibrationMode::default(), + per_stack_clocking: false, + voltage_domains: vec![Bzm2VoltageDomain { + domain_id: 0, + asic_ids: vec![0], + voltage_offset_mv: 0, + max_power_w: None, + }], + asics: vec![Bzm2AsicTopology { + asic_id: 0, + domain_id: 0, + pll_count: 2, + alive: true, + active_engine_count: NOMINAL_ACTIVE_ENGINE_COUNT, + missing_engines: Vec::new(), + }], + saved_operating_point: Some(Bzm2SavedOperatingPoint { + board_voltage_mv: 17_500, + board_throughput_ths: 42.0, + per_domain_voltage_mv: BTreeMap::new(), + per_asic_engine_topology: BTreeMap::new(), + per_asic_pll_mhz: stored, + }), + domain_measurements: vec![Bzm2DomainMeasurement { + domain_id: 0, + measured_voltage_mv: Some(17_480), + measured_power_w: Some(320.0), + }], + asic_measurements: vec![Bzm2AsicMeasurement { + asic_id: 0, + temperature_c: Some(72.0), + throughput_ths: Some(40.0), + average_pass_rate: Some(0.98), + pll_pass_rates: [Some(0.98), Some(0.98)], + }], + constraints: Bzm2CalibrationConstraints::default(), + force_retune: false, + }); + + assert!(plan.reuse_saved_operating_point); + assert!(!plan.needs_retune); + assert_eq!(plan.asic_plans[0].pll_frequencies_mhz, [1_075.0, 1_075.0]); + } + + #[test] + fn planner_requests_retune_when_throughput_drops() { + let planner = Bzm2CalibrationPlanner; + let mut stored = BTreeMap::new(); + stored.insert(0, [1_150.0, 1_150.0]); + let plan = planner.plan(&Bzm2BoardCalibrationInput { + operating_class: Bzm2OperatingClass::Generic, + site_temp_c: 20.0, + target_mode: Bzm2PerformanceMode::Standard, + mode: Bzm2CalibrationMode::default(), + per_stack_clocking: false, + voltage_domains: vec![Bzm2VoltageDomain { + domain_id: 0, + asic_ids: vec![0], + voltage_offset_mv: 0, + max_power_w: None, + }], + asics: vec![Bzm2AsicTopology { + asic_id: 0, + domain_id: 0, + pll_count: 2, + alive: true, + active_engine_count: NOMINAL_ACTIVE_ENGINE_COUNT, + missing_engines: Vec::new(), + }], + saved_operating_point: Some(Bzm2SavedOperatingPoint { + board_voltage_mv: 17_500, + board_throughput_ths: 50.0, + per_domain_voltage_mv: BTreeMap::new(), + per_asic_engine_topology: BTreeMap::new(), + per_asic_pll_mhz: stored, + }), + domain_measurements: vec![], + asic_measurements: vec![Bzm2AsicMeasurement { + asic_id: 0, + temperature_c: Some(74.0), + throughput_ths: Some(20.0), + average_pass_rate: Some(0.94), + pll_pass_rates: [Some(0.94), Some(0.94)], + }], + constraints: Bzm2CalibrationConstraints::default(), + force_retune: false, + }); + + assert!(!plan.reuse_saved_operating_point); + assert!(plan.needs_retune); + } + + #[test] + fn planner_normalizes_saved_throughput_by_active_engine_capacity() { + let planner = Bzm2CalibrationPlanner; + let mut stored = BTreeMap::new(); + stored.insert(0, [1_075.0, 1_075.0]); + let plan = planner.plan(&Bzm2BoardCalibrationInput { + operating_class: Bzm2OperatingClass::Generic, + site_temp_c: 15.0, + target_mode: Bzm2PerformanceMode::Standard, + mode: Bzm2CalibrationMode::default(), + per_stack_clocking: false, + voltage_domains: vec![Bzm2VoltageDomain { + domain_id: 0, + asic_ids: vec![0], + voltage_offset_mv: 0, + max_power_w: None, + }], + asics: vec![Bzm2AsicTopology { + asic_id: 0, + domain_id: 0, + pll_count: 2, + alive: true, + active_engine_count: NOMINAL_ACTIVE_ENGINE_COUNT / 2, + missing_engines: vec![Bzm2SavedEngineCoordinate { row: 0, col: 1 }], + }], + saved_operating_point: Some(Bzm2SavedOperatingPoint { + board_voltage_mv: 17_500, + board_throughput_ths: 42.0, + per_domain_voltage_mv: BTreeMap::new(), + per_asic_engine_topology: BTreeMap::from([( + 0, + Bzm2SavedEngineTopology { + active_engine_count: NOMINAL_ACTIVE_ENGINE_COUNT, + missing_engines: Vec::new(), + }, + )]), + per_asic_pll_mhz: stored, + }), + domain_measurements: vec![], + asic_measurements: vec![Bzm2AsicMeasurement { + asic_id: 0, + temperature_c: Some(70.0), + throughput_ths: Some(21.0), + average_pass_rate: Some(0.98), + pll_pass_rates: [Some(0.98), Some(0.98)], + }], + constraints: Bzm2CalibrationConstraints::default(), + force_retune: false, + }); + + assert!(plan.reuse_saved_operating_point); + assert!(!plan.needs_retune); + assert!( + plan.notes + .iter() + .any(|note| note.contains("active engine capacity")) + ); + } + + #[test] + fn multi_domain_plan_scales_to_large_topology() { + let planner = Bzm2CalibrationPlanner; + let domains: Vec = (0..25) + .map(|domain_id| Bzm2VoltageDomain { + domain_id, + asic_ids: (0..4).map(|offset| domain_id * 4 + offset).collect(), + voltage_offset_mv: if domain_id % 2 == 0 { 0 } else { 25 }, + max_power_w: Some(450.0), + }) + .collect(); + let asics: Vec = (0..100) + .map(|asic_id| Bzm2AsicTopology { + asic_id, + domain_id: asic_id / 4, + pll_count: 2, + alive: true, + active_engine_count: NOMINAL_ACTIVE_ENGINE_COUNT, + missing_engines: Vec::new(), + }) + .collect(); + let domain_measurements: Vec = (0..25) + .map(|domain_id| Bzm2DomainMeasurement { + domain_id, + measured_voltage_mv: Some(17_450), + measured_power_w: Some(if domain_id == 3 { 500.0 } else { 300.0 }), + }) + .collect(); + let asic_measurements: Vec = (0..100) + .map(|asic_id| Bzm2AsicMeasurement { + asic_id, + temperature_c: Some(if asic_id == 13 { 101.0 } else { 74.0 }), + throughput_ths: Some(0.4), + average_pass_rate: Some(if asic_id % 9 == 0 { 0.93 } else { 0.98 }), + pll_pass_rates: [Some(0.97), Some(0.98)], + }) + .collect(); + + let plan = planner.plan(&Bzm2BoardCalibrationInput { + operating_class: Bzm2OperatingClass::ExtendedHeadroom, + site_temp_c: 10.0, + target_mode: Bzm2PerformanceMode::Standard, + mode: Bzm2CalibrationMode::default(), + per_stack_clocking: true, + voltage_domains: domains, + asics, + saved_operating_point: None, + domain_measurements, + asic_measurements, + constraints: Bzm2CalibrationConstraints::default(), + force_retune: false, + }); + + assert_eq!(plan.domain_plans.len(), 25); + assert_eq!(plan.asic_plans.len(), 100); + assert!( + plan.domain_plans + .iter() + .find(|domain| domain.domain_id == 3) + .unwrap() + .guarded + ); + assert!( + plan.asic_plans + .iter() + .find(|asic| asic.asic_id == 13) + .unwrap() + .pll_frequencies_mhz[0] + < plan.initial_frequency_mhz + 1.0 + ); + assert!(plan.notes.iter().any(|note| note.contains("domain-first"))); + } +} diff --git a/mujina-miner/src/tuning/mod.rs b/mujina-miner/src/tuning/mod.rs new file mode 100644 index 00000000..d26352aa --- /dev/null +++ b/mujina-miner/src/tuning/mod.rs @@ -0,0 +1 @@ +pub mod blockscale;