Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,10 @@ NTFY_TOPIC=
# Binance Host (default: binance.com, use binance.us for US)
# BINANCE_WS_HOST=stream.binance.com
# BINANCE_API_HOST=api.binance.com
# BINANCE_BNB_VALUATION_SYMBOL=BNBUSDT
# BINANCE_FEE_LOOKUP_MAX_CHECKS=20
# BINANCE_SLIPPAGE_WARN_PCT=0.01
# BINANCE_SLIPPAGE_WARN_TTL_SEC=3600

# Alpha Vantage (optional, for Python sentiment lab)
# ALPHA_VANTAGE_API_KEY=
Expand Down
6 changes: 6 additions & 0 deletions apps/neko-trade/Sources/DCTradingViewer/Models/Models.swift
Original file line number Diff line number Diff line change
Expand Up @@ -207,6 +207,8 @@ struct BotStatus {
let markSymbol: String
let checkpointHealth: String
let checkpointError: String
let exchangeHealth: String
let exchangeError: String
let resourceHealth: String
let resourceError: String
let resourceRssMb: Double
Expand Down Expand Up @@ -234,6 +236,10 @@ struct BotStatus {
!checkpointHealth.isEmpty && checkpointHealth != "OK"
}

var exchangeNeedsAttention: Bool {
!exchangeHealth.isEmpty && exchangeHealth != "OK"
}

var resourceNeedsAttention: Bool {
!resourceHealth.isEmpty && resourceHealth != "OK"
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -280,6 +280,8 @@ final class TursoClient {
markSymbol: getOptionalString(row, result.cols, "mark_symbol") ?? "",
checkpointHealth: getOptionalString(row, result.cols, "checkpoint_health") ?? "OK",
checkpointError: getOptionalString(row, result.cols, "checkpoint_error") ?? "",
exchangeHealth: getOptionalString(row, result.cols, "exchange_health") ?? "OK",
exchangeError: getOptionalString(row, result.cols, "exchange_error") ?? "",
resourceHealth: getOptionalString(row, result.cols, "resource_health") ?? "OK",
resourceError: getOptionalString(row, result.cols, "resource_error") ?? "",
resourceRssMb: getDouble(row, result.cols, "resource_rss_mb"),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,7 @@ struct DashboardView: View {
}

private func botNeedsAttention(_ bs: BotStatus) -> Bool {
!bs.isLive || bs.checkpointNeedsAttention || bs.resourceNeedsAttention
!bs.isLive || bs.checkpointNeedsAttention || bs.exchangeNeedsAttention || bs.resourceNeedsAttention
}

private func botWarningButton(_ bs: BotStatus) -> some View {
Expand Down Expand Up @@ -198,6 +198,9 @@ struct DashboardView: View {
if bs.checkpointNeedsAttention {
issues.append("checkpoint \(bs.checkpointHealth)")
}
if bs.exchangeNeedsAttention {
issues.append("exchange \(bs.exchangeHealth)")
}
if bs.resourceNeedsAttention {
issues.append("resource \(bs.resourceHealth)")
}
Expand Down
31 changes: 31 additions & 0 deletions apps/neko-trade/Sources/DCTradingViewer/Views/SettingsView.swift
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,8 @@ struct SettingsView: View {
.padding(.vertical, 10)
}

exchangeStatusRow(bs)

cardDivider(statusColor)

HStack {
Expand Down Expand Up @@ -303,6 +305,35 @@ struct SettingsView: View {
.padding(.vertical, 10)
}

private func exchangeStatusRow(_ bs: BotStatus) -> some View {
let exchangeColor: Color = bs.exchangeNeedsAttention ? .orange : .green
let health = bs.exchangeHealth.isEmpty ? "OK" : bs.exchangeHealth
let detail = bs.exchangeError.isEmpty ? "OK" : bs.exchangeError

return VStack(alignment: .leading, spacing: 8) {
Divider()
.overlay(exchangeColor.opacity(0.25))

HStack(alignment: .top, spacing: 8) {
Image(systemName: bs.exchangeNeedsAttention ? "exclamationmark.triangle.fill" : "checkmark.seal.fill")
.font(.system(.body, design: .monospaced, weight: .semibold))
.foregroundStyle(exchangeColor)
Text("EXCHANGE \(health)")
.font(.system(.body, design: .monospaced, weight: .semibold))
.foregroundStyle(exchangeColor)
Spacer()
}
.padding(.horizontal)

Text(detail)
.font(.system(.caption, design: .monospaced))
.foregroundStyle(.secondary)
.lineLimit(2)
.padding(.horizontal)
}
.padding(.vertical, 10)
}

private func resourceMetric(_ title: String, _ value: String) -> some View {
VStack(alignment: .leading, spacing: 2) {
Text(title.uppercased())
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,18 @@ final class AccountingTests: XCTestCase {
quoteAsset: quoteAsset,
markSymbol: markSymbol,
checkpointHealth: "OK",
checkpointError: ""
checkpointError: "",
exchangeHealth: "OK",
exchangeError: "",
resourceHealth: "OK",
resourceError: "",
resourceRssMb: 0,
resourceDiskFreeMb: 0,
resourceDiskUsedPct: 0,
resourceFeedGapSec: 0,
resourceWsLagSec: 0,
resourceHttpErrors: 0,
resourceHttpMaxMs: 0
)
}
}
27 changes: 18 additions & 9 deletions services/dctrading-bot/AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ BTC algorithmic trading bot using Directional Change (DC) theory. Zig 0.16 produ
- `live_loop.zig` — Extracted core order flow logic: pending order tracking, trailing stop, strategy signals, buy/sell submission, capital_reserved, and ledger vtable. Shared by `runLive()` and integration tests.
- `tick_source.zig` — `TickSource` vtable interface + `SimFeed` (replays ticks from array). For testing.
- `sim_exchange.zig` — `SimExchange` implementing Exchange vtable with configurable fill delay, slippage, partial fills, cancel races, failure injection, order log. For testing.
- `integration_tests.zig` — 32 end-to-end scenarios using LiveLoop + SimExchange + mock ledger.
- `tests.zig` — 182 tests covering DC detector, strategy, checkpoint, regime transitions, JSON parsing, capital accounting, double-entry transfers, exchange interface, funding rate filter, non-blocking order flow, capital_reserved, resource monitoring, integration scenarios.
- `integration_tests.zig` — 34 end-to-end scenarios using LiveLoop + SimExchange + mock ledger.
- `tests.zig` — 195 tests covering DC detector, strategy, checkpoint, regime transitions, JSON parsing, capital accounting, double-entry transfers, exchange interface, funding rate filter, non-blocking order flow, capital_reserved, resource monitoring, Binance Spot parsing, integration scenarios.
- CLI `checkpoint:migrate [path] [backups]` migrates checkpoint primary/backups offline to the current DCTRADE5 layout after writing `.pre-migrate` copies.

### Scripts (`scripts/`)
Expand All @@ -31,21 +31,30 @@ BTC algorithmic trading bot using Directional Change (DC) theory. Zig 0.16 produ
- `setup-aws-iam.sh` — Standalone IAM instance profile creation for CloudWatch. Creates the role, attaches `CloudWatchAgentServerPolicy`, creates the instance profile, and waits for propagation.
- `switch-to-gcp.sh` — Stop local bot, start GCP Tokyo instance + systemd service. Copies binary plus checkpoint primary/local backups, excluding temp files.
- `switch-to-local.sh` — Stop cloud bot + instance, download checkpoint primary/local backups, start local bot in tmux. Defaults to AWS; use `CLOUD_TARGET=gcp` for the legacy GCP path.
- `nuke.sh` — Destructive reset for Turso state, local + remote checkpoint primary/backups, and Alpaca positions. Uses `CLOUD_TARGET` to stop and clear remote state.
- `nuke.sh` — Destructive reset for Turso state, local + remote checkpoint primary/backups, and exchange positions. Uses `CLOUD_TARGET` to stop and clear remote state.

### Key Patterns
- **HTTP calls**: All modules use shared `HttpClient` (native `std.http.Client`). Exception: `feed.zig` bootstrap uses `popen("curl")` for Binance REST, and `telegram.zig` shutdown uses curl fallback.
- **Async writes**: Turso and Telegram fire-and-forget via `std.Thread.spawn` + `detach()`. Context struct heap-allocated, freed in worker.
- **Sync reads**: Startup queries (capital, position, trade count) are blocking HTTP calls.
- **Exchange selection**: Runtime-selectable via `EXCHANGE` env var. `alpaca` (paper trading) and `binance_spot` (live Spot trading) are supported. The exchange vtable pattern keeps `live_loop.zig` and `strategy.zig` venue-agnostic.
- **Spot position model**: On Binance Spot, account balance is NOT position. The bot tracks its own position via Turso `transfers` (immutable fill history). `getPosition()` replays buy/sell transfers chronologically to compute `qty` and `entry_price`. `/api/v3/account` is used only for discrepancy alerts.
- **Checkpoint**: Binary file with magic number validation. DCTRADE5 writes 27 f64 scalars, including `funding_avg`, `funding_avg_updated_at`, and `funding_latest_time`, plus two ring buffers (vol, MA). DCTRADE4 and earlier DCTRADE5 funding-cache files are still accepted and migrated on the next save. Saved every minute through an atomic temp-file swap. Live mode keeps rotated local backups (`dctrading.checkpoint.bak.N`) and falls back to the newest valid backup if the primary checkpoint cannot be loaded.
- **Checkpoint startup guard**: Live mode prints cwd/checkpoint path and refuses to bootstrap if any local checkpoint primary/backup exists but none can be loaded and Turso restore fails. Use `checkpoint:migrate` or manually promote a known-good backup instead.
- **Regime**: `enum { bull, sideways, bear }`. Encoded as 0/1/2 in checkpoint scalar[8].

### Environment Variables
| Variable | Required | Used By |
|----------|----------|---------|
| `ALPACA_API_KEY` | Yes | alpaca.zig |
| `ALPACA_API_SECRET` | Yes | alpaca.zig |
| `EXCHANGE` | No | main.zig (default: `alpaca`; options: `alpaca`, `binance_spot`) |
| `ALPACA_API_KEY` | Yes (if EXCHANGE=alpaca) | alpaca.zig |
| `ALPACA_API_SECRET` | Yes (if EXCHANGE=alpaca) | alpaca.zig |
| `BINANCE_API_KEY` | Yes (if EXCHANGE=binance_spot) | binance_spot.zig |
| `BINANCE_API_SECRET` | Yes (if EXCHANGE=binance_spot) | binance_spot.zig |
| `BINANCE_BNB_VALUATION_SYMBOL` | No | binance_spot.zig (default: BNBUSDT; override for testnet/alternate quote) |
| `BINANCE_FEE_LOOKUP_MAX_CHECKS` | No | binance_spot.zig (default: 20 checks before fallback/manual reconciliation log) |
| `BINANCE_SLIPPAGE_WARN_PCT` | No | live_loop.zig (default: 0.01 = 1% post-fill warning) |
| `BINANCE_SLIPPAGE_WARN_TTL_SEC` | No | live_loop.zig/main.zig (default: 3600s; app-visible slippage warning retention) |
| `TRADING_SYMBOL` | No | main.zig/alpaca.zig/feed.zig (default: BTC/USD; Binance maps USD quote to USDT) |
| `TURSO_URL` | No | turso.zig |
| `TURSO_TOKEN` | No | turso.zig |
Expand All @@ -64,8 +73,8 @@ BTC algorithmic trading bot using Directional Change (DC) theory. Zig 0.16 produ
| `RESOURCE_FEED_GAP_WARN_SEC` | No | resource_monitor.zig (default: 180) |
| `RESOURCE_WS_LAG_WARN_SEC` | No | resource_monitor.zig (default: 180) |
| `RESOURCE_HTTP_LATENCY_WARN_MS` | No | resource_monitor.zig (default: 5000) |
| `BINANCE_WS_HOST` | No | feed.zig (default: stream.binance.com) |
| `BINANCE_API_HOST` | No | feed.zig (default: api.binance.com) |
| `BINANCE_WS_HOST` | No | feed.zig / binance_spot.zig (default: stream.binance.com) |
| `BINANCE_API_HOST` | No | feed.zig / binance_spot.zig (default: api.binance.com; testnet: testnet.binance.vision) |
| `BOT_INSTANCE` | No | main.zig (default: "local") |
| `CHECKPOINT_BACKUP_RETENTION` | No | main.zig (default: 5 local rotated backups; 0 disables) |
| `CHECKPOINT_REMOTE_BACKUP_INTERVAL` | No | main.zig/Turso (default: 3600 seconds; 0 disables) |
Expand All @@ -90,14 +99,14 @@ BTC algorithmic trading bot using Directional Change (DC) theory. Zig 0.16 produ
- `transfers` — Immutable append-only transfer log. Two-phase (pending/posted/voided). Codes: 1=deposit, 2=buy, 3=sell, 4=fee, 5=pnl. Atomic BEGIN/COMMIT pipelines.
- Fee routing: adapter-provided `commission_asset` routes fees to the paying asset account (`USD`/`USDT` → cash, `BTC` → btc_position, `BNB` → bnb), even when commission is zero. Transfer `amount` is historical quote-currency value at fill time; native fee quantity is stored in transfer `size`, with the fill-time asset quote valuation rate in `price`. `strategy.size` remains whatever the exchange adapter reports as fill quantity. `fee_pct` is for backtest/simulation estimates and legacy fills with no commission metadata, not for Alpaca paper fills.
- `equity_log` — Periodic snapshots (every 5 min + on trades): capital, equity, unrealized, regime, price.
- `bot_status` — Single row (id=1): regime, position, equity, version (DCTRADE5@instance), active symbol metadata, checkpoint health/error, and latest resource health/metrics for Neko.
- `bot_status` — Single row (id=1): regime, position, equity, version (DCTRADE5@instance), active symbol metadata, checkpoint health/error, exchange health/error, and latest resource health/metrics for Neko.
- `resource_log` — Periodic process/feed/HTTP resource snapshots used for dashboard status and diagnostics. Resource degradation is app-visible only; Telegram/ntfy stays reserved for trading, checkpoint, funding, startup/shutdown events.
- `checkpoint_backups` — Single remote checkpoint snapshot (id=1): base64-encoded DCTRADE5 checkpoint, byte length, checksum, tick count, and update time. Used only if local primary/backups cannot be loaded.
### Build
```bash
zig build -Doptimize=ReleaseFast # macOS arm64
zig build -Doptimize=ReleaseFast -Dtarget=x86_64-linux # cloud Linux
zig build test # 182 tests
zig build test # 195 tests
./zig-out/bin/dctrading checkpoint:migrate dctrading.checkpoint 5
```

Expand Down
55 changes: 49 additions & 6 deletions services/dctrading-bot/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ Parameters: λ=0.07, 60-day MA, 3% buffer, 2% trailing stop (72h vol lookback)
## Architecture

```
Binance WebSocket → Zig Bot → Alpaca Paper Trading
Binance WebSocket → Zig Bot → Alpaca Paper Trading / Binance Spot
Turso DB ← Neko Trade App (SwiftUI)
Expand All @@ -38,6 +38,7 @@ Single static binary. No Python, no Docker, no runtime dependencies (except curl
| `dc_detector.zig` | Streaming DC event detector |
| `exchange.zig` | Exchange vtable interface for sync and async order flow |
| `alpaca.zig` | Alpaca paper trading (sync/async orders, position queries) |
| `binance_spot.zig` | Binance Spot live trading (HMAC-signed orders, fills, balances) |
| `live_loop.zig` | Shared live order-flow engine and ledger interface used by production and simulation tests |
| `sim_exchange.zig` | Configurable simulated exchange for integration tests |
| `tick_source.zig` | Tick source interface and simulated feed |
Expand All @@ -47,26 +48,36 @@ Single static binary. No Python, no Docker, no runtime dependencies (except curl
| `http_client.zig` | Shared HTTP client (std.http.Client wrapper + request metrics) |
| `resource_monitor.zig` | Process, disk, feed, and HTTP resource health snapshots |
| `types.zig` | Tick, Trade, DC event types |
| `tests.zig` | 182 tests |
| `tests.zig` | 195 tests |

## Setup

### Prerequisites
- [Zig 0.16](https://ziglang.org/download/)
- Alpaca paper trading account (free): https://app.alpaca.markets
- Exchange account (one of):
- **Alpaca** paper trading (free): https://app.alpaca.markets
- **Binance** Spot (live): https://www.binance.com — API key + secret required
- Turso database (free tier): https://turso.tech
- Telegram bot (optional): @BotFather
- ntfy.sh (optional): https://ntfy.sh

### Environment Variables

```bash
# Required
# Exchange selection (default: alpaca)
export EXCHANGE=alpaca # or: binance_spot

# Alpaca (required if EXCHANGE=alpaca)
export ALPACA_API_KEY=PK...
export ALPACA_API_SECRET=...

# Binance Spot (required if EXCHANGE=binance_spot)
export BINANCE_API_KEY=...
export BINANCE_API_SECRET=...
# Required API permissions: read + spot trading. Do not enable withdrawals.

# Trading pair (optional; default: BTC/USD)
# Alpaca uses BTC/USD; Binance feed/funding maps this to BTCUSDT
# Alpaca uses BTC/USD; Binance Spot uses BTCUSDT automatically
export TRADING_SYMBOL=BTC/USD

# Turso (optional but recommended)
Expand All @@ -79,8 +90,14 @@ export TELEGRAM_CHAT_ID=...
export NTFY_TOPIC=your-topic

# Binance host (default: stream.binance.com / api.binance.com)
# For testnet: testnet.binance.vision (both WS and API must match).
# BNB valuation may need a different symbol on testnet.
export BINANCE_WS_HOST=stream.binance.com
export BINANCE_API_HOST=api.binance.com
export BINANCE_BNB_VALUATION_SYMBOL=BNBUSDT
export BINANCE_FEE_LOOKUP_MAX_CHECKS=20
export BINANCE_SLIPPAGE_WARN_PCT=0.01
export BINANCE_SLIPPAGE_WARN_TTL_SEC=3600

# Funding filter (default: 0.0001 = 0.010%; 0 disables)
# Funding updates are cached, checkpointed, checked hourly, and sent to Telegram/ntfy when Binance publishes a new funding print.
Expand Down Expand Up @@ -137,6 +154,30 @@ zig build -Doptimize=ReleaseFast -Dtarget=x86_64-linux
Known-good historical simulation outputs are recorded in
[`docs/DCTRADING_SIMULATION_BASELINES.md`](../../docs/DCTRADING_SIMULATION_BASELINES.md).

### Binance Spot Accounting Edge Cases

The live Spot adapter records exact exchange quote notional when Binance returns
`cummulativeQuoteQty`, and fee transfers are routed to the asset that paid the
commission. The following edge cases are intentionally documented because they
require operator awareness or follow-up work:

- **Partially filled then canceled orders:** Binance can return a terminal
`CANCELED` order with non-zero `executedQty`. The current order-status model
only has `filled` or `cancelled`, so this path should be reconciled manually
until partial-cancel settlement is implemented.
- **Mixed commission assets:** If an order's fills are charged in different
assets, the adapter does not collapse them into one fee transfer. It leaves
the order pending until the configured lookup limit, then marks exchange
health as `EXCHANGE_RECONCILE` for manual accounting.
- **Fee lookup lag:** `/api/v3/myTrades` can lag `/api/v3/order`. The bot keeps
the fill pending while fee metadata is missing. After
`BINANCE_FEE_LOOKUP_MAX_CHECKS`, it posts using the configured fee fallback
and surfaces `EXCHANGE_RECONCILE` in Neko.
- **Slippage warnings:** Fill prices are compared to the strategy signal price.
Warnings persist in `bot_status.exchange_health` for
`BINANCE_SLIPPAGE_WARN_TTL_SEC` seconds so the dashboard does not miss a
one-tick warning.

### AWS Tokyo Deployment

```bash
Expand Down Expand Up @@ -196,7 +237,7 @@ CLOUD_TARGET=gcp ./scripts/switch-to-local.sh
### Nuke

```bash
# Nuke everything — stops remote bot, drops Turso tables, deletes checkpoints, closes Alpaca positions
# Nuke everything — stops remote bot, drops Turso tables, deletes checkpoints, closes exchange positions
CLOUD_TARGET=aws ./scripts/nuke.sh

# Or skip remote cleanup and only nuke local state + Turso + Alpaca
Expand Down Expand Up @@ -242,6 +283,8 @@ Binary checkpoint (DCTRADE5) saves full strategy state every minute:
- If any local checkpoint file exists but none can be loaded and Turso restore also fails, live mode refuses to bootstrap so it cannot overwrite possibly recoverable checkpoint state
- Checkpoint problems are surfaced in `bot_status.checkpoint_health` and
`bot_status.checkpoint_error`; Telegram/ntfy sends a warning when health first enters a degraded state
- Binance exchange problems that need app review are surfaced in
`bot_status.exchange_health` and `bot_status.exchange_error`

## Resource Monitoring

Expand Down
3 changes: 2 additions & 1 deletion services/dctrading-bot/src/alpaca.zig
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,8 @@ pub const Alpaca = struct {

/// Submit order without waiting for fill. Returns PendingOrder with order ID.
/// Typically completes in ~200ms (just the POST, no polling).
pub fn submitOrderAsync(self: *const Alpaca, side: Side, qty: f64) ?PendingOrder {
pub fn submitOrderAsync(self: *const Alpaca, side: Side, qty: f64, signal_price: f64) ?PendingOrder {
_ = signal_price;
var body_buf: [256]u8 = undefined;
const side_str = if (side == .buy) "buy" else "sell";
const body = std.fmt.bufPrint(&body_buf,
Expand Down
Loading