bryceweiner commited on
Commit
ad0694f
·
verified ·
1 Parent(s): 6570567

release 1 (7ab2355a0c)

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .env.example +40 -5
  2. .gitattributes +1 -0
  3. CHANGELOG.md +131 -0
  4. Dockerfile +40 -12
  5. README.md +96 -33
  6. deploy/bootstrap-host.sh +433 -38
  7. deploy/check-host.sh +106 -0
  8. deploy/custody.env.example +26 -0
  9. deploy/docker-compose.vault.yml +55 -11
  10. deploy/fly +9 -0
  11. deploy/fly-aide.service +8 -0
  12. deploy/fly-aide.timer +9 -0
  13. deploy/fly-audit-watch.service +8 -0
  14. deploy/fly-audit-watch.timer +10 -0
  15. deploy/fly_audit_watch.py +109 -0
  16. deploy/fly_update.py +151 -51
  17. deploy/signer.env.example +4 -0
  18. deploy/update.json.example +6 -1
  19. deploy/vault.env.example +22 -12
  20. docker-compose.yml +7 -0
  21. docker/bootstrap.sh +42 -8
  22. docker/signer-requirements.txt +51 -0
  23. docs/vault/RUNBOOK.md +424 -109
  24. docs/vault/SPEC.md +233 -57
  25. fly_trader/__main__.py +159 -1
  26. fly_trader/agent/fly_live.py +16 -8
  27. fly_trader/agent/fly_session.py +280 -75
  28. fly_trader/agent/handover.py +12 -4
  29. fly_trader/agent/minute_engine.py +102 -45
  30. fly_trader/agent/paper_trading.py +8 -5
  31. fly_trader/agent/rails.py +65 -43
  32. fly_trader/agent/selector_session.py +45 -25
  33. fly_trader/agent/sizing.py +29 -20
  34. fly_trader/brain/connectome.py +27 -0
  35. fly_trader/brain/geometry.py +42 -3
  36. fly_trader/brain/pathways.py +4 -3
  37. fly_trader/brain/plastic.py +7 -1
  38. fly_trader/chain/cluster_guard.py +15 -0
  39. fly_trader/chain/jupiter_swap.py +1 -1
  40. fly_trader/chain/rpc.py +6 -0
  41. fly_trader/chain/withdraw.py +129 -0
  42. fly_trader/config.py +186 -8
  43. fly_trader/db/schema.py +198 -36
  44. fly_trader/execution/broker_live.py +46 -41
  45. fly_trader/execution/broker_paper.py +5 -3
  46. fly_trader/execution/ledger.py +14 -7
  47. fly_trader/kalshi/__init__.py +1 -0
  48. fly_trader/kalshi/client.py +342 -0
  49. fly_trader/kalshi/data.py +164 -0
  50. fly_trader/kalshi/decisions.py +149 -0
.env.example CHANGED
@@ -2,10 +2,14 @@
2
  # Keep values bare: Docker Compose keeps a trailing "# comment" as part of an empty value, so comments live on their own lines.
3
 
4
  # --- keys (never logged) ---
5
- # Helius: https://dashboard.helius.dev (RPC for balances and transactions)
6
  HELIUS_API_KEY=
7
- # Jupiter: https://portal.jup.ag (swaps and token stats)
8
  JUPITER_API_KEY=
 
 
 
 
9
 
10
  # --- database (docker-compose.yml sets the container's URL; this one is for running outside Docker) ---
11
  DATABASE_URL=postgresql:///fly_trader
@@ -15,7 +19,7 @@ DATABASE_URL=postgresql:///fly_trader
15
  LIVE_ENABLED=1
16
  # What you fund the wallet with; also the paper warm-up bankroll.
17
  CAPITAL_SOL=0.5
18
- # Hard cap per position.
19
  MAX_POSITION_SOL=0.05
20
  # Per position, of the bankroll minus the gas reserve.
21
  MAX_POSITION_FRACTION=0.10
@@ -26,6 +30,13 @@ MIN_POSITION_SOL=0.02
26
  # Never spent (fees and token-account rent).
27
  GAS_RESERVE_SOL=0.05
28
  KELLY_FRACTION=0.25
 
 
 
 
 
 
 
29
 
30
  # --- rails ---
31
  # 20 % below the wallet's peak: stop entering ...
@@ -41,18 +52,42 @@ HANDOVER_BEAT_SELECTOR=0
41
  # A restart keeps the paper book, so the warm-up count survives it.
42
  RESET_ON_START=0
43
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
44
  # --- runtime ---
45
  SOLANA_CLUSTER=mainnet-beta
46
  # auto: an NVIDIA GPU when the container has one (docker-compose.cuda.yml), else the CPU. Or cpu | cuda | cuda:1.
47
  DEVICE=auto
48
  # Batch sizes are derived from the device's free memory; on a shared GPU cap what the fly may use, in GB.
49
  # DEVICE_MEMORY_GB=
 
50
  DATA_DIR=data
51
  LOG_DIR=logs
52
 
53
  # --- staying current ---
54
- # Every UPDATE_INTERVAL_S the container asks Hugging Face whether HF_REPO has a new release; if so it downloads the new
55
- # code and model, restarts itself (about a minute; open positions are untouched) and keeps the previous release for a
 
56
  # rollback if the new one fails to start. AUTO_UPDATE=0 pins this install to what it was built from.
57
  AUTO_UPDATE=1
58
  HF_REPO=bryceweiner/fly-trader
 
2
  # Keep values bare: Docker Compose keeps a trailing "# comment" as part of an empty value, so comments live on their own lines.
3
 
4
  # --- keys (never logged) ---
5
+ # Helius: https://dashboard.helius.dev (RPC for balances and transactions; also the signer's, unless SIGNER_RPC_URL is set)
6
  HELIUS_API_KEY=
7
+ # Jupiter: https://portal.jup.ag (swaps, through Jupiter's own Metis routes only, and token stats)
8
  JUPITER_API_KEY=
9
+ # The bot key: `docker compose exec fly fly-trader wallet new` appends BOT_PRIVATE_KEY=... to this file. Do not add an
10
+ # empty BOT_PRIVATE_KEY= line yourself: wallet new refuses when the name is already here.
11
+ # Optional: a second Solana RPC from another provider for the signer, which simulates every swap before signing it.
12
+ # SIGNER_RPC_URL=
13
 
14
  # --- database (docker-compose.yml sets the container's URL; this one is for running outside Docker) ---
15
  DATABASE_URL=postgresql:///fly_trader
 
19
  LIVE_ENABLED=1
20
  # What you fund the wallet with; also the paper warm-up bankroll.
21
  CAPITAL_SOL=0.5
22
+ # The fixed size for a model without a sizing table (the shipped fly has one); must be above 0 to trade live.
23
  MAX_POSITION_SOL=0.05
24
  # Per position, of the bankroll minus the gas reserve.
25
  MAX_POSITION_FRACTION=0.10
 
30
  # Never spent (fees and token-account rent).
31
  GAS_RESERVE_SOL=0.05
32
  KELLY_FRACTION=0.25
33
+ # The signer's own limits (defaults shown): SOL per buy, SOL of buys per day, swaps per minute (buys and sells), priority
34
+ # fee per swap, and the least value a swap may return for what it costs, at the signer's own Jupiter prices.
35
+ # SIGNER_MAX_BUY_SOL=1
36
+ # SIGNER_DAILY_BUY_SOL=10
37
+ # SIGNER_SWAPS_PER_MIN=6
38
+ # SIGNER_MAX_PRIORITY_SOL=0.005
39
+ # SIGNER_MIN_VALUE_RATIO=0.6
40
 
41
  # --- rails ---
42
  # 20 % below the wallet's peak: stop entering ...
 
52
  # A restart keeps the paper book, so the warm-up count survives it.
53
  RESET_ON_START=0
54
 
55
+ # --- Robinhood Chain memecoins (Pons launches graduated into Uniswap v4; money in ETH) ---
56
+ # Off by default. 1 = index Robinhood Chain (the rh_stream worker) and paper-trade it in ETH beside Solana. Indexing needs
57
+ # an Envio HyperSync token (ENVIO_API_TOKEN below; the free tier works, slower) -- the public RPC is far too slow for
58
+ # the backfill -- and disk for the history: ~30 GB from 2026-08-04. RH_START_DAY a ~40 days back indexes much less
59
+ # (the RH models need about a month of history per wallet); the RH books trade once the stream reaches the present.
60
+ RH_ENABLED=0
61
+ # ENVIO_API_TOKEN=
62
+ # RH_START_DAY=2026-08-04
63
+ # Real RH transactions stay off: the RH books trade paper until you set 1 and add the key and address below.
64
+ RH_LIVE_ENABLED=0
65
+ # 1 = the rehearsal chain (46630); the guard refuses any other chain id than the one selected here.
66
+ RH_TESTNET=0
67
+ # RPCs (defaults shown): logs from HyperSync (the official RPC when HyperSync fails), transactions from the official RPC,
68
+ # block headers from PublicNode. A QuickNode Robinhood Chain endpoint, if you have one, serves the transactions.
69
+ # RH_RPC_URL=https://rpc.mainnet.chain.robinhood.com
70
+ # RH_RPC_URL_BLOCKS=https://robinhood-rpc.publicnode.com
71
+ # QUICKNODE_RH_API_URL=
72
+ # The address RH_BOT_PRIVATE_KEY must derive to (a guard, not a secret). Needed only for live.
73
+ RH_BOT_ADDRESS=
74
+ # RH_BOT_PRIVATE_KEY (hex) goes in this .env only, on its own line, when you enable live. It is read from here at run
75
+ # time and never baked into the image. Do not leave an empty RH_BOT_PRIVATE_KEY= line.
76
+
77
  # --- runtime ---
78
  SOLANA_CLUSTER=mainnet-beta
79
  # auto: an NVIDIA GPU when the container has one (docker-compose.cuda.yml), else the CPU. Or cpu | cuda | cuda:1.
80
  DEVICE=auto
81
  # Batch sizes are derived from the device's free memory; on a shared GPU cap what the fly may use, in GB.
82
  # DEVICE_MEMORY_GB=
83
+ # docker-compose.yml sets these to /app/data and /app/logs, your ./data and ./logs
84
  DATA_DIR=data
85
  LOG_DIR=logs
86
 
87
  # --- staying current ---
88
+ # Every UPDATE_INTERVAL_S the container asks Hugging Face whether HF_REPO has a new release. It installs one only if it is
89
+ # signed by a key in release/allowed_signers and newer than the installed one, and every file matches the signed
90
+ # manifest; then it restarts itself (about a minute; open positions are untouched) and keeps the previous release for a
91
  # rollback if the new one fails to start. AUTO_UPDATE=0 pins this install to what it was built from.
92
  AUTO_UPDATE=1
93
  HF_REPO=bryceweiner/fly-trader
.gitattributes CHANGED
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ models/selectors/selector_20261003T201413Z.skops filter=lfs diff=lfs merge=lfs -text
CHANGELOG.md ADDED
@@ -0,0 +1,131 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Changelog
2
+
3
+ Releases are identified by date. The Docker distribution (branch `distribution`, Hugging Face `bryceweiner/fly-trader`)
4
+ ships signed releases built from this development line.
5
+
6
+ ## 2026-10-11: Robinhood Chain, and a fly that starts as its selector
7
+
8
+ Not yet published to Hugging Face. Ships fly #157 (network) with selector #154, both trained on Solana and Robinhood
9
+ Chain rows together.
10
+
11
+ ### Upgrading
12
+
13
+ - **Docker installs from before October 2026 must rebuild once:** `git pull && docker compose up -d --build`. The old
14
+ image cannot verify signed releases, so it stops updating after this release. Data, the fly's learned state and
15
+ open positions are kept.
16
+ - **Robinhood Chain is off by default** in the distribution. To turn it on, set `RH_ENABLED=1` and `ENVIO_API_TOKEN` (an
17
+ Envio HyperSync token; the free tier works) in `.env`.
18
+ - Indexing the full history from 2026-08-04 takes about 30 GB.
19
+ - Setting `RH_START_DAY` about 40 days back is enough to trade.
20
+ - **New settings:**
21
+ - `FLY_SCORE` (default `teacher`), `FLY_TEACHER_GATE` (default on), `FLY_LEARNED_VETO` (default 0.02) and `FLY_SIZING`
22
+ (default `kelly`).
23
+ - Robinhood Chain: `ENVIO_API_TOKEN`, `RH_*`, and optionally `QUICKNODE_RH_API_URL` for transactions.
24
+
25
+ ### The fly
26
+
27
+ - **It scores as its selector plus what it has learned** (`FLY_SCORE=teacher`).
28
+ - On the first day its score equals the selector's exactly, verified live to 3e-8 on every row.
29
+ - Its learnable mushroom-body pathway (the plastic KC→MBON synapses) adds a correction, learned from realized returns,
30
+ that starts at zero.
31
+ - It therefore learns only where the selector is wrong.
32
+ - The earlier fly was a distilled copy of the selector, accurate overall but not at the buy line: it ranked the
33
+ selector's top 1 % of candidates only about 0.6-correlated.
34
+ - **It buys only what the selector buys** (`FLY_TEACHER_GATE`).
35
+ - The selector's trigger, buy line, win filter, dump veto and fail-closed rules decide every row.
36
+ - The fly's learning can drop a selector trade (a learned veto: its score more than 0.02 below its starting score) but
37
+ can never add one.
38
+ - Ungated, the fly had bought rows the selector rejected; those rows averaged −25.9 % per trade.
39
+ - **It sizes each buy exactly as the selector sizes it** (`FLY_SIZING=kelly`): the selector's score, buy line and Kelly
40
+ table for that row.
41
+ - **It learns from the selector's trades,** live and in the replay, after a restart too. It no longer learns from rows
42
+ above its own buy line.
43
+ - **It never trades without its teacher.** A fly whose selector cannot be loaded enters nothing. A release install now
44
+ points the fly at the shipped selector's new database id. Before that fix, the Docker boot test found a fresh install
45
+ running the fly ungated.
46
+ - **Replay verdict** (2026-09-24 to 10-09, the fly against the selector itself):
47
+ - Solana +3.78 % per trade over 308 trades.
48
+ - Robinhood Chain +2.66 % over 1,477 trades (selector +2.63 %).
49
+ - Random picks lost 4.4–4.6 %.
50
+ - **Smaller changes:**
51
+ - The fly calibrates on days its teacher never saw.
52
+ - A NaN mean can no longer pass the deploy gate.
53
+ - A second chain's teacher gets 28 days of its own data before the replay starts.
54
+ - `fly-replay --chains` re-judges one chain and keeps the others' verdicts.
55
+
56
+ ### Robinhood Chain (new)
57
+
58
+ - **Pons memecoins traded beside pump.fun.** It trades tokens launched on Pons V2 after they graduate into Uniswap v4 pools
59
+ with the Pons hook.
60
+ - Each chain has its own books in its own unit (Robinhood Chain in ETH), with its own bankroll, kill switch, handover
61
+ and verdict.
62
+ - One selector and one fly cover both chains. A chain trades only if its own replay verdict passed.
63
+ - **Its own indexer.** It reads launches, graduations, swaps, the base assets' prices and 1-minute rows.
64
+ - Logs come from Envio HyperSync, in one query per pass.
65
+ - Minutes land about 15–25 s after they close (they took 35–95 s at first).
66
+ - If HyperSync fails, the official RPC carries the feed until it recovers.
67
+ - RH minutes up to 240 s old are still traded (`RH_TRADE_LAG_S`).
68
+ - **Live trading** (`RH_LIVE_ENABLED=0` by default):
69
+ - Swaps route through KyberSwap, with a Uniswap v4 fallback.
70
+ - One transaction is in flight at a time, booked from measured balances.
71
+ - It recovers after a crash and is reconciled against the chain.
72
+ - It has its own circuit breaker (circuit 3).
73
+ - `fly-trader rh-smoke` runs a tiny mainnet round trip.
74
+ - A QuickNode endpoint (`QUICKNODE_RH_API_URL`), if set, carries transactions. Its free plan caps log queries at 5
75
+ blocks, so logs stay on HyperSync.
76
+ - **Costs:** the Pons hook fee and creator tax (0.99 % per side at the median), price impact and gas.
77
+ - **Console:**
78
+ - A chain picker switches the Overview and Trades pages.
79
+ - A Robinhood Chain page shows the indexer, quote-asset prices and the ETH wallet.
80
+ - Safety & wallet shows both chains' bot wallets in USD and withdraws only to your own wallets.
81
+
82
+ ### $FLY vault
83
+
84
+ - Robinhood Chain profit is paid to lockers in ETH, beside the SOL pot.
85
+ - **Custody:** a Squads treasury with spending limits and a key-isolated signer container. Claims are paid from a
86
+ separate payout wallet, and code releases are delayed.
87
+ - **Nothing published helps trade ahead of the fly.** The trading wallet's address, open positions and settlement
88
+ timing are not shown.
89
+ - **Audits:** red-team tests, mutation testing, Echidna and Halmos proofs.
90
+ - Fixed: a claim double-pay on resume.
91
+ - Added: Kyber calldata and fee checks, and relay griefing limits.
92
+
93
+ ### Docker distribution
94
+
95
+ - Runs the development line's code again, merged instead of hand-ported (it had fallen about 35 commits behind).
96
+ - **Signed releases only.**
97
+ - The container installs a release only if its manifest is signed by a key in `release/allowed_signers`, its number
98
+ is higher than the installed one, and every file matches its hash.
99
+ - Releases carry no `.git*` file and no model they do not name.
100
+ - Startup removes model files an earlier release left behind.
101
+ - **Models are accepted whether or not Robinhood Chain is on.** Before release, an install with RH off would have
102
+ rejected the combined models and crash-looped at boot.
103
+ - The Robinhood Chain reference table ships in the seed.
104
+ - Kalshi ships dormant: no worker starts, and its pages appear only once `KALSHI_API_KEY_ID` is set.
105
+
106
+ ### Kalshi prediction markets (development line)
107
+
108
+ - Under development, not traded in the distribution. Covers:
109
+ - an outcome-blind event corpus
110
+ - a selector judged by a weekly sign test
111
+ - maker and taker arms
112
+ - a visual fly
113
+
114
+ ### Fixes
115
+
116
+ - **Schema changes no longer deadlock live workers.** An unchanged schema is skipped, and a changed one waits at most
117
+ 5 s for a lock, then retries.
118
+ - **The RH paper books had made no trades.** A day's wallet-skill table did not exist yet when its minutes were scored.
119
+ Tables are now written ahead of their day, and the stream falls back to the newest one.
120
+ - **HyperSync could stall for 76 s.** A dead server address hung new connections; connections now time out in 5 s and
121
+ are kept alive.
122
+ - **The RH price scan ran about 270 queries per pass.** It now reads every price cursor at once.
123
+
124
+ ## 2026-09-24: Fly #107 taught by selector #100
125
+
126
+ - The selector was retrained on real fees, and the fly was bootstrapped from it.
127
+ - The container keeps itself current from Hugging Face.
128
+
129
+ ## 2026-09-22: First distribution
130
+
131
+ - Fire-and-forget trading in Docker, with the fly, its teacher selector and their connectome.
Dockerfile CHANGED
@@ -1,28 +1,56 @@
1
  # fly-trader: the trading engine and its console, on the CPU (a Linux container has no Metal GPU; the fly infers fine on CPU).
2
- FROM python:3.13-slim
 
 
 
 
 
 
3
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4
  ENV PYTHONUNBUFFERED=1 \
5
  UV_PROJECT_ENVIRONMENT=/app/.venv \
 
 
6
  PATH="/app/.venv/bin:${PATH}"
7
 
8
- RUN apt-get update && apt-get install -y --no-install-recommends postgresql-client ca-certificates curl age \
9
- && rm -rf /var/lib/apt/lists/*
10
- COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv
 
 
11
 
12
  WORKDIR /app
13
  COPY . .
14
  # Which torch: TORCH_INDEX=cpu (default: any machine, ~200 MB) or cu126 / cu130 for an NVIDIA GPU (the CUDA runtime is
15
  # inside the wheel; the host needs only the driver and the NVIDIA Container Toolkit -- see docker-compose.cuda.yml).
16
- # Installed first so the project's own pin is already satisfied and PyPI's 3 GB CUDA default is never pulled by accident.
17
- # Editable install: the package IS /app/fly_trader, so config.REPO_ROOT is /app -- the mounted .env the wallet writes
18
- # to, the console's static directory, config/. A plain install ran a copy under site-packages and looked there instead.
19
  ARG TORCH_INDEX=cpu
20
- RUN uv venv /app/.venv \
21
- && uv pip install --index-url "https://download.pytorch.org/whl/${TORCH_INDEX}" "torch>=2.12" \
22
- && uv pip install -e . \
23
- && chmod +x /app/docker/entrypoint.sh
 
 
 
24
 
25
  EXPOSE 8501
26
- # healthy = the database answers and the console's worker threads run (the vault server's updater waits on this)
27
  HEALTHCHECK --interval=30s --timeout=20s --start-period=300s --retries=5 CMD fly-trader health || exit 1
28
  ENTRYPOINT ["/app/docker/entrypoint.sh"]
 
1
  # fly-trader: the trading engine and its console, on the CPU (a Linux container has no Metal GPU; the fly infers fine on CPU).
2
+ # Two images from one file:
3
+ # default (last stage, "brain"): the engine, console and vault worker. It holds NO wallet key.
4
+ # --target signer: the only holder of the hot keys (fly_trader/signer): standard library + solders + httpx, nothing
5
+ # else of fly_trader, so none of the brain's dependencies ever run next to a key.
6
+ # Base images are pinned by digest and every Python dependency by hash (uv.lock, docker/signer-requirements.txt): a
7
+ # release builds exactly what was reviewed, whatever the registries serve later.
8
+ ARG PYTHON_IMAGE=python:3.13-slim@sha256:7c61056e61ac89e852de05f3dc6fa51a6dd2181797bceed46aa725dd7cb2cd3b
9
 
10
+ # ---------------------------------------------------------------- the signer
11
+ FROM ${PYTHON_IMAGE} AS signer
12
+ ENV PYTHONUNBUFFERED=1 PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=/app
13
+ RUN apt-get update && apt-get install -y --no-install-recommends age && rm -rf /var/lib/apt/lists/* \
14
+ && groupadd -g 10000 fly && useradd -u 10002 -g 10000 -M -s /usr/sbin/nologin signer
15
+ COPY docker/signer-requirements.txt /tmp/req.txt
16
+ RUN pip install --no-cache-dir --require-hashes --only-binary=:all: -r /tmp/req.txt && rm /tmp/req.txt
17
+ WORKDIR /app
18
+ COPY fly_trader/__init__.py fly_trader/__init__.py
19
+ COPY fly_trader/signer fly_trader/signer
20
+ USER 10002:10000
21
+ HEALTHCHECK --interval=30s --timeout=10s --start-period=20s --retries=3 CMD python -m fly_trader.signer ping || exit 1
22
+ CMD ["python", "-m", "fly_trader.signer", "serve"]
23
+
24
+ # ---------------------------------------------------------------- the brain (default)
25
+ FROM ${PYTHON_IMAGE} AS brain
26
  ENV PYTHONUNBUFFERED=1 \
27
  UV_PROJECT_ENVIRONMENT=/app/.venv \
28
+ UV_COMPILE_BYTECODE=1 \
29
+ HOME=/tmp \
30
  PATH="/app/.venv/bin:${PATH}"
31
 
32
+ # openssh-client: ssh-keygen verifies signed releases (the self-updater, fly_trader/ops/autoupdate.py)
33
+ RUN apt-get update && apt-get install -y --no-install-recommends postgresql-client ca-certificates curl age openssh-client \
34
+ && rm -rf /var/lib/apt/lists/* \
35
+ && groupadd -g 10000 fly && useradd -u 10001 -g 10000 -M -s /usr/sbin/nologin fly
36
+ COPY --from=ghcr.io/astral-sh/uv:0.12.19@sha256:04d046b13e60d6bcec73cbc5e1cad25d680dea90c8573340950a0ac2d1aef424 /uv /usr/local/bin/uv
37
 
38
  WORKDIR /app
39
  COPY . .
40
  # Which torch: TORCH_INDEX=cpu (default: any machine, ~200 MB) or cu126 / cu130 for an NVIDIA GPU (the CUDA runtime is
41
  # inside the wheel; the host needs only the driver and the NVIDIA Container Toolkit -- see docker-compose.cuda.yml).
42
+ # It names the project extra that pins torch to that index. --frozen: exactly uv.lock with its hashes, never a fresh
43
+ # resolution. The project installs editable: the package IS /app/fly_trader, so config.REPO_ROOT is /app.
 
44
  ARG TORCH_INDEX=cpu
45
+ ENV TORCH_INDEX=${TORCH_INDEX}
46
+ RUN uv sync --frozen --no-dev --extra "${TORCH_INDEX}" \
47
+ && chmod +x /app/docker/entrypoint.sh \
48
+ && mkdir -p /app/fly_trader/ui/static/brain && chown -R 10001:10000 /app/fly_trader/ui/static/brain
49
+ # The vault server runs this unprivileged user and never updates itself (AUTO_UPDATE=0). The self-hosted compose file
50
+ # runs the container as root instead: its self-updater replaces /app, and it writes the bind-mounted data/, logs/, .env.
51
+ USER 10001:10000
52
 
53
  EXPOSE 8501
54
+ # healthy = the database answers, the console's worker threads run and (on the vault server) the signer answers
55
  HEALTHCHECK --interval=30s --timeout=20s --start-period=300s --retries=5 CMD fly-trader health || exit 1
56
  ENTRYPOINT ["/app/docker/entrypoint.sh"]
README.md CHANGED
@@ -6,6 +6,7 @@ tags:
6
  - solana
7
  - memecoin
8
  - pump.fun
 
9
  - trading
10
  - connectome
11
  - drosophila
@@ -14,12 +15,12 @@ tags:
14
 
15
  # fly-trader — distribution
16
 
17
- **A fruit-fly connectome that trades Solana memecoins with your money.**
18
 
19
  > ## ⚠️ Read this first
20
  >
21
- > This software signs real transactions from a wallet you fund. It buys tokens that graduated from pump.fun, holds them
22
- > for about two hours, and sells. **It can lose everything in that wallet, quickly.** The model was trained on a few
23
  > months of data; the market it trades is adversarial and changes. Nothing here is a claim of profit, and nothing here is
24
  > financial advice. Run it only with money you can lose entirely, on a wallet used for nothing else. MIT licence — no
25
  > warranty of any kind. See [LICENSE](LICENSE).
@@ -32,10 +33,16 @@ connectome they run on. The training pipeline lives on [`master`](https://github
32
  - Every minute it scores every PumpSwap token that graduated from pump.fun at least six hours ago, from live 1-minute
33
  candles (a token needs that much history before its features mean anything).
34
  - The fly — the FlyWire *Drosophila* central brain (41,756 neurons) as a rate network on its real signed synapses —
35
- predicts each token's net return over the next 120 minutes. It was taught by a gradient-boosted selector; it then keeps
36
- learning from what its own trades earn, through plasticity at its mushroom body's KC→MBON synapses.
37
- - It buys when the predicted return clears its own buy line, sizes the position from measured certainty (a quarter
38
- of the growth-optimal bet, capped), holds 120 minutes, sells.
 
 
 
 
 
 
39
  - **It starts on paper.** After 20 closed paper trades it switches itself to the wallet and stays there. Before that,
40
  and whenever the wallet is unfunded or a key is missing, it trades paper and says so on the console.
41
  - **Rails:** 20 % below the wallet's peak it stops entering and sells every open position (the kill switch); three
@@ -46,13 +53,23 @@ connectome they run on. The training pipeline lives on [`master`](https://github
46
 
47
  ## Requirements
48
 
49
- - Docker (Desktop on macOS/Windows, or Engine + Compose v2 on Linux). ~2 GB of disk for the image and database.
 
50
  - A [Helius](https://dashboard.helius.dev) API key (free tier is enough) and a [Jupiter](https://portal.jup.ag) API key.
 
 
51
  - 0.5 SOL you are prepared to lose. Not more: the caps below are sized for 0.5.
52
  - Any 64-bit CPU: the fly scores about as fast on 16 CPU threads as on a GPU (its forward pass is a memory-bound
53
  scatter over a million synapses). An NVIDIA GPU is used when you start it with the CUDA override (see below). Apple
54
  GPUs are not reachable from inside Docker.
55
 
 
 
 
 
 
 
 
56
  ## Start trading
57
 
58
  1. Download this branch and open a terminal in it:
@@ -67,13 +84,15 @@ connectome they run on. The training pipeline lives on [`master`](https://github
67
  ```bash
68
  docker compose up -d --build
69
  ```
70
- The first start builds the image (several minutes), creates the database, installs the models and starts the market
71
- feed and the trading engine.
 
72
  4. Create the bot wallet — it prints the public key and writes the secret into your `.env`:
73
  ```bash
74
  docker compose exec fly fly-trader wallet new
75
  ```
76
- **Back up `.env` now.** It is the only copy of that key.
 
77
  5. Send **0.5 SOL** to the printed address, then restart the engine so it picks up the key:
78
  ```bash
79
  docker compose restart fly
@@ -92,7 +111,7 @@ scores anything); then it trades paper until 20 trades have closed; then it trad
92
  | Clear a tripped kill switch | Safety & wallet → **Clear kill switch**, or `docker compose exec fly fly-trader reset-circuit --kill` (re-bases the peak) |
93
  | Stop everything | `docker compose down` — **open positions stay open in the wallet**; start again and it sells them on schedule, or sell them yourself |
94
  | See the logs | `docker compose logs -f fly` |
95
- | Start again from nothing | `docker compose down -v` (deletes the database) and delete `data/` |
96
 
97
  The console listens on `127.0.0.1` only. Do not expose it: it can start and stop the engine and clear the rails.
98
 
@@ -100,15 +119,17 @@ The console listens on `127.0.0.1` only. Do not expose it: it can start and stop
100
 
101
  | Path | What |
102
  |---|---|
103
- | `models/policies/fly_*.pt` | the fly (`brain_snapshots` #62): taught by selector #59, two strategies (`ev`, `capitulation`), 120-minute hold, its own buy lines |
104
- | `models/selectors/selector_*.joblib` | the selector that taught it (#59), kept so the console can show the paper selector alongside |
105
  | `models/connectome/` | the FlyWire FAFB v783 central brain as the fly runs on it (14 MB), with its calibrated scale |
106
- | `models/wallet_skill/` | the fitted wallet-skill inputs: `config.json` (the horizon and lookback the model was trained with) and the name of the newest per-day table. The table itself (~300 MB) is on Hugging Face; the container fetches it once at first start |
107
- | `seed/seed.sql` | the database rows that admit the fly to trade: the model records, the passed replay verdict, autostart |
 
108
  | `.env.example` | every knob, set for 0.5 SOL |
109
- | `docker-compose.yml`, `Dockerfile`, `docker/entrypoint.sh` | one Postgres, one engine + console |
 
110
 
111
- The fly's replay verdict shipped here: over the 28 evaluation days it made +1.52 % per trade after real costs (profit
112
  factor 1.13) against −5.27 % for random picks of the same tokens. That is a backtest on one corpus, not a forecast.
113
 
114
  ## Tuning
@@ -118,14 +139,40 @@ Edit `.env`, then `docker compose restart fly`. The ones that matter:
118
  | Variable | Shipped | Meaning |
119
  |---|---|---|
120
  | `CAPITAL_SOL` | 0.5 | what you funded; also the paper warm-up bankroll |
121
- | `MAX_POSITION_SOL` / `MAX_POSITION_FRACTION` | 0.05 / 0.10 | per-position caps (the smaller wins) |
 
122
  | `GAS_RESERVE_SOL` | 0.05 | never spent |
123
  | `KILL_SWITCH_DRAWDOWN` / `KILL_SWITCH_LIQUIDATE` | 0.20 / 1 | drawdown from peak that halts and sells everything |
124
  | `HANDOVER_MIN_TRADES` | 20 | closed paper trades before it goes live |
125
  | `LIVE_ENABLED` | 1 | set 0 to run paper only |
126
 
127
- Scaling the wallet up scales the position caps with it (`MAX_POSITION_FRACTION`), but `MAX_POSITION_SOL` is a hard
128
- ceiling: raise it deliberately or not at all.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
129
 
130
  ## NVIDIA GPU
131
 
@@ -139,31 +186,47 @@ docker compose -f docker-compose.yml -f docker-compose.cuda.yml up -d --build
139
 
140
  The image is built with the CUDA 12.6 torch wheel (`TORCH_INDEX: cu126` in `docker-compose.cuda.yml`; `cu130` for the
141
  newest drivers) and the container is given every GPU. `DEVICE=auto` in `.env` then resolves to the GPU; the console's
142
- Model page shows which device the fly is on. Batch sizes are derived from the card's free memory, so an 8 GB card works;
143
  on a shared card set `DEVICE_MEMORY_GB` in `.env` to what the fly may use.
144
 
145
  ## Updates
146
 
147
- The container keeps itself current. Every hour it asks Hugging Face whether
148
- [`bryceweiner/fly-trader`](https://huggingface.co/bryceweiner/fly-trader) has a new release; when it has, it downloads
149
- the new code and model, restarts (about a minute; the paper books, the fly's learned state and any open positions are
150
- untouched -- open positions are sold on schedule by the restarted engine) and keeps the previous release beside the new
151
- one. If the new release does not come up within five minutes it goes back to the previous one and never tries that
152
- release again. `docker compose logs -f fly` shows what it did. `AUTO_UPDATE=0` in `.env` turns this off; then the
 
 
 
 
 
153
  "Refreshing the model" steps below are the manual way.
154
 
 
 
 
 
 
 
 
 
 
 
155
  ## Refreshing the model
156
 
157
  The model is a snapshot of one training run. To ship a newer one, on a `master` checkout with its database, after a
158
  passed replay:
159
 
160
  ```bash
161
- .venv/bin/python tools/make_seed.py --out dist
162
- cp -R dist/models/* models/ && cp -R dist/seed/* seed/
163
  ```
164
 
165
- Commit `models/` and `seed/` on this branch. The seed is idempotent: an install that already has these rows is not
166
- changed by it, so pull the branch, `docker compose up -d --build`, and the new model is in place.
 
 
167
 
168
  ## Also on Hugging Face
169
 
@@ -174,6 +237,6 @@ changed by it, so pull the branch, `docker compose up -d --build`, and the new m
174
 
175
  FlyWire consortium, Dorkenwald et al., *Neuronal wiring diagram of an adult brain*, Nature 2024; annotations Schlegel
176
  et al. 2024; synapse signs Shiu et al. 2024. Connectome export: [snedea/flybrain](https://github.com/snedea/flybrain)
177
- (MIT). Market data: [PumpAPI](https://pumpapi.io); routing: [Jupiter](https://jup.ag).
178
 
179
  [MIT](LICENSE).
 
6
  - solana
7
  - memecoin
8
  - pump.fun
9
+ - robinhood-chain
10
  - trading
11
  - connectome
12
  - drosophila
 
15
 
16
  # fly-trader — distribution
17
 
18
+ **A fruit-fly connectome that trades memecoins with your money: on Solana, and on Robinhood Chain if you turn it on.**
19
 
20
  > ## ⚠️ Read this first
21
  >
22
+ > This software signs real transactions from a wallet you fund. It buys tokens that graduated from pump.fun (and, if
23
+ > you turn Robinhood Chain on, from Pons), holds them for two to four hours, and sells. **It can lose everything in that wallet, quickly.** The model was trained on a few
24
  > months of data; the market it trades is adversarial and changes. Nothing here is a claim of profit, and nothing here is
25
  > financial advice. Run it only with money you can lose entirely, on a wallet used for nothing else. MIT licence — no
26
  > warranty of any kind. See [LICENSE](LICENSE).
 
33
  - Every minute it scores every PumpSwap token that graduated from pump.fun at least six hours ago, from live 1-minute
34
  candles (a token needs that much history before its features mean anything).
35
  - The fly — the FlyWire *Drosophila* central brain (41,756 neurons) as a rate network on its real signed synapses —
36
+ starts as the gradient-boosted selector that taught it: it scores every token with the selector's own prediction of
37
+ its net return. Its mushroom body's KC→MBON synapses then learn, from what the trades earn, a correction to that
38
+ prediction, and can veto a selector buy they have learned to distrust.
39
+ - It buys only what the selector buys (the selector's strategy, buy line and filters), sizes the position exactly as the
40
+ selector does (a quarter of the growth-optimal bet for its score band, capped), holds two to four hours (the
41
+ strategy's hold), sells.
42
+ - **Every transaction is checked before it is signed** (`fly_trader/signer`): swaps go only through Jupiter's own
43
+ routes, each one is simulated first and refused if it would spend more than ordered, deliver the token anywhere but
44
+ your wallet, touch another holding, or return far less value than it costs; buys are capped per trade, per day and per
45
+ minute.
46
  - **It starts on paper.** After 20 closed paper trades it switches itself to the wallet and stays there. Before that,
47
  and whenever the wallet is unfunded or a key is missing, it trades paper and says so on the console.
48
  - **Rails:** 20 % below the wallet's peak it stops entering and sells every open position (the kill switch); three
 
53
 
54
  ## Requirements
55
 
56
+ - Docker (Desktop on macOS/Windows, or Engine + Compose v2 on Linux). ~2 GB of disk for the image and database
57
+ (Robinhood Chain, if you turn it on, adds ~30 GB for its full history; see below).
58
  - A [Helius](https://dashboard.helius.dev) API key (free tier is enough) and a [Jupiter](https://portal.jup.ag) API key.
59
+ Optional: a second Solana RPC from another provider as `SIGNER_RPC_URL`, so the checks above do not rely on Helius
60
+ alone.
61
  - 0.5 SOL you are prepared to lose. Not more: the caps below are sized for 0.5.
62
  - Any 64-bit CPU: the fly scores about as fast on 16 CPU threads as on a GPU (its forward pass is a memory-bound
63
  scatter over a million synapses). An NVIDIA GPU is used when you start it with the CUDA override (see below). Apple
64
  GPUs are not reachable from inside Docker.
65
 
66
+ > ### ⚠️ Robinhood Chain needs an Envio HyperSync API token
67
+ >
68
+ > Robinhood Chain is off by default. To trade it, create a token in the [Envio dashboard](https://envio.dev/app/api-tokens)
69
+ > (personal accounts get one free), put it in `.env` as `ENVIO_API_TOKEN`, and set `RH_ENABLED=1`. Its feed reads the
70
+ > chain's logs from [Envio HyperSync](https://envio.dev/pricing/hypersync); the public Robinhood Chain RPC is far too slow
71
+ > to index its history. Without a token, leave Robinhood Chain off: Solana trades either way.
72
+
73
  ## Start trading
74
 
75
  1. Download this branch and open a terminal in it:
 
84
  ```bash
85
  docker compose up -d --build
86
  ```
87
+ The first start builds the image (several minutes), creates the database, downloads the wallet-skill table (~300 MB,
88
+ checked against its pinned sha256), installs the models and starts the market feed and the trading engine. The
89
+ Processes page also lists a `vault` worker that stops at once: it runs only on the hosted $FLY vault server.
90
  4. Create the bot wallet — it prints the public key and writes the secret into your `.env`:
91
  ```bash
92
  docker compose exec fly fly-trader wallet new
93
  ```
94
+ **Back up `.env` now.** It is the only copy of that key. (`wallet new` refuses if `.env` already has a
95
+ `BOT_PRIVATE_KEY=` line, even an empty one.)
96
  5. Send **0.5 SOL** to the printed address, then restart the engine so it picks up the key:
97
  ```bash
98
  docker compose restart fly
 
111
  | Clear a tripped kill switch | Safety & wallet → **Clear kill switch**, or `docker compose exec fly fly-trader reset-circuit --kill` (re-bases the peak) |
112
  | Stop everything | `docker compose down` — **open positions stay open in the wallet**; start again and it sells them on schedule, or sell them yourself |
113
  | See the logs | `docker compose logs -f fly` |
114
+ | Start again from nothing | `docker compose down -v` (deletes the database) and `sudo rm -rf data/` (the container writes it as root; it also holds the signer's ledger) |
115
 
116
  The console listens on `127.0.0.1` only. Do not expose it: it can start and stop the engine and clear the rails.
117
 
 
119
 
120
  | Path | What |
121
  |---|---|
122
+ | `models/policies/fly_*.pt` | the fly (#107 on the training machine; a fresh install gives it its own id): taught by selector #100, one strategy (`capitulation`), 240-minute hold, its own buy line. Loaded weights-only: the file cannot run code |
123
+ | `models/selectors/selector_*.skops` | the selector that taught it (#100), kept so the console can show the paper selector alongside. Saved with skops, not pickle |
124
  | `models/connectome/` | the FlyWire FAFB v783 central brain as the fly runs on it (14 MB), with its calibrated scale |
125
+ | `models/wallet_skill/` | the fitted wallet-skill inputs: `config.json` (the horizon and lookback the model was trained with), the name of the newest per-day table (`TABLE`) and its sha256 (`TABLE.sha256`). The table itself (~300 MB) is on Hugging Face; the container downloads it once and uses it only if the hash matches |
126
+ | `seed/release_seed.json` (`seed/seed.sql` is the fallback) | the database rows that admit the fly to trade: the model records (by file hash), the passed replay verdict, autostart |
127
+ | `release/allowed_signers` | the release keys this install trusts for automatic updates (see Updates) |
128
  | `.env.example` | every knob, set for 0.5 SOL |
129
+ | `docker-compose.yml`, `Dockerfile`, `docker/` | one Postgres, one engine + console (the Dockerfile's default stage; its `signer` stage is for the vault server) |
130
+ | `deploy/` | the hosted $FLY vault server: a separate signer container, host hardening, the signed-release updater ([docs/vault/RUNBOOK.md](docs/vault/RUNBOOK.md)) |
131
 
132
+ The replay verdict shipped here: over the 29 evaluation days it made +1.52 % per trade after real costs (profit
133
  factor 1.13) against −5.27 % for random picks of the same tokens. That is a backtest on one corpus, not a forecast.
134
 
135
  ## Tuning
 
139
  | Variable | Shipped | Meaning |
140
  |---|---|---|
141
  | `CAPITAL_SOL` | 0.5 | what you funded; also the paper warm-up bankroll |
142
+ | `MAX_POSITION_FRACTION` | 0.10 | the most of the bankroll one position may take |
143
+ | `MAX_POSITION_SOL` | 0.05 | the fixed size for a model without a sizing table (the shipped fly has one); must be above 0 to trade live |
144
  | `GAS_RESERVE_SOL` | 0.05 | never spent |
145
  | `KILL_SWITCH_DRAWDOWN` / `KILL_SWITCH_LIQUIDATE` | 0.20 / 1 | drawdown from peak that halts and sells everything |
146
  | `HANDOVER_MIN_TRADES` | 20 | closed paper trades before it goes live |
147
  | `LIVE_ENABLED` | 1 | set 0 to run paper only |
148
 
149
+ Scaling the wallet up scales the positions with it (`MAX_POSITION_FRACTION`). The signer has its own limits that do
150
+ not scale: at most `SIGNER_MAX_BUY_SOL` (1) per buy, `SIGNER_DAILY_BUY_SOL` (10) of buys a day and
151
+ `SIGNER_SWAPS_PER_MIN` (6) swaps a minute; raise them in `.env` deliberately.
152
+
153
+ ## Robinhood Chain (Pons) memecoins
154
+
155
+ The same engine also trades Robinhood Chain: tokens launched on Pons that graduated into Uniswap v4 pools.
156
+
157
+ - **Off by default; turn it on with `RH_ENABLED=1` and `ENVIO_API_TOKEN` in `.env`** (an Envio HyperSync token; the
158
+ free tier works). The `rh_stream` worker then starts with the others and indexes Pons launches, graduations and swaps
159
+ into 1-minute rows: a backfill from `RH_START_DAY` (default 2026-08-04, ~30 GB; ~40 days back is enough to trade),
160
+ then live, with RH minutes landing ~15-25 s after they close. If HyperSync fails, the official RPC carries the feed
161
+ until it is back.
162
+ - **Paper, in ETH.** The Robinhood Chain books
163
+ (`paper_rh_selector`, `paper_rh_fly`) keep their money in native ETH (`RH_CAPITAL_ETH`, 5 SOL's worth at build time)
164
+ and pay Pons' hook fee, price impact and gas. They have their own kill switch and circuit (Safety & wallet →
165
+ Robinhood Chain trading rails) and their own handover. The console's chain picker switches Overview and Trades
166
+ between the two chains, and the **Robinhood Chain** page shows the indexer, the quote-asset prices and the ETH wallet.
167
+ - **One model for both chains.** The shipped selector and fly were trained on both chains' rows together, and the
168
+ replay judged each chain on its own trades; a chain's books trade only where its verdict passed. With RH off the same
169
+ models trade Solana alone.
170
+ - **Live** needs `RH_LIVE_ENABLED=1`, `RH_BOT_PRIVATE_KEY` (in `.env` only, never in the image) and the address it
171
+ derives to as `RH_BOT_ADDRESS`, plus a funded wallet (ETH on Robinhood Chain, chain 4663). Then
172
+ `docker compose restart fly`. As on Solana, the fly trades the wallet only after it has won the RH paper race.
173
+ - `RH_ENABLED=0` turns all of it off.
174
+
175
+ RH training data does not ship: an install that turns RH on builds its own from the stream, as Solana does.
176
 
177
  ## NVIDIA GPU
178
 
 
186
 
187
  The image is built with the CUDA 12.6 torch wheel (`TORCH_INDEX: cu126` in `docker-compose.cuda.yml`; `cu130` for the
188
  newest drivers) and the container is given every GPU. `DEVICE=auto` in `.env` then resolves to the GPU; the console's
189
+ Model & training page shows which device the fly is on. Batch sizes are derived from the card's free memory, so an 8 GB card works;
190
  on a shared card set `DEVICE_MEMORY_GB` in `.env` to what the fly may use.
191
 
192
  ## Updates
193
 
194
+ The container keeps itself current, with **signed releases only**. Every hour it asks Hugging Face whether
195
+ [`bryceweiner/fly-trader`](https://huggingface.co/bryceweiner/fly-trader) has a new release. It installs one only if:
196
+ - its `release.json` is signed by a key in this install's `release/allowed_signers`
197
+ - its release number is higher than the installed one
198
+ - every file matches the size and sha256 the signed manifest lists
199
+
200
+ Then it restarts (about a minute; the paper books, the fly's learned state and any open positions are untouched --
201
+ open positions are sold on schedule by the restarted engine) and keeps the previous release beside the new one. If the
202
+ new release does not come up within five minutes, it goes back to the previous one and never tries that release again.
203
+ An unsigned or wrongly signed commit is refused and logged. While `release/allowed_signers` lists no key, nothing is
204
+ installed. `docker compose logs -f fly` shows what it did. `AUTO_UPDATE=0` in `.env` turns this off; then the
205
  "Refreshing the model" steps below are the manual way.
206
 
207
+ **Installed before October 2026? Rebuild once.** The updater in those images installed whatever Hugging Face held,
208
+ unverified; it may install this release by itself, but only a rebuilt image can check signatures (it needs `ssh-keygen`),
209
+ so it stops at this release. From your install directory:
210
+
211
+ ```bash
212
+ git pull && docker compose up -d --build
213
+ ```
214
+
215
+ Your data, the fly's learned state and open positions are kept. Robinhood Chain stays off until you turn it on.
216
+
217
  ## Refreshing the model
218
 
219
  The model is a snapshot of one training run. To ship a newer one, on a `master` checkout with its database, after a
220
  passed replay:
221
 
222
  ```bash
223
+ .venv/bin/python tools/make_seed.py --out ../fly-trader-dist
 
224
  ```
225
 
226
+ Then publish it as a signed release with `tools/publish_release.py` (docs/vault/RUNBOOK.md, section 4.1): installs
227
+ with automatic updates pick it up within the hour. For a manual install, commit `models/` and `seed/` on this branch;
228
+ every start runs `fly-trader apply-release`, which reuses rows it already has (same file hash) and registers new
229
+ models, so pull the branch, `docker compose up -d --build`, and the new model is in place.
230
 
231
  ## Also on Hugging Face
232
 
 
237
 
238
  FlyWire consortium, Dorkenwald et al., *Neuronal wiring diagram of an adult brain*, Nature 2024; annotations Schlegel
239
  et al. 2024; synapse signs Shiu et al. 2024. Connectome export: [snedea/flybrain](https://github.com/snedea/flybrain)
240
+ (MIT). Market data: [PumpAPI](https://pumpapi.io); routing: [Jupiter](https://jup.ag) (Metis routes only).
241
 
242
  [MIT](LICENSE).
deploy/bootstrap-host.sh CHANGED
@@ -1,64 +1,459 @@
1
  #!/usr/bin/env bash
2
- # One-time setup of the vault fly's server (Ubuntu 24.04, e.g. Netcup RS 2000 G12). Run as root:
3
- # bash bootstrap-host.sh "<your ssh-ed25519 RELEASE public key line>"
4
- # It installs Docker, locks the box down (SSH keys only, firewall: SSH in, nothing else), installs the updater with
5
- # YOUR release key as the only trusted signer, and lays out /srv/fly. It does not create the wallet or start the fly.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6
  set -euo pipefail
7
- RELEASE_PUBKEY="${1:?usage: bootstrap-host.sh \"ssh-ed25519 AAAA... release-key\"}"
 
 
 
 
 
 
 
 
 
 
 
8
 
9
- apt-get update
10
- apt-get install -y ca-certificates curl ufw unattended-upgrades python3 openssh-client
 
 
 
 
11
  install -m 0755 -d /etc/apt/keyrings
12
  curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
13
  echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu $(. /etc/os-release && echo "$VERSION_CODENAME") stable" > /etc/apt/sources.list.d/docker.list
14
- apt-get update
15
- apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
16
- dpkg-reconfigure -f noninteractive unattended-upgrades
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
17
 
18
- # SSH: keys only (make sure your key is in ~/.ssh/authorized_keys BEFORE running this)
19
- sed -i 's/^#\?PasswordAuthentication .*/PasswordAuthentication no/; s/^#\?PermitRootLogin .*/PermitRootLogin prohibit-password/' /etc/ssh/sshd_config
20
- systemctl reload ssh || systemctl reload sshd
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
21
  ufw default deny incoming
22
- ufw default allow outgoing
23
- ufw allow OpenSSH
 
 
 
 
 
 
 
 
 
 
 
 
 
 
24
  ufw --force enable
25
 
26
- # layout
27
- install -d -m 0750 /srv/fly /srv/fly/data /srv/fly/logs /srv/fly/releases
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
28
  install -d -m 0700 /srv/fly/secrets
29
  install -d -m 0755 /etc/fly /usr/local/lib/fly
30
- echo "bryce ${RELEASE_PUBKEY}" > /etc/fly/allowed_signers
 
 
 
 
 
 
 
 
31
  chmod 0644 /etc/fly/allowed_signers
32
- # the trading wallet: an ed25519 seed made HERE, never copied off this box in plaintext (the fly's backup job writes an
33
- # age-encrypted copy for you). fly_trader.chain.keys accepts a base58 32-byte seed.
34
- if [ ! -e /srv/fly/secrets/bot_key ]; then
35
- umask 077
36
- openssl genpkey -algorithm ed25519 -out /srv/fly/secrets/bot_key.pem
37
- python3 - <<'PY'
38
- import subprocess
 
 
 
 
 
 
 
 
 
 
 
 
 
 
39
  A = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"
40
  def b58(b):
41
  n = int.from_bytes(b, "big"); s = ""
42
  while n: n, r = divmod(n, 58); s = A[r] + s
43
  return "1" * (len(b) - len(b.lstrip(b"\0"))) + s
44
- priv = subprocess.run(["openssl", "pkey", "-in", "/srv/fly/secrets/bot_key.pem", "-outform", "DER"], check=True, capture_output=True).stdout[-32:]
45
- pub = subprocess.run(["openssl", "pkey", "-in", "/srv/fly/secrets/bot_key.pem", "-pubout", "-outform", "DER"], check=True, capture_output=True).stdout[-32:]
46
- open("/srv/fly/secrets/bot_key", "x").write(b58(priv) + "\n")
47
- print("vault wallet address:", b58(pub))
 
48
  PY
49
- shred -u /srv/fly/secrets/bot_key.pem
50
- chmod 0400 /srv/fly/secrets/bot_key
51
- fi
52
- HERE="$(cd "$(dirname "$0")" && pwd)"
 
 
 
 
53
  install -m 0755 "$HERE/fly_update.py" /usr/local/lib/fly/fly_update.py
 
54
  [ -f /etc/fly/update.json ] || install -m 0600 "$HERE/update.json.example" /etc/fly/update.json
 
 
 
 
 
 
 
 
 
 
55
  [ -f /srv/fly/vault.env ] || install -m 0600 "$HERE/vault.env.example" /srv/fly/vault.env
 
 
56
  install -m 0644 "$HERE/docker-compose.vault.yml" /srv/fly/docker-compose.yml
57
  install -m 0644 "$HERE/fly-update.service" /etc/systemd/system/fly-update.service
58
  install -m 0644 "$HERE/fly-update.timer" /etc/systemd/system/fly-update.timer
 
 
 
 
 
59
  systemctl daemon-reload
60
- echo
61
- echo "Next (see deploy/VAULT.md):"
62
- echo " 1. edit /srv/fly/vault.env and /etc/fly/update.json (Telegram)"
63
- echo " 2. systemctl enable --now fly-update.timer # pulls and builds the first release"
64
- echo " 3. create the wallet key, then restart: see VAULT.md 'wallet'"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  #!/usr/bin/env bash
2
+ # One-time setup of the vault fly's server: a dedicated machine (e.g. a mini PC at home) on a fresh Ubuntu Server 24.04
3
+ # with full-disk encryption, hardened along the lines of the CIS Ubuntu 24.04 benchmark for a single-purpose host that
4
+ # holds a hot wallet. Run with sudo, in two passes:
5
+ #
6
+ # 1. sudo TS_AUTHKEY=tskey-auth-... RECOVERY_PUBKEY="ssh-ed25519 AAAA... recovery" EXPECT_MANIFEST=<sha256 of release.json> \
7
+ # TPM_UNLOCK=1 ADMIN_USER=fly-admin bash bootstrap-host.sh "<release public key line: Secretive's ecdsa-sha2-nistp256 ...>"
8
+ # then, from your Mac, log in over Tailscale: ssh fly-admin@fly-vault (MagicDNS name or the 100.x address)
9
+ # 2. sudo CLOSE_PUBLIC_SSH=1 bash bootstrap-host.sh "<same release key>"
10
+ # removes the SSH rule for the local network / internet: after it the machine accepts SSH only over Tailscale and
11
+ # has no other open port.
12
+ #
13
+ # TPM_UNLOCK=1 binds the root disk's LUKS encryption to the TPM (clevis, PCR 7 = the Secure Boot state), so an
14
+ # unattended security reboot comes back up without a passphrase; it asks once for the disk passphrase. Without it the
15
+ # machine waits at every boot for the passphrase on its console, and automatic reboots are turned off.
16
+ #
17
+ # ADMIN_USER (default fly-admin) becomes the only account that can log in over SSH (key only, sudo without a
18
+ # password; root login is switched off). If it has no ~/.ssh/authorized_keys yet, root's are copied to it first, and
19
+ # the script stops before touching SSH if neither exists. Keep this root session open until you have logged in as
20
+ # ADMIN_USER from a second terminal. deploy/check-host.sh verifies every setting afterwards.
21
+ #
22
+ # RELEASE_PUBKEY / RECOVERY_PUBKEY: the two keys whose signatures the updater accepts (principals bryce and
23
+ # bryce-recovery), valid for the release namespace only. EXPECT_MANIFEST pins the first release the updater may
24
+ # install (tools/publish_release.py prints it as the manifest sha256); until it is set the updater installs nothing.
25
+ # TS_AUTHKEY: a one-off, pre-authorised key tagged tag:fly from the Tailscale admin console; used once, not stored.
26
+ #
27
+ # Idempotent: safe to run again (keys and edited config files are never overwritten).
28
  set -euo pipefail
29
+ RELEASE_PUBKEY="${1:?usage: ADMIN_USER=fly-admin bootstrap-host.sh \"ecdsa-sha2-nistp256 AAAA... release-key\"}"
30
+ RECOVERY_PUBKEY="${RECOVERY_PUBKEY:-}"
31
+ EXPECT_MANIFEST="${EXPECT_MANIFEST:-}"
32
+ TS_AUTHKEY="${TS_AUTHKEY:-}"
33
+ CLOSE_PUBLIC_SSH="${CLOSE_PUBLIC_SSH:-0}"
34
+ TPM_UNLOCK="${TPM_UNLOCK:-0}"
35
+ ADMIN_USER="${ADMIN_USER:-fly-admin}"
36
+ # container users (the images' USER lines); with userns-remap each is REMAP_UID + id on the host
37
+ BRAIN_UID=10001; SIGNER_UID=10002; FLY_GID=10000
38
+ HERE="$(cd "$(dirname "$0")" && pwd)"
39
+ export DEBIAN_FRONTEND=noninteractive
40
+ step() { printf '\n== %s\n' "$*"; }
41
 
42
+ # ---------------------------------------------------------------- 1. packages
43
+ step "packages"
44
+ apt-get update -q
45
+ apt-get install -y -q --no-install-recommends ca-certificates curl ufw unattended-upgrades python3 openssh-client openssl \
46
+ fail2ban python3-systemd auditd audispd-plugins aide aide-common lynis apparmor-utils systemd-timesyncd needrestart \
47
+ libpam-tmpdir debsums apt-show-versions apt-listchanges acct sysstat mokutil
48
  install -m 0755 -d /etc/apt/keyrings
49
  curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
50
  echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu $(. /etc/os-release && echo "$VERSION_CODENAME") stable" > /etc/apt/sources.list.d/docker.list
51
+ apt-get update -q
52
+ apt-get install -y -q docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
53
+ # Tailscale: SSH reaches this box only over your tailnet (WireGuard); its repo key is pinned like Docker's
54
+ curl -fsSL "https://pkgs.tailscale.com/stable/ubuntu/$(. /etc/os-release && echo "$VERSION_CODENAME").noarmor.gpg" -o /etc/apt/keyrings/tailscale.gpg
55
+ echo "deb [signed-by=/etc/apt/keyrings/tailscale.gpg] https://pkgs.tailscale.com/stable/ubuntu $(. /etc/os-release && echo "$VERSION_CODENAME") main" > /etc/apt/sources.list.d/tailscale.list
56
+ apt-get update -q
57
+ apt-get install -y -q tailscale
58
+ # nothing on this box needs snaps; fewer daemons, fewer holes
59
+ if dpkg -l snapd >/dev/null 2>&1; then apt-get purge -y -q snapd; fi
60
+ # no mail server: nothing here sends mail (alerts go to Telegram), and an MTA is one more listening daemon
61
+ if dpkg -l postfix 2>/dev/null | grep -q '^ii'; then apt-get purge -y -q postfix; fi
62
+
63
+ # ---------------------------------------------------------------- 1b. the machine itself
64
+ step "machine: wireless off, DMA protection, disk encryption"
65
+ # wired Ethernet only: Wi-Fi and Bluetooth drivers can never load (a radio is a way in that bypasses the router)
66
+ printf 'install cfg80211 /bin/false\ninstall mac80211 /bin/false\ninstall bluetooth /bin/false\ninstall btusb /bin/false\n' \
67
+ > /etc/modprobe.d/61-fly-no-wireless.conf
68
+ command -v rfkill >/dev/null && rfkill block all 2>/dev/null || true
69
+ systemctl disable --now bluetooth.service wpa_supplicant.service 2>/dev/null || true
70
+ # USB4 / Thunderbolt / OCuLink devices can read memory directly; the IOMMU confines them
71
+ if [ -z "$(ls -A /sys/kernel/iommu_groups 2>/dev/null)" ]; then
72
+ echo "WARNING: the IOMMU is off: enable it (AMD-Vi / IOMMU) in the firmware settings"
73
+ fi
74
+ # the root filesystem must sit on LUKS: the wallet keys are files on this disk
75
+ ROOT_LUKS="$(lsblk -rno NAME,TYPE -s "$(findmnt -no SOURCE /)" 2>/dev/null | awk '$2=="crypt"{found=1} found && $2=="part"{print "/dev/"$1; exit}')"
76
+ if [ -z "$ROOT_LUKS" ]; then
77
+ echo "WARNING: the root filesystem is not on an encrypted (LUKS) disk: reinstall with 'Encrypt the LVM group with LUKS'"
78
+ elif [ "$TPM_UNLOCK" = 1 ]; then
79
+ apt-get install -y -q --no-install-recommends clevis clevis-luks clevis-tpm2 clevis-initramfs
80
+ if ! clevis luks list -d "$ROOT_LUKS" 2>/dev/null | grep -q tpm2; then
81
+ echo "binding $ROOT_LUKS to the TPM (PCR 7); enter the disk passphrase when asked"
82
+ clevis luks bind -d "$ROOT_LUKS" tpm2 '{"pcr_ids":"7"}'
83
+ update-initramfs -u -k all
84
+ fi
85
+ fi
86
+
87
+ # ---------------------------------------------------------------- 2. patching
88
+ step "automatic security updates (reboot 04:00 UTC when a kernel needs it)"
89
+ cat > /etc/apt/apt.conf.d/20auto-upgrades <<'EOF'
90
+ APT::Periodic::Update-Package-Lists "1";
91
+ APT::Periodic::Unattended-Upgrade "1";
92
+ APT::Periodic::AutocleanInterval "7";
93
+ EOF
94
+ REBOOT=true # a disk that needs a typed passphrase at boot: no unattended reboots
95
+ if [ -n "${ROOT_LUKS:-}" ] && ! { command -v clevis >/dev/null && clevis luks list -d "$ROOT_LUKS" 2>/dev/null | grep -q tpm2; }; then
96
+ REBOOT=false
97
+ fi
98
+ cat > /etc/apt/apt.conf.d/52fly-unattended <<EOF
99
+ Unattended-Upgrade::Automatic-Reboot "$REBOOT";
100
+ Unattended-Upgrade::Automatic-Reboot-Time "04:00";
101
+ Unattended-Upgrade::Remove-Unused-Dependencies "true";
102
+ EOF
103
+ systemctl enable --now unattended-upgrades.service
104
+ timedatectl set-timezone UTC
105
+ systemctl enable --now systemd-timesyncd.service # the relay refuses the fly's requests if the clock is > 5 min off
106
+
107
+ # ---------------------------------------------------------------- 3. admin account
108
+ step "admin account: $ADMIN_USER"
109
+ id "$ADMIN_USER" >/dev/null 2>&1 || useradd -m -s /bin/bash "$ADMIN_USER"
110
+ passwd -l "$ADMIN_USER" >/dev/null # no password at all: SSH key only
111
+ home="$(getent passwd "$ADMIN_USER" | cut -d: -f6)"
112
+ install -d -m 0700 -o "$ADMIN_USER" -g "$ADMIN_USER" "$home/.ssh"
113
+ if [ ! -s "$home/.ssh/authorized_keys" ]; then
114
+ if [ -s /root/.ssh/authorized_keys ]; then
115
+ install -m 0600 -o "$ADMIN_USER" -g "$ADMIN_USER" /root/.ssh/authorized_keys "$home/.ssh/authorized_keys"
116
+ else
117
+ echo "STOP: neither $ADMIN_USER nor root has ~/.ssh/authorized_keys; add your key first" >&2; exit 1
118
+ fi
119
+ fi
120
+ echo "$ADMIN_USER ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/90-fly-admin
121
+ chmod 0440 /etc/sudoers.d/90-fly-admin
122
+ visudo -cf /etc/sudoers.d/90-fly-admin >/dev/null
123
+ # NOT in the docker group: that group is root without sudo's log line; docker runs through sudo
124
+ gpasswd -d "$ADMIN_USER" docker >/dev/null 2>&1 || true
125
+
126
+ # ---------------------------------------------------------------- 4. SSH
127
+ step "SSH: $ADMIN_USER only, key only, modern crypto"
128
+ cat > /etc/ssh/sshd_config.d/10-fly-hardening.conf <<EOF
129
+ PermitRootLogin no
130
+ PasswordAuthentication no
131
+ KbdInteractiveAuthentication no
132
+ PubkeyAuthentication yes
133
+ AuthenticationMethods publickey
134
+ AllowUsers $ADMIN_USER
135
+ MaxAuthTries 3
136
+ MaxSessions 4
137
+ LoginGraceTime 20
138
+ ClientAliveInterval 300
139
+ ClientAliveCountMax 2
140
+ PermitEmptyPasswords no
141
+ PermitUserEnvironment no
142
+ X11Forwarding no
143
+ AllowAgentForwarding no
144
+ # local forwarding stays on: it is how you reach the console (ssh -L 8501:127.0.0.1:8501)
145
+ AllowTcpForwarding local
146
+ GatewayPorts no
147
+ PermitTunnel no
148
+ Compression no
149
+ TCPKeepAlive no
150
+ MaxStartups 10:30:60
151
+ LogLevel VERBOSE
152
+ KexAlgorithms sntrup761x25519-sha512@openssh.com,curve25519-sha256,curve25519-sha256@libssh.org
153
+ Ciphers chacha20-poly1305@openssh.com,aes256-gcm@openssh.com,aes128-gcm@openssh.com
154
+ MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com
155
+ HostKeyAlgorithms ssh-ed25519,rsa-sha2-512,rsa-sha2-256
156
+ # client keys: ed25519, FIDO/hardware-backed keys, and P-256 (Secure Enclave keys from Secretive; the Ledger later)
157
+ PubkeyAcceptedAlgorithms sk-ssh-ed25519@openssh.com,ssh-ed25519,sk-ecdsa-sha2-nistp256@openssh.com,ecdsa-sha2-nistp256
158
+ EOF
159
+ # older lines in the main file must not win: Include is at the top, and first match wins in sshd
160
+ sed -i 's/^\(PermitRootLogin\|PasswordAuthentication\|KbdInteractiveAuthentication\|X11Forwarding\)/# fly: see sshd_config.d\/10-fly-hardening.conf\n#&/' /etc/ssh/sshd_config
161
+ install -d -m 0755 /run/sshd # 24.04 starts sshd on demand; the check needs this dir
162
+ sshd -t
163
+ systemctl reload ssh 2>/dev/null || systemctl restart ssh
164
 
165
+ step "fail2ban: SSH brute force"
166
+ cat > /etc/fail2ban/jail.d/fly-sshd.local <<'EOF'
167
+ [sshd]
168
+ enabled = true
169
+ backend = systemd
170
+ maxretry = 4
171
+ findtime = 10m
172
+ bantime = 1h
173
+ bantime.increment = true
174
+ EOF
175
+ systemctl enable --now fail2ban
176
+ systemctl restart fail2ban
177
+
178
+ # ---------------------------------------------------------------- 5. Tailscale + firewall
179
+ step "Tailscale"
180
+ systemctl enable --now tailscaled
181
+ if [ -n "$TS_AUTHKEY" ] && ! tailscale status >/dev/null 2>&1; then
182
+ tailscale up --auth-key="$TS_AUTHKEY" --hostname=fly-vault --advertise-tags=tag:fly --ssh=false --accept-dns=false
183
+ fi
184
+ unset TS_AUTHKEY
185
+ TS_UP=0; tailscale status >/dev/null 2>&1 && TS_UP=1
186
+
187
+ step "firewall: SSH over Tailscale only (plus LAN/public SSH until CLOSE_PUBLIC_SSH=1); out: HTTPS, HTTP (apt), DNS, NTP, WireGuard"
188
+ ufw --force reset >/dev/null
189
  ufw default deny incoming
190
+ ufw default deny outgoing
191
+ ufw default deny routed
192
+ ufw allow in on tailscale0 to any port 22 proto tcp comment 'ssh over the tailnet'
193
+ if [ "$CLOSE_PUBLIC_SSH" = 1 ]; then
194
+ [ "$TS_UP" = 1 ] || { echo "STOP: Tailscale is not up; closing public SSH now would lock you out" >&2; exit 1; }
195
+ echo "public SSH closed: log in with ssh $ADMIN_USER@fly-vault (Tailscale)"
196
+ else
197
+ ufw limit in 22/tcp comment 'ssh (rate limited) -- removed by CLOSE_PUBLIC_SSH=1'
198
+ fi
199
+ ufw allow out 443/tcp comment 'https: RPCs, relay, HF, Telegram, apt, Tailscale control'
200
+ ufw allow out 80/tcp comment 'http: apt mirrors'
201
+ ufw allow out 53 comment 'dns'
202
+ ufw allow out 123/udp comment 'ntp'
203
+ ufw allow out 41641/udp comment 'tailscale wireguard'
204
+ ufw allow out 3478/udp comment 'tailscale stun'
205
+ ufw logging low
206
  ufw --force enable
207
 
208
+ # Docker writes its own iptables rules and bypasses ufw; the DOCKER-USER chain is where the host gets a say. Containers
209
+ # may only talk to each other, reach DNS/HTTPS out, and nothing on the internet may open a connection to them.
210
+ cat > /usr/local/sbin/fly-docker-firewall <<'EOF'
211
+ #!/bin/sh
212
+ set -e
213
+ iptables -N DOCKER-USER 2>/dev/null || true
214
+ iptables -F DOCKER-USER
215
+ iptables -A DOCKER-USER -m conntrack --ctstate ESTABLISHED,RELATED -j RETURN
216
+ iptables -A DOCKER-USER -i docker0 -o docker0 -j RETURN
217
+ iptables -A DOCKER-USER -i br-+ -o br-+ -j RETURN
218
+ for p in 443 80 53; do # 80: apt mirrors during image builds (apt checks its own signatures)
219
+ iptables -A DOCKER-USER -i br-+ -p tcp --dport $p -j RETURN
220
+ iptables -A DOCKER-USER -i docker0 -p tcp --dport $p -j RETURN
221
+ done
222
+ iptables -A DOCKER-USER -i br-+ -p udp --dport 53 -j RETURN
223
+ iptables -A DOCKER-USER -i docker0 -p udp --dport 53 -j RETURN
224
+ iptables -A DOCKER-USER -i br-+ -j DROP
225
+ iptables -A DOCKER-USER -i docker0 -j DROP
226
+ iptables -A DOCKER-USER -m conntrack --ctstate NEW -j DROP
227
+ EOF
228
+ chmod 0750 /usr/local/sbin/fly-docker-firewall
229
+ cat > /etc/systemd/system/fly-docker-firewall.service <<'EOF'
230
+ [Unit]
231
+ Description=fly-trader: restrict container traffic (DOCKER-USER chain)
232
+ After=docker.service
233
+ Requires=docker.service
234
+ PartOf=docker.service
235
+ [Service]
236
+ Type=oneshot
237
+ RemainAfterExit=yes
238
+ ExecStart=/usr/local/sbin/fly-docker-firewall
239
+ [Install]
240
+ WantedBy=docker.service
241
+ EOF
242
+
243
+ # ---------------------------------------------------------------- 6. kernel
244
+ step "kernel and network settings"
245
+ cat > /etc/sysctl.d/60-fly-hardening.conf <<'EOF'
246
+ net.ipv4.conf.all.accept_redirects = 0
247
+ net.ipv4.conf.default.accept_redirects = 0
248
+ net.ipv4.conf.all.secure_redirects = 0
249
+ net.ipv4.conf.default.secure_redirects = 0
250
+ net.ipv4.conf.all.send_redirects = 0
251
+ net.ipv4.conf.default.send_redirects = 0
252
+ net.ipv4.conf.all.accept_source_route = 0
253
+ net.ipv4.conf.default.accept_source_route = 0
254
+ net.ipv4.conf.all.rp_filter = 1
255
+ net.ipv4.conf.default.rp_filter = 1
256
+ net.ipv4.conf.all.log_martians = 1
257
+ net.ipv4.conf.default.log_martians = 1
258
+ net.ipv4.icmp_echo_ignore_broadcasts = 1
259
+ net.ipv4.icmp_ignore_bogus_error_responses = 1
260
+ net.ipv4.tcp_syncookies = 1
261
+ net.ipv6.conf.all.accept_redirects = 0
262
+ net.ipv6.conf.default.accept_redirects = 0
263
+ net.ipv6.conf.all.accept_source_route = 0
264
+ net.ipv6.conf.default.accept_source_route = 0
265
+ kernel.kptr_restrict = 2
266
+ kernel.dmesg_restrict = 1
267
+ kernel.unprivileged_bpf_disabled = 1
268
+ net.core.bpf_jit_harden = 2
269
+ kernel.yama.ptrace_scope = 2
270
+ kernel.randomize_va_space = 2
271
+ kernel.sysrq = 0
272
+ fs.protected_hardlinks = 1
273
+ fs.protected_symlinks = 1
274
+ fs.protected_fifos = 2
275
+ fs.protected_regular = 2
276
+ fs.suid_dumpable = 0
277
+ EOF
278
+ sysctl -q --system
279
+ # core dumps could write a key to disk
280
+ cat > /etc/security/limits.d/60-fly-nocore.conf <<'EOF'
281
+ * hard core 0
282
+ EOF
283
+ mkdir -p /etc/systemd/coredump.conf.d
284
+ printf '[Coredump]\nStorage=none\nProcessSizeMax=0\n' > /etc/systemd/coredump.conf.d/60-fly.conf
285
+ aa-status --enabled # AppArmor must be on (Docker's default profile uses it)
286
+ # protocols, filesystems and devices this server never uses: not loadable at all
287
+ for m in dccp sctp rds tipc usb-storage firewire-core cramfs freevxfs hfs hfsplus jffs2 udf; do
288
+ echo "install $m /bin/false"
289
+ done > /etc/modprobe.d/60-fly-blacklist.conf
290
+ # new files readable by their group at most, never by everyone
291
+ sed -i 's/^UMASK.*/UMASK\t\t027/' /etc/login.defs
292
+ printf 'Authorized access only. Activity on this system is logged.\n' > /etc/issue
293
+ cp /etc/issue /etc/issue.net
294
+ sed -i 's/^ENABLED=.*/ENABLED="true"/' /etc/default/sysstat 2>/dev/null || true
295
+ systemctl enable --now acct sysstat >/dev/null 2>&1 || true
296
+
297
+ # ---------------------------------------------------------------- 7. Docker
298
+ step "Docker: user namespaces, no new privileges, bounded logs"
299
+ install -d -m 0755 /etc/docker
300
+ [ -f /etc/docker/daemon.json ] || cat > /etc/docker/daemon.json <<'EOF'
301
+ {
302
+ "userns-remap": "default",
303
+ "no-new-privileges": true,
304
+ "icc": false,
305
+ "live-restore": true,
306
+ "userland-proxy": false,
307
+ "log-driver": "json-file",
308
+ "log-opts": {"max-size": "50m", "max-file": "5"}
309
+ }
310
+ EOF
311
+ systemctl restart docker
312
+ systemctl daemon-reload
313
+ systemctl enable --now fly-docker-firewall.service
314
+ # with user namespaces, root inside a container is this unprivileged uid on the host
315
+ REMAP_UID="$(awk -F: '$1=="dockremap"{print $2; exit}' /etc/subuid)"
316
+
317
+ # ---------------------------------------------------------------- 8. layout, keys, updater
318
+ step "fly layout and wallets"
319
+ install -d -m 0750 /srv/fly
320
+ install -d -m 0755 /srv/fly/releases # public release files (from HF); the remapped container root reads them
321
+ H_BRAIN=$((REMAP_UID + BRAIN_UID)); H_SIGNER=$((REMAP_UID + SIGNER_UID)); H_FLY=$((REMAP_UID + FLY_GID))
322
+ install -d -m 0750 -o "$H_BRAIN" -g "$H_FLY" /srv/fly/data /srv/fly/logs
323
+ install -d -m 0700 -o "$H_SIGNER" -g "$H_SIGNER" /srv/fly/signer # the signer's own ledger (policy state)
324
+ install -d -m 0750 -o "$H_SIGNER" -g "$H_FLY" /srv/fly/run # its socket: the brain may connect, nothing else
325
  install -d -m 0700 /srv/fly/secrets
326
  install -d -m 0755 /etc/fly /usr/local/lib/fly
327
+ # release signers: valid for the release namespace only, so these keys sign nothing else this box would accept. A re-run
328
+ # without RECOVERY_PUBKEY keeps the recovery signer already installed (so closing SSH or updating the scripts never drops it).
329
+ if [ -z "$RECOVERY_PUBKEY" ] && [ -f /etc/fly/allowed_signers ]; then
330
+ RECOVERY_PUBKEY="$(awk '$1=="bryce-recovery"{ $1=""; $2=""; sub(/^ */, ""); print; exit }' /etc/fly/allowed_signers)"
331
+ fi
332
+ {
333
+ echo "bryce namespaces=\"fly-trader-release\" ${RELEASE_PUBKEY}"
334
+ if [ -n "$RECOVERY_PUBKEY" ]; then echo "bryce-recovery namespaces=\"fly-trader-release\" ${RECOVERY_PUBKEY}"; fi
335
+ } > /etc/fly/allowed_signers
336
  chmod 0644 /etc/fly/allowed_signers
337
+ [ -n "$RECOVERY_PUBKEY" ] || echo "WARNING: no RECOVERY_PUBKEY: losing the release key would mean re-bootstrapping the updater"
338
+ install -m 0755 "$HERE/fly" /usr/local/bin/fly # sudo fly <command>: fly-trader inside the running container
339
+ # the database password: made here, root-only, handed to both database and brain by compose
340
+ if [ ! -s /srv/fly/db.env ]; then
341
+ pw="$(openssl rand -hex 24)"
342
+ ( umask 077; printf 'POSTGRES_PASSWORD=%s\nDATABASE_URL=postgresql://fly:%s@db:5432/fly_trader\n' "$pw" "$pw" > /srv/fly/db.env )
343
+ unset pw
344
+ fi
345
+ chmod 0600 /srv/fly/db.env
346
+ # two hot keys, each an ed25519 seed made HERE and never copied off this box in plaintext (the signer writes
347
+ # age-encrypted backups for you): the trading key (bot_key: the trading wallet; refills itself from the treasury under
348
+ # L1) and the payout key (payout_key: holds nothing; pays claims from the treasury under L2). Only the signer container
349
+ # mounts them. Their public keys are printed below, once; later: sudo fly signer status.
350
+ make_key() {
351
+ local name="$1"
352
+ if [ ! -e "/srv/fly/secrets/$name" ]; then
353
+ umask 077
354
+ openssl genpkey -algorithm ed25519 -out "/srv/fly/secrets/$name.pem"
355
+ python3 - "$name" <<'PY'
356
+ import subprocess, sys
357
+ name = sys.argv[1]
358
  A = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"
359
  def b58(b):
360
  n = int.from_bytes(b, "big"); s = ""
361
  while n: n, r = divmod(n, 58); s = A[r] + s
362
  return "1" * (len(b) - len(b.lstrip(b"\0"))) + s
363
+ pem = f"/srv/fly/secrets/{name}.pem"
364
+ priv = subprocess.run(["openssl", "pkey", "-in", pem, "-outform", "DER"], check=True, capture_output=True).stdout[-32:]
365
+ pub = subprocess.run(["openssl", "pkey", "-in", pem, "-pubout", "-outform", "DER"], check=True, capture_output=True).stdout[-32:]
366
+ open(f"/srv/fly/secrets/{name}", "x").write(b58(priv) + "\n")
367
+ print(f"{name}: {b58(pub)}")
368
  PY
369
+ shred -u "/srv/fly/secrets/$name.pem"
370
+ fi
371
+ # readable only by the signer container's user (remapped), not by the brain container or any host account
372
+ chown "$H_SIGNER:$H_SIGNER" "/srv/fly/secrets/$name"
373
+ chmod 0400 "/srv/fly/secrets/$name"
374
+ }
375
+ make_key bot_key # the trading hot key: holds at most the float; refills itself from the Squads treasury (L1)
376
+ make_key payout_key # the payout key: holds nothing; pays claims from the treasury within its weekly cap (L2)
377
  install -m 0755 "$HERE/fly_update.py" /usr/local/lib/fly/fly_update.py
378
+ install -m 0755 "$HERE/check-host.sh" /usr/local/lib/fly/check-host.sh
379
  [ -f /etc/fly/update.json ] || install -m 0600 "$HERE/update.json.example" /etc/fly/update.json
380
+ if [ -n "$EXPECT_MANIFEST" ]; then # pin the first release: the updater refuses anything else until one is installed
381
+ python3 - "$EXPECT_MANIFEST" <<'PY'
382
+ import json, re, sys
383
+ pin = sys.argv[1].strip().lower()
384
+ if not re.fullmatch(r"[0-9a-f]{64}", pin):
385
+ raise SystemExit("EXPECT_MANIFEST must be a sha256 hex digest")
386
+ p = "/etc/fly/update.json"; c = json.load(open(p)); c["expect_manifest_sha256"] = pin
387
+ open(p, "w").write(json.dumps(c, indent=1) + "\n")
388
+ PY
389
+ fi
390
  [ -f /srv/fly/vault.env ] || install -m 0600 "$HERE/vault.env.example" /srv/fly/vault.env
391
+ [ -f /srv/fly/custody.env ] || install -m 0600 "$HERE/custody.env.example" /srv/fly/custody.env
392
+ [ -f /srv/fly/signer.env ] || install -m 0600 "$HERE/signer.env.example" /srv/fly/signer.env
393
  install -m 0644 "$HERE/docker-compose.vault.yml" /srv/fly/docker-compose.yml
394
  install -m 0644 "$HERE/fly-update.service" /etc/systemd/system/fly-update.service
395
  install -m 0644 "$HERE/fly-update.timer" /etc/systemd/system/fly-update.timer
396
+ install -m 0755 "$HERE/fly_audit_watch.py" /usr/local/lib/fly/fly_audit_watch.py
397
+ install -m 0644 "$HERE/fly-audit-watch.service" /etc/systemd/system/fly-audit-watch.service
398
+ install -m 0644 "$HERE/fly-audit-watch.timer" /etc/systemd/system/fly-audit-watch.timer
399
+ install -m 0644 "$HERE/fly-aide.service" /etc/systemd/system/fly-aide.service
400
+ install -m 0644 "$HERE/fly-aide.timer" /etc/systemd/system/fly-aide.timer
401
  systemctl daemon-reload
402
+ systemctl enable --now fly-audit-watch.timer # a key read by anything but the signer goes to Telegram within a minute
403
+ systemctl enable --now fly-aide.timer # the daily file-integrity check reports to Telegram (there is no mail here)
404
+
405
+ # ---------------------------------------------------------------- 9. audit and integrity
406
+ step "audit log: key reads and config changes"
407
+ cat > /etc/audit/rules.d/60-fly.rules <<'EOF'
408
+ -w /srv/fly/secrets -p rwa -k fly_keys
409
+ -w /etc/fly -p wa -k fly_conf
410
+ -w /srv/fly/vault.env -p rwa -k fly_conf
411
+ -w /srv/fly/db.env -p rwa -k fly_conf
412
+ -w /srv/fly/custody.env -p wa -k fly_conf
413
+ -w /srv/fly/signer.env -p rwa -k fly_conf
414
+ -w /srv/fly/signer -p wa -k fly_signer
415
+ -w /usr/local/lib/fly -p wa -k fly_updater
416
+ -w /etc/ssh/sshd_config -p wa -k sshd
417
+ -w /etc/ssh/sshd_config.d -p wa -k sshd
418
+ -w /etc/sudoers -p wa -k sudoers
419
+ -w /etc/sudoers.d -p wa -k sudoers
420
+ -w /etc/docker -p wa -k docker_conf
421
+ -w /etc/systemd/system -p wa -k systemd_units
422
+ -w /etc/passwd -p wa -k identity
423
+ -w /etc/shadow -p wa -k identity
424
+ -w /etc/group -p wa -k identity
425
+ EOF
426
+ augenrules --load >/dev/null
427
+ systemctl enable --now auditd
428
+
429
+ step "file integrity baseline (AIDE, checked daily; takes a few minutes)"
430
+ echo 'Checksums = sha512' > /etc/aide/aide.conf.d/10_fly_checksums
431
+ cat > /etc/aide/aide.conf.d/90_fly_ignore <<'EOF'
432
+ !/srv/fly/data
433
+ !/srv/fly/logs
434
+ !/srv/fly/releases
435
+ !/var/lib/docker
436
+ !/var/lib/containerd
437
+ EOF
438
+ [ -f /var/lib/aide/aide.db ] || { aideinit -y -f >/dev/null 2>&1 && cp /var/lib/aide/aide.db.new /var/lib/aide/aide.db; }
439
+
440
+ # ---------------------------------------------------------------- 10. logs
441
+ step "persistent journal"
442
+ mkdir -p /etc/systemd/journald.conf.d
443
+ printf '[Journal]\nStorage=persistent\nSystemMaxUse=2G\n' > /etc/systemd/journald.conf.d/60-fly.conf
444
+ systemctl restart systemd-journald
445
+
446
+ step "done"
447
+ if [ "$CLOSE_PUBLIC_SSH" = 1 ]; then
448
+ next="Verify: sudo /usr/local/lib/fly/check-host.sh (every control should pass now)"
449
+ else
450
+ next="From your Mac, BEFORE closing this session: ssh $ADMIN_USER@fly-vault (over Tailscale; root login is now off)
451
+ Then run this script again with CLOSE_PUBLIC_SSH=1 to remove public SSH."
452
+ fi
453
+ cat <<EOF
454
+ $next
455
+ Next (docs/vault/RUNBOOK.md, sections 4.3 and 4.4):
456
+ 1. the treasury on https://fly-trader.app/owner.html with your owner wallet (the two keys printed above)
457
+ 2. sudoedit /srv/fly/{custody,signer,vault}.env and /etc/fly/update.json (Telegram)
458
+ 3. sudo systemctl enable --now fly-update.timer # pulls, verifies and starts the pinned first release
459
+ EOF
deploy/check-host.sh ADDED
@@ -0,0 +1,106 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env bash
2
+ # Verify the vault server's hardening (bootstrap-host.sh). Run as root: sudo /usr/local/lib/fly/check-host.sh
3
+ # Prints PASS/FAIL per control and a Lynis hardening index; exits 1 if any control fails.
4
+ set -uo pipefail
5
+ fail=0
6
+ pass() { printf ' PASS %s\n' "$1"; }
7
+ bad() { printf ' FAIL %s\n' "$1"; fail=1; }
8
+ check() { local what="$1"; shift; if "$@" >/dev/null 2>&1; then pass "$what"; else bad "$what"; fi; }
9
+ sshd_is() { sshd -T 2>/dev/null | grep -qix "$1 $2"; }
10
+ sysctl_is() { [ "$(sysctl -n "$1" 2>/dev/null)" = "$2" ]; }
11
+
12
+ echo "SSH"
13
+ check "root login off" sshd_is permitrootlogin no
14
+ check "passwords off" sshd_is passwordauthentication no
15
+ check "keyboard-interactive off" sshd_is kbdinteractiveauthentication no
16
+ check "only the admin account" sh -c "sshd -T | grep -qi '^allowusers '"
17
+ check "max 3 auth tries" sshd_is maxauthtries 3
18
+ check "X11 / agent forwarding off" sh -c "sshd -T | grep -qix 'x11forwarding no' && sshd -T | grep -qix 'allowagentforwarding no'"
19
+ check "only local TCP forwarding" sshd_is allowtcpforwarding local
20
+ check "no weak ciphers" sh -c "! sshd -T | grep -i '^ciphers' | grep -Eqi 'cbc|3des|arcfour'"
21
+ check "fail2ban guards sshd" sh -c "fail2ban-client status sshd"
22
+ check "client keys: ed25519/P-256/FIDO only" sh -c "sshd -T | grep -i '^pubkeyacceptedalgorithms' | grep -vqi rsa"
23
+ check "admin not in the docker group" sh -c "! id -nG \"\$(sshd -T | awk 'tolower(\$1)==\"allowusers\"{print \$2}')\" | grep -qw docker"
24
+
25
+ echo "Machine"
26
+ root_luks() { lsblk -rno NAME,TYPE -s "$(findmnt -no SOURCE /)" | awk '$2=="crypt"{f=1} f && $2=="part"{print "/dev/"$1; exit}'; }
27
+ check "root filesystem encrypted (LUKS)" sh -c "[ -n \"\$(lsblk -rno TYPE -s \$(findmnt -no SOURCE /) | grep -x crypt)\" ]"
28
+ check "unattended reboots unlock (TPM) or are off" sh -c "grep -q 'Automatic-Reboot \"false\"' /etc/apt/apt.conf.d/52fly-unattended || clevis luks list -d $(root_luks) 2>/dev/null | grep -q tpm2"
29
+ check "IOMMU on (USB4/OCuLink DMA)" sh -c "[ -n \"\$(ls -A /sys/kernel/iommu_groups)\" ]"
30
+ check "Secure Boot on" sh -c "mokutil --sb-state 2>/dev/null | grep -qi 'SecureBoot enabled'"
31
+ check "no Wi-Fi or Bluetooth" sh -c "! ls /sys/class/net/*/wireless >/dev/null 2>&1 && [ -z \"\$(ls -A /sys/class/bluetooth 2>/dev/null)\" ]"
32
+
33
+ echo "Tailscale"
34
+ check "tailscaled up" sh -c "tailscale status >/dev/null"
35
+ check "Tailnet Lock enabled" sh -c "tailscale lock status 2>/dev/null | grep -qi 'lock is enabled'"
36
+ check "no public SSH (tailnet only)" sh -c "! ufw status | grep -E '^22/tcp' | grep -v tailscale0 | grep -q ."
37
+
38
+ echo "Firewall"
39
+ check "ufw active" sh -c "ufw status | grep -q 'Status: active'"
40
+ check "incoming denied by default" sh -c "ufw status verbose | grep -q 'deny (incoming)'"
41
+ check "outgoing denied by default" sh -c "ufw status verbose | grep -q 'deny (outgoing)'"
42
+ check "only SSH listens publicly" sh -c "! ss -Hltn | awk '{print \$4}' | grep -vE '^(127\\.|\\[::1\\]|\\[::ffff:127|100\\.|\\[fd7a:)' | grep -vE ':22\$' | grep -q ."
43
+ check "container traffic restricted" sh -c "iptables -S DOCKER-USER | grep -q -- '-j DROP'"
44
+ check "console bound to localhost" sh -c "! ss -Hltn | grep -E ':8501 ' | grep -vqE '127\\.0\\.0\\.1:8501'"
45
+
46
+ echo "Kernel"
47
+ for kv in net.ipv4.conf.all.accept_redirects=0 net.ipv4.conf.all.send_redirects=0 net.ipv4.conf.all.accept_source_route=0 \
48
+ net.ipv4.conf.all.rp_filter=1 net.ipv4.tcp_syncookies=1 kernel.kptr_restrict=2 kernel.dmesg_restrict=1 \
49
+ kernel.unprivileged_bpf_disabled=1 kernel.yama.ptrace_scope=2 kernel.randomize_va_space=2 fs.suid_dumpable=0 \
50
+ fs.protected_symlinks=1 fs.protected_hardlinks=1; do
51
+ check "${kv%%=*} = ${kv#*=}" sysctl_is "${kv%%=*}" "${kv#*=}"
52
+ done
53
+ check "core dumps off" sh -c "grep -qs 'Storage=none' /etc/systemd/coredump.conf.d/60-fly.conf"
54
+ check "AppArmor enforcing" aa-status --enabled
55
+
56
+ echo "Docker"
57
+ check "user namespace remapping" sh -c "docker info 2>/dev/null | grep -qi 'userns'"
58
+ check "no-new-privileges by default" sh -c "grep -q '\"no-new-privileges\": true' /etc/docker/daemon.json"
59
+ check "log size bounded" sh -c "grep -q 'max-size' /etc/docker/daemon.json"
60
+ check "live restore" sh -c "docker info 2>/dev/null | grep -qi 'Live Restore Enabled: true'"
61
+
62
+ echo "Secrets"
63
+ for k in bot_key payout_key; do
64
+ f=/srv/fly/secrets/$k
65
+ check "$k exists, mode 400" sh -c "[ -f $f ] && [ \"\$(stat -c %a $f)\" = 400 ]"
66
+ check "$k not owned by a login user" sh -c "[ \"\$(stat -c %U $f)\" != root ] || true; ! getent passwd \"\$(stat -c %U $f)\" | grep -qE ':/bin/(ba)?sh\$'"
67
+ done
68
+ check "secrets dir only root" sh -c "[ \"\$(stat -c %a /srv/fly/secrets)\" = 700 ]"
69
+ check "signer up (it refuses equal keys)" sh -c "docker inspect fly-fly-signer-1 --format '{{.State.Health.Status}}' | grep -q healthy"
70
+ check "release signer pinned" sh -c "[ -s /etc/fly/allowed_signers ] && [ \"\$(stat -c %U /etc/fly/allowed_signers)\" = root ]"
71
+ check "updater owned by root" sh -c "[ \"\$(stat -c %U /usr/local/lib/fly/fly_update.py)\" = root ] && ! [ -w /usr/local/lib/fly/fly_update.py -a \"\$(stat -c %a /usr/local/lib/fly/fly_update.py)\" != 755 ]"
72
+ for e in vault custody signer db; do
73
+ check "$e.env mode 600" sh -c "[ \"\$(stat -c %a /srv/fly/$e.env)\" = 600 ]"
74
+ done
75
+ check "no default database password" sh -c "! grep -qE 'POSTGRES_PASSWORD(:|=) *fly\$' /srv/fly/docker-compose.yml /srv/fly/db.env"
76
+ check "release signers namespaced" sh -c "! grep -v 'namespaces=\"fly-trader-release\"' /etc/fly/allowed_signers | grep -q ."
77
+ check "first release pinned (no TOFU)" sh -c "grep -Eq '\"expect_manifest_sha256\": *\"[0-9a-f]{64}\"' /etc/fly/update.json || [ -s /srv/fly/state.json ]"
78
+
79
+ echo "Key isolation"
80
+ check "keys mounted into the signer only" sh -c "! docker inspect fly-fly-1 --format '{{range .Mounts}}{{.Source}} {{end}}' | grep -q /srv/fly/secrets"
81
+ check "signer has the keys" sh -c "docker inspect fly-fly-signer-1 --format '{{range .Mounts}}{{.Source}} {{end}}' | grep -q /srv/fly/secrets/bot_key"
82
+ check "signer filesystem read-only" sh -c "docker inspect fly-fly-signer-1 --format '{{.HostConfig.ReadonlyRootfs}}' | grep -q true"
83
+ check "brain runs as uid 10001" sh -c "docker inspect fly-fly-1 --format '{{.Config.User}}' | grep -q '^10001'"
84
+ check "database has no route out" sh -c "docker network inspect fly_internal --format '{{.Internal}}' | grep -q true"
85
+
86
+ echo "Monitoring"
87
+ check "auditd running" systemctl is-active auditd
88
+ check "key reads audited" sh -c "auditctl -l | grep -q fly_keys"
89
+ check "key-read watcher running" systemctl is-active fly-audit-watch.timer
90
+ check "daily AIDE report to Telegram" systemctl is-active fly-aide.timer
91
+ check "AIDE baseline present" test -f /var/lib/aide/aide.db
92
+ check "journal persistent" test -d /var/log/journal
93
+ check "automatic security updates" sh -c "systemctl is-enabled unattended-upgrades && grep -q 'Automatic-Reboot' /etc/apt/apt.conf.d/52fly-unattended"
94
+ check "clock synchronised" sh -c "timedatectl show -p NTPSynchronized --value | grep -q yes"
95
+ check "no snapd" sh -c "! dpkg -l snapd 2>/dev/null | grep -q '^ii'"
96
+
97
+ echo "Lynis"
98
+ if command -v lynis >/dev/null; then
99
+ lynis audit system --quick --no-colors >/tmp/lynis.out 2>&1
100
+ printf ' hardening index: %s\n' "$(grep -o 'Hardening index : [0-9]*' /tmp/lynis.out | grep -o '[0-9]*$' || echo '?')"
101
+ grep -E '^\s+\* ' /var/log/lynis-report.dat >/dev/null 2>&1 || true
102
+ fi
103
+
104
+ echo
105
+ [ "$fail" = 0 ] && echo "all controls pass" || echo "SOME CONTROLS FAILED"
106
+ exit $fail
deploy/custody.env.example ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # /srv/fly/custody.env (mode 600): the treasury setup and the caps, read by BOTH the signer and the brain.
2
+ # Nothing secret: all of it is visible on chain or harmless. Comments on their own lines only.
3
+ # Create the multisig and its limits on https://fly-trader.app/owner.html with your owner wallet (docs/vault/RUNBOOK.md
4
+ # section 4.3); the page prints the three lines below. After changing any value: sudo python3 /usr/local/lib/fly/fly_update.py restart
5
+
6
+ # the Squads v4 multisig whose vault 0 is the treasury (members: your owner wallets only; never a server key)
7
+ VAULT_MULTISIG=
8
+ # L1: SOL, daily, member = the trading key, the ONLY destination = the trading wallet. A new limit has a new address.
9
+ VAULT_LIMIT_TRADING=
10
+ # L2: SOL, weekly, member = the payout key, any destination (claims)
11
+ VAULT_LIMIT_PAYOUT=
12
+ # the most SOL the trading wallet keeps (positions are sized on the whole book, spent from this float); also the largest
13
+ # single top-up the signer signs. Moves smaller than TOPUP_MIN_SOL are skipped.
14
+ TREASURY_FLOAT_SOL=2.0
15
+ TOPUP_MIN_SOL=0.1
16
+ # the fixed size for a model without a sizing table (sizing is otherwise MAX_POSITION_FRACTION of the book); must be > 0
17
+ MAX_POSITION_SOL=0.5
18
+ # the signer refuses any buy larger than this (a bound on a compromised brain, not sizing)
19
+ SIGNER_MAX_BUY_SOL=0.6
20
+ # the signer's own limits: SOL bought per day, swaps per minute, priority fee per swap, the worst value a swap may return
21
+ SIGNER_DAILY_BUY_SOL=10
22
+ SIGNER_SWAPS_PER_MIN=6
23
+ SIGNER_MAX_PRIORITY_SOL=0.005
24
+ SIGNER_MIN_VALUE_RATIO=0.6
25
+ # your age1... public key (NOT the release key): the signer encrypts the two hot keys to it, the brain the nightly ledger
26
+ VAULT_BACKUP_RECIPIENT=
deploy/docker-compose.vault.yml CHANGED
@@ -1,47 +1,91 @@
1
  # The hosted $FLY vault fly (docs/vault/SPEC.md). Installed by bootstrap-host.sh as /srv/fly/docker-compose.yml and
2
- # driven by the updater (fly_update.py): FLY_RELEASE picks the image tag it built from a verified release.
3
- # Nothing listens on a public interface: the console is on 127.0.0.1 (reach it with an SSH tunnel) and the fly only
4
- # makes outbound connections (Solana RPC, Jupiter, pumpapi, Robinhood Chain RPC, the site's relay, Telegram).
 
 
 
 
 
 
 
 
 
5
  services:
6
  db:
7
- image: postgres:17
 
8
  environment:
9
  POSTGRES_USER: fly
10
- POSTGRES_PASSWORD: fly
11
  POSTGRES_DB: fly_trader
12
  volumes:
13
  - pgdata:/var/lib/postgresql/data
 
14
  healthcheck:
15
  test: ["CMD-SHELL", "pg_isready -U fly -d fly_trader"]
16
  interval: 5s
17
  timeout: 3s
18
  retries: 30
19
  restart: unless-stopped
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
20
  logging: {driver: json-file, options: {max-size: "20m", max-file: "5"}}
 
21
  fly:
22
- image: fly-trader:${FLY_RELEASE:?set by the updater}
23
- env_file: /srv/fly/vault.env
24
  environment:
25
- DATABASE_URL: postgresql://fly:fly@db:5432/fly_trader
26
  DATA_DIR: /app/data
27
  LOG_DIR: /app/logs
28
- BOT_PRIVATE_KEY_FILE: /run/secrets/bot_key
29
  VAULT_ENABLED: "1"
 
30
  volumes:
31
  - /srv/fly/data:/app/data
32
  - /srv/fly/logs:/app/logs
33
- - /srv/fly/secrets/bot_key:/run/secrets/bot_key:ro
34
  - ${FLY_RELEASES:-/srv/fly/releases}:/releases:ro
 
35
  ports:
36
  - "127.0.0.1:8501:8501"
37
  depends_on:
38
  db:
39
  condition: service_healthy
 
 
40
  restart: unless-stopped
41
  stop_grace_period: 150s
42
  security_opt: ["no-new-privileges:true"]
43
  cap_drop: [ALL]
44
- mem_limit: 14g
45
  logging: {driver: json-file, options: {max-size: "50m", max-file: "5"}}
 
 
 
 
 
 
46
  volumes:
47
  pgdata:
 
1
  # The hosted $FLY vault fly (docs/vault/SPEC.md). Installed by bootstrap-host.sh as /srv/fly/docker-compose.yml and
2
+ # driven by the updater (fly_update.py): FLY_RELEASE picks the image tags it built from a verified release.
3
+ # Nothing listens on a public interface: the console is on 127.0.0.1 (reach it with an SSH tunnel over Tailscale) and
4
+ # the containers only make outbound connections.
5
+ #
6
+ # fly-signer the ONLY container that can read the two hot keys; it checks and signs, over a Unix socket in
7
+ # /srv/fly/run (a directory only it and the brain's group can enter). Read-only filesystem, no capabilities.
8
+ # fly the brain: engine, console, vault worker, data feeds. No key is mounted into it.
9
+ # db Postgres on an internal network (no route out), its password generated by bootstrap-host.sh.
10
+ #
11
+ # Settings: /srv/fly/custody.env (public: the treasury multisig, its limits, the float and caps; read by both),
12
+ # /srv/fly/signer.env (the signer's own RPC URL), /srv/fly/vault.env (the brain's API keys, relay, Telegram, backups),
13
+ # /srv/fly/db.env (the database password). All root-only (0600) on the host.
14
  services:
15
  db:
16
+ image: postgres:17@sha256:d74eeac9a635390a49bc21bd49fccd973de707e2a53a76ac49b552b8712ec46f
17
+ env_file: /srv/fly/db.env
18
  environment:
19
  POSTGRES_USER: fly
 
20
  POSTGRES_DB: fly_trader
21
  volumes:
22
  - pgdata:/var/lib/postgresql/data
23
+ networks: [internal]
24
  healthcheck:
25
  test: ["CMD-SHELL", "pg_isready -U fly -d fly_trader"]
26
  interval: 5s
27
  timeout: 3s
28
  retries: 30
29
  restart: unless-stopped
30
+ security_opt: ["no-new-privileges:true"]
31
+ logging: {driver: json-file, options: {max-size: "20m", max-file: "5"}}
32
+
33
+ fly-signer:
34
+ image: fly-signer:${FLY_RELEASE:-none}
35
+ env_file: [/srv/fly/custody.env, /srv/fly/signer.env]
36
+ environment:
37
+ BOT_PRIVATE_KEY_FILE: /run/secrets/bot_key
38
+ PAYOUT_PRIVATE_KEY_FILE: /run/secrets/payout_key
39
+ SIGNER_SOCKET: /run/fly-signer/sock
40
+ SIGNER_DB: /srv/signer/ledger.sqlite
41
+ volumes:
42
+ - /srv/fly/secrets/bot_key:/run/secrets/bot_key:ro
43
+ - /srv/fly/secrets/payout_key:/run/secrets/payout_key:ro
44
+ - /srv/fly/signer:/srv/signer
45
+ - /srv/fly/run:/run/fly-signer
46
+ networks: [egress] # its own RPC and Jupiter's price API; nothing reaches in
47
+ read_only: true
48
+ tmpfs: [/tmp]
49
+ restart: unless-stopped
50
+ security_opt: ["no-new-privileges:true"]
51
+ cap_drop: [ALL]
52
+ mem_limit: 512m
53
+ pids_limit: 64
54
  logging: {driver: json-file, options: {max-size: "20m", max-file: "5"}}
55
+
56
  fly:
57
+ image: fly-trader:${FLY_RELEASE:-none}
58
+ env_file: [/srv/fly/custody.env, /srv/fly/vault.env, /srv/fly/db.env]
59
  environment:
 
60
  DATA_DIR: /app/data
61
  LOG_DIR: /app/logs
62
+ SIGNER_SOCKET: /run/fly-signer/sock
63
  VAULT_ENABLED: "1"
64
+ AUTO_UPDATE: "0" # releases come through the host updater (signed, 24 h hold on code), never from inside
65
  volumes:
66
  - /srv/fly/data:/app/data
67
  - /srv/fly/logs:/app/logs
68
+ - /srv/fly/run:/run/fly-signer
69
  - ${FLY_RELEASES:-/srv/fly/releases}:/releases:ro
70
+ networks: [internal, egress]
71
  ports:
72
  - "127.0.0.1:8501:8501"
73
  depends_on:
74
  db:
75
  condition: service_healthy
76
+ fly-signer:
77
+ condition: service_healthy
78
  restart: unless-stopped
79
  stop_grace_period: 150s
80
  security_opt: ["no-new-privileges:true"]
81
  cap_drop: [ALL]
82
+ mem_limit: 48g # a dedicated 128 GB box; lower it on a smaller one (16 GB: 14g)
83
  logging: {driver: json-file, options: {max-size: "50m", max-file: "5"}}
84
+
85
+ networks:
86
+ internal:
87
+ internal: true # the database has no route to anything but its two clients
88
+ egress: {}
89
+
90
  volumes:
91
  pgdata:
deploy/fly ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/sh
2
+ # Run a fly-trader command inside the running brain container, e.g. sudo fly vault status
3
+ # sudo fly signer <cmd> runs python -m fly_trader.signer <cmd> in the signer container (status, clear-panic)
4
+ set -e
5
+ if [ "$1" = "signer" ]; then
6
+ shift
7
+ exec docker exec -i fly-fly-signer-1 python -m fly_trader.signer "$@"
8
+ fi
9
+ exec docker exec -i fly-fly-1 fly-trader "$@"
deploy/fly-aide.service ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ [Unit]
2
+ Description=fly-trader vault: daily file-integrity check (AIDE), findings to Telegram
3
+
4
+ [Service]
5
+ Type=oneshot
6
+ Nice=10
7
+ ExecStart=/usr/bin/python3 /usr/local/lib/fly/fly_audit_watch.py aide
8
+ TimeoutStartSec=3600
deploy/fly-aide.timer ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ [Unit]
2
+ Description=Run the AIDE check daily
3
+
4
+ [Timer]
5
+ OnCalendar=*-*-* 05:30:00 UTC
6
+ Persistent=true
7
+
8
+ [Install]
9
+ WantedBy=timers.target
deploy/fly-audit-watch.service ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ [Unit]
2
+ Description=fly-trader vault: alert when anything but the signer reads a wallet key
3
+ After=auditd.service
4
+
5
+ [Service]
6
+ Type=oneshot
7
+ ExecStart=/usr/bin/python3 /usr/local/lib/fly/fly_audit_watch.py run
8
+ TimeoutStartSec=120
deploy/fly-audit-watch.timer ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ [Unit]
2
+ Description=Check the key-read audit log every minute
3
+
4
+ [Timer]
5
+ OnBootSec=1min
6
+ OnUnitActiveSec=1min
7
+ AccuracySec=5s
8
+
9
+ [Install]
10
+ WantedBy=timers.target
deploy/fly_audit_watch.py ADDED
@@ -0,0 +1,109 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Tell the operator when anything but the signer reads a wallet key (runs on the HOST, root, every minute).
3
+
4
+ auditd logs every access to /srv/fly/secrets (key ``fly_keys``, bootstrap-host.sh). The signer container's Python is
5
+ the only reader there should ever be (plus AIDE's daily integrity scan); a read by anything else -- a shell, cat, a
6
+ process in the brain container -- is the earliest sign of a compromise, so it goes to Telegram at once, with the
7
+ program, user and time. ``ausearch --checkpoint`` makes each event reported once.
8
+
9
+ fly_audit_watch.py run (the systemd timer, every minute)
10
+ fly_audit_watch.py aide (fly-aide.timer, daily: AIDE's integrity check; changes go to Telegram, since
11
+ this host has no mail. After reviewing expected changes: sudo aide --update and
12
+ sudo cp /var/lib/aide/aide.db.new /var/lib/aide/aide.db)
13
+ fly_audit_watch.py test (send a test message)
14
+ """
15
+ from __future__ import annotations
16
+
17
+ import json
18
+ import re
19
+ import subprocess
20
+ import sys
21
+ import urllib.request
22
+ from pathlib import Path
23
+
24
+ CONF = Path("/etc/fly/update.json") # the updater's Telegram settings
25
+ CHECKPOINT = Path("/var/lib/fly/audit.checkpoint")
26
+ SIGNER_EXE = {"/usr/local/bin/python3.13", "/usr/local/bin/python3"} # the signer image's interpreter
27
+ ALLOWED_EXE = {"/usr/bin/aide", "/usr/sbin/aide"} # AIDE hashes every file daily
28
+ SIGNER_UID_OFFSET = 10002 # the signer container's user; on the host: dockremap base + this
29
+
30
+ FIELD = re.compile(r'(\w+)=("[^"]*"|\S+)')
31
+
32
+
33
+ def remap_base() -> int:
34
+ for line in Path("/etc/subuid").read_text().splitlines():
35
+ name, start, _ = line.split(":")
36
+ if name == "dockremap":
37
+ return int(start)
38
+ return -1
39
+
40
+
41
+ def events(raw: str) -> list[dict]:
42
+ """SYSCALL records of ``ausearch --raw`` output as dicts (pure)."""
43
+ out = []
44
+ for line in raw.splitlines():
45
+ if not line.startswith("type=SYSCALL"):
46
+ continue
47
+ out.append({k: v.strip('"') for k, v in FIELD.findall(line)})
48
+ return out
49
+
50
+
51
+ def suspicious(evs: list[dict], signer_uid: int) -> list[dict]:
52
+ """Reads that are not the signer's interpreter as the signer's user, nor AIDE (pure)."""
53
+ bad = []
54
+ for e in evs:
55
+ exe, uid = e.get("exe", ""), int(e.get("uid", "-1")) if e.get("uid", "").lstrip("-").isdigit() else -1
56
+ if exe in SIGNER_EXE and uid == signer_uid:
57
+ continue
58
+ if exe in ALLOWED_EXE and uid == 0:
59
+ continue
60
+ bad.append(e)
61
+ return bad
62
+
63
+
64
+ def alert(text: str) -> None:
65
+ c = json.loads(CONF.read_text()) if CONF.exists() else {}
66
+ tok, chat = c.get("telegram_bot_token"), c.get("telegram_chat_id")
67
+ print("ALERT", text, flush=True)
68
+ if tok and chat:
69
+ body = json.dumps({"chat_id": chat, "text": "[fly host] " + text}).encode()
70
+ urllib.request.urlopen(urllib.request.Request(f"https://api.telegram.org/bot{tok}/sendMessage", data=body,
71
+ headers={"Content-Type": "application/json"}), timeout=10)
72
+
73
+
74
+ def run() -> int:
75
+ CHECKPOINT.parent.mkdir(parents=True, exist_ok=True)
76
+ r = subprocess.run(["ausearch", "-k", "fly_keys", "--raw", "--checkpoint", str(CHECKPOINT), "--start", "checkpoint"],
77
+ capture_output=True, text=True)
78
+ if r.returncode not in (0, 1): # 1: no matches; 10/11/12: checkpoint problems -> start fresh next run
79
+ CHECKPOINT.unlink(missing_ok=True)
80
+ return 0
81
+ bad = suspicious(events(r.stdout), remap_base() + SIGNER_UID_OFFSET)
82
+ if bad:
83
+ lines = [f"{e.get('exe')} (uid {e.get('uid')}, comm {e.get('comm')}, syscall {e.get('syscall')})" for e in bad[:10]]
84
+ alert(f"WALLET KEY READ by something other than the signer ({len(bad)} event(s)):\n" + "\n".join(lines)
85
+ + "\nIf this was not you: revoke L1 and L2 in the owner wallet now, then investigate.")
86
+ return len(bad)
87
+
88
+
89
+ def aide() -> int:
90
+ """AIDE's exit code: 0 no change, 1-7 added/removed/changed files, >= 14 an error."""
91
+ r = subprocess.run(["aide", "--check", "--config", "/etc/aide/aide.conf"], capture_output=True, text=True)
92
+ if r.returncode == 0:
93
+ return 0
94
+ summary = [l for l in r.stdout.splitlines() if l.strip() and (l.startswith(("Added", "Removed", "Changed", "f", "d", "l"))
95
+ or "entries" in l)][:25]
96
+ what = "an ERROR" if r.returncode >= 14 else "file changes"
97
+ alert(f"AIDE found {what} (exit {r.returncode}). If you did not make them (a release, apt upgrades), investigate:\n"
98
+ + "\n".join(summary) + "\nFull report: sudo aide --check")
99
+ return r.returncode
100
+
101
+
102
+ if __name__ == "__main__":
103
+ cmd = sys.argv[1] if len(sys.argv) > 1 else "run"
104
+ if cmd == "test":
105
+ alert("audit watch test message")
106
+ elif cmd == "aide":
107
+ aide()
108
+ else:
109
+ run()
deploy/fly_update.py CHANGED
@@ -3,7 +3,8 @@
3
 
4
  Runs on the HOST (root, systemd timer, standard library only), never inside the container, because it rebuilds and
5
  restarts the container. Everything it trusts comes from files on this host that no release can change:
6
- ``/etc/fly/allowed_signers`` (Bryce's release key) and ``/etc/fly/update.json``.
 
7
 
8
  One round:
9
  1. poll the HF model repo's commit sha; nothing to do if it is the installed one
@@ -12,13 +13,19 @@ One round:
12
  no forks)
13
  3. download exactly the listed paths (strict path check), verify size + sha256 of each, into a fresh directory
14
  4. compare file classes with the installed release:
15
- only model/seed -> ``docker compose exec fly fly-trader apply-release /releases/<seq>`` (hot swap)
16
- code -> wait for ``ready-for-restart``, build ``fly-trader:<seq>`` from that directory, recreate,
17
- wait for healthy, else roll back to the previous tag
18
- infra -> stop and ask: ``fly_update.py approve <seq>`` (compose/Dockerfile/entrypoint can grant
19
- host access, so a human looks first)
20
-
21
- fly_update.py run | approve <seq> | status | verify <dir>
 
 
 
 
 
 
22
  """
23
  from __future__ import annotations
24
 
@@ -37,13 +44,17 @@ from pathlib import Path
37
  CONF = Path(os.environ.get("FLY_UPDATE_CONF", "/etc/fly/update.json"))
38
  PATH_RE = re.compile(r"^(?!.*\.\.)(?!/)[A-Za-z0-9_.\-/]+$")
39
  INFRA = ("Dockerfile", "docker-compose.yml", "docker-compose.cuda.yml", ".dockerignore", "docker/", "deploy/")
 
 
 
40
 
41
 
42
  def conf() -> dict:
43
  c = json.loads(CONF.read_text())
44
  c.setdefault("repo", "bryceweiner/fly-trader")
45
  c.setdefault("allowed_signers", "/etc/fly/allowed_signers")
46
- c.setdefault("principal", "bryce")
 
47
  c.setdefault("namespace", "fly-trader-release")
48
  c.setdefault("root", "/srv/fly")
49
  c.setdefault("project", "fly")
@@ -95,13 +106,19 @@ def classify(path: str) -> str:
95
  return "code"
96
 
97
 
98
- def verify_signature(c: dict, manifest: bytes, sig: bytes) -> None:
 
 
 
99
  with tempfile.TemporaryDirectory() as d:
100
  sp = Path(d) / "release.json.sig"; sp.write_bytes(sig)
101
- r = subprocess.run(["ssh-keygen", "-Y", "verify", "-f", c["allowed_signers"], "-I", c["principal"], "-n", c["namespace"], "-s", str(sp)],
102
- input=manifest, capture_output=True)
103
- if r.returncode != 0:
104
- raise SystemExit(f"release signature does NOT verify: {r.stderr.decode(errors='replace').strip()[:300]}")
 
 
 
105
 
106
 
107
  def check_manifest(m: dict) -> None:
@@ -113,6 +130,8 @@ def check_manifest(m: dict) -> None:
113
  raise SystemExit(f"refusing path {p!r}")
114
  if f.get("class") != classify(p):
115
  raise SystemExit(f"{p}: class {f.get('class')} does not match {classify(p)}")
 
 
116
  if not re.fullmatch(r"[0-9a-f]{64}", f.get("sha256", "")) or not isinstance(f.get("size"), int):
117
  raise SystemExit(f"{p}: bad hash or size")
118
 
@@ -146,19 +165,27 @@ def save_state(c: dict, st: dict) -> None:
146
 
147
 
148
  def compose(c: dict, *args: str, check: bool = True, env: dict | None = None) -> subprocess.CompletedProcess:
149
- e = {**os.environ, **(env or {})}
 
 
150
  return subprocess.run(["docker", "compose", "-p", c["project"], "-f", str(Path(c["root"]) / "docker-compose.yml"), *args],
151
  cwd=c["root"], check=check, capture_output=True, text=True, env=e)
152
 
153
 
154
- def fetch(c: dict, sha: str) -> tuple[dict, Path, bytes]:
155
  base = f"https://huggingface.co/{c['repo']}/resolve/{sha}/"
156
  raw = http(base + "release.json"); sig = http(base + "release.json.sig")
157
- verify_signature(c, raw, sig)
158
  m = json.loads(raw)
159
  check_manifest(m)
160
  dest = Path(c["root"]) / "releases" / str(m["seq"])
161
  if dest.exists():
 
 
 
 
 
 
162
  shutil.rmtree(dest)
163
  part = dest.with_name(dest.name + ".part")
164
  if part.exists():
@@ -170,7 +197,7 @@ def fetch(c: dict, sha: str) -> tuple[dict, Path, bytes]:
170
  (part / "release.json").write_bytes(raw); (part / "release.json.sig").write_bytes(sig)
171
  verify_dir(part, m)
172
  part.rename(dest)
173
- return m, dest, raw
174
 
175
 
176
  def installed_manifest(c: dict, st: dict) -> dict | None:
@@ -212,82 +239,155 @@ def healthy(c: dict, limit_s: int = 900) -> bool:
212
  return False
213
 
214
 
215
- def deploy_code(c: dict, st: dict, m: dict, d: Path, infra: bool) -> bool:
 
216
  tag = f"fly-trader:{m['seq']}"
217
- subprocess.run(["docker", "build", "-t", tag, str(d)], check=True, capture_output=True)
218
- if infra: # approved: the release's compose file becomes the running one
219
- shutil.copy2(d / "deploy" / "docker-compose.vault.yml", Path(c["root"]) / "docker-compose.yml")
220
- if not wait_ready(c):
 
 
 
 
221
  alert(c, f"release {m['seq']}: the fly never became ready for a restart (orders/claims/settlement in flight); will retry")
222
- return False
 
 
 
223
  prev = st.get("image")
224
- compose(c, "up", "-d", env={"FLY_RELEASE": str(m["seq"]), "FLY_RELEASES": str(Path(c["root"]) / "releases")})
 
225
  if healthy(c):
226
- return True
227
  alert(c, f"release {m['seq']} did not become healthy; rolling back to {prev}")
 
 
228
  if prev:
229
- compose(c, "up", "-d", env={"FLY_RELEASE": str(prev), "FLY_RELEASES": str(Path(c["root"]) / "releases")})
230
- return False
 
231
 
 
 
232
 
233
- def apply(c: dict, st: dict, m: dict, d: Path, raw: bytes, hf_sha: str, approved: bool = False) -> None:
 
 
 
 
234
  old = installed_manifest(c, st)
235
- if old is not None and m["seq"] <= old["seq"]:
236
- raise SystemExit(f"release seq {m['seq']} is not newer than installed {old['seq']} (rollback refused)")
237
- if old is not None and m.get("prev_sha256") != st.get("manifest_sha256"):
238
- raise SystemExit("release does not chain from the installed one (prev_sha256 mismatch)")
 
 
 
 
 
 
 
 
 
239
  classes = changed_classes(old, m)
240
- if "infra" in classes and not approved:
241
- st["pending"] = {"seq": m["seq"], "hf_sha": hf_sha}
 
 
 
 
 
 
 
 
 
 
 
242
  save_state(c, st)
243
- alert(c, f"release {m['seq']} changes Docker/compose/entrypoint files: review, then run `fly_update.py approve {m['seq']}`")
244
- return
 
 
245
  if classes & {"code", "infra"} or st.get("image") is None:
246
- if not deploy_code(c, st, m, d, "infra" in classes):
 
247
  return
 
 
248
  st["image"] = m["seq"]
249
  elif classes & {"model", "seed"}:
250
  r = compose(c, "exec", "-T", "fly", "fly-trader", "apply-release", f"/releases/{m['seq']}", check=False)
251
  if r.returncode != 0:
252
- alert(c, f"release {m['seq']}: apply-release failed: {(r.stderr or r.stdout)[-300:]}")
253
- return
254
- st.update({"seq": m["seq"], "manifest_sha256": hashlib.sha256(raw).hexdigest(), "hf_sha": hf_sha, "pending": None})
 
 
 
255
  save_state(c, st)
256
- alert(c, f"release {m['seq']} applied ({', '.join(sorted(classes))})")
257
 
258
 
259
- def run(approve_seq: int | None = None) -> None:
260
  c = conf()
 
261
  Path(c["root"], "releases").mkdir(parents=True, exist_ok=True)
262
  st = load_state(c)
263
- info = json.loads(http(f"https://huggingface.co/api/models/{c['repo']}"))
264
- sha = info["sha"]
265
  if approve_seq is not None:
266
- pend = st.get("pending") or {}
267
  if pend.get("seq") != approve_seq:
268
  raise SystemExit(f"no pending release {approve_seq}")
269
  sha = pend["hf_sha"]
270
- elif sha in (st.get("hf_sha"), st.get("hf_sha_rejected")) or (st.get("pending") or {}).get("hf_sha") == sha:
271
- return # installed, already rejected (alerted once), or waiting for approval
 
 
 
 
 
 
272
  try:
273
- m, d, raw = fetch(c, sha)
274
- apply(c, st, m, d, raw, sha, approved=approve_seq is not None)
275
  except SystemExit as e:
 
 
 
276
  st["hf_sha_rejected"] = sha; save_state(c, st)
277
  alert(c, f"release at {sha[:10]} rejected: {e}")
278
  raise
279
 
280
 
 
 
 
 
 
 
 
 
 
 
 
281
  def main(argv: list[str]) -> None:
282
  cmd = argv[1] if len(argv) > 1 else "run"
283
  if cmd == "run":
284
  run()
285
  elif cmd == "approve":
286
  run(int(argv[2]))
 
 
 
 
 
 
 
 
287
  elif cmd == "status":
288
  print(json.dumps(load_state(conf()), indent=1))
289
  elif cmd == "verify": # used by tools/publish_release.py on the Mac before uploading
290
- d = Path(argv[2]); c = {"allowed_signers": argv[3], "principal": argv[4], "namespace": "fly-trader-release"}
291
  raw = (d / "release.json").read_bytes()
292
  verify_signature(c, raw, (d / "release.json.sig").read_bytes())
293
  m = json.loads(raw); check_manifest(m); verify_dir(d, m)
 
3
 
4
  Runs on the HOST (root, systemd timer, standard library only), never inside the container, because it rebuilds and
5
  restarts the container. Everything it trusts comes from files on this host that no release can change:
6
+ ``/etc/fly/allowed_signers`` (Bryce's release keys, restricted to the release namespace) and ``/etc/fly/update.json``
7
+ (which also pins the very first release by its manifest hash: no trust on first use).
8
 
9
  One round:
10
  1. poll the HF model repo's commit sha; nothing to do if it is the installed one
 
13
  no forks)
14
  3. download exactly the listed paths (strict path check), verify size + sha256 of each, into a fresh directory
15
  4. compare file classes with the installed release:
16
+ only model/seed -> ``docker compose exec fly fly-trader apply-release /releases/<seq>`` (hot swap; models
17
+ cannot run code, fly_trader/train/model_io.py); unhealthy after it -> the previous release's
18
+ models are re-applied
19
+ code -> held for ``code_delay_s`` (24 h) with a Telegram notice, so a stolen signing key cannot ship
20
+ code faster than Bryce can ``veto`` it; ``approve`` applies it at once. Then: wait for
21
+ ``ready-for-restart``, build ``fly-trader:<seq>`` from that directory, recreate, wait for
22
+ healthy, else roll back to the previous tag and compose file
23
+ infra -> held until ``approve`` (compose/Dockerfile/entrypoint can grant host access, so a human
24
+ looks first)
25
+
26
+ fly_update.py run | approve <seq> | veto <seq> | status | restart | verify <dir> <allowed_signers> [principal ...]
27
+
28
+ ``restart`` recreates the containers on the installed release (after you edit /srv/fly/*.env).
29
  """
30
  from __future__ import annotations
31
 
 
44
  CONF = Path(os.environ.get("FLY_UPDATE_CONF", "/etc/fly/update.json"))
45
  PATH_RE = re.compile(r"^(?!.*\.\.)(?!/)[A-Za-z0-9_.\-/]+$")
46
  INFRA = ("Dockerfile", "docker-compose.yml", "docker-compose.cuda.yml", ".dockerignore", "docker/", "deploy/")
47
+ # what models/ and seed/ may hold: nothing a loader could execute (fly_trader/train/model_io.SAFE_SUFFIXES)
48
+ MODEL_SUFFIXES = {".pt", ".skops", ".npz", ".json", ".bin", ".parquet", ".txt", ".sha256", ""}
49
+ SEED_SUFFIXES = {".json", ".sql"}
50
 
51
 
52
  def conf() -> dict:
53
  c = json.loads(CONF.read_text())
54
  c.setdefault("repo", "bryceweiner/fly-trader")
55
  c.setdefault("allowed_signers", "/etc/fly/allowed_signers")
56
+ c.setdefault("principals", [c.get("principal") or "bryce", "bryce-recovery"])
57
+ c.setdefault("code_delay_s", 86400)
58
  c.setdefault("namespace", "fly-trader-release")
59
  c.setdefault("root", "/srv/fly")
60
  c.setdefault("project", "fly")
 
106
  return "code"
107
 
108
 
109
+ def verify_signature(c: dict, manifest: bytes, sig: bytes) -> str:
110
+ """The principal whose key signed ``manifest`` (any of ``principals``), else SystemExit."""
111
+ principals = c.get("principals") or [c["principal"]]
112
+ err = ""
113
  with tempfile.TemporaryDirectory() as d:
114
  sp = Path(d) / "release.json.sig"; sp.write_bytes(sig)
115
+ for who in principals:
116
+ r = subprocess.run(["ssh-keygen", "-Y", "verify", "-f", c["allowed_signers"], "-I", who, "-n", c["namespace"], "-s", str(sp)],
117
+ input=manifest, capture_output=True)
118
+ if r.returncode == 0:
119
+ return who
120
+ err = r.stderr.decode(errors="replace").strip()
121
+ raise SystemExit(f"release signature does NOT verify: {err[:300]}")
122
 
123
 
124
  def check_manifest(m: dict) -> None:
 
130
  raise SystemExit(f"refusing path {p!r}")
131
  if f.get("class") != classify(p):
132
  raise SystemExit(f"{p}: class {f.get('class')} does not match {classify(p)}")
133
+ if f["class"] == "model" and Path(p).suffix not in MODEL_SUFFIXES or f["class"] == "seed" and Path(p).suffix not in SEED_SUFFIXES:
134
+ raise SystemExit(f"{p}: file type not allowed in {f['class']}/ (a model or seed must not be able to run code)")
135
  if not re.fullmatch(r"[0-9a-f]{64}", f.get("sha256", "")) or not isinstance(f.get("size"), int):
136
  raise SystemExit(f"{p}: bad hash or size")
137
 
 
165
 
166
 
167
  def compose(c: dict, *args: str, check: bool = True, env: dict | None = None) -> subprocess.CompletedProcess:
168
+ """docker compose on the installed release (FLY_RELEASE = the image tag this host runs, unless ``env`` says otherwise)."""
169
+ installed_tag = load_state(c).get("image")
170
+ e = {**os.environ, "FLY_RELEASE": str(installed_tag) if installed_tag is not None else "none", **(env or {})}
171
  return subprocess.run(["docker", "compose", "-p", c["project"], "-f", str(Path(c["root"]) / "docker-compose.yml"), *args],
172
  cwd=c["root"], check=check, capture_output=True, text=True, env=e)
173
 
174
 
175
+ def fetch(c: dict, sha: str) -> tuple[dict, Path, bytes, str]:
176
  base = f"https://huggingface.co/{c['repo']}/resolve/{sha}/"
177
  raw = http(base + "release.json"); sig = http(base + "release.json.sig")
178
+ who = verify_signature(c, raw, sig)
179
  m = json.loads(raw)
180
  check_manifest(m)
181
  dest = Path(c["root"]) / "releases" / str(m["seq"])
182
  if dest.exists():
183
+ try: # already downloaded (a held or retried release): re-verify, reuse
184
+ if (dest / "release.json").read_bytes() == raw:
185
+ verify_dir(dest, m)
186
+ return m, dest, raw, who
187
+ except (SystemExit, OSError):
188
+ pass
189
  shutil.rmtree(dest)
190
  part = dest.with_name(dest.name + ".part")
191
  if part.exists():
 
197
  (part / "release.json").write_bytes(raw); (part / "release.json.sig").write_bytes(sig)
198
  verify_dir(part, m)
199
  part.rename(dest)
200
+ return m, dest, raw, who
201
 
202
 
203
  def installed_manifest(c: dict, st: dict) -> dict | None:
 
239
  return False
240
 
241
 
242
+ def deploy_code(c: dict, st: dict, m: dict, d: Path, infra: bool) -> str:
243
+ """'ok', 'busy' (not ready for a restart: retried next round) or 'failed' (rolled back)."""
244
  tag = f"fly-trader:{m['seq']}"
245
+ root = Path(c["root"]); live, backup = root / "docker-compose.yml", root / "docker-compose.yml.prev"
246
+ try: # the brain (default stage) and the key-holding signer, same release
247
+ subprocess.run(["docker", "build", "-t", tag, str(d)], check=True, capture_output=True)
248
+ subprocess.run(["docker", "build", "--target", "signer", "-t", f"fly-signer:{m['seq']}", str(d)], check=True, capture_output=True)
249
+ except subprocess.CalledProcessError as e:
250
+ alert(c, f"release {m['seq']}: image build failed: {(e.stderr or b'').decode(errors='replace')[-300:]}")
251
+ return "failed"
252
+ if st.get("image") is not None and not wait_ready(c): # the first install has nothing running to wait for
253
  alert(c, f"release {m['seq']}: the fly never became ready for a restart (orders/claims/settlement in flight); will retry")
254
+ return "busy"
255
+ if infra: # approved: the release's compose file becomes the running one
256
+ shutil.copy2(live, backup)
257
+ shutil.copy2(d / "deploy" / "docker-compose.vault.yml", live)
258
  prev = st.get("image")
259
+ env = {"FLY_RELEASES": str(root / "releases")}
260
+ compose(c, "up", "-d", check=False, env={**env, "FLY_RELEASE": str(m["seq"])})
261
  if healthy(c):
262
+ return "ok"
263
  alert(c, f"release {m['seq']} did not become healthy; rolling back to {prev}")
264
+ if infra and backup.exists():
265
+ shutil.copy2(backup, live)
266
  if prev:
267
+ compose(c, "up", "-d", check=False, env={**env, "FLY_RELEASE": str(prev)})
268
+ return "failed"
269
+
270
 
271
+ def _when(ts: float) -> str:
272
+ return time.strftime("%Y-%m-%d %H:%M UTC", time.gmtime(ts))
273
 
274
+
275
+ def apply(c: dict, st: dict, m: dict, d: Path, raw: bytes, hf_sha: str, approved: bool = False, who: str = "?",
276
+ now: float | None = None) -> None:
277
+ now = time.time() if now is None else now
278
+ msha = hashlib.sha256(raw).hexdigest()
279
  old = installed_manifest(c, st)
280
+ if old is None: # the first release is pinned by hand at bootstrap (no TOFU)
281
+ pin = c.get("expect_manifest_sha256")
282
+ if not pin:
283
+ raise SystemExit("no release installed and update.json pins none (expect_manifest_sha256): refusing to trust the first one seen")
284
+ if msha != pin:
285
+ raise SystemExit(f"first release's manifest {msha[:12]} is not the pinned {str(pin)[:12]}")
286
+ else:
287
+ if m["seq"] <= old["seq"]:
288
+ raise SystemExit(f"release seq {m['seq']} is not newer than installed {old['seq']} (rollback refused)")
289
+ pend = st.get("pending") or {}
290
+ chain = {st.get("manifest_sha256"), pend.get("manifest_sha256"), *(st.get("vetoed") or [])} - {None}
291
+ if m.get("prev_sha256") not in chain:
292
+ raise SystemExit("release does not chain from the installed one (prev_sha256 mismatch)")
293
  classes = changed_classes(old, m)
294
+ if old is not None and classes & {"code", "infra"}:
295
+ pend = st.get("pending") or {}
296
+ if pend.get("manifest_sha256") != msha: # newly seen: the delay starts now (a replaced pending one restarts it)
297
+ pend = {"seq": m["seq"], "hf_sha": hf_sha, "manifest_sha256": msha, "classes": sorted(classes), "signer": who,
298
+ "first_seen": now, "due_at": now + float(c.get("code_delay_s", 86400)), "approved": False}
299
+ st["pending"] = pend
300
+ what = ("changes Docker/compose/entrypoint files: review it, then `sudo python3 /usr/local/lib/fly/fly_update.py approve "
301
+ f"{m['seq']}`" if "infra" in classes else
302
+ f"changes code: it applies at {_when(pend['due_at'])} unless you run `sudo python3 /usr/local/lib/fly/fly_update.py veto "
303
+ f"{m['seq']}` (`approve {m['seq']}` applies it now)")
304
+ alert(c, f"release {m['seq']} signed by {who} {what}")
305
+ if approved:
306
+ pend["approved"] = True
307
  save_state(c, st)
308
+ if "infra" in classes and not pend["approved"]:
309
+ return
310
+ if not (pend["approved"] or now >= pend["due_at"]):
311
+ return
312
  if classes & {"code", "infra"} or st.get("image") is None:
313
+ res = deploy_code(c, st, m, d, "infra" in classes)
314
+ if res == "busy":
315
  return
316
+ if res == "failed":
317
+ raise SystemExit("the new image did not come up healthy (rolled back)")
318
  st["image"] = m["seq"]
319
  elif classes & {"model", "seed"}:
320
  r = compose(c, "exec", "-T", "fly", "fly-trader", "apply-release", f"/releases/{m['seq']}", check=False)
321
  if r.returncode != 0:
322
+ raise SystemExit(f"apply-release failed: {(r.stderr or r.stdout)[-300:]}")
323
+ if not healthy(c, 300):
324
+ if old is not None:
325
+ compose(c, "exec", "-T", "fly", "fly-trader", "apply-release", f"/releases/{old['seq']}", check=False)
326
+ raise SystemExit("unhealthy after the model release; the previous release's models were re-applied")
327
+ st.update({"seq": m["seq"], "manifest_sha256": msha, "hf_sha": hf_sha, "pending": None})
328
  save_state(c, st)
329
+ alert(c, f"release {m['seq']} applied ({', '.join(sorted(classes))}), signed by {who}")
330
 
331
 
332
+ def run(approve_seq: int | None = None, now: float | None = None) -> None:
333
  c = conf()
334
+ now = time.time() if now is None else now
335
  Path(c["root"], "releases").mkdir(parents=True, exist_ok=True)
336
  st = load_state(c)
337
+ pend = st.get("pending") or {}
 
338
  if approve_seq is not None:
 
339
  if pend.get("seq") != approve_seq:
340
  raise SystemExit(f"no pending release {approve_seq}")
341
  sha = pend["hf_sha"]
342
+ else:
343
+ head = json.loads(http(f"https://huggingface.co/api/models/{c['repo']}"))["sha"]
344
+ if head not in (st.get("hf_sha"), st.get("hf_sha_rejected"), pend.get("hf_sha")):
345
+ sha = head # a new release
346
+ elif pend and (pend.get("approved") or now >= pend.get("due_at", float("inf"))):
347
+ sha = pend["hf_sha"] # a held release whose time came (or a retry after 'busy')
348
+ else:
349
+ return # installed, rejected (alerted once), or still held
350
  try:
351
+ m, d, raw, who = fetch(c, sha)
352
+ apply(c, st, m, d, raw, sha, approved=approve_seq is not None, who=who, now=now)
353
  except SystemExit as e:
354
+ st = load_state(c)
355
+ if (st.get("pending") or {}).get("hf_sha") == sha:
356
+ st["pending"] = None
357
  st["hf_sha_rejected"] = sha; save_state(c, st)
358
  alert(c, f"release at {sha[:10]} rejected: {e}")
359
  raise
360
 
361
 
362
+ def veto(seq: int) -> None:
363
+ c = conf(); st = load_state(c); pend = st.get("pending") or {}
364
+ if pend.get("seq") != seq:
365
+ raise SystemExit(f"no pending release {seq}")
366
+ st["vetoed"] = [*(st.get("vetoed") or []), pend["manifest_sha256"]][-20:]
367
+ st["hf_sha_rejected"] = pend["hf_sha"]; st["pending"] = None
368
+ save_state(c, st)
369
+ alert(c, f"release {seq} VETOED: it will never apply. If you did not publish it, your release key is compromised: "
370
+ "remove it from /etc/fly/allowed_signers and rotate.")
371
+
372
+
373
  def main(argv: list[str]) -> None:
374
  cmd = argv[1] if len(argv) > 1 else "run"
375
  if cmd == "run":
376
  run()
377
  elif cmd == "approve":
378
  run(int(argv[2]))
379
+ elif cmd == "veto":
380
+ veto(int(argv[2]))
381
+ elif cmd == "restart":
382
+ c = conf()
383
+ if load_state(c).get("image") is None:
384
+ raise SystemExit("nothing installed yet")
385
+ r = compose(c, "up", "-d", "--force-recreate", check=False, env={"FLY_RELEASES": str(Path(c["root"]) / "releases")})
386
+ print((r.stdout or "") + (r.stderr or ""))
387
  elif cmd == "status":
388
  print(json.dumps(load_state(conf()), indent=1))
389
  elif cmd == "verify": # used by tools/publish_release.py on the Mac before uploading
390
+ d = Path(argv[2]); c = {"allowed_signers": argv[3], "principals": argv[4:] or ["bryce"], "namespace": "fly-trader-release"}
391
  raw = (d / "release.json").read_bytes()
392
  verify_signature(c, raw, (d / "release.json.sig").read_bytes())
393
  m = json.loads(raw); check_manifest(m); verify_dir(d, m)
deploy/signer.env.example ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ # /srv/fly/signer.env (mode 600): the signer's own settings. Only the signer container reads this file.
2
+ # REQUIRED: the signer does not start without it. Use a different provider from the brain's Helius: it simulates every
3
+ # swap and reads the treasury itself. (Its value check reads Jupiter's public price API, lite-api.jup.ag, no key.)
4
+ SIGNER_RPC_URL=
deploy/update.json.example CHANGED
@@ -1,10 +1,15 @@
1
  {
2
  "repo": "bryceweiner/fly-trader",
3
  "allowed_signers": "/etc/fly/allowed_signers",
4
- "principal": "bryce",
 
 
 
5
  "namespace": "fly-trader-release",
6
  "root": "/srv/fly",
7
  "project": "fly",
 
 
8
  "telegram_bot_token": "",
9
  "telegram_chat_id": ""
10
  }
 
1
  {
2
  "repo": "bryceweiner/fly-trader",
3
  "allowed_signers": "/etc/fly/allowed_signers",
4
+ "principals": [
5
+ "bryce",
6
+ "bryce-recovery"
7
+ ],
8
  "namespace": "fly-trader-release",
9
  "root": "/srv/fly",
10
  "project": "fly",
11
+ "code_delay_s": 86400,
12
+ "expect_manifest_sha256": "",
13
  "telegram_bot_token": "",
14
  "telegram_chat_id": ""
15
  }
deploy/vault.env.example CHANGED
@@ -1,16 +1,18 @@
1
- # /srv/fly/vault.env for the hosted $FLY vault fly (mode 600). Comments on their own lines only (Compose keeps a
2
- # trailing "# comment" as part of the value). The wallet key is NOT here: it is the file /srv/fly/secrets/bot_key.
 
3
 
4
- # --- keys ---
5
  HELIUS_API_KEY=
6
  JUPITER_API_KEY=
 
 
7
 
8
- # --- money: the vault wallet (5 SOL principal) ---
9
  LIVE_ENABLED=1
10
  SOLANA_CLUSTER=mainnet-beta
 
11
  CAPITAL_SOL=5
12
- # only used when the fly has no sizing table yet; the Kelly caps below govern (~0.47 SOL per position on 5 SOL)
13
- MAX_POSITION_SOL=0.5
14
  MAX_POSITION_FRACTION=0.10
15
  MAX_POOL_SHARE=0.02
16
  MIN_POSITION_SOL=0.02
@@ -22,15 +24,21 @@ HANDOVER_DAYS=0
22
  HANDOVER_MIN_TRADES=20
23
  HANDOVER_BEAT_SELECTOR=0
24
  RESET_ON_START=0
 
25
  DEVICE=cpu
26
- # leave memory for Postgres and the wallet-skill table (16 GB host)
27
- DEVICE_MEMORY_GB=6
28
 
29
  # --- the vault ---
30
- # SOL from these addresses is your deposit (principal); from anyone else it is profit for the lockers
 
31
  FUNDING_ADDRESSES=
32
  RH_CHAIN_ID=4663
33
  RH_RPC_URL=https://rpc.mainnet.chain.robinhood.com
 
 
 
 
34
  # from contracts/deployments/4663.json after you deploy
35
  VAULT_ADDRESS=
36
  VAULT_TIMELOCK=
@@ -47,9 +55,11 @@ VAULT_SITE_URI=https://fly-trader.app/vault.html
47
  # --- alerts ---
48
  TELEGRAM_BOT_TOKEN=
49
  TELEGRAM_CHAT_ID=
 
 
 
 
50
 
51
- # --- encrypted backups (key once, ledger nightly) ---
52
- # your age1... or ssh-ed25519 public key
53
- VAULT_BACKUP_RECIPIENT=
54
  VAULT_BACKUP_BUCKET=bryceweiner/fly-vault-backups
55
  VAULT_BACKUP_HF_TOKEN=
 
1
+ # /srv/fly/vault.env for the hosted $FLY vault fly's brain (mode 600). Comments on their own lines only (Compose keeps
2
+ # a trailing "# comment" as part of the value). No key is here or anywhere the brain can read: the two hot keys are
3
+ # files only the signer container mounts. Treasury and cap settings both containers share are in custody.env.
4
 
5
+ # --- API keys ---
6
  HELIUS_API_KEY=
7
  JUPITER_API_KEY=
8
+ # a second, independent Solana RPC (Triton, QuickNode, Alchemy...): money-moving reads must agree with it
9
+ SOLANA_CHECK_RPC_URL=
10
 
11
+ # --- money: the book = the trading wallet (a float, TREASURY_FLOAT_SOL in custody.env) + the Squads treasury ---
12
  LIVE_ENABLED=1
13
  SOLANA_CLUSTER=mainnet-beta
14
+ # your principal (what you send from FUNDING_ADDRESSES)
15
  CAPITAL_SOL=5
 
 
16
  MAX_POSITION_FRACTION=0.10
17
  MAX_POOL_SHARE=0.02
18
  MIN_POSITION_SOL=0.02
 
24
  HANDOVER_MIN_TRADES=20
25
  HANDOVER_BEAT_SELECTOR=0
26
  RESET_ON_START=0
27
+ # the CPU: the fly's scoring is memory-bound, and 16+ cores score as fast as a GPU; no GPU driver stack in the container
28
  DEVICE=cpu
29
+ # what the fly's batches may use, leaving room for Postgres and the wallet-skill table (128 GB host; 16 GB host: 6)
30
+ DEVICE_MEMORY_GB=24
31
 
32
  # --- the vault ---
33
+ # comma-separated; SOL from these addresses (to the treasury or the trading wallet) is your deposit (principal); from
34
+ # anyone else it is profit for the lockers. At least your Solflare owner address.
35
  FUNDING_ADDRESSES=
36
  RH_CHAIN_ID=4663
37
  RH_RPC_URL=https://rpc.mainnet.chain.robinhood.com
38
+ # Robinhood Chain memecoin trading and the vault's ETH pot stay off on the hosted server: its signer container holds no
39
+ # RH key, and the brain holds no wallet key at all.
40
+ RH_ENABLED=0
41
+ RH_LIVE_ENABLED=0
42
  # from contracts/deployments/4663.json after you deploy
43
  VAULT_ADDRESS=
44
  VAULT_TIMELOCK=
 
55
  # --- alerts ---
56
  TELEGRAM_BOT_TOKEN=
57
  TELEGRAM_CHAT_ID=
58
+ # your own Telegram user id: only its /panic and /status are obeyed (message @userinfobot to learn it)
59
+ TELEGRAM_ADMIN_USER_ID=
60
+ # dead-man switch: a healthchecks.io check URL (its Telegram integration tells you when pings stop)
61
+ HEALTHCHECK_URL=
62
 
63
+ # --- encrypted backups (hot keys once, by the signer; ledger nightly). The recipient key is in custody.env ---
 
 
64
  VAULT_BACKUP_BUCKET=bryceweiner/fly-vault-backups
65
  VAULT_BACKUP_HF_TOKEN=
docker-compose.yml CHANGED
@@ -18,12 +18,19 @@ services:
18
 
19
  fly:
20
  build: .
 
 
 
 
 
21
  env_file: .env
22
  environment:
23
  DATABASE_URL: postgresql://fly:fly@db:5432/fly_trader
24
  DATA_DIR: /app/data
25
  LOG_DIR: /app/logs
26
  DEVICE: auto # the CPU in this image; an NVIDIA GPU with docker-compose.cuda.yml
 
 
27
  volumes:
28
  - ./data:/app/data # models, the fly's plastic state, its hourly snapshots: survives restarts and rebuilds
29
  - ./logs:/app/logs
 
18
 
19
  fly:
20
  build: .
21
+ # root inside the container: the self-updater (AUTO_UPDATE=1) replaces the code in /app, and data/, logs/ and .env
22
+ # are folders and a file of yours bind-mounted from here (the vault server's compose runs an unprivileged user and
23
+ # never self-updates)
24
+ user: "0:0"
25
+ # every key (BOT_PRIVATE_KEY, and RH_BOT_PRIVATE_KEY for Robinhood Chain live) comes from .env at run time, never the image
26
  env_file: .env
27
  environment:
28
  DATABASE_URL: postgresql://fly:fly@db:5432/fly_trader
29
  DATA_DIR: /app/data
30
  LOG_DIR: /app/logs
31
  DEVICE: auto # the CPU in this image; an NVIDIA GPU with docker-compose.cuda.yml
32
+ # Robinhood Chain is opt-in: set RH_ENABLED=1 in .env (it needs ENVIO_API_TOKEN and room for its history)
33
+ RH_ENABLED: ${RH_ENABLED:-0}
34
  volumes:
35
  - ./data:/app/data # models, the fly's plastic state, its hourly snapshots: survives restarts and rebuilds
36
  - ./logs:/app/logs
docker/bootstrap.sh CHANGED
@@ -1,24 +1,58 @@
1
  #!/bin/sh
2
  # What every boot (and every self-update) does before the console starts: schema, the shipped models into the data
3
- # volume, the wallet-skill table, the neuron geometry, the seed rows that admit the fly. Nothing here overwrites a
4
- # running install: copies skip files that exist, the seed is ON CONFLICT DO NOTHING.
5
  set -e
6
  cd /app
7
  until pg_isready -d "$DATABASE_URL" >/dev/null 2>&1; do echo "fly-trader: waiting for postgres"; sleep 2; done
8
  fly-trader init-db
 
 
 
 
 
 
 
 
 
 
 
 
9
  for sub in policies selectors connectome; do
10
  mkdir -p "data/brain/$sub"
11
  for f in models/$sub/*; do [ -e "$f" ] || continue; [ -e "data/brain/$sub/$(basename "$f")" ] || cp "$f" "data/brain/$sub/"; done
12
  done
 
 
 
13
  mkdir -p data/corpus/wallet_skill
14
- for f in models/wallet_skill/*; do [ -e "$f" ] || continue; [ -e "data/corpus/wallet_skill/$(basename "$f")" ] || cp "$f" data/corpus/wallet_skill/; done
 
 
 
 
15
  T=$(cat models/wallet_skill/TABLE 2>/dev/null || true)
 
16
  if [ -n "$T" ] && [ ! -e "data/corpus/wallet_skill/$T" ]; then
17
- echo "fly-trader: fetching the wallet-skill table $T (~300 MB, once)"
18
- curl -fsSL --retry 5 -o "data/corpus/wallet_skill/$T.part" "https://huggingface.co/${HF_REPO:-bryceweiner/fly-trader}/resolve/main/models/wallet_skill/$T" \
19
- && mv "data/corpus/wallet_skill/$T.part" "data/corpus/wallet_skill/$T" \
20
- || echo "fly-trader: WARNING could not fetch the wallet-skill table; the paper selector will not trade and the fly's skill inputs read as zero until it is present"
 
 
 
 
 
 
 
 
21
  fi
 
 
22
  mkdir -p fly_trader/ui/static/brain && cp models/geometry/* fly_trader/ui/static/brain/ 2>/dev/null || true
23
- psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -q -f seed/seed.sql
 
 
 
 
24
  echo "fly-trader: device $(python -c 'from fly_trader.brain import device; print(device.describe(device.resolve()))' 2>/dev/null || echo unknown)"
 
1
  #!/bin/sh
2
  # What every boot (and every self-update) does before the console starts: schema, the shipped models into the data
3
+ # volume, the wallet-skill table, the neuron geometry, the rows that admit the fly. Nothing here overwrites a running
4
+ # install: copies skip files that exist, and apply-release reuses rows it already has.
5
  set -e
6
  cd /app
7
  until pg_isready -d "$DATABASE_URL" >/dev/null 2>&1; do echo "fly-trader: waiting for postgres"; sleep 2; done
8
  fly-trader init-db
9
+ # models/ holds exactly this release's fly and selector: an earlier release's (the updater before signed releases never
10
+ # deleted a file) would be copied below, and a pickled .joblib makes apply-release refuse the whole release
11
+ if [ -f seed/release_seed.json ]; then
12
+ python - <<'PY'
13
+ import json, pathlib
14
+ keep = {s["file"] for s in json.loads(pathlib.Path("seed/release_seed.json").read_text())["snapshots"]}
15
+ for sub in ("policies", "selectors"):
16
+ for f in pathlib.Path("models", sub).glob("*"):
17
+ if f.is_file() and f.name not in keep:
18
+ f.unlink(); print(f"fly-trader: removed {f} (not in this release)")
19
+ PY
20
+ fi
21
  for sub in policies selectors connectome; do
22
  mkdir -p "data/brain/$sub"
23
  for f in models/$sub/*; do [ -e "$f" ] || continue; [ -e "data/brain/$sub/$(basename "$f")" ] || cp "$f" "data/brain/$sub/"; done
24
  done
25
+ # The wallet-skill inputs: the fitted config and the newest table. The table (~300 MB) is too big for git; it comes from
26
+ # a verified release (the vault server's /releases), or is downloaded from Hugging Face and used only if its sha256 is
27
+ # the one pinned in models/wallet_skill/TABLE.sha256. Nothing unverified reaches the engine.
28
  mkdir -p data/corpus/wallet_skill
29
+ for f in models/wallet_skill/*; do
30
+ [ -e "$f" ] || continue
31
+ case "$(basename "$f")" in TABLE|TABLE.sha256) continue;; esac
32
+ [ -e "data/corpus/wallet_skill/$(basename "$f")" ] || cp "$f" data/corpus/wallet_skill/
33
+ done
34
  T=$(cat models/wallet_skill/TABLE 2>/dev/null || true)
35
+ WANT=$(cat models/wallet_skill/TABLE.sha256 2>/dev/null || true)
36
  if [ -n "$T" ] && [ ! -e "data/corpus/wallet_skill/$T" ]; then
37
+ src=$(ls -d /releases/*/models/wallet_skill/"$T" 2>/dev/null | tail -1 || true)
38
+ if [ -n "$src" ]; then
39
+ cp "$src" "data/corpus/wallet_skill/$T"
40
+ elif [ -n "$WANT" ]; then
41
+ echo "fly-trader: fetching the wallet-skill table $T (~300 MB, once; checked against its pinned sha256)"
42
+ if curl -fsSL --retry 5 -o "data/corpus/wallet_skill/$T.part" "https://huggingface.co/${HF_REPO:-bryceweiner/fly-trader}/resolve/main/models/wallet_skill/$T" \
43
+ && [ "$(sha256sum "data/corpus/wallet_skill/$T.part" | cut -d' ' -f1)" = "$WANT" ]; then
44
+ mv "data/corpus/wallet_skill/$T.part" "data/corpus/wallet_skill/$T"
45
+ else
46
+ rm -f "data/corpus/wallet_skill/$T.part"
47
+ fi
48
+ fi
49
  fi
50
+ [ -z "$T" ] || [ -e "data/corpus/wallet_skill/$T" ] || \
51
+ echo "fly-trader: WARNING the wallet-skill table $T is missing or did not match its sha256; the paper selector will not trade and the fly's skill inputs read as zero until it is present (data/corpus/wallet_skill/$T)"
52
  mkdir -p fly_trader/ui/static/brain && cp models/geometry/* fly_trader/ui/static/brain/ 2>/dev/null || true
53
+ if [ -f seed/release_seed.json ]; then
54
+ fly-trader apply-release /app --bootstrap # the shipped models, registered by file hash (fresh ids), idempotently
55
+ else
56
+ psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -q -f seed/seed.sql
57
+ fi
58
  echo "fly-trader: device $(python -c 'from fly_trader.brain import device; print(device.describe(device.resolve()))' 2>/dev/null || echo unknown)"
docker/signer-requirements.txt ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ anyio==4.15.1 \
2
+ --hash=sha256:6152fdbbf9a77fdec97731721bebf7c4c44f7c29b424b0065826173efc7ed101 \
3
+ --hash=sha256:9f28306018cbd6d329e64a36d58256edff76dd996fe423bc957326e578b82a94
4
+ # via httpx
5
+ base58==2.1.1 \
6
+ --hash=sha256:11a36f4d3ce51dfc1043f3218591ac4eb1ceb172919cebe05b52a5bcc8d245c2 \
7
+ --hash=sha256:c5d0cb3f5b6e81e8e35da5754388ddcc6d0d14b6c6a132cb93d69ed580a7278c
8
+ # via -r /private/tmp/claude-501/-Users-bryce-Documents-code-fly-trader/28aad393-6f38-484a-a2b2-a06a8f112cf1/scratchpad/signer.in
9
+ certifi==2026.7.22 \
10
+ --hash=sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775 \
11
+ --hash=sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55
12
+ # via
13
+ # httpcore
14
+ # httpx
15
+ h11==0.16.0 \
16
+ --hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \
17
+ --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86
18
+ # via httpcore
19
+ httpcore==1.0.9 \
20
+ --hash=sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55 \
21
+ --hash=sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8
22
+ # via httpx
23
+ httpx==0.28.1 \
24
+ --hash=sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc \
25
+ --hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad
26
+ # via -r /private/tmp/claude-501/-Users-bryce-Documents-code-fly-trader/28aad393-6f38-484a-a2b2-a06a8f112cf1/scratchpad/signer.in
27
+ idna==3.20 \
28
+ --hash=sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44 \
29
+ --hash=sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c
30
+ # via
31
+ # anyio
32
+ # httpx
33
+ jsonalias==0.1.1 \
34
+ --hash=sha256:64f04d935397d579fc94509e1fcb6212f2d081235d9d6395bd10baedf760a769 \
35
+ --hash=sha256:a56d2888e6397812c606156504e861e8ec00e188005af149f003c787db3d3f18
36
+ # via solders
37
+ solders==0.29.0 \
38
+ --hash=sha256:00524baf575ae5edb2ba43cdb1d80491c59efdc16b48fec96a6f929c0401d0e4 \
39
+ --hash=sha256:09e80c545c728f434f627d3b4714675a0f654c4d387f3339d54f7d09d59f3f1c \
40
+ --hash=sha256:62e8a5bb4bc8628d5fe49faa56aca149dce284a9f8bf5f3c56c4721e561be84b \
41
+ --hash=sha256:785841623bdd7b8dedc68e7c794b54327d3b03e1ab9697eda90a139ca3c98cfe \
42
+ --hash=sha256:9d2e011ff1d5f82a7de81d6eb89f54d002189f50ee48d4a15d7b51849e04b1c3 \
43
+ --hash=sha256:c17098cab72a1c202ec4cc0ce3ecbff4296c9de97000e05944337ba5ddb681a8 \
44
+ --hash=sha256:f36db7b43088866081b5d37ff62668551e30dd136b9ee58a5fb524fcd828082d
45
+ # via -r /private/tmp/claude-501/-Users-bryce-Documents-code-fly-trader/28aad393-6f38-484a-a2b2-a06a8f112cf1/scratchpad/signer.in
46
+ typing-extensions==4.16.0 \
47
+ --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \
48
+ --hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5
49
+ # via
50
+ # anyio
51
+ # solders
docs/vault/RUNBOOK.md CHANGED
@@ -1,120 +1,435 @@
1
  # $FLY vault: launch runbook
2
 
3
- These are the steps only you can do, because they need your keys or accounts. They are in order. The agent never
4
- handles your keys.
5
-
6
- ## 0. Accounts and keys (once)
7
- - **Reown project id.** Create it at dashboard.reown.com and allowlist `fly-trader.app`, plus `localhost:5173` for the
8
- rehearsal.
9
- - **Telegram bot.** Create one with @BotFather; you need its token and your chat id.
10
- - **Release-signing key (Mac).** Run `ssh-keygen -t ed25519 -f ~/.ssh/fly_release`. The public key goes to the server's
11
- allowed signers in step 3.
12
- - **Backup key.** This is the age or ssh-ed25519 public key the backups are encrypted to. You can use
13
- `~/.ssh/fly_release.pub` or an age key.
14
- - **HF storage bucket.** Create a private bucket named `bryceweiner/fly-vault-backups`, then a fine-grained token that
15
- can write only to that bucket.
16
- - **Hardware-wallet EVM account.** This account owns the vault.
17
-
18
- ## 1. Rehearsal (testnet 46630 + Solana devnet)
19
- 1. Deploy the contracts to the testnet:
20
- ```
21
- cd contracts
22
- forge script script/DeployTestnet.s.sol --rpc-url https://rpc.testnet.chain.robinhood.com --ledger --broadcast
23
- ```
24
- The addresses land in `contracts/deployments/46630.json`.
25
- 2. Build the site against the testnet:
26
- ```
27
- cd web
28
- VITE_NETWORK=testnet VITE_REOWN_PROJECT_ID=… VITE_VAULT_ADDRESS=… VITE_TIMELOCK_ADDRESS=… VITE_FLY_ADDRESS=<MockFLY> npm run dev
29
- ```
30
- Run a local relay (see `web/relay/README.md`) with `domain: localhost:5173`, `chain_id: 46630`, `sol_chain: devnet`.
31
- 3. Run the fly (Docker or the Mac) with these settings:
32
- - `VAULT_ENABLED=1`, `VAULT_CLUSTER=devnet`, `VAULT_SOLANA_RPC_URL=https://api.devnet.solana.com`
33
- - `VAULT_PERIOD_S=900`
34
- - `RH_CHAIN_ID=46630`, `RH_RPC_URL=https://rpc.testnet.chain.robinhood.com`
35
- - `VAULT_SITE_DOMAIN=localhost:5173`, `VAULT_SITE_URI=http://localhost:5173/vault.html`
36
- - `LIVE_ENABLED=0`
37
- - a throwaway devnet key, funded from the devnet faucet
38
- 4. Work through the checklist:
39
- - lock from two wallets at different times, then request and cancel a withdrawal
40
- - send devnet SOL from an address that is not a funding address (it counts as profit)
41
- - wait for a 15-minute settlement
42
- - claim from both wallets, then replay a used nonce
43
- - kill the fly in the middle of a claim and restart it
44
- - `fly-trader vault status`
45
-
46
- ## 2. Mainnet contracts
47
- 1. Deploy:
48
- ```
49
- cd contracts
50
- FLY_TOKEN=0x2fC7f9E2911f20b2C4660d2AEf808aa91bDdb3D3 OWNER=<hardware account> \
51
- forge script script/Deploy.s.sol --rpc-url https://rpc.mainnet.chain.robinhood.com --ledger --broadcast
52
- ```
53
- The script refuses to run on chain 4663 unless the delays are 8 days (timelock) and 7 days (withdrawal).
54
- 2. Verify the contracts on Blockscout. `contracts/README.md` has a manual fallback if Cloudflare blocks forge.
55
- 3. Record the implementation code hash:
56
- ```
57
- cast keccak $(cast code <impl> --rpc-url https://rpc.mainnet.chain.robinhood.com)
58
- ```
59
- This value becomes `VAULT_IMPL_CODEHASHES` in step 3.
60
-
61
- ## 3. Server (Netcup RS 2000 G12, Ubuntu 24.04)
62
- 1. Put your SSH key on the server. Then copy the `deploy/` folder from the distribution branch and run:
63
- `sudo bash deploy/bootstrap-host.sh "$(cat ~/.ssh/fly_release.pub)"`
64
- It prints the **vault wallet address**. The key is created on the server and stays there.
65
- 2. Edit `/srv/fly/vault.env`. Fill in:
66
- - `HELIUS_API_KEY`, `JUPITER_API_KEY`
67
- - `FUNDING_ADDRESSES`
68
- - `VAULT_ADDRESS`, `VAULT_TIMELOCK`, `VAULT_START_BLOCK`, `VAULT_IMPL_CODEHASHES`
69
- - `RELAY_SECRET`
70
- - `TELEGRAM_*`
71
- - `VAULT_BACKUP_*`
72
- 3. Edit `/etc/fly/update.json`. Only the Telegram fields need filling.
73
- 4. Publish the first release from the Mac. Keep a distribution worktree at `../fly-trader-dist`, then run:
74
- `.venv/bin/python tools/publish_release.py --worktree ../fly-trader-dist --key ~/.ssh/fly_release --models`
75
- 5. On the server, start the updater:
76
- ```
77
- sudo systemctl enable --now fly-update.timer
78
- journalctl -u fly-update -f
79
- ```
80
- It builds the image and starts the fly, which warms up on paper.
81
- 6. To open the console:
82
- ```
83
- ssh -L 8501:127.0.0.1:8501 <server>
84
- ```
85
- Then browse to http://localhost:8501/vault.
86
-
87
- ## 4. Site (Gandi)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
88
  1. Build the upload bundle:
89
  ```
90
- cd web
91
- VITE_REOWN_PROJECT_ID=… VITE_VAULT_ADDRESS=… VITE_TIMELOCK_ADDRESS=… bash scripts/bundle_gandi.sh
 
 
 
 
 
 
 
 
 
 
 
 
92
  ```
93
- 2. Upload the contents of `build/gandi/` so they replace `site/` entirely.
94
- 3. Put `relay.json` next to `wsgi.py`, using the same secret as `RELAY_SECRET`, then restart the instance.
95
- 4. Probe the relay from the Mac:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
96
  ```
97
- FLY_RELAY_SECRET=<secret> python web/relay/hmacauth.py https://fly-trader.app/api/fly/probe k1
98
  ```
99
- Check the output against the checklist in `web/relay/README.md`. The things that must hold: the database path is
100
- writable, WAL works, and nothing caches `/api`.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
101
 
102
  ## 5. Go live
103
- 1. Send 5 SOL to the vault wallet from an address listed in `FUNDING_ADDRESSES`.
104
- 2. Check the deposit was counted: `fly-trader vault status` (inside the container) should show `deposits: 5000000000`.
105
- If it came from another address, run `fly-trader vault reclassify <sig> deposit`.
106
- 3. The fly trades live after the handover: at least 20 paper trades.
107
- 4. Settlements run every Monday at 00:00 UTC. Claims open from the first one.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
108
 
109
  ## Day to day
110
- - **New models or code:** run `publish_release.py`, with `--models` to refresh the models. The server applies the
111
- release within 5 minutes. A release that changes Docker or compose files waits until you run
112
- `sudo python3 /usr/local/lib/fly/fly_update.py approve <seq>` on the server.
113
- - **Take principal out:** `docker compose -p fly -f /srv/fly/docker-compose.yml exec fly fly-trader vault withdraw --sol X --to <funding address>`.
114
- - **A halt alert** (unknown transaction or anomaly):
115
- 1. Find the cause on the console's `$FLY vault` page.
116
- 2. Run `fly-trader vault resume`.
117
- 3. Run `fly-trader resume-entries`.
118
- - **Restore from backups:**
119
- - Key: `age -d -i ~/.ssh/fly_release keys/<pubkey>.age`
120
- - Ledger: `age -d … ledger/<day>.dump.age | pg_restore`
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  # $FLY vault: launch runbook
2
 
3
+ The steps only you can do, because they need your keys, accounts or devices, in the order to do them. The agent never
4
+ handles your keys. How the system works is in [SPEC.md](SPEC.md); this file is what to type.
5
+
6
+ **Who holds what** (until the Ledger is back; the "Ledger" section moves everything to it without redeploying):
7
+
8
+ | Key | Where | Can do |
9
+ |---|---|---|
10
+ | Solana owner | Solflare on your phone | Everything with the treasury: create/remove its spending limits, take principal out |
11
+ | EVM owner | MetaMask | Pause new $FLY locks; propose upgrades (public for 8 days); hand the roles to the Ledger |
12
+ | SSH | Secretive on the Mac (Secure Enclave, Touch ID) | Log in to the server over Tailscale |
13
+ | Release signing | Secretive (Touch ID) + an offline recovery key | Sign releases; the server still holds code changes 24 h |
14
+ | Backup decryption | an age key, offline | Read the encrypted backups |
15
+ | Trading / payout | on the server, readable only by the signer container | Refill the float (L1, per day), pay claims (L2, per week) |
16
+
17
+ A full takeover of the server can lose at most the trading float plus what is left of this day's L1 and this week's
18
+ L2, and only until you revoke them from your phone ("Emergency").
19
+
20
+ Conventions below:
21
+ - `$` lines run on the **Mac** in the fly-trader checkout (the repo root). Commands for a subfolder run in a subshell,
22
+ `$ (cd web && …)`, so you are always back at the root.
23
+ - `server$` lines run on the **server** as `fly-admin`.
24
+ - `sudo fly <command>` runs `fly-trader <command>` inside the running fly container; `sudo fly signer <command>` runs
25
+ in the signer container.
26
+
27
+ ## 0. Once: tools, repos, accounts, keys
28
+
29
+ ### 0.1 Mac tools
30
+ ```
31
+ $ brew install age node uv git
32
+ $ brew install --cask secretive tailscale
33
+ $ curl -L https://foundry.paradigm.xyz | bash && foundryup # forge, cast
34
+ $ sh -c "$(curl -sSfL https://release.anza.xyz/stable/install)" # solana CLI + solana-test-validator
35
+ $ uv sync --extra dev # the Python environment (.venv)
36
+ $ (cd web && npm ci)
37
+ $ (cd contracts && forge soldeer install)
38
+ $ git worktree add ../fly-trader-dist distribution # the branch the server runs
39
+ ```
40
+
41
+ ### 0.2 Accounts
42
+ - **2FA everywhere.** Use a hardware key or TOTP on Gandi, Hugging Face, GitHub,
43
+ your Tailscale identity provider and your Apple ID. Set a Telegram two-step password.
44
+ - **Phone wallets.** Write the Solflare and MetaMask seed phrases on paper and keep them offline. The phone needs a
45
+ passcode and biometrics. Keep a little SOL in Solflare (about 0.05 SOL pays the treasury's setup and any later
46
+ change) and a little ETH on Robinhood Chain in MetaMask (for `pause()`).
47
+ - **Reown project id.** Create it at dashboard.reown.com. Allowlist `fly-trader.app`, and `localhost:5173` for local
48
+ testing.
49
+ - **Telegram.**
50
+ 1. Create a bot with @BotFather and note its token.
51
+ 2. Send `/start` to your new bot; it cannot message you before that.
52
+ 3. Message @userinfobot to learn your user id. In a private chat with the bot, the chat id is that same number.
53
+ Only that user's `/panic` and `/status` count.
54
+ - **Hugging Face.**
55
+ 1. Create a **public** model repo `bryceweiner/fly-trader`: the server and self-hosted installs read releases
56
+ without logging in.
57
+ 2. Create a **private** storage bucket `bryceweiner/fly-vault-backups` for the encrypted backups.
58
+ 3. Create two tokens: one with write access to the model repo, for publishing from the Mac; one fine-grained,
59
+ write-only to the bucket, for the server's backups.
60
+ 4. `$ uv run hf auth login` with the first token.
61
+ - **GitHub.** Push access to the fly-trader repo: publishing pushes the `distribution` branch.
62
+ - **Second Solana RPC.** Get an API URL from a provider other than Helius (Triton, QuickNode or Alchemy). It is used
63
+ twice:
64
+ - `SOLANA_CHECK_RPC_URL`: settlement balances and halts must agree with it.
65
+ - `SIGNER_RPC_URL`: the signer's own reads and simulations.
66
+ - **healthchecks.io.** Create a check with period 5 minutes and grace 10 minutes, and connect its Telegram
67
+ integration. Its ping URL is `HEALTHCHECK_URL`.
68
+
69
+ ### 0.3 Tailscale
70
+ 1. Install Tailscale on the Mac, log in, and turn on **MagicDNS** in the admin console, so `fly-vault` resolves.
71
+ 2. In the admin console's access controls, add:
72
+ ```
73
+ "tagOwners": {"tag:fly": ["autogroup:admin"]},
74
+ "acls": [{"action": "accept", "src": ["<your Tailscale login>"], "dst": ["tag:fly:22"]}]
75
+ ```
76
+ Add no rule whose source is `tag:fly`: the server needs to reach nothing on your tailnet.
77
+ 3. Turn on **Tailnet Lock** from the Mac:
78
+ ```
79
+ $ tailscale lock status # shows this Mac's tlpub:… key
80
+ $ tailscale lock init --gen-disablements 3 --confirm tlpub:<that key>
81
+ ```
82
+ Store the printed disablement secrets offline, like a seed phrase.
83
+ 4. Create a one-off, pre-authorised auth key tagged `tag:fly` in the admin console. Sign it so the server may join the
84
+ locked tailnet, and keep the output (this is `TS_AUTHKEY` in section 4.2):
85
+ ```
86
+ $ tailscale lock sign tskey-auth-…
87
+ ```
88
+
89
+ ### 0.4 Your keys (Secretive, recovery, backup)
90
+ 1. In Secretive create two keys, `fly-ssh` and `fly-release`, each requiring Touch ID. Save their public keys as
91
+ `~/.ssh/fly-ssh.pub` and `~/.ssh/fly-release.pub`.
92
+ 2. Put the path of Secretive's agent in your shell profile (`~/.zshrc`), since several steps use it:
93
+ `export SECRETIVE=~/Library/Containers/com.maxgoedjen.Secretive.SecretAgent/Data/socket.ssh`.
94
+ Then tell ssh to use Secretive for the server, in `~/.ssh/config`:
95
+ ```
96
+ Host fly-vault
97
+ User fly-admin
98
+ IdentityAgent ~/Library/Containers/com.maxgoedjen.Secretive.SecretAgent/Data/socket.ssh
99
+ ```
100
+ 3. The offline recovery release signer:
101
+ ```
102
+ $ mkdir -p ~/fly-keys
103
+ $ ssh-keygen -t ed25519 -f ~/fly-keys/fly-release-recovery -C fly-release-recovery
104
+ $ age -p ~/fly-keys/fly-release-recovery > ~/fly-keys/fly-release-recovery.age && rm ~/fly-keys/fly-release-recovery
105
+ ```
106
+ Put `fly-release-recovery.age` on a USB stick and in your password manager. Keep `fly-release-recovery.pub`.
107
+ 4. The backup key: `$ age-keygen -o ~/fly-keys/fly-backup.key`. Store it like the recovery key; its public line
108
+ (`age1…`, printed and at the top of the file) is `VAULT_BACKUP_RECIPIENT`. It is deliberately not the release key.
109
+ 5. Trust both release keys in the distribution, so that self-hosted installs accept your releases:
110
+ ```
111
+ $ echo "bryce namespaces=\"fly-trader-release\" $(cat ~/.ssh/fly-release.pub)" >> ../fly-trader-dist/release/allowed_signers
112
+ $ echo "bryce-recovery namespaces=\"fly-trader-release\" $(cat ~/fly-keys/fly-release-recovery.pub)" >> ../fly-trader-dist/release/allowed_signers
113
+ $ git -C ../fly-trader-dist status # must be clean apart from this file (no merge in progress)
114
+ $ git -C ../fly-trader-dist commit -m "Release keys" release/allowed_signers
115
+ ```
116
+
117
+ ## 1. Rehearse locally (no real money)
118
+ - **Custody against the real Squads program:**
119
+ `$ tools/custody_e2e.sh <your Helius mainnet RPC URL>`. It dumps the program from mainnet (read-only) and runs it on
120
+ a throwaway validator on ports 11xxx. It checks:
121
+ - the owner page's code creates the treasury and both limits, changes L2 and revokes them
122
+ - the signer refills through L1 and pays a claim through L2 up to its cap
123
+ - a stolen key cannot send L1 anywhere but the trading wallet (refused on chain)
124
+ - **The whole vault on this Mac:** `$ .venv/bin/python tools/vault_demo.py up`. It runs anvil, the contracts, the relay,
125
+ the vault worker, a treasury on a local validator and the site at http://localhost:5173. The file's docstring lists
126
+ the wallet settings and commands (`fund-evm`, `profit`, `status`, `down`).
127
+ - **Optional: RH testnet + Solana devnet.**
128
+ 1. Deploy the contracts:
129
+ ```
130
+ $ cast wallet new ~/.foundry/keystores rehearsal # fund it from faucet.testnet.chain.robinhood.com
131
+ $ (cd contracts && forge script script/DeployTestnet.s.sol --rpc-url robinhood_testnet --account rehearsal \
132
+ --sender $(cast wallet address --account rehearsal) --broadcast)
133
+ $ cp contracts/deployments/46630.json ~/fly-keys/testnet-deployment.json
134
+ ```
135
+ The addresses land in `contracts/deployments/46630.json`: `vault`, `timelock`, `token` (MockFLY), `blockNumber`.
136
+ Keep the copy: `vault_demo.py` deploys to anvil, which uses the same chain id, and overwrites that file.
137
+ 2. Run the site against it:
138
+ Run a relay as in `web/relay/README.md` (e.g. `tools/vault_demo.py` shows how it starts one on port 8612), with
139
+ `"domain": "localhost:5173"`, `"uri": "http://localhost:5173/vault.html"`, `"chain_id": 46630`,
140
+ `"sol_chain": "devnet"`. Then:
141
+ ```
142
+ $ (cd web && VITE_NETWORK=testnet VITE_REOWN_PROJECT_ID=… VITE_VAULT_ADDRESS=<vault> VITE_TIMELOCK_ADDRESS=<timelock> \
143
+ VITE_FLY_ADDRESS=<token> VITE_SOLANA_RPC=https://api.devnet.solana.com VITE_RELAY_PROXY=http://127.0.0.1:8612 npm run dev)
144
+ ```
145
+ 3. Make two throwaway devnet keys (`$ solana-keygen new -o /tmp/devnet-trading.json`, the same for payout; fund
146
+ the trading one at faucet.solana.com). Create the treasury: http://localhost:5173/owner.html with Solflare on
147
+ devnet, as in section 4.3, with those two public keys.
148
+ 4. Run the vault worker on the Mac with these settings in the environment:
149
+ - `VAULT_ENABLED=1`, `VAULT_CLUSTER=devnet`, `VAULT_SOLANA_RPC_URL=https://api.devnet.solana.com`,
150
+ `SIGNER_RPC_URL=https://api.devnet.solana.com`, `VAULT_PERIOD_S=900`, `LIVE_ENABLED=0`
151
+ - `BOT_PRIVATE_KEY_FILE=/tmp/devnet-trading.json`, `PAYOUT_PRIVATE_KEY_FILE=/tmp/devnet-payout.json`
152
+ - `VAULT_MULTISIG`, `VAULT_LIMIT_TRADING`, `VAULT_LIMIT_PAYOUT` (from the owner page)
153
+ - `RH_CHAIN_ID=46630`, `RH_RPC_URL=https://rpc.testnet.chain.robinhood.com`, `VAULT_ADDRESS`,
154
+ `VAULT_TIMELOCK`, `VAULT_START_BLOCK`
155
+ - `VAULT_SITE_DOMAIN=localhost:5173`, `VAULT_SITE_URI=http://localhost:5173/vault.html`
156
+ - `RELAY_URL=http://127.0.0.1:8612`, `RELAY_SECRET` (the relay's), and a separate `DATABASE_URL`
157
+ (`createdb fly_vault_rehearsal; DATABASE_URL=postgresql:///fly_vault_rehearsal uv run fly-trader init-db`)
158
+
159
+ ```
160
+ $ uv run python -c "from fly_trader.vault import worker; worker.main()"
161
+ ```
162
+
163
+ ## 2. Mainnet contracts (EVM owner = MetaMask)
164
+ 1. Deploy from a throwaway keystore. It pays the gas and holds no role afterwards:
165
+ ```
166
+ $ cast wallet new ~/.foundry/keystores deployer # fund with a little ETH on Robinhood Chain
167
+ $ (cd contracts && FLY_TOKEN=0x2fC7f9E2911f20b2C4660d2AEf808aa91bDdb3D3 OWNER=<your MetaMask address> \
168
+ forge script script/Deploy.s.sol --rpc-url robinhood --account deployer \
169
+ --sender $(cast wallet address --account deployer) --broadcast \
170
+ --verify --verifier blockscout --verifier-url https://robinhoodchain.blockscout.com/api/)
171
+ ```
172
+ The script refuses to run on chain 4663 unless the token is the real $FLY and the delays are 8 days (timelock) and
173
+ 7 days (withdrawal). It writes `contracts/deployments/4663.json`: `vault`, `implementation`, `timelock`,
174
+ `blockNumber`. You need all four below.
175
+ 2. Check that the deployer holds nothing. Each of these must print `false`:
176
+ ```
177
+ $ D=$(cast wallet address --account deployer)
178
+ $ R=https://rpc.mainnet.chain.robinhood.com
179
+ $ cast call <timelock> "hasRole(bytes32,address)(bool)" $(cast keccak PROPOSER_ROLE) $D --rpc-url $R
180
+ $ cast call <timelock> "hasRole(bytes32,address)(bool)" 0x0000000000000000000000000000000000000000000000000000000000000000 $D --rpc-url $R
181
+ $ cast call <vault> "hasRole(bytes32,address)(bool)" $(cast keccak PAUSER_ROLE) $D --rpc-url $R
182
+ ```
183
+ Then delete `~/.foundry/keystores/deployer`.
184
+ 3. Check the contracts show as verified on Blockscout; `contracts/README.md` has a manual fallback if Cloudflare
185
+ blocked forge's verification.
186
+ 4. Record the implementation code hash; it becomes `VAULT_IMPL_CODEHASHES`:
187
+ `$ cast keccak $(cast code <implementation> --rpc-url https://rpc.mainnet.chain.robinhood.com)`
188
+ 5. From your phone (MetaMask), you use Blockscout's page for each contract, under "Write proxy" / "Write contract":
189
+ - **Pause new locks:** FlyVault, `pause()`; later `unpause()`. Withdrawals always stay open.
190
+ - **Cancel a timelock operation you did not schedule:** Timelock, `cancel(<id>)`. The Telegram alert gives the full
191
+ id.
192
+
193
+ ## 3. Site (Gandi)
194
+ The owner page is part of the site, and you need it for the treasury, so the site goes up before the server.
195
  1. Build the upload bundle:
196
  ```
197
+ $ (cd web && VITE_REOWN_PROJECT_ID=… VITE_VAULT_ADDRESS=<vault> VITE_TIMELOCK_ADDRESS=<timelock> bash scripts/bundle_gandi.sh)
198
+ ```
199
+ The bundle lands in `build/gandi/` at the repo root: `wsgi.py`, `site-headers.json`, `relay/` and `site/`.
200
+ `owner.html` is included; it is not linked anywhere, it is noindex, and it can do nothing without your owner wallet.
201
+ 2. Upload the contents of `build/gandi/` over Gandi SFTP (or git) into the instance's deploy directory (where
202
+ `wsgi.py` lives), replacing `site/` entirely.
203
+ 3. Make the relay secret: `$ python3 -c "import secrets;print(secrets.token_hex(32))"`. The fly gets the same value as
204
+ `RELAY_SECRET`.
205
+ 4. Upload `relay.json` next to `wsgi.py` (never into `site/`, never into git). The database goes on the instance's
206
+ persistent disk, outside the deploy directory:
207
+ ```json
208
+ {"keys": {"k1": "<the secret>"}, "db_path": "/srv/data/home/relay/relay.sqlite3", "domain": "fly-trader.app",
209
+ "uri": "https://fly-trader.app/vault.html", "chain_id": 4663, "sol_chain": "mainnet", "claim_ttl_s": 900,
210
+ "min_lamports": 2000000, "client_ip_header": null}
211
  ```
212
+ Restart the instance. If a later deploy replaces the directory, upload `relay.json` again.
213
+ 5. Probe the relay and work through the 7-point checklist in `web/relay/README.md` (writable, WAL, client IP header,
214
+ no caching, clock):
215
+ `$ FLY_RELAY_SECRET=<the secret> python3 web/relay/hmacauth.py https://fly-trader.app/api/fly/probe k1`
216
+
217
+ ## 4. Server (a dedicated machine at home, Ubuntu Server 24.04)
218
+
219
+ ### 4.1 Publish the first release
220
+ 1. The distribution worktree must hold exactly what you want to ship. Code is ported there from master by hand (or with
221
+ a merge); publishing does not port it. Check it is clean: `$ git -C ../fly-trader-dist status`.
222
+ 2. `--models` refreshes the models from this Mac; use it for the first release. That needs:
223
+ - a deployable fly (`fly-trader train-fly-selector`) that passed `fly-trader fly-replay`
224
+ - a deployed selector (pinned, or the newest that qualifies)
225
+ - fitted wallet skill (`fly-trader fit-wallet-skill`) with its tables, and a built connectome
226
+ Without `--models` the release carries the models already in the worktree, and publishing refuses if the
227
+ wallet-skill table named in `models/wallet_skill/TABLE` is not there.
228
+ 3. Publish. `SSH_AUTH_SOCK` points `ssh-keygen` at Secretive, so Touch ID confirms the signature:
229
  ```
230
+ $ SSH_AUTH_SOCK=$SECRETIVE .venv/bin/python tools/publish_release.py --worktree ../fly-trader-dist --key ~/.ssh/fly-release.pub --models
231
  ```
232
+ It runs the tests, commits, signs, verifies with the server's own verifier, pushes the `distribution` branch and
233
+ uploads one Hugging Face commit. It refuses unless the key is in the worktree's `release/allowed_signers` (0.4 step
234
+ 5). Note the printed **manifest sha256**; it pins this first release on the server.
235
+ 4. **Do not publish again until the server has installed this release** (section 4.4). A new server accepts only the
236
+ pinned manifest, and it permanently refuses any other head it sees first.
237
+
238
+ ### 4.2 The machine and its bootstrap
239
+ A dedicated machine at home (the reference is a GMKtec EVO-X3: Ryzen AI Max+ 395, 128 GB, 2 TB NVMe, 2.5 GbE). It
240
+ runs nothing but the vault fly. The fly runs on its CPU; the Radeon GPU is not used (no GPU driver stack in reach of
241
+ the keys).
242
+
243
+ 1. **Firmware** (press Del or F7 at power-on for the setup screen):
244
+ - Update the BIOS from the vendor's support site first.
245
+ - Set a supervisor (admin) password.
246
+ - Turn on the firmware TPM (AMD fTPM / "Security Device Support"), Secure Boot (standard keys) and the IOMMU
247
+ (AMD-Vi / "IOMMU: Enabled").
248
+ - Turn off Wi-Fi and Bluetooth if the menu offers it (the bootstrap also blocks their drivers).
249
+ - "Restore on AC power loss": Power On.
250
+ - Boot order: the internal disk first. After installing (step 2), turn off booting from USB.
251
+ 2. **A clean Ubuntu Server 24.04 LTS install** from a USB stick made from ubuntu.com's image. Erase the preinstalled
252
+ Windows; some mini-PC vendors have shipped malware in theirs.
253
+ - Storage: "Use an entire disk", "Set up this disk as an LVM group", **"Encrypt the LVM group with LUKS"**. Keep
254
+ the passphrase in your password manager.
255
+ - Profile: server name `fly-vault`, user **`fly-admin`**, any password (the bootstrap locks it).
256
+ - SSH: "Install OpenSSH server", with password authentication (only until the bootstrap).
257
+ - No featured snaps.
258
+ 3. **Network:** plug it into the router by cable, on its own VLAN or guest port if the router can. Forward no ports to
259
+ it, and turn off UPnP on the router. Put it on a UPS: during a power or internet outage the fly cannot sell,
260
+ refill or pay, and healthchecks.io tells you it went quiet.
261
+ 4. From the Mac, on the same network (the machine's address is on its console, `ip -4 addr`):
262
+ ```
263
+ $ ssh-copy-id -o IdentityAgent=$SECRETIVE -f -i ~/.ssh/fly-ssh.pub fly-admin@<LAN address>
264
+ $ scp -o IdentityAgent=$SECRETIVE -r ../fly-trader-dist/deploy ~/.ssh/fly-release.pub ~/fly-keys/fly-release-recovery.pub fly-admin@<LAN address>:
265
+ $ ssh -o IdentityAgent=$SECRETIVE fly-admin@<LAN address>
266
+ ```
267
+ 5. The first pass. `TPM_UNLOCK=1` binds the disk encryption to the TPM, so security-update reboots come back up on
268
+ their own; it asks once for the disk passphrase. Without it, every boot waits for the passphrase at the console,
269
+ and automatic reboots are switched off.
270
+ ```
271
+ server$ sudo TS_AUTHKEY=<signed key from 0.3 step 4> RECOVERY_PUBKEY="$(cat fly-release-recovery.pub)" \
272
+ EXPECT_MANIFEST=<manifest sha256 from 4.1> TPM_UNLOCK=1 ADMIN_USER=fly-admin bash deploy/bootstrap-host.sh "$(cat fly-release.pub)"
273
+ ```
274
+ It hardens the box (roughly CIS level 1):
275
+ - **The machine:** Wi-Fi and Bluetooth drivers can never load; it warns if the IOMMU is off or the disk is not
276
+ encrypted.
277
+ - **SSH:** only `fly-admin`, key only (ed25519, P-256, FIDO), root off.
278
+ - **Firewall:** in and out; containers reach only HTTPS, HTTP (apt) and DNS.
279
+ - **Kernel and Docker:** user namespaces, no new privileges.
280
+ - **Monitoring:** fail2ban; an audit log of every read of the wallet keys, with a Telegram alert when anything but
281
+ the signer reads them; a daily AIDE file-integrity check (05:30 UTC) that reports to Telegram when files changed;
282
+ automatic security updates (reboot 04:00 UTC when needed and the TPM unlocks the disk).
283
+ - **Keys:** it makes the trading and payout keys and prints them, **once**, as `bot_key: <address>` (the trading
284
+ key) and `payout_key: <address>`. Copy both lines now. Later, once the containers run:
285
+ `server$ sudo fly signer status` (as `"trading"` and `"payout"`).
286
+ 6. **Keep this session open.** From a second terminal on the Mac, log in over Tailscale: `$ ssh fly-vault`. Only when
287
+ that works, run the second pass in the first session. It removes SSH from the local network, leaving Tailscale as
288
+ the only way in:
289
+ ```
290
+ server$ sudo CLOSE_PUBLIC_SSH=1 bash deploy/bootstrap-host.sh "$(cat fly-release.pub)"
291
+ ```
292
+ The recovery key already installed is kept. After this the machine has no open port except SSH over Tailscale.
293
+ 7. Physical access is now the way around every other control: keep the machine somewhere only you reach. The disk is
294
+ useless without the TPM on this board (or the passphrase), and the firmware password and USB-boot-off stop anyone
295
+ booting another system on it. Unbinding the TPM (a new board, a firmware reset) means typing the passphrase at the
296
+ console once and re-running step 5 with `TPM_UNLOCK=1`.
297
+
298
+ ### 4.3 The treasury (your phone)
299
+ 1. Open `https://fly-trader.app/owner.html` in Solflare's browser and connect Solflare.
300
+ 2. Paste the trading key (`bot_key`) and the payout key (`payout_key`), leave the multisig field empty, and tap
301
+ "Read the treasury".
302
+ 3. Tap "Create the treasury multisig": you are its only member, threshold 1, no time lock.
303
+ 4. Tap "Create L1" (2 SOL a day, to the trading wallet only) and "Create L2" (1 SOL a week, claims).
304
+ 5. Every check the page shows must pass. It prints the three lines for `custody.env`. **Save the multisig, L1 and L2
305
+ addresses in your password manager.** The page remembers them only in that browser, and the Revoke button needs
306
+ them.
307
+
308
+ ### 4.4 Settings, start, check
309
+ 1. Fill in the settings with `sudoedit`: `server$ sudoedit /srv/fly/custody.env`, and so on. The files are root-only
310
+ (mode 600).
311
+ - **`/srv/fly/custody.env`:** the three lines from the owner page (`VAULT_MULTISIG`, `VAULT_LIMIT_TRADING`,
312
+ `VAULT_LIMIT_PAYOUT`) and `VAULT_BACKUP_RECIPIENT` (your `age1…`). The float and caps have working defaults.
313
+ - **`/srv/fly/signer.env`:** `SIGNER_RPC_URL` (the second provider). Required: the signer does not start without it.
314
+ - **`/srv/fly/vault.env`:**
315
+ - `HELIUS_API_KEY`, `JUPITER_API_KEY`, `SOLANA_CHECK_RPC_URL`
316
+ - `CAPITAL_SOL` (your principal, e.g. 5)
317
+ - `FUNDING_ADDRESSES`: the addresses you will send principal from, comma-separated; at least your Solflare address
318
+ - `VAULT_ADDRESS`, `VAULT_TIMELOCK`, `VAULT_START_BLOCK` (`blockNumber`), `VAULT_IMPL_CODEHASHES`, all from step 2
319
+ - `RELAY_SECRET` (the relay's secret from step 3)
320
+ - `TELEGRAM_BOT_TOKEN`, `TELEGRAM_CHAT_ID`, `TELEGRAM_ADMIN_USER_ID`, `HEALTHCHECK_URL`
321
+ - `VAULT_BACKUP_BUCKET`, `VAULT_BACKUP_HF_TOKEN` (the bucket-only token)
322
+ - **`/etc/fly/update.json`:** `telegram_bot_token` and `telegram_chat_id` (the host updater, the key-read watcher and
323
+ AIDE report through these). `expect_manifest_sha256` was set by the bootstrap.
324
+
325
+ Everything must be filled in before the next step: a first release that does not come up healthy is refused for
326
+ good. If that happens: fix the cause, publish a new release, and put its manifest sha256 into
327
+ `expect_manifest_sha256` in `/etc/fly/update.json`.
328
+ 2. Start the updater. It installs the pinned release, builds the brain and signer images and starts them (the first
329
+ build takes several minutes):
330
+ ```
331
+ server$ sudo systemctl enable --now fly-update.timer
332
+ server$ sudo systemctl start fly-update.service # the first round now, instead of in 2 minutes
333
+ server$ sudo journalctl -u fly-update -f
334
+ ```
335
+ 3. Check everything:
336
+ - `server$ sudo /usr/local/lib/fly/check-host.sh`: every control must pass (keys only in the signer, Tailnet Lock on,
337
+ no public SSH…)
338
+ - `server$ sudo fly vault custody`: no problems
339
+ - `server$ sudo python3 /usr/local/lib/fly/fly_audit_watch.py test`: the Telegram test message arrives
340
+ 4. The console: `$ ssh -L 8501:127.0.0.1:8501 fly-vault`, then http://localhost:8501/vault.
341
 
342
  ## 5. Go live
343
+ 1. From a funding address, send **0.05 SOL to the trading wallet** (the `bot_key` address). It pays the fees of its own
344
+ refills and of claims.
345
+ 2. Send your principal to the **treasury** (the vault 0 address on the owner page) from a funding address. The fly
346
+ refills its 2 SOL float from it through L1 on its own.
347
+ 3. `server$ sudo fly vault status`: `"ledger": {"deposits": …}` must equal the SOL you sent from funding addresses. If
348
+ some came from another address, it counts as profit: `server$ sudo fly vault reclassify <signature> deposit`.
349
+ 4. The fly trades live after the handover (at least 20 paper trades). Settlements run every Monday 00:00 UTC; claims
350
+ open from the first one.
351
+ 5. When claims are larger than L2 allows in a week, Telegram says a claim is waiting. Raise L2 as in "Day to day".
352
+ Waiting claims never fail.
353
+
354
+ ## Emergency
355
+ - **Something is wrong (possible compromise):**
356
+ 1. On your phone, open the owner page and tap **Revoke L1 and L2 now**. From then on the server cannot move any SOL
357
+ out of the treasury; trading keeps only its float.
358
+ 2. Send `/panic` to the bot. The fly halts; the signer refuses buys, top-ups and claims; open positions are sold; and
359
+ the float goes back to the treasury, all but the 0.3 SOL gas reserve.
360
+ 3. If $FLY locking should stop too: MetaMask, Blockscout, FlyVault `pause()`.
361
+ - **A key-read, halt, AIDE or setup-change alert:** look before resuming. Resuming is SSH-only:
362
+ ```
363
+ server$ sudo fly vault resume
364
+ server$ sudo fly reset-circuit --kill
365
+ server$ sudo fly resume-entries
366
+ server$ sudo fly signer clear-panic
367
+ ```
368
+ - **Rebuild after losing the server:**
369
+ 1. On the owner page, revoke L1 and L2 of the old keys.
370
+ 2. Publish nothing; take the manifest sha256 of the current release:
371
+ `$ curl -sL https://huggingface.co/bryceweiner/fly-trader/resolve/main/release.json | shasum -a 256`.
372
+ 3. New server: sections 4.2 (with that pin) to 4.4. On the owner page, clear the old L1/L2 fields and create new
373
+ limits for the new keys; put the new addresses in `custody.env`.
374
+ 4. Restore the ledger. Download the newest `ledger/<day>.dump.age` from the backup bucket, then:
375
+ ```
376
+ $ age -d -i ~/fly-keys/fly-backup.key <day>.dump.age > ledger.dump
377
+ $ scp ledger.dump fly-vault: && rm ledger.dump
378
+ server$ sudo docker stop fly-fly-1 # the brain must not write while the tables are replaced
379
+ server$ sudo docker exec -i fly-db-1 pg_restore -U fly -d fly_trader --clean < ledger.dump && rm ledger.dump
380
+ server$ sudo docker exec fly-db-1 psql -U fly -d fly_trader -c "DELETE FROM vault_kv WHERE key IN ('backup_key_done','custody','halt','panic')"
381
+ server$ sudo python3 /usr/local/lib/fly/fly_update.py restart
382
+ ```
383
+ 5. The old trading wallet's float and positions: `$ age -d -i ~/fly-keys/fly-backup.key keys/trading-<address>.age`
384
+ gives its secret (base58, 64 bytes), which you can import into a wallet to move them to the treasury.
385
 
386
  ## Day to day
387
+ - **Changing a limit** (e.g. raising L2 after a big settlement): on the owner page, "Set L2 to this amount". A new
388
+ limit has a **new address**, so put it in `/srv/fly/custody.env` and restart:
389
+ `server$ sudo python3 /usr/local/lib/fly/fly_update.py restart`. Until then claims wait. The same applies to any L1
390
+ change.
391
+ - **After editing any `/srv/fly/*.env`:** `server$ sudo python3 /usr/local/lib/fly/fly_update.py restart`.
392
+ - **New models or code:** publish as in 4.1 step 3 (`--models` to refresh the models).
393
+ - Model-only releases apply within 5 minutes; they cannot carry code (weights-only torch files, skops selectors).
394
+ - A release that changes **code** is held 24 h, and Telegram says so. To apply it now:
395
+ `server$ sudo python3 /usr/local/lib/fly/fly_update.py approve <seq>`. To stop it:
396
+ `server$ sudo python3 /usr/local/lib/fly/fly_update.py veto <seq>`. If you did not publish it, the release key is
397
+ compromised: remove it from `/etc/fly/allowed_signers` and rotate.
398
+ - A release that changes `deploy/`, `docker/`, the Dockerfile, `.dockerignore` or a compose file waits for
399
+ `approve`. It replaces only the compose file. For changed host scripts (bootstrap, `fly_update.py`,
400
+ `check-host.sh`, the watchers), copy them over and re-run the bootstrap **with `CLOSE_PUBLIC_SSH=1`** (without it
401
+ the bootstrap opens public SSH again); your recovery key is kept:
402
+ ```
403
+ $ scp -r ../fly-trader-dist/deploy ~/.ssh/fly-release.pub fly-vault:
404
+ server$ sudo CLOSE_PUBLIC_SSH=1 bash deploy/bootstrap-host.sh "$(cat fly-release.pub)"
405
+ ```
406
+ - To sign with the recovery key: decrypt it (`$ age -d ~/fly-keys/fly-release-recovery.age > /tmp/k && chmod 600
407
+ /tmp/k`), pass `--key /tmp/k`, then `rm /tmp/k`.
408
+ - **Take principal out:** `server$ sudo fly vault withdraw` prints how much you may take and how. You send it from the
409
+ treasury in the Squads app with your owner wallet; the fly books and reports it.
410
+ - **Watch it:** `/status` to the bot at any time. A daily report arrives at 09:00 UTC. If it does not, or
411
+ healthchecks.io alerts, the server or the fly is down. `server$ sudo fly vault status` gives the full ledger.
412
+ - **After an AIDE alert** you expected (a release, apt upgrades): `server$ sudo aide --update` then
413
+ `server$ sudo cp /var/lib/aide/aide.db.new /var/lib/aide/aide.db`.
414
+ - **Backups:** hot keys once, ledger nightly, both age-encrypted in the bucket. Until `VAULT_BACKUP_*` and the recipient
415
+ are set, Telegram reminds you daily. To read them:
416
+ - Hot keys: `$ age -d -i ~/fly-keys/fly-backup.key keys/<name>-<address>.age`
417
+ - Ledger: see "Rebuild", step 4.
418
+
419
+ ## Ledger (when you're home): no redeploys
420
+ - **Solana:** in the Squads app, add the Ledger as a member, then remove Solflare (or keep both at threshold 2; the
421
+ owner page then proposes and approves, and the other member executes in the Squads app). L1 and L2 stay as they are.
422
+ - **EVM:** move MetaMask's roles to the Ledger through the timelock (8 days):
423
+ 1. Print the calldata:
424
+ ```
425
+ $ (cd contracts && VAULT=<vault> TIMELOCK=<timelock> OLD_OWNER=<MetaMask> NEW_OWNER=<Ledger> MODE=calldata \
426
+ forge script script/HandOver.s.sol --rpc-url robinhood)
427
+ ```
428
+ 2. Send the printed "schedule" data from MetaMask to the timelock (Blockscout "Write contract", `scheduleBatch`, or
429
+ any raw-data send).
430
+ 3. After 8 days, send the "execute" data from any wallet.
431
+ - **SSH and releases:** add the Ledger's SSH key (ledger-agent) to `~fly-admin/.ssh/authorized_keys`. For releases,
432
+ make the Ledger's key the release key: re-run the bootstrap (as in "Day to day") with the Ledger's public key as the
433
+ argument (the bootstrap rewrites `/etc/fly/allowed_signers` from its arguments; a line added by hand would be lost),
434
+ and replace the `bryce` line in the distribution's `release/allowed_signers` (publish that change while the old key
435
+ still signs). Then remove the Secretive keys.
docs/vault/SPEC.md CHANGED
@@ -1,11 +1,12 @@
1
  # $FLY vault — interface spec v1
2
 
3
- The contract between the four parts of the vault system. Change it here first, then in code.
4
 
5
  - **FlyVault** (`contracts/`, Robinhood Chain): holds locked $FLY.
6
  - **The vault fly** (`fly_trader/vault/`, the hosted server): trades, keeps the ledger, settles weekly and pays claims.
 
7
  - **The relay** (`web/relay/`, on the site's Gandi app): passes stats and claims between the fly and browsers.
8
- - **The site** (`web/`): the Trading and Vault pages.
9
 
10
  Units, everywhere:
11
  - SOL amounts are integer **lamports**.
@@ -19,18 +20,18 @@ Solidity `0.8.24+` on OpenZeppelin Contracts Upgradeable v5, using UUPS and ERC-
19
  (`fly.storage.FlyVault`). The constructor calls `_disableInitializers()`.
20
 
21
  ```solidity
22
- uint256 public immutable WITHDRAW_DELAY; // constructor arg of the implementation: 7 days on mainnet (Deploy.s.sol refuses
23
- // anything else on chain 4663); shorter on the testnet rehearsal. Changing it = an upgrade.
24
  bytes32 public constant PAUSER_ROLE = keccak256("PAUSER_ROLE");
25
  bytes32 public constant UPGRADER_ROLE = keccak256("UPGRADER_ROLE");
26
 
27
- function initialize(address fly, address pauser, address timelock) external initializer;
28
  // DEFAULT_ADMIN_ROLE + UPGRADER_ROLE -> timelock ; PAUSER_ROLE -> pauser
29
 
30
- function lock(uint256 amount) external; // whenNotPaused; reverts unless received == amount
31
  function requestWithdrawal(uint256 amount) external returns (uint256 id); // never paused
32
  function cancelRequest(uint256 id) external; // whenNotPaused (it re-locks)
33
- function withdraw(uint256 id) external; // never paused; block.timestamp >= readyAt
34
  function pause() external; function unpause() external; // PAUSER_ROLE
35
 
36
  function token() external view returns (address);
@@ -47,16 +48,29 @@ event WithdrawRequested(address indexed user, uint256 indexed id, uint256 amount
47
  event RequestCancelled(address indexed user, uint256 indexed id, uint256 amount, uint256 lockedAfter);
48
  event Withdrawn(address indexed user, uint256 indexed id, uint256 amount);
49
  // plus OZ: Paused(address), Unpaused(address), Upgraded(address indexed implementation), RoleGranted/RoleRevoked
 
 
 
 
50
  ```
51
 
52
  - **Request ids** start at 1 and are global.
53
  - **`lockedAfter`** is the user's earning balance after the event, so the indexer can resync to an absolute value.
54
- - **Timelock.** Upgrades go through an OZ `TimelockController`:
55
- - minDelay = 8 days
56
- - proposer = canceller = the operator's hardware account
57
  - executor = anyone (`address(0)`)
58
- - admin = `address(0)`
 
 
59
  - **No rescue function for FLY.** The token is set only in `initialize`.
 
 
 
 
 
 
 
60
 
61
  ## 2. Claim messages (v1)
62
 
@@ -66,13 +80,15 @@ newline. Vectors: `tests/vectors/claim_v1.json`. Python (`fly_trader/vault/claim
66
 
67
  Fields:
68
  - `domain`: e.g. `fly-trader.app`, or `localhost:5173` on staging
69
- - `uri`: e.g. `https://fly-trader.app/vault.html`
 
70
  - `chain_id`: `4663`, or `46630` on testnet
71
  - `sol_chain`: `mainnet` or `devnet`
72
  - `evm`: EIP-55 checksummed
73
  - `sol`: base58 pubkey
74
  - `nonce`: 32 lowercase hex characters, issued by the relay
75
- - `issued_at`, `expires_at`: RFC 3339 `YYYY-MM-DDTHH:MM:SSZ`; expiry = issue + 900 s
 
76
  - `request_id`: always `fly-vault-claim-v1`
77
 
78
  EVM text, signed with `personal_sign` (EIP-191); the signature is `0x…` hex, 65 bytes for an EOA and up to 1024 bytes
@@ -108,60 +124,95 @@ Request ID: fly-vault-claim-v1
108
  ```
109
 
110
  What the fly checks:
111
- - **The texts.** It re-renders both texts from the fields and never uses text supplied by the relay.
 
 
 
112
  - **The EVM signature.**
113
- - Low-s, and v ∈ {27, 28} (0/1 are normalized).
114
- - Recovered by ecrecover. If the address has code, it instead calls EIP-1271 `isValidSignature(hashMessage(text), sig) == 0x1626ba7e`.
115
- - **The Solana signature.** ed25519 over the text.
116
- - **The nonce.** Unique per EVM address and not expired.
117
- - **The amount.** Owed ≥ `CLAIM_MIN_LAMPORTS` (2,000,000).
118
-
119
- A valid claim pays **all** SOL owed to the EVM address, sent to `sol`.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
120
 
121
  ## 3. Relay HTTP API (v1)
122
 
123
  The base is `{origin}/api`. Every response is JSON with `Cache-Control: no-store`. Errors look like `{"error": "<text>"}`
124
- with status 400, 401, 404, 409, 413, 429 or 503.
 
125
 
126
  ### Public
127
 
128
  - **`GET /api/stats`** returns the latest snapshot (§4), or 503 before the first push.
129
  - **`GET /api/history?kind=nav|trades|flows|settlements&before=<cursor>&limit=<1..500, default 100>`** returns
130
- `{"kind", "items": [...], "next": <cursor|null>}`, newest first. The cursor is opaque; send it back as `before`.
131
- - **`GET /api/account?evm=0x…`** returns the account (§4.3). An unknown address gets all zeros.
 
 
 
132
  - **`GET /api/claim/challenge?evm=0x…&sol=<base58>`**:
133
  - Returns `{"nonce", "issued_at", "expires_at", "domain", "uri", "chain_id", "sol_chain", "request_id", "min_lamports"}`.
134
- - The relay stores the challenge and checks the syntax of `evm` and `sol`.
135
  - **`POST /api/claim`** with body `{"nonce", "evm", "sol", "evm_sig", "sol_sig"}` (≤ 4 KB):
136
  - Returns `202 {"id", "status": "received"}`.
137
  - Checks, all cheap and none cryptographic:
138
  - the nonce is known, unexpired, unused, and issued for this `evm` + `sol`
139
  - the signatures have valid syntax: `evm_sig` matches `^0x([0-9a-fA-F]{2}){65,1024}$`; `sol_sig` is base58 of 64 bytes
140
- - the published account's owed ≥ min
141
- - no claim for this EVM address is in flight
142
- - 409 if the nonce was already used or a claim is in flight.
143
- - **`GET /api/claim/<id>`** returns `{"id", "status", "reason", "lamports", "tx", "created_at", "updated_at"}`.
 
 
 
 
144
  - Status is one of `received | verified | waiting_liquidity | sending | paid | rejected | failed`.
145
- - **Rate limits (token buckets):**
 
146
  - challenge: 30 per hour per IP
147
- - claim: 10 per hour per IP and 5 per hour per EVM address
148
- - reads: 600 per hour per IP.
 
 
 
149
 
150
  ### Fly only (HMAC)
151
 
152
  - **`POST /api/fly/push`**: upserts.
153
  - Body: `{"stats": {…§4…}?, "history": {"nav": [...], "trades": [...], "flows": [...], "settlements": [...]}?, "accounts": [ …§4.3… ]?}`.
154
- - History items upsert by `id` (nav by `ts`); accounts upsert by `evm`.
 
 
 
155
  - Returns `{"ok": true}`. Body ≤ 2 MB.
156
- - **`GET /api/fly/claims?after=<id>&limit=<1..50>`**:
157
  - Returns `{"claims": [{"id", "nonce", "evm", "sol", "evm_sig", "sol_sig", "domain", "uri", "chain_id", "sol_chain", "issued_at", "expires_at", "created_at"}]}`.
158
- - Only claims in status `received`, in id order.
159
- - **`POST /api/fly/claims/result`**: body `{"results": [{"id", "status", "reason"?, "lamports"?, "tx"?}]}` returns `{"ok": true}`.
160
- - **`GET /api/fly/probe`** returns `{"python", "db_path", "writable", "wal", "pid", "remote_addr", "headers": {...}}`.
 
 
 
161
 
162
  HMAC headers:
163
  - `X-Fly-Key`: the key id
164
- - `X-Fly-Ts`: unix seconds
165
  - `X-Fly-Nonce`: 32 hex characters
166
  - `X-Fly-Sig`: hex HMAC-SHA256(key = the UTF-8 bytes of the configured secret string, canonical)
167
 
@@ -169,12 +220,15 @@ The canonical string:
169
  ```
170
  FLY-RELAY-1\n{METHOD}\n{PATH}\n{QUERY}\n{TS}\n{NONCE}\n{sha256hex(body)}
171
  ```
172
- - `PATH` excludes the query string.
173
  - `QUERY` is `&`.join(`quote(k,safe='')=quote(v,safe='')` for (k, v) in sorted(parse_qsl(qs, keep_blank_values=True))).
174
  - The body is the raw bytes; an empty body hashes as sha256 of `b""`.
175
- - The timestamp must be within ±300 s. Nonces are remembered for 15 minutes, and a reused nonce gets 401.
 
176
 
177
- Relay config: `relay.json`, kept next to `wsgi.py` and never inside `site/`:
 
 
178
  ```json
179
  {"keys": {"k1": "<64 hex>"}, "db_path": "<writable path>", "domain": "fly-trader.app", "uri": "https://fly-trader.app/vault.html",
180
  "chain_id": 4663, "sol_chain": "mainnet", "claim_ttl_s": 900, "min_lamports": 2000000, "client_ip_header": null}
@@ -182,33 +236,56 @@ Relay config: `relay.json`, kept next to `wsgi.py` and never inside `site/`:
182
 
183
  ## 4. Payloads
184
 
185
- ### 4.1 Stats snapshot (`stats`, pushed every 60 s)
186
  ```json
187
  {
188
- "v": 1, "ts": 0, "cluster": "mainnet",
189
- "fly": {"state": "starting|paper|live|halted", "wallet": "<base58>", "handover": false, "kill_switch": false,
190
  "entries_paused": false, "model": {"fly": 62, "selector": 59, "release": 3}},
191
- "wallet": {"native": 0, "token_accounts": 0, "open_cost": 0, "positions_value": 0, "exit_cost": 0, "nav": 0},
192
  "ledger": {"deposits": 0, "withdrawals": 0, "claims_paid": 0, "realized": 0, "booked_realized": 0,
193
  "allocated": 0, "reserved": 0, "pot": 0},
194
  "prices": {"sol_usd": 0.0, "fly_usd": 0.0, "ts": 0},
195
  "vault": {"address": "0x…", "chain_id": 4663, "total_locked": "0", "total_pending": "0", "earners": 0,
196
  "finalized_block": 0, "paused": false, "impl": "0x…", "upgrade_scheduled": null},
197
- "settlement": {"next_at": 0, "last": null},
198
- "positions": [{"mint": "", "symbol": "", "opened_at": 0, "cost": 0, "value": 0, "entry_price": 0.0, "mark_price": 0.0, "hold_min": 0}],
199
- "index": {"value": 1.0, "peak": 1.0, "drawdown": 0.0}
200
  }
201
  ```
202
- - `wallet.nav` = native + token_accounts + positions_value − exit_cost.
203
- - `ledger.realized` is R. `pot` = max(0, R − allocated). `reserved` = allocated − claims_paid.
204
- - `settlement.last` is a settlements item (§4.2) or null.
205
- - `vault.upgrade_scheduled` is `{"eta": ts, "id": "0x…"}` or null.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
206
 
207
  ### 4.2 History items
208
- - `nav`: `{"ts", "nav", "index", "sol_usd"}`
209
- - `trades`: `{"id", "mint", "symbol", "opened_at", "closed_at", "cost", "proceeds", "realized", "exit_kind"}`
210
- - `flows`: `{"id", "ts", "signature", "direction": "in|out", "kind": "deposit|profit|withdrawal|claim", "counterparty", "lamports"}`
211
- - `settlements`: `{"id", "period_start", "period_end", "realized", "pot", "allocated", "carried", "earners", "total_weight": "<str>", "status"}`
 
 
 
 
 
212
 
213
  ### 4.3 Account
214
  ```json
@@ -216,15 +293,114 @@ Relay config: `relay.json`, kept next to `wsgi.py` and never inside `site/`:
216
  "allocations": [{"period_end": 0, "lamports": 0, "weight": "<str>", "share": 0.0}],
217
  "claims": [{"id": 0, "status": "", "lamports": 0, "sol": "", "tx": "", "created_at": 0, "updated_at": 0}]}
218
  ```
219
- `owed` = allocated − claimed − in_flight, where in_flight counts claims in `verified | waiting_liquidity | sending`.
 
 
 
220
 
221
  ## 5. Networks
222
 
223
  | | mainnet | rehearsal |
224
  |---|---|---|
 
225
  | EVM chain | Robinhood Chain 4663, `https://rpc.mainnet.chain.robinhood.com` | RH testnet 46630, `https://rpc.testnet.chain.robinhood.com` |
226
  | Explorer | `https://robinhoodchain.blockscout.com` | `https://explorer.testnet.chain.robinhood.com` |
227
  | $FLY | `0x2fC7f9E2911f20b2C4660d2AEf808aa91bDdb3D3` | `MockFLY` (deployed for the rehearsal) |
 
 
228
  | Solana | mainnet-beta | devnet |
 
 
229
  | Kyber | `https://aggregator-api.kyberswap.com/robinhood/api/v1/`, router `0x6131B5fae19EA4f9D964eAc0408E4408b66337b5` | — |
230
  | Chart | GeckoTerminal network `robinhood`, pool `0xe6925f7bdedf22c2714c749d0ff208ce9e4bc1540938fa2b9ab2379486322edd` | fixtures |
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  # $FLY vault — interface spec v1
2
 
3
+ The contract between the parts of the vault system. Change it here first, then in code.
4
 
5
  - **FlyVault** (`contracts/`, Robinhood Chain): holds locked $FLY.
6
  - **The vault fly** (`fly_trader/vault/`, the hosted server): trades, keeps the ledger, settles weekly and pays claims.
7
+ - **The signer** (`fly_trader/signer/`, its own container on the server): the only holder of the two hot keys (§6).
8
  - **The relay** (`web/relay/`, on the site's Gandi app): passes stats and claims between the fly and browsers.
9
+ - **The site** (`web/`): the Trading, Vault and (unlisted) Owner pages.
10
 
11
  Units, everywhere:
12
  - SOL amounts are integer **lamports**.
 
20
  (`fly.storage.FlyVault`). The constructor calls `_disableInitializers()`.
21
 
22
  ```solidity
23
+ uint256 public immutable WITHDRAW_DELAY; // constructor arg of the implementation: 7 days on mainnet, 300 s by default
24
+ // on the rehearsal (DeployTestnet.s.sol). Changing it = an upgrade.
25
  bytes32 public constant PAUSER_ROLE = keccak256("PAUSER_ROLE");
26
  bytes32 public constant UPGRADER_ROLE = keccak256("UPGRADER_ROLE");
27
 
28
+ function initialize(address fly, address pauser, address timelock) external initializer; // reverts on a zero address
29
  // DEFAULT_ADMIN_ROLE + UPGRADER_ROLE -> timelock ; PAUSER_ROLE -> pauser
30
 
31
+ function lock(uint256 amount) external; // whenNotPaused, nonReentrant; reverts unless received == amount
32
  function requestWithdrawal(uint256 amount) external returns (uint256 id); // never paused
33
  function cancelRequest(uint256 id) external; // whenNotPaused (it re-locks)
34
+ function withdraw(uint256 id) external; // never paused, nonReentrant; block.timestamp >= readyAt
35
  function pause() external; function unpause() external; // PAUSER_ROLE
36
 
37
  function token() external view returns (address);
 
48
  event RequestCancelled(address indexed user, uint256 indexed id, uint256 amount, uint256 lockedAfter);
49
  event Withdrawn(address indexed user, uint256 indexed id, uint256 amount);
50
  // plus OZ: Paused(address), Unpaused(address), Upgraded(address indexed implementation), RoleGranted/RoleRevoked
51
+
52
+ error ZeroAddress(); error ZeroAmount(); error TransferAmountMismatch(uint256 expected, uint256 received);
53
+ error InsufficientLocked(uint256 requested, uint256 available); error UnknownRequest(uint256 id);
54
+ error NotRequestOwner(uint256 id); error RequestNotPending(uint256 id); error WithdrawalNotReady(uint256 id, uint64 readyAt);
55
  ```
56
 
57
  - **Request ids** start at 1 and are global.
58
  - **`lockedAfter`** is the user's earning balance after the event, so the indexer can resync to an absolute value.
59
+ - **Timelock.** Upgrades and role changes go through an OZ `TimelockController`:
60
+ - minDelay = 8 days on mainnet (600 s by default on the rehearsal)
61
+ - proposer = canceller = the owner (MetaMask until the Ledger takes over; `OWNER` in the deploy)
62
  - executor = anyone (`address(0)`)
63
+ - admin = `address(0)`: the timelock administers itself, so role changes also wait the full delay
64
+ - **Mainnet deploy rules.** On chain 4663 the deploy (`FlyVaultStack.enforceChainRules`) refuses anything but the real
65
+ $FLY, an 8-day timelock and a 7-day withdraw delay. `PAUSER` defaults to `OWNER`.
66
  - **No rescue function for FLY.** The token is set only in `initialize`.
67
+ - **The fly indexes** the vault's events and the timelock's `CallScheduled`, `CallExecuted` and `Cancelled` for
68
+ operations aimed at the vault or the timelock (an alert and `vault.upgrade_scheduled`, §4.1).
69
+ - **Scripts.**
70
+ - `HandOver.s.sol`: one timelock batch that grants PROPOSER, CANCELLER and PAUSER to a new owner and revokes them
71
+ from the old one. `MODE=schedule|execute`, or `MODE=calldata` to print the calldata for a browser wallet.
72
+ - `ScheduleUpgrade.s.sol` rehearses an upgrade locally and schedules `upgradeToAndCall`; `ExecuteUpgrade.s.sol` runs
73
+ it after the delay.
74
 
75
  ## 2. Claim messages (v1)
76
 
 
80
 
81
  Fields:
82
  - `domain`: e.g. `fly-trader.app`, or `localhost:5173` on staging
83
+ - `uri`: e.g. `https://fly-trader.app/vault.html`, or `http://localhost:5173/vault.html` on staging; https (http only
84
+ for localhost/127.0.0.1), no spaces or line breaks
85
  - `chain_id`: `4663`, or `46630` on testnet
86
  - `sol_chain`: `mainnet` or `devnet`
87
  - `evm`: EIP-55 checksummed
88
  - `sol`: base58 pubkey
89
  - `nonce`: 32 lowercase hex characters, issued by the relay
90
+ - `issued_at`, `expires_at`: RFC 3339 `YYYY-MM-DDTHH:MM:SSZ`; expiry = issue + 900 s. The fly accepts no other TTL, so
91
+ the relay's `claim_ttl_s` must stay 900.
92
  - `request_id`: always `fly-vault-claim-v1`
93
 
94
  EVM text, signed with `personal_sign` (EIP-191); the signature is `0x…` hex, 65 bytes for an EOA and up to 1024 bytes
 
124
  ```
125
 
126
  What the fly checks:
127
+ - **The texts.** Both are re-rendered from the fields; relay-supplied text is never used. `domain`/`uri` must equal
128
+ `VAULT_SITE_DOMAIN`/`VAULT_SITE_URI`, `chain_id` must equal `RH_CHAIN_ID`, and `sol_chain` must match `VAULT_CLUSTER`.
129
+ - **Freshness.** A claim is rejected if the fly first stored it more than 3600 s after `expires_at`, or if `issued_at`
130
+ is more than 300 s in the future. The relay already refuses expired nonces at submission.
131
  - **The EVM signature.**
132
+ - ecrecover first: exactly 65 bytes, 0 < r < n, low-s, v ∈ {27, 28} (0/1 are normalized).
133
+ - Only if that does not recover the address and the address has code: EIP-1271 `isValidSignature(hashMessage(text), sig)`
134
+ (65..1024 bytes, 200,000 gas cap) must return exactly `0x1626ba7e`.
135
+ - **The Solana signature.** 64 bytes, ed25519 over the UTF-8 text; `sol` must be an on-curve pubkey.
136
+ - **The nonce.** Unique per EVM address.
137
+ - **One in flight.** A claim is rejected while another claim for the address is `verified | waiting_liquidity |
138
+ sending`.
139
+ - **The amount.** Owed ≥ `CLAIM_MIN_LAMPORTS` (2,000,000), read when the claim moves to `sending`; the amount is fixed
140
+ then. The relay's `min_lamports` and the site's `claimMinLamports` must hold the same value.
141
+ - **Halts.** Nothing is verified or paid while the vault is halted (claims are still pulled from the relay).
142
+
143
+ A valid claim pays **all** SOL owed to the EVM address, sent to `sol` from the **treasury** (Squads vault 0) under
144
+ spending limit L2 (§6): the payout key signs `spendingLimitUse` and the trading key pays the fee. Nothing is ever paid
145
+ from the trading wallet's own SOL. Neither the treasury, the trading wallet nor any payout signature is published.
146
+
147
+ **Statuses:** `received → verified → (waiting_liquidity) → sending → paid | rejected | failed`.
148
+ - `waiting_liquidity`: L2 has too little left this week, the treasury is short, or the signer is in panic. The claim
149
+ retries every loop and alerts the operator; it never fails for these reasons.
150
+ - `failed`: four unlanded attempts, a failure on chain, or a signer policy refusal.
151
+ - The payment's signature and last valid block height are stored before broadcast; a claim is re-signed only once its
152
+ blockhash has expired unlanded, and the signer keeps its own once-per-claim ledger (§6).
153
 
154
  ## 3. Relay HTTP API (v1)
155
 
156
  The base is `{origin}/api`. Every response is JSON with `Cache-Control: no-store`. Errors look like `{"error": "<text>"}`
157
+ with status 400, 401, 404, 405 (with `Allow`), 409, 413, 429 (with `Retry-After`), 500 or 503. 503 means either
158
+ `relay not configured` (relay.json missing or invalid) or `storage unavailable` (SQLite error, with `Retry-After: 5`).
159
 
160
  ### Public
161
 
162
  - **`GET /api/stats`** returns the latest snapshot (§4), or 503 before the first push.
163
  - **`GET /api/history?kind=nav|trades|flows|settlements&before=<cursor>&limit=<1..500, default 100>`** returns
164
+ `{"kind", "items": [...], "next": <cursor|null>}`, newest first by (time, id). The time is nav `ts`, trades
165
+ `closed_at`, flows `ts`, settlements `period_end`. The cursor is opaque; send it back as `before`. An unknown `kind`,
166
+ a bad `limit` or a bad cursor gets 400.
167
+ - **`GET /api/account?evm=0x…`** returns the account (§4.3). `evm` is accepted in any case. An unknown address gets all
168
+ zeros; the fly pushes only addresses with at least one allocation.
169
  - **`GET /api/claim/challenge?evm=0x…&sol=<base58>`**:
170
  - Returns `{"nonce", "issued_at", "expires_at", "domain", "uri", "chain_id", "sol_chain", "request_id", "min_lamports"}`.
171
+ - The relay stores the challenge; a bad `evm` or `sol` gets 400 (checked before the rate limit).
172
  - **`POST /api/claim`** with body `{"nonce", "evm", "sol", "evm_sig", "sol_sig"}` (≤ 4 KB):
173
  - Returns `202 {"id", "status": "received"}`.
174
  - Checks, all cheap and none cryptographic:
175
  - the nonce is known, unexpired, unused, and issued for this `evm` + `sol`
176
  - the signatures have valid syntax: `evm_sig` matches `^0x([0-9a-fA-F]{2}){65,1024}$`; `sol_sig` is base58 of 64 bytes
177
+ - the published account's owed ≥ `min_lamports`
178
+ - no claim for this EVM address that the fly has verified is in flight (`verified | waiting_liquidity |
179
+ sending`). An unverified `received` claim does not block, since it may be a stranger's junk; the fly pays
180
+ each address at most once whatever the relay accepts.
181
+ - 400 for bad syntax, an unknown, expired or mismatched nonce, or nothing to claim; 409 if the nonce was already used
182
+ or a verified claim is in flight; 429 per the limits below.
183
+ - **`GET /api/claim/<id>`** (the relay's id, 1-18 digits) returns `{"id", "status", "reason", "lamports", "tx",
184
+ "created_at", "updated_at"}`, or 404.
185
  - Status is one of `received | verified | waiting_liquidity | sending | paid | rejected | failed`.
186
+ - `tx` is always null: the fly never reports the payout signature.
187
+ - **Rate limits (token buckets; a 429 carries `Retry-After`):**
188
  - challenge: 30 per hour per IP
189
+ - claim: 10 per hour per IP, charged after the syntax checks; and 5 per hour per EVM address and IP, charged only
190
+ after every other check passes (so junk from one client cannot lock a holder out)
191
+ - reads (stats, history, account, claim/<id>): 600 per hour per IP
192
+ - IPv6 clients are bucketed per /64. Behind a proxy, `client_ip_header` names the header to read, and the relay takes
193
+ its rightmost entry (the one the proxy appended). The fly's HMAC routes are not rate-limited.
194
 
195
  ### Fly only (HMAC)
196
 
197
  - **`POST /api/fly/push`**: upserts.
198
  - Body: `{"stats": {…§4…}?, "history": {"nav": [...], "trades": [...], "flows": [...], "settlements": [...]}?, "accounts": [ …§4.3… ]?}`.
199
+ - Unknown top-level keys get 400. History items upsert by `id` (nav by `ts`); each needs an integer time field (nav
200
+ `ts`; trades `closed_at` or `opened_at`; settlements `period_end` or `period_start`; flows `ts`) and, except nav, an
201
+ `id` (an integer ≥ 0 or a string ≤ 200 characters). Accounts upsert by `evm` and need an integer `owed`. `stats` is
202
+ stored as is; NaN and Infinity become null.
203
  - Returns `{"ok": true}`. Body ≤ 2 MB.
204
+ - **`GET /api/fly/claims?after=<id, default 0>&limit=<1..50, default 50>`**:
205
  - Returns `{"claims": [{"id", "nonce", "evm", "sol", "evm_sig", "sol_sig", "domain", "uri", "chain_id", "sol_chain", "issued_at", "expires_at", "created_at"}]}`.
206
+ - Only claims in status `received`, in id order. `evm` is lowercase; `created_at` is unix seconds.
207
+ - **`POST /api/fly/claims/result`**: body `{"results": [{"id", "status", "reason"?, "lamports"?, "tx"?}]}` returns
208
+ `{"ok": true}`. `status` must be one of the seven statuses; `reason` is cut to 500 characters; omitted fields keep
209
+ their value; unknown ids are ignored.
210
+ - **`GET /api/fly/probe`** returns `{"python", "db_path", "writable", "wal", "pid", "remote_addr", "headers": {...},
211
+ "sqlite", "time", "client_ip", "multiprocess", "multithread"}` (headers: every `HTTP_*` except Cookie).
212
 
213
  HMAC headers:
214
  - `X-Fly-Key`: the key id
215
+ - `X-Fly-Ts`: unix seconds (1-15 digits)
216
  - `X-Fly-Nonce`: 32 hex characters
217
  - `X-Fly-Sig`: hex HMAC-SHA256(key = the UTF-8 bytes of the configured secret string, canonical)
218
 
 
220
  ```
221
  FLY-RELAY-1\n{METHOD}\n{PATH}\n{QUERY}\n{TS}\n{NONCE}\n{sha256hex(body)}
222
  ```
223
+ - `PATH` starts at `/api/…` and excludes the query string.
224
  - `QUERY` is `&`.join(`quote(k,safe='')=quote(v,safe='')` for (k, v) in sorted(parse_qsl(qs, keep_blank_values=True))).
225
  - The body is the raw bytes; an empty body hashes as sha256 of `b""`.
226
+ - The timestamp must be within ±300 s. Nonces are remembered for 15 minutes (case-insensitively), and a reused nonce
227
+ gets 401. The body is verified before the nonce is recorded.
228
 
229
+ Relay config: `relay.json`, kept next to `wsgi.py` and never inside `site/`, or wherever `FLY_RELAY_CONFIG` points. It
230
+ is re-read when it changes; a relative `db_path` is resolved against its folder. Secrets must be at least 32 characters
231
+ (64 hex recommended).
232
  ```json
233
  {"keys": {"k1": "<64 hex>"}, "db_path": "<writable path>", "domain": "fly-trader.app", "uri": "https://fly-trader.app/vault.html",
234
  "chain_id": 4663, "sol_chain": "mainnet", "claim_ttl_s": 900, "min_lamports": 2000000, "client_ip_header": null}
 
236
 
237
  ## 4. Payloads
238
 
239
+ ### 4.1 Stats snapshot (`stats`, pushed every vault-worker loop, about every 10 s)
240
  ```json
241
  {
242
+ "v": 1, "ts": 0, "book": "live", "cluster": "mainnet",
243
+ "fly": {"state": "starting|paper|live|halted", "handover": false, "kill_switch": false,
244
  "entries_paused": false, "model": {"fly": 62, "selector": 59, "release": 3}},
245
+ "wallet": {"native": 0, "nav": 0},
246
  "ledger": {"deposits": 0, "withdrawals": 0, "claims_paid": 0, "realized": 0, "booked_realized": 0,
247
  "allocated": 0, "reserved": 0, "pot": 0},
248
  "prices": {"sol_usd": 0.0, "fly_usd": 0.0, "ts": 0},
249
  "vault": {"address": "0x…", "chain_id": 4663, "total_locked": "0", "total_pending": "0", "earners": 0,
250
  "finalized_block": 0, "paused": false, "impl": "0x…", "upgrade_scheduled": null},
251
+ "settlement": {"last": null}
 
 
252
  }
253
  ```
254
+ - **Nothing that helps trade ahead of the fly.** `ts` is rounded down to the hour; `wallet.native` and `wallet.nav`
255
+ come from the last 5-minute mark and are rounded to 0.1 SOL (NAV history points likewise), so one buy cannot be
256
+ matched to an on-chain trade; `ledger.realized` is R at that same mark. Open positions, the performance index and
257
+ the next settlement time are not published; a trade appears in `trades` history once it closes. No address of the
258
+ fly's (trading wallet, treasury, keys) is published, nor anything that leads to one: no transaction signatures and no
259
+ deposit senders.
260
+ - `book` is `VAULT_BOOK` (`live`, or a paper book such as `paper_fly` on a dry run). `cluster` is `mainnet` or
261
+ `VAULT_CLUSTER` as is.
262
+ - `fly.state`: `halted` if the vault is halted or the kill switch is on; `live` after the handover once the live wallet
263
+ reports; `paper` once the fly reports status; else `starting`. `fly.model` = `{fly, selector, release}`, each may be
264
+ null.
265
+ - `wallet.native` = trading wallet + treasury lamports; `wallet.nav` = native + open positions' value − exit cost.
266
+ - `ledger`: `realized` is R (§7); `pot` = max(0, R − allocated); `reserved` = max(0, allocated − claims_paid);
267
+ `booked_realized` is the book's summed closed-trade profit; `deposits`, `withdrawals` (fees included) and
268
+ `claims_paid` are running totals. On a paper book `realized` is closed-trade profit since the vault started and
269
+ `deposits` is `CAPITAL_SOL`.
270
+ - `prices`: `sol_usd` from Jupiter Price v3, `fly_usd` from the GeckoTerminal pool, cached 60 s (a failed fetch keeps
271
+ the last value); `ts` is the older of the two fetch times, or 0.
272
+ - `vault`: `total_locked`, `total_pending`, `earners` and `paused` come from events in finalized blocks (about 18 minutes
273
+ behind the head), up to `finalized_block`. `address` is null until `VAULT_ADDRESS` is set.
274
+ - `settlement.last` is the newest `allocated` settlement (§4.2), or null.
275
+ - `vault.upgrade_scheduled` is `{"eta": ts, "id": "0x…"}` or null: the soonest pending timelock operation whose target is the
276
+ vault (upgrade, role change) or the timelock itself (`updateDelay`, proposer changes). `eta` is the block time of
277
+ `CallScheduled` plus its `delay`.
278
 
279
  ### 4.2 History items
280
+ - `nav`: `{"ts", "nav", "sol_usd"}`: one point per 5-minute boundary (`ts`), from the newest consistent mark at or before
281
+ it, nav rounded to 0.1 SOL; `sol_usd` is the price at push time.
282
+ - `trades`: `{"id", "mint", "symbol", "opened_at", "closed_at", "cost", "proceeds", "realized", "exit_kind"}`: closed
283
+ positions of `VAULT_BOOK` only; `proceeds` = cost + realized; `exit_kind` is the forced-exit kind or `hold`.
284
+ - `flows`: `{"id", "ts", "direction": "in|out", "kind": "deposit|profit|withdrawal|claim", "lamports"}`. Internal moves
285
+ (`sweep`, `topup`) and `unknown_outbound`/`anomaly` rows are never published.
286
+ - `settlements`: `{"id", "period_start", "period_end", "realized", "pot", "allocated", "carried", "earners",
287
+ "total_weight": "<str>", "status"}`: `status` is always `allocated`; `carried` = pot − allocated; `total_weight` is
288
+ the sum of wei·seconds; `realized` is R at the snapshot.
289
 
290
  ### 4.3 Account
291
  ```json
 
293
  "allocations": [{"period_end": 0, "lamports": 0, "weight": "<str>", "share": 0.0}],
294
  "claims": [{"id": 0, "status": "", "lamports": 0, "sol": "", "tx": "", "created_at": 0, "updated_at": 0}]}
295
  ```
296
+ - `owed` = allocated − claimed − in_flight. `in_flight` counts claims from `sending` on (a `verified` claim has no amount
297
+ yet).
298
+ - `allocations`: the 52 most recent; `share` = weight / that settlement's total weight.
299
+ - `claims`: the 20 most recent; `id` is the relay's claim id (as in `GET /api/claim/<id>`); `tx` is always `""`.
300
 
301
  ## 5. Networks
302
 
303
  | | mainnet | rehearsal |
304
  |---|---|---|
305
+ | Site build | `VITE_NETWORK=mainnet` (default) | `VITE_NETWORK=testnet` |
306
  | EVM chain | Robinhood Chain 4663, `https://rpc.mainnet.chain.robinhood.com` | RH testnet 46630, `https://rpc.testnet.chain.robinhood.com` |
307
  | Explorer | `https://robinhoodchain.blockscout.com` | `https://explorer.testnet.chain.robinhood.com` |
308
  | $FLY | `0x2fC7f9E2911f20b2C4660d2AEf808aa91bDdb3D3` | `MockFLY` (deployed for the rehearsal) |
309
+ | USDG | `0x5fc5360D0400a0Fd4f2af552ADD042D716F1d168` (6 decimals) | — |
310
+ | Multicall3 | `0xcA11bde05977b3631167028862bE2a173976CA11` | same |
311
  | Solana | mainnet-beta | devnet |
312
+ | Squads v4 | `SQDS4ep65T869zMMBKyuUq6aD6EgTu8psMjkvj52pCf` | same |
313
+ | Settlement period | 7 days (`VAULT_PERIOD_S=604800`) | 900 s |
314
  | Kyber | `https://aggregator-api.kyberswap.com/robinhood/api/v1/`, router `0x6131B5fae19EA4f9D964eAc0408E4408b66337b5` | — |
315
  | Chart | GeckoTerminal network `robinhood`, pool `0xe6925f7bdedf22c2714c749d0ff208ce9e4bc1540938fa2b9ab2379486322edd` | fixtures |
316
+
317
+ - The fly's defaults are mainnet. A rehearsal sets `RH_CHAIN_ID=46630`, `RH_RPC_URL`, `VAULT_CLUSTER=devnet`,
318
+ `VAULT_SOLANA_RPC_URL`, `VAULT_PERIOD_S=900` and `VAULT_SITE_DOMAIN`/`VAULT_SITE_URI`.
319
+ - The site's Solana RPC (the Owner page) is WalletConnect's unless `VITE_SOLANA_RPC` is set.
320
+
321
+ ## 6. Custody (Solana)
322
+
323
+ The fly's SOL is one book in two accounts:
324
+ - **The treasury** is vault 0 of a Squads v4 multisig. Its members are the owner's wallets only (Solflare until the
325
+ Ledger takes over); threshold 1 and no time lock while there is one member.
326
+ - **The trading wallet** is a hot key that keeps a float of about `TREASURY_FLOAT_SOL` (default 2 SOL).
327
+
328
+ | Spending limit | Member | Destinations | Period | Used for |
329
+ |---|---|---|---|---|
330
+ | L1 | trading key | the trading wallet only | Day | refilling the float (`topup` flows, memo `fly-topup`) |
331
+ | L2 | payout key | any | Week | claims (`spendingLimitUse` from the treasury, memo `fly-vault-claim:<fly claim id>:<nonce>`) |
332
+
333
+ - **Rebalancing** (every minute, `vault/payout.py`): first the treasury must hold everything owed plus its 890,880-lamport
334
+ rent minimum, sweeping from the trading wallet if it is short; then the trading wallet is brought to the float, by a
335
+ top-up under L1 when low or a sweep when high. Moves under `TOPUP_MIN_SOL` (0.1) are skipped, except a sweep the
336
+ treasury needs. A sweep never takes the trading wallet below `GAS_RESERVE_SOL` + fee. Under `/panic` the float target
337
+ is 0.
338
+ - **Flows.** A trading → treasury transfer is a `sweep`, a treasury → trading one a `topup`: both internal, excluded
339
+ from R, the index and published history. SOL or wSOL arriving at either account from `FUNDING_ADDRESSES` is a
340
+ `deposit`; from anyone else it is `profit` (a gift). A treasury outflow signed by a multisig member is a `withdrawal`
341
+ (the owner's principal; alerted, louder when the destination is not a funding address). A transaction one of our
342
+ keys signed that no table knows is `unknown_outbound`; lamports leaving without our or an owner's signature are an
343
+ `anomaly`. Both halt the vault (§7), unless the second RPC provider sees the transaction differently.
344
+ - **Accounting.** R, NAV and the performance index count both accounts together. Positions are sized on the bankroll
345
+ = trading free SOL + open positions at cost + (treasury − owed); buys spend only the trading wallet's own SOL, less
346
+ any owed SOL the treasury does not hold.
347
+ - **The signer** (`fly_trader/signer`) is the only holder of the two hot keys. It builds and signs:
348
+ - Jupiter swaps: SOL ↔ one token only; Metis routes only (`excludeRouters=jupiterz,dflow,okx`); static rules on every
349
+ instruction (programs, the trading wallet's roles, a priority-fee cap of `SIGNER_MAX_PRIORITY_SOL`, 0.005); a
350
+ simulation, through its own RPC, of the trading wallet's SOL and every token account it owns (a buy may spend at
351
+ most the ordered SOL plus fees and must deliver the token to the trading wallet; a sell may spend at most the
352
+ ordered tokens and must return SOL; no other holding may shrink); a value floor of `SIGNER_MIN_VALUE_RATIO` (0.6)
353
+ at the signer's own Jupiter prices. Buys are capped per trade (`SIGNER_MAX_BUY_SOL`, default `MAX_POSITION_SOL`) and
354
+ per day (`SIGNER_DAILY_BUY_SOL`, 10); all swaps per minute (`SIGNER_SWAPS_PER_MIN`, 6).
355
+ - top-ups under L1 (at most `TREASURY_FLOAT_SOL` each; L1's destinations must be exactly the trading wallet) and
356
+ sweeps (a plain transfer to the treasury)
357
+ - claims under L2, once per claim id, with the same destination and amount; re-signed only after the earlier
358
+ signature failed or its blockhash expired
359
+ - closes of empty token accounts of the trading wallet (batches of 10)
360
+ - the hot keys, age-encrypted to `VAULT_BACKUP_RECIPIENT`, for the off-box backup
361
+ - after `panic`: no buys, top-ups or claims; `clear-panic` is a CLI command inside its container only
362
+
363
+ On the server the brain reaches it over a Unix socket (one JSON request per line, 256 KB max, one at a time) and never
364
+ sees a key. The signer keeps its own SQLite ledger. Without `SIGNER_SOCKET` (self-hosting, tests) the same signer runs
365
+ inside the brain's process.
366
+ - **Custody checks** (every minute, `vault/custody.py`), alerted: neither server key is a multisig member; no config
367
+ authority; L1 is a SOL limit with member = the trading key and sole destination = the trading wallet; L2 is a SOL
368
+ limit with member = the payout key. Any change to members, threshold, time lock or the limits is reported.
369
+ - **The owner** changes the limits from `owner.html` (`web/src/lib/squads.ts`). Every new limit has a fresh address:
370
+ the server's `VAULT_LIMIT_TRADING` / `VAULT_LIMIT_PAYOUT` must be updated to it (the page prints the lines). A
371
+ limit's amount is changed by removing it and adding a new one, in two config transactions: the program rejects both
372
+ in one.
373
+
374
+ ## 7. Settlement, halts, kill switch
375
+
376
+ - **Schedule.** Period ends fall at `VAULT_EPOCH + k·VAULT_PERIOD_S` (345600 = Monday 1970-01-05 00:00 UTC; 7 days).
377
+ The first settlement covers the period before the first end after the vault started; missed periods are settled as
378
+ one window. Settlements advance one step per minute, in period order.
379
+ - **Two phases.**
380
+ 1. Snapshot at the period end T1: both accounts at a finalized slot that contains every transaction the fly sent,
381
+ read under the wallet lock. The second RPC provider (`SOLANA_CHECK_RPC_URL`) must report the same balance, or the
382
+ settlement pauses, alerts and retries. Flows are scanned up to that slot.
383
+ 2. Allocate once Robinhood Chain is finalized past T1, and only if the vault's implementation code hash is in
384
+ `VAULT_IMPL_CODEHASHES` (else the vault halts).
385
+
386
+ Statuses: `snapshotted | allocated | failed`.
387
+ - **Formulas.**
388
+ ```
389
+ R = N + K + C − D + Wd + P N: both accounts' SOL; K: wSOL and empty token-account rent; C: open positions at
390
+ cost; D: deposits; Wd: withdrawals incl. fees; P: claims paid
391
+ pot = max(0, R − A) A: everything ever allocated
392
+ out = min(pot, N − (A − P) − GAS_RESERVE_SOL)
393
+ weight_i = ∫ lockedAfter_i dt over [T0, T1) (wei·s, from FlyVault events)
394
+ alloc_i = floor(out · weight_i / Σ weight); the remainder, dust and empty weeks carry forward (A grows only by alloc)
395
+ ```
396
+ - **Halts.** The vault halts on `unknown_outbound`, `anomaly`, an unknown implementation code hash, or Telegram
397
+ `/panic` (the operator's user only). A halt stops claim processing, settlement and new entries. `/panic` also trips
398
+ the kill switch with liquidation, sets the float target to 0 and puts the signer in panic. Resume is SSH-only:
399
+ `fly-trader vault resume` (clears halt and panic), `fly-trader reset-circuit --kill`, `fly-trader resume-entries`,
400
+ and the signer's `clear-panic`.
401
+ - **Kill switch.** A flow-adjusted index `I_t = I_{t−1}·(W_t − F_t)/W_{t−1}` over consistent NAV marks of both
402
+ accounts, where F is every deposit/profit in and withdrawal/claim out since the previous mark. Its peak uses marks at
403
+ least 600 s old, over 30 days. It trips at `KILL_SWITCH_DRAWDOWN` (default 0.30; the vault server sets 0.20) and, with
404
+ `KILL_SWITCH_LIQUIDATE`, sells every open position.
405
+ - **Principal withdrawal.** Only the owner, from the Squads app. The cap `fly-trader vault withdraw` prints is
406
+ min(D − Wd + min(0, R − A), treasury − owed − 890,880).
fly_trader/__main__.py CHANGED
@@ -144,7 +144,8 @@ def _cmd_fly_replay(args):
144
  from .logging_setup import setup
145
  from .train import fly_replay
146
  setup("train"); _use_device(args)
147
- out = fly_replay.main(days=args.days, start_day=args.start_day if args.start_day is not None else fly_replay.DEFAULT_START)
 
148
  print(json.dumps({k: out.get(k) for k in ("S", "passed", "reason", "alpha", "half_life_days", "evaluation", "random", "frozen", "plastic_beats_frozen")}, indent=1, default=str))
149
 
150
 
@@ -183,6 +184,123 @@ def _cmd_worker(args):
183
  procs.worker_cli(args.action, args.name)
184
 
185
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
186
  def build_parser() -> argparse.ArgumentParser:
187
  p = argparse.ArgumentParser(prog="fly-trader")
188
  sub = p.add_subparsers(dest="command", required=True)
@@ -211,6 +329,7 @@ def build_parser() -> argparse.ArgumentParser:
211
  fr.add_argument("--start-day", type=int, default=None,
212
  help="day index of the bootstrap (the selector teaches on the days before it); default: train/fly_replay.DEFAULT_START, "
213
  "the earliest day that leaves the teacher's walk-forward enough days in each half")
 
214
  fr.set_defaults(fn=_cmd_fly_replay)
215
  bc = sub.add_parser("backtest-corpus"); bc.add_argument("--max-tokens", type=int, default=None); bc.add_argument("--days", type=int, default=None)
216
  bc.add_argument("--fees", default="0.003,0.0055"); bc.add_argument("--only", default=None); bc.add_argument("--universe", default="graduation", choices=["graduation", "mature"]); bc.set_defaults(fn=_cmd_backtest_corpus)
@@ -229,9 +348,42 @@ def build_parser() -> argparse.ArgumentParser:
229
  ar.add_argument("dir"); ar.add_argument("--bootstrap", action="store_true"); ar.set_defaults(fn=_cmd_apply_release)
230
  sub.add_parser("health", help="exit 0 when the database answers and workers run (container HEALTHCHECK)").set_defaults(fn=_cmd_health)
231
  sub.add_parser("ready-for-restart", help="exit 0 when no order, claim payment or settlement is in flight").set_defaults(fn=_cmd_ready)
 
 
232
  from .vault import cli as vault_cli
233
  vault_cli.add(sub)
234
  wk = sub.add_parser("worker"); wk.add_argument("action", choices=["start", "stop", "status"]); wk.add_argument("name", nargs="?"); wk.set_defaults(fn=_cmd_worker)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
235
  return p
236
 
237
 
@@ -248,6 +400,12 @@ def _cmd_health(args) -> None:
248
  raise SystemExit(0 if ok else 1)
249
 
250
 
 
 
 
 
 
 
251
  def _cmd_ready(args) -> None:
252
  from .ops import release
253
  ok, why = release.ready_for_restart(); print(why)
 
144
  from .logging_setup import setup
145
  from .train import fly_replay
146
  setup("train"); _use_device(args)
147
+ out = fly_replay.main(days=args.days, start_day=args.start_day if args.start_day is not None else fly_replay.DEFAULT_START,
148
+ only_chains=tuple(args.chains.split(",")) if args.chains else None)
149
  print(json.dumps({k: out.get(k) for k in ("S", "passed", "reason", "alpha", "half_life_days", "evaluation", "random", "frozen", "plastic_beats_frozen")}, indent=1, default=str))
150
 
151
 
 
184
  procs.worker_cli(args.action, args.name)
185
 
186
 
187
+ def _cmd_kalshi_history(args):
188
+ from .kalshi import history
189
+ history.main()
190
+
191
+
192
+ def _cmd_kalshi_build(args):
193
+ from .kalshi import mature
194
+ mature.main()
195
+
196
+
197
+ def _cmd_kalshi_train_selector(args):
198
+ from .kalshi import selector
199
+ from .logging_setup import setup
200
+ setup("kalshi_train")
201
+ out = selector.main(days=args.days, holdout_days=args.holdout_days)
202
+ print(json.dumps({k: out.get(k) for k in ("snapshot_id", "deployable", "deploy_reason")}, indent=1, default=str))
203
+
204
+
205
+ def _cmd_kalshi_train_fly(args):
206
+ from .kalshi import fly
207
+ from .logging_setup import setup
208
+ setup("kalshi_train"); _use_device(args)
209
+ out = fly.main(days=args.days, epochs=args.epochs)
210
+ print(json.dumps({k: out.get(k) for k in ("snapshot_id", "S", "lines", "calibration", "diagnostics", "gates_ok", "gate_failures", "teacher")}, indent=1, default=str))
211
+
212
+
213
+ def _cmd_kalshi_fly_replay(args):
214
+ from .kalshi import fly_replay
215
+ from .logging_setup import setup
216
+ setup("kalshi_train"); _use_device(args)
217
+ out = fly_replay.main(days=args.days, start_day=args.start_day if args.start_day is not None else fly_replay.DEFAULT_START,
218
+ only_chains=tuple(args.chains.split(",")) if args.chains else None)
219
+ print(json.dumps({k: out.get(k) for k in ("S", "passed", "reason", "alpha", "half_life_days", "evaluation", "random", "frozen", "plastic_beats_frozen")}, indent=1, default=str))
220
+
221
+
222
+ def _cmd_kalshi_train(args):
223
+ from .kalshi import pipeline
224
+ pipeline.main()
225
+
226
+
227
+ def _cmd_kalshi_stream(args):
228
+ from .kalshi import stream
229
+ stream.main()
230
+
231
+
232
+ def _cmd_kalshi_run(args):
233
+ from .kalshi import engine
234
+ engine.main()
235
+
236
+
237
+ def _cmd_kalshi_subaccount(args):
238
+ from .kalshi import client
239
+ if args.action == "create":
240
+ print(f"subaccount {client.subaccount_create()} created and written to .env")
241
+ else:
242
+ print(json.dumps(client.rest().subaccount_balances(), indent=1, default=str))
243
+
244
+
245
+ def _cmd_kalshi_fund(args):
246
+ from .kalshi import client
247
+ print(json.dumps(client.fund(args.usd), indent=1, default=str))
248
+
249
+
250
+ def _cmd_rh_backfill(args):
251
+ from .rh import stream
252
+ stream.main()
253
+
254
+
255
+ def _cmd_rh_status(args):
256
+ from .rh import stream
257
+ print(json.dumps(stream.status(), indent=1, default=str))
258
+
259
+
260
+ def _cmd_rh_probe(args):
261
+ from .rh import probe
262
+ print(json.dumps(probe.run(sample=args.sample), indent=1, default=str))
263
+
264
+
265
+ def _cmd_rh_wallet(args):
266
+ from .rh import smoke
267
+ if args.action == "new":
268
+ print(f"RH bot wallet {smoke.new_key()} written to .env (RH_BOT_PRIVATE_KEY, RH_BOT_ADDRESS). Fund it with ETH on Robinhood Chain.")
269
+ else:
270
+ from . import config
271
+ from .rh.rpc import RhRpc
272
+ addr = (config.RH_BOT_ADDRESS or "").lower()
273
+ print(json.dumps({"address": addr or None, "eth": RhRpc().get_balance(addr) / 1e18 if addr else None,
274
+ "live_prerequisites_missing": config.rh_live_prerequisites_missing()}, indent=1))
275
+
276
+
277
+ def _cmd_rh_smoke(args):
278
+ if not args.yes:
279
+ raise SystemExit("rh-smoke sends real Robinhood Chain transactions: pass --yes")
280
+ from .rh import smoke
281
+ print(json.dumps(smoke.run(args.size, args.cap), indent=1, default=str))
282
+
283
+
284
+ def _cmd_kalshi_status(args):
285
+ from .kalshi import client
286
+ print(json.dumps(client.status(), indent=1, default=str))
287
+
288
+
289
+ def _cmd_kalshi_pause(args):
290
+ from .agent import rails
291
+ rails.set_entries_paused(True, rails.KALSHI_CIRCUIT)
292
+
293
+
294
+ def _cmd_kalshi_resume(args):
295
+ from .agent import rails
296
+ rails.set_entries_paused(False, rails.KALSHI_CIRCUIT)
297
+
298
+
299
+ def _cmd_kalshi_reset_circuit(args):
300
+ from .agent import rails
301
+ rails.reset_circuit(kill=args.kill, circuit_id=rails.KALSHI_CIRCUIT)
302
+
303
+
304
  def build_parser() -> argparse.ArgumentParser:
305
  p = argparse.ArgumentParser(prog="fly-trader")
306
  sub = p.add_subparsers(dest="command", required=True)
 
329
  fr.add_argument("--start-day", type=int, default=None,
330
  help="day index of the bootstrap (the selector teaches on the days before it); default: train/fly_replay.DEFAULT_START, "
331
  "the earliest day that leaves the teacher's walk-forward enough days in each half")
332
+ fr.add_argument("--chains", default=None, help="store the verdict of these chains only (comma list, e.g. rh); the others keep theirs")
333
  fr.set_defaults(fn=_cmd_fly_replay)
334
  bc = sub.add_parser("backtest-corpus"); bc.add_argument("--max-tokens", type=int, default=None); bc.add_argument("--days", type=int, default=None)
335
  bc.add_argument("--fees", default="0.003,0.0055"); bc.add_argument("--only", default=None); bc.add_argument("--universe", default="graduation", choices=["graduation", "mature"]); bc.set_defaults(fn=_cmd_backtest_corpus)
 
348
  ar.add_argument("dir"); ar.add_argument("--bootstrap", action="store_true"); ar.set_defaults(fn=_cmd_apply_release)
349
  sub.add_parser("health", help="exit 0 when the database answers and workers run (container HEALTHCHECK)").set_defaults(fn=_cmd_health)
350
  sub.add_parser("ready-for-restart", help="exit 0 when no order, claim payment or settlement is in flight").set_defaults(fn=_cmd_ready)
351
+ mc = sub.add_parser("models", help="convert: rewrite this machine's own .joblib selectors as .skops (pickles can run code)")
352
+ mc.add_argument("action", choices=["convert"]); mc.set_defaults(fn=_cmd_models)
353
  from .vault import cli as vault_cli
354
  vault_cli.add(sub)
355
  wk = sub.add_parser("worker"); wk.add_argument("action", choices=["start", "stop", "status"]); wk.add_argument("name", nargs="?"); wk.set_defaults(fn=_cmd_worker)
356
+ # ---- Kalshi prediction markets (fly_trader/kalshi) ----
357
+ ks = sub.add_parser("kalshi-subaccount", help="create the fly's dedicated Kalshi subaccount (writes KALSHI_SUBACCOUNT to .env) or list balances")
358
+ ks.add_argument("action", choices=["create", "list"]); ks.set_defaults(fn=_cmd_kalshi_subaccount)
359
+ kf = sub.add_parser("kalshi-fund", help="move dollars from the primary Kalshi account into the fly's subaccount"); kf.add_argument("--usd", type=float, required=True); kf.set_defaults(fn=_cmd_kalshi_fund)
360
+ sub.add_parser("rh-backfill", help="the Robinhood Chain index from Pons V2's deployment to the head, then live (the rh_stream worker)").set_defaults(fn=_cmd_rh_backfill)
361
+ sub.add_parser("rh-status", help="Robinhood Chain index cursors, row counts and stream status").set_defaults(fn=_cmd_rh_status)
362
+ rp = sub.add_parser("rh-probe", help="read-only check of the Robinhood Chain path: contracts, Kyber / v4 routability and gas on recent Pons graduates")
363
+ rp.add_argument("--sample", type=int, default=12); rp.set_defaults(fn=_cmd_rh_probe)
364
+ rw = sub.add_parser("rh-wallet", help="new: create the RH bot key in .env (refuses to replace one); show: its address, ETH and what live trading still needs")
365
+ rw.add_argument("action", choices=["new", "show"]); rw.set_defaults(fn=_cmd_rh_wallet)
366
+ rs = sub.add_parser("rh-smoke", help="three tiny real RH round trips (Kyber, v4 fallback, non-ETH quote) booked and reconciled to the wei")
367
+ rs.add_argument("--size", type=float, default=0.002); rs.add_argument("--cap", type=float, default=0.01); rs.add_argument("--yes", action="store_true")
368
+ rs.set_defaults(fn=_cmd_rh_smoke)
369
+ sub.add_parser("kalshi-status", help="exchange status, subaccount balance, resting orders, positions, settlements").set_defaults(fn=_cmd_kalshi_status)
370
+ sub.add_parser("kalshi-history", help="the Kalshi corpus worker: dataset seed, settled markets, candles, trades").set_defaults(fn=_cmd_kalshi_history)
371
+ sub.add_parser("kalshi-build", help="build the Kalshi feature parts from the corpus").set_defaults(fn=_cmd_kalshi_build)
372
+ kts = sub.add_parser("kalshi-train-selector", help="fit the Kalshi strategy stack walk-forward and save it"); kts.add_argument("--days", type=int, default=None)
373
+ kts.add_argument("--holdout-days", type=int, default=None, help="last days withheld from every fit and scored once (default 21)")
374
+ kts.set_defaults(fn=_cmd_kalshi_train_selector)
375
+ ktf = sub.add_parser("kalshi-train-fly", help="bootstrap the Kalshi fly (optic lobes) from the deployable Kalshi selector")
376
+ ktf.add_argument("--days", type=int, default=None); ktf.add_argument("--epochs", type=int, default=None); ktf.add_argument("--device", default=None)
377
+ ktf.set_defaults(fn=_cmd_kalshi_train_fly)
378
+ kfr = sub.add_parser("kalshi-fly-replay", help="bootstrap the Kalshi fly once, let it learn from settlements over the corpus, judge it")
379
+ kfr.add_argument("--days", type=int, default=None); kfr.add_argument("--device", default=None); kfr.add_argument("--start-day", type=int, default=None)
380
+ kfr.set_defaults(fn=_cmd_kalshi_fly_replay)
381
+ sub.add_parser("kalshi-train", help="the Kalshi training pipeline worker (selector weekly, fly bootstrap when needed)").set_defaults(fn=_cmd_kalshi_train)
382
+ sub.add_parser("kalshi-stream", help="the Kalshi live feed worker (websocket → kalshi_minutes / kalshi_quotes)").set_defaults(fn=_cmd_kalshi_stream)
383
+ sub.add_parser("kalshi-run", help="the Kalshi trading engine worker (the visual fly's paper arms, live mirror behind KALSHI_LIVE_ENABLED)").set_defaults(fn=_cmd_kalshi_run)
384
+ sub.add_parser("kalshi-pause-entries").set_defaults(fn=_cmd_kalshi_pause)
385
+ sub.add_parser("kalshi-resume-entries").set_defaults(fn=_cmd_kalshi_resume)
386
+ kr = sub.add_parser("kalshi-reset-circuit"); kr.add_argument("--kill", action="store_true"); kr.set_defaults(fn=_cmd_kalshi_reset_circuit)
387
  return p
388
 
389
 
 
400
  raise SystemExit(0 if ok else 1)
401
 
402
 
403
+ def _cmd_models(args) -> None:
404
+ from .train import model_io
405
+ for r in model_io.convert_legacy():
406
+ print(f"selector #{r['id']} -> {r['path']}")
407
+
408
+
409
  def _cmd_ready(args) -> None:
410
  from .ops import release
411
  ok, why = release.ready_for_restart(); print(why)
fly_trader/agent/fly_live.py CHANGED
@@ -79,15 +79,18 @@ class LiveMirror:
79
  opens = ledger.open_positions(conn, BOOK); held_mints = {p["mint"] for p in opens} | set(inflight)
80
  c = rails.load_circuit(conn)
81
  blocked = "kill switch" if c.kill_switch else "circuit tripped" if c.tripped else "paused" if c.entries_paused else None
82
- reserved = 0.0
83
- if config.VAULT_ENABLED: # SOL owed to vault lockers stays in the wallet but is never traded
84
  from ..vault import nav as vault_nav
85
- reserved = vault_nav.reserved_lamports(conn) / config.LAMPORTS_PER_SOL
86
- bankroll = sol_free + sum(float(p["cost_sol"]) for p in opens) - reserved; cash = sol_free - reserved; n_enter = 0; skipped = []
 
 
87
  for e in entries:
88
  if e["mint"] in held_mints:
89
  skipped.append((e["mint"], "held")); continue
90
- size, why = sizing.size_position(e["score"], e.get("threshold", line), e.get("table") or table or [], bankroll, cash, e["info"]["resq"])
 
91
  if blocked or size <= 0:
92
  skipped.append((e["mint"], blocked or why)); continue
93
  if self.worker.submit(ExecRequest(decision_id=e["decision_id"], mint=e["mint"], pool=e["info"]["pool"], side="buy", amount_in=int(size * config.LAMPORTS_PER_SOL),
@@ -109,13 +112,18 @@ class LiveMirror:
109
  "ON CONFLICT (beat_id, book) DO NOTHING",
110
  (beat_id, BOOK, m1, sol_free, gross_v - exit_cost, exit_cost, wealth, peak, (1.0 - wealth / peak) if peak > 0 else 0.0, exposure, len(opens)))
111
  if config.VAULT_ENABLED: # deposits, withdrawals and payouts are not performance
112
- vault_nav.mark(conn, m1, snap.lamports, int(round(gross_v * config.LAMPORTS_PER_SOL)),
 
113
  int(round(exit_cost * config.LAMPORTS_PER_SOL)))
114
  ix = vault_nav.status(conn, since=rebase)
115
- killed = rails.check_drawdown(conn, ix["value"], ix["peak"])
116
  else:
117
  killed = rails.check_drawdown(conn, wealth, peak)
118
- n_liq = self.liquidate(conn, ctx, run_id, beat_id) if killed and config.KILL_SWITCH_LIQUIDATE else 0
 
 
 
 
119
  gap = self.gap(conn)
120
  if ctx.m1_epoch - self.last_sweep >= SWEEP_S:
121
  self.last_sweep = ctx.m1_epoch; self.sweep(conn, snap, inflight, m1)
 
79
  opens = ledger.open_positions(conn, BOOK); held_mints = {p["mint"] for p in opens} | set(inflight)
80
  c = rails.load_circuit(conn)
81
  blocked = "kill switch" if c.kill_switch else "circuit tripped" if c.tripped else "paused" if c.entries_paused else None
82
+ reserved = treasury = 0.0
83
+ if config.VAULT_ENABLED: # SOL owed to vault lockers is never traded; the treasury's idle SOL is bankroll
84
  from ..vault import nav as vault_nav
85
+ reserved = vault_nav.reserved_in_trading(conn) / config.LAMPORTS_PER_SOL
86
+ treasury = vault_nav.treasury_free(conn) / config.LAMPORTS_PER_SOL
87
+ # sized on the whole book, spent only from the trading wallet's own float (cash)
88
+ bankroll = sol_free + sum(float(p["cost_sol"]) for p in opens) - reserved + treasury; cash = sol_free - reserved; n_enter = 0; skipped = []
89
  for e in entries:
90
  if e["mint"] in held_mints:
91
  skipped.append((e["mint"], "held")); continue
92
+ size, why = sizing.size_position(e["score"], e.get("threshold", line), e.get("table") or table or [], bankroll, cash, e["info"]["resq"],
93
+ flat=config.FLY_SIZING == "flat")
94
  if blocked or size <= 0:
95
  skipped.append((e["mint"], blocked or why)); continue
96
  if self.worker.submit(ExecRequest(decision_id=e["decision_id"], mint=e["mint"], pool=e["info"]["pool"], side="buy", amount_in=int(size * config.LAMPORTS_PER_SOL),
 
112
  "ON CONFLICT (beat_id, book) DO NOTHING",
113
  (beat_id, BOOK, m1, sol_free, gross_v - exit_cost, exit_cost, wealth, peak, (1.0 - wealth / peak) if peak > 0 else 0.0, exposure, len(opens)))
114
  if config.VAULT_ENABLED: # deposits, withdrawals and payouts are not performance
115
+ from ..vault import payout as vault_payout # trading wallet + treasury are one book: a top-up or sweep is no loss
116
+ vault_nav.mark(conn, m1, snap.lamports + vault_payout.cached_balance(), int(round(gross_v * config.LAMPORTS_PER_SOL)),
117
  int(round(exit_cost * config.LAMPORTS_PER_SOL)))
118
  ix = vault_nav.status(conn, since=rebase)
119
+ killed = rails.check_drawdown(conn, ix["value"], ix["peak"], unit="index")
120
  else:
121
  killed = rails.check_drawdown(conn, wealth, peak)
122
+ panic = False
123
+ if config.VAULT_ENABLED: # Telegram /panic: sell everything whatever KILL_SWITCH_LIQUIDATE says
124
+ from ..vault import state as vault_state
125
+ panic = bool(vault_state.get("panic"))
126
+ n_liq = self.liquidate(conn, ctx, run_id, beat_id) if killed and (config.KILL_SWITCH_LIQUIDATE or panic) else 0
127
  gap = self.gap(conn)
128
  if ctx.m1_epoch - self.last_sweep >= SWEEP_S:
129
  self.last_sweep = ctx.m1_epoch; self.sweep(conn, snap, inflight, m1)
fly_trader/agent/fly_session.py CHANGED
@@ -35,7 +35,7 @@ from pathlib import Path
35
  import numpy as np
36
  import torch
37
 
38
- from .. import config
39
  from ..brain import activity, device as brain_device, plastic
40
  from ..brain.connectome import current_connectome_path
41
  from ..db.apilog import record_event
@@ -43,12 +43,12 @@ from ..db.connection import transaction
43
  from ..execution import ledger
44
  from ..execution.broker_paper import PaperBroker
45
  from ..ops.reset import activity_dir, fly_state_dir
46
- from ..train import fly_calibrate, fly_governance as gov, fly_selector
47
  from ..train.decisions import X_COLS
48
  from . import handover, paper_trading
49
 
50
  log = logging.getLogger(__name__)
51
- BOOK, KIND = "paper_fly", "fly"
52
  BAR_KEEP = 270 # minutes of bars kept per watched mint (the longest strategy hold, 240, + next-open window + margin)
53
  NEXT_OPEN_S = 120.0 # entry at the next traded minute's open when it comes within two minutes (decisions.build)
54
  JUMP = 50.0
@@ -90,37 +90,57 @@ def resolve_label(bars: list, t: float, horizon_s: float) -> float | None:
90
  return float(exit_px / entry * (1 - rows[0][3]) * (1 - rows[j][3]) - 1)
91
 
92
 
93
- def try_start(live: bool = False) -> tuple["FlyBook | None", str]:
94
- """A FlyBook when the fly may trade, else (None, why)."""
 
 
 
 
 
 
 
 
 
 
95
  from ..train import fly_replay
96
  v = fly_replay.verdict()
97
  if not v:
98
  return None, "the fly's replay has not run on the current definitions (fly-trader fly-replay)"
99
- if not v.get("passed"):
100
- return None, f"the fly's replay did not pass: {v.get('reason')}"
 
101
  with transaction() as conn:
102
  boot = fly_selector.latest_deployable(conn)
103
  if boot is None:
104
  return None, "no deployable fly bootstrap (the training pipeline bootstraps one)"
105
  try:
106
- return FlyBook(boot, v, live=live), "ok"
107
  except Exception as e:
108
  log.exception("fly book failed to start")
109
  return None, f"fly book failed to start: {type(e).__name__}: {e}"
110
 
111
 
112
- class FlyBook:
113
- name = "fly"
 
 
 
 
 
114
 
115
- def __init__(self, boot: dict, verdict: dict, live: bool = False):
 
116
  self.live = live; self.done = False; self.mirror = None
117
  self.verdict = verdict
118
- self.state_path = fly_state_dir() / "state.pt"
 
 
 
119
  self._load_bootstrap(boot)
120
- self.run_id = str(uuid.uuid4()); self.beat_no = 0; self.broker = PaperBroker(BOOK)
121
  with transaction() as conn:
122
  conn.execute("INSERT INTO runs (run_id, kind, config, brain_snapshot_id, status) VALUES (%s,%s,%s,%s,'running')",
123
- (self.run_id, KIND, json.dumps({"channels": self.cfg, "book": BOOK}, default=str), self.boot_id))
124
  self._restore_tags(conn)
125
  record_event("info", "fly", "fly session started", {"run_id": self.run_id, "bootstrap": self.boot_id, "channels": self.cfg, "lines": self.lines["plastic"],
126
  "pending": len(self.pending), "learning_frozen": self.learning_frozen})
@@ -128,54 +148,126 @@ class FlyBook:
128
  self.lines["plastic"], len(self.pending))
129
 
130
  # ---- model and state ----
131
- def _load_bootstrap(self, boot: dict) -> None:
132
- self.boot_id = int(boot["id"])
133
- self.fly = fly_selector.load(boot["path"])
134
- try:
135
- self.connectome_name = current_connectome_path().name # stamped on the activity files the console draws
136
- except Exception:
137
- self.connectome_name = ""
138
- self.capture_ok = True
139
- self.idx = np.asarray([X_COLS.index(c) for c in self.fly.cols], dtype=int)
140
- self.names = list(self.fly.strategies); self.holds = {k: float(self.fly.rules[k]["hold_min"]) * 60.0 for k in self.names}
141
- self.H = max(self.holds.values())
142
- per = self.verdict.get("per_strategy") or {self.names[0]: {"alpha": self.verdict.get("alpha", 0.0), "half_life_days": self.verdict.get("half_life_days")}}
143
- self.cfg = {k: (float((per.get(k) or {}).get("alpha") or 0.0), (per.get(k) or {}).get("half_life_days")) for k in self.names}
144
- self.alpha = self.cfg[self.names[0]][0]; self.half_life = float(self.cfg[self.names[0]][1]) if self.cfg[self.names[0]][1] is not None else math.inf
145
- net = self.fly.net
146
- self.bank = plastic.PlasticBank(net, [(self.alpha, self.half_life)], fly_selector.SCALE, learn=net.learn.cpu().numpy(), read=net.read.cpu().numpy())
147
- for j, k in enumerate(self.names): # each strategy's channel learns and forgets at its own replay-chosen rate
148
- a, h = self.cfg[k]; cols = self.bank.learn[j]
149
- self.bank.alpha[0, cols] = a; self.bank.half_life_s[0, cols] = float(h) * 86400.0 if h is not None else float("inf")
 
 
 
 
 
 
 
 
 
150
  self.pending = plastic.PendingTags(net.n_kc, net.k_active)
151
  self.bars: dict[str, deque] = {}; self.watch: dict[str, int] = {}
152
- self.lines = {"plastic": dict(self.fly.lines), "frozen": dict(self.fly.lines)}
153
- self.sizing = {"plastic": {k: list(v) for k, v in self.fly.sizings.items()}, "frozen": {k: list(v) for k, v in self.fly.sizings.items()}}
 
 
 
 
 
154
  self.applied_through = -math.inf; self.learning_frozen = False; self.checks_paused_until = {k: 0.0 for k in self.names}
155
  self.race_started_at: float | None = None; self.last_day: int | None = None; self.last_hour: int | None = None
156
  self.rollbacks: dict = {k: [] for k in self.names}; self.nu_set = False; self.hour_stats = self._empty_stats(); self.last_checks: dict = {}
157
  s = self._read_state()
 
 
 
 
 
158
  if s and s.get("bootstrap_id") == self.boot_id and s.get("cfg") == self.cfg:
159
- self.bank.load_state(s["bank"]); self.nu_set = True
 
 
160
  self.lines, self.sizing = s["lines"], s["sizing"]
161
  self.applied_through, self.learning_frozen, self.checks_paused_until = s["applied_through"], s["learning_frozen"], s["checks_paused_until"]
162
  self.race_started_at, self.last_day, self.last_hour, self.rollbacks = s["race_started_at"], s["last_day"], s["last_hour"], s["rollbacks"]
163
  elif s:
164
  self.race_started_at = s.get("race_started_at") # a new bootstrap keeps racing the same book
165
  with transaction() as conn:
166
- conn.execute("UPDATE fly_scored SET state = 'dropped', x = NULL WHERE state = 'pending' AND bootstrap_id IS DISTINCT FROM %s", (self.boot_id,))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
167
 
168
  def _read_state(self) -> dict | None:
169
  if not self.state_path.exists():
170
  return None
171
  try:
172
- return torch.load(self.state_path, map_location="cpu", weights_only=False)
173
  except Exception:
174
  log.exception("fly state unreadable; starting from the bootstrap")
175
  return None
176
 
 
 
 
 
 
 
 
 
 
 
177
  def _state(self) -> dict:
178
- return {"bootstrap_id": self.boot_id, "cfg": self.cfg, "bank": self.bank.state(), "lines": self.lines, "sizing": self.sizing,
179
  "applied_through": self.applied_through, "learning_frozen": self.learning_frozen, "checks_paused_until": self.checks_paused_until,
180
  "race_started_at": self.race_started_at, "last_day": self.last_day, "last_hour": self.last_hour, "rollbacks": self.rollbacks,
181
  "saved_at": time.time()}
@@ -186,7 +278,8 @@ class FlyBook:
186
  torch.save(self._state(), tmp); os.replace(tmp, self.state_path)
187
 
188
  def _restore_tags(self, conn) -> None:
189
- rows = conn.execute("SELECT ts, mint, strategy, x, score, line FROM fly_scored WHERE state = 'pending' AND bootstrap_id = %s ORDER BY ts", (self.boot_id,)).fetchall()
 
190
  rows = [r for r in rows if r["strategy"] in self.holds]
191
  applied = [(r["ts"], r["mint"], r["strategy"]) for r in rows if r["ts"].timestamp() + self.holds[r["strategy"]] + LABEL_LAG_S <= self.applied_through]
192
  if applied:
@@ -198,14 +291,34 @@ class FlyBook:
198
  groups.setdefault((r["ts"].timestamp(), r["strategy"]), []).append(r)
199
  for (t, name), rs in sorted(groups.items()):
200
  j = self.names.index(name)
201
- Y, u0, k = self.fly.parts_all(np.asarray([r["x"] for r in rs], dtype=np.float32))
202
- w = plastic.row_weights(torch.tensor([[float(r["score"]) for r in rs]]), torch.tensor([float(rs[0]["line"])]))
203
- self.pending.push([(t, r["mint"], name) for r in rs], np.full(len(rs), t), t + self.holds[name] + LABEL_LAG_S, Y[:, j], u0, k, w, s=j)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
204
  for r in rs:
205
  self.watch[r["mint"]] = self.watch.get(r["mint"], 0) + 1
206
 
207
  def maybe_reload(self) -> bool:
208
- """A newer deployable bootstrap (a re-bootstrap) replaces this one at once: D = 0, pending tags dropped, the book carries on."""
 
 
 
 
 
 
209
  from ..train import fly_replay
210
  with transaction() as conn:
211
  boot = fly_selector.latest_deployable(conn)
@@ -231,7 +344,7 @@ class FlyBook:
231
  q.append(b)
232
 
233
  def open_mints(self, conn) -> set[str]:
234
- return {p["mint"] for p in ledger.open_positions(conn, BOOK)}
235
 
236
  def on_minute(self, ctx) -> dict:
237
  conn, t_now = ctx.conn, ctx.t_start
@@ -254,11 +367,12 @@ class FlyBook:
254
  self.race_started_at = ctx.m1_epoch
255
  self.beat_no += 1
256
  d = scored["decision"]
257
- st = paper_trading.trade_minute(ctx, book=BOOK, run_id=self.run_id, beat_no=self.beat_no, broker=self.broker, kind=KIND, mints=ctx.mints, infos=ctx.infos,
258
  scores=d["score"], threshold=d["threshold"], table=None, horizon_s=self.holds[self.names[0]], holds=d["hold_s"],
259
- strategies=d["strategy"], tables=d["tables"])
 
260
  out.update({k: v for k, v in st.items() if k != "entries"}); out["stage"] = "trading"
261
- if self.live and handover.state(conn) is not None:
262
  out["live"] = self._live(ctx, st)
263
  else:
264
  out["stage"] = "learning (not trading this minute)" if ctx.trade else "catching up"
@@ -273,12 +387,15 @@ class FlyBook:
273
  cur.executemany("UPDATE fly_scored SET label = %s, state = %s, resolved_at = now(), x = NULL WHERE ts = %s AND mint = %s AND strategy = %s", learned.pop("keys"))
274
  learned.pop("keys", None)
275
  out["checks"] = self.last_checks; out["updated_at"] = datetime.now(timezone.utc).isoformat()
276
- conn.execute("INSERT INTO ui_settings (key, value) VALUES ('fly_status', %s) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()",
277
- (json.dumps(out, default=str),))
278
  return out
279
 
280
  def _live(self, ctx, st: dict) -> dict:
281
  """The fly holds the seat: mirror this minute on the bot wallet (agent/fly_live.py) while signing is allowed."""
 
 
 
282
  from ..chain.cluster_guard import signing_allowed
283
  if not signing_allowed():
284
  return {"stage": "live gated: signing is not allowed (LIVE_ENABLED=1 and the live prerequisites)"}
@@ -343,21 +460,37 @@ class FlyBook:
343
  self._capture(H, trig, ctx.t_start); del H
344
  if not self.nu_set:
345
  self.bank.estimate_nu(Y[:, 0], u0, k); self.nu_set = True
346
- t = ctx.t_start; ts = datetime.fromtimestamp(t, timezone.utc); rows = []
347
- V = np.full((n, len(self.names)), -np.inf)
 
 
 
 
 
348
  for j, name in enumerate(self.names):
349
  loc = np.flatnonzero(trig[:, j])
 
 
350
  if not len(loc):
351
  continue
352
  li = torch.as_tensor(loc, device=Y.device)
 
353
  with torch.no_grad():
354
- sc, _ = self.bank.predict(Y[li, j], u0[li], k[li], s=np.full(len(loc), j)); sc = sc[0]
355
- fr = self.bank.frozen(Y[li, j], u0[li], s=np.full(len(loc), j))
356
- v, f = sc.float().cpu().numpy(), fr.float().cpu().numpy(); V[loc, j] = v
357
  line, fline = self.lines["plastic"][name], self.lines["frozen"][name]
358
- rows += [(ts, ctx.mints[i], name, int(self.holds[name] // 60), ctx.X[i].tolist(), float(v[q]), float(f[q]), line, fline, self.boot_id) for q, i in enumerate(loc)]
359
- w = plastic.row_weights(sc[None], torch.tensor([line], dtype=sc.dtype, device=sc.device))
360
- self.pending.push([(t, ctx.mints[i], name) for i in loc], np.full(len(loc), t), t + self.holds[name] + LABEL_LAG_S, Y[li, j], u0[li], k[li], w, s=j)
 
 
 
 
 
 
 
 
361
  for i in loc:
362
  m = ctx.mints[i]; self.watch[m] = self.watch.get(m, 0) + 1
363
  q_ = self.bars.setdefault(m, deque(maxlen=BAR_KEEP)); b = ctx.bars.get(m)
@@ -365,12 +498,64 @@ class FlyBook:
365
  q_.append(b)
366
  if rows:
367
  with conn.cursor() as cur:
368
- cur.executemany("INSERT INTO fly_scored (ts, mint, strategy, hold_min, x, score, frozen_score, line, frozen_line, bootstrap_id) "
369
- "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT (ts, mint, strategy) DO NOTHING", rows)
370
- self.fly.lines = dict(self.lines["plastic"]); self.fly.sizings = {k: list(v) for k, v in self.sizing["plastic"].items()}
 
 
 
 
 
 
 
 
371
  d = fly_selector.fly_decide(self.fly, ctx.X, X_COLS, np.where(np.isfinite(V), V, -np.inf))
372
  return {"decision": d, "picks": int(sum(1 for x in d["strategy"] if x is not None))}
373
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
374
  def _capture(self, H: torch.Tensor, trig: np.ndarray, t: float) -> None:
375
  """The console's brain view: this minute's mean activity over the rows some strategy's trigger fired on (every
376
  row when none did), one file per minute (brain/activity.py). Never allowed to interrupt scoring."""
@@ -387,25 +572,39 @@ class FlyBook:
387
  # ---- daily and hourly ----
388
  def _resolved(self, conn, since_resolved: float, cols: str, name: str) -> list[dict]:
389
  """A strategy's resolved rows whose labels became known after ``since_resolved`` (epoch s)."""
390
- return conn.execute(f"SELECT {cols} FROM fly_scored WHERE state = 'resolved' AND label IS NOT NULL AND bootstrap_id = %s AND strategy = %s AND ts >= %s ORDER BY ts",
391
- (self.boot_id, name, datetime.fromtimestamp(since_resolved - self.holds[name] - LABEL_LAG_S, timezone.utc))).fetchall()
 
392
 
393
  def _calibrate(self, conn, now: float) -> None:
394
  day = datetime.fromtimestamp(now, timezone.utc).date()
 
395
  for name in self.names:
396
- rows = self._resolved(conn, now - fly_calibrate.WINDOW_DAYS * 86400.0, "ts, mint, score, frozen_score, label", name)
 
 
 
 
 
 
 
397
  if not rows:
398
  continue
399
  ts = np.array([r["ts"].timestamp() for r in rows]); mint = np.array([r["mint"] for r in rows]); lab = np.array([r["label"] for r in rows], dtype=np.float64)
 
 
 
400
  for arm, col in (("plastic", "score"), ("frozen", "frozen_score")):
401
  cal = fly_calibrate.calibrate(ts, mint, self.holds[name], np.array([r[col] for r in rows], dtype=np.float64), lab,
402
- prev_line=self.lines[arm][name], prev_sizing=self.sizing[arm][name])
403
  if cal.changed:
404
- record_event("info", "fly", f"{name} {arm} line {self.lines[arm][name] * 100:+.2f}% → {cal.line * 100:+.2f}%", {"trades": cal.trades, "mean": cal.mean, "total": cal.total})
 
405
  self.lines[arm][name], self.sizing[arm][name] = cal.line, cal.sizing
406
  conn.execute("INSERT INTO fly_calibrations (day, arm, line, sizing, trades, total, mean, window_days) VALUES (%s,%s,%s,%s,%s,%s,%s,%s) "
407
  "ON CONFLICT (day, arm) DO UPDATE SET line = EXCLUDED.line, sizing = EXCLUDED.sizing, trades = EXCLUDED.trades, total = EXCLUDED.total, mean = EXCLUDED.mean",
408
- (day, f"{arm}:{name}", cal.line, json.dumps(cal.sizing), cal.trades, cal.total, cal.mean, fly_calibrate.WINDOW_DAYS))
 
409
 
410
  def _drift(self, j: int) -> float:
411
  return float(self.bank.drift(j)[0]) if self.bank.learn.shape[0] > 1 else float(self.bank.drift()[0])
@@ -427,6 +626,10 @@ class FlyBook:
427
  return out
428
 
429
  def _hourly(self, conn, now: float) -> None:
 
 
 
 
430
  hour = datetime.fromtimestamp(math.floor(now / 3600) * 3600 - 3600, timezone.utc)
431
  checks = self._checks(conn, now); self.last_checks = checks
432
  h = self.hour_stats; first = checks.get(self.names[0]) or {}
@@ -447,7 +650,7 @@ class FlyBook:
447
  d = fly_state_dir(); d.mkdir(parents=True, exist_ok=True)
448
  path = d / f"snap_{datetime.fromtimestamp(now, timezone.utc).strftime('%Y%m%dT%H%M')}.pt"
449
  torch.save({"bank": self.bank.state(), "bootstrap_id": self.boot_id, "lines": self.lines, "sizing": self.sizing}, path)
450
- note = {"bootstrap_id": self.boot_id, "data": fly_selector.FLY_VERSION, "good": good, "checks": checks,
451
  "drift": max((c["drift"] for c in checks.values()), default=0.0), "ic24": {k: c.get("ic") for k, c in checks.items()}}
452
  return int(conn.execute("INSERT INTO brain_snapshots (run_id, path, kind, note) VALUES (%s,%s,'fly_plastic',%s) RETURNING id",
453
  (self.run_id, str(path), json.dumps(note, default=str))).fetchone()["id"])
@@ -476,11 +679,11 @@ class FlyBook:
476
  (datetime.fromtimestamp(now - ROLLBACK_MIN_AGE_S, timezone.utc),)).fetchall():
477
  n = json.loads(r["note"] or "{}")
478
  ok = n.get("good") or not ((n.get("checks") or {}).get(name) or {}).get("triggers")
479
- if ok and n.get("bootstrap_id") == self.boot_id and Path(r["path"]).exists():
480
  target = r; break
481
  if target is not None:
482
  snap = plastic.PlasticBank(self.fly.net, [(0.0, math.inf)], fly_selector.SCALE, learn=self.bank.learn.cpu().numpy(), read=self.bank.read.cpu().numpy())
483
- st = torch.load(target["path"], map_location="cpu", weights_only=False)["bank"]
484
  snap.D = torch.zeros_like(snap.D); snap.D[:, snap.mask.bool()] = st["D"].to(snap.dev).float()
485
  self.bank.D[0][:, cols] = snap.D[0][:, cols]
486
  else:
@@ -519,15 +722,17 @@ class FlyBook:
519
  whole race), closed ``config.HANDOVER_MIN_TRADES`` paper trades and, when ``config.HANDOVER_BEAT_SELECTOR``,
520
  realized at least the paper selector's P&L over that window."""
521
  days = config.HANDOVER_DAYS
522
- if handover.state(conn) is not None or self.race_started_at is None or now - self.race_started_at < days * 86400.0:
523
  return
524
  since = datetime.fromtimestamp(self.race_started_at if days <= 0 else now - days * 86400.0, timezone.utc)
 
525
  pnl = {b: conn.execute("SELECT COALESCE(sum(realized_sol), 0) AS s, count(*) AS n FROM positions WHERE book = %s AND status = 'closed' AND closed_at >= %s",
526
- (b, since)).fetchone() for b in (BOOK, "paper_selector")}
527
- fly_pnl, sel_pnl, n = float(pnl[BOOK]["s"]), float(pnl["paper_selector"]["s"]), int(pnl[BOOK]["n"])
528
  if n >= config.HANDOVER_MIN_TRADES and (fly_pnl >= sel_pnl or not config.HANDOVER_BEAT_SELECTOR):
529
  detail = {"at": datetime.fromtimestamp(now, timezone.utc).isoformat(), "fly_pnl_sol": fly_pnl, "selector_pnl_sol": sel_pnl, "fly_trades": n,
530
- "selector_trades": int(pnl["paper_selector"]["n"]), "days": days, "beat_selector": config.HANDOVER_BEAT_SELECTOR, "bootstrap": self.boot_id}
531
- handover.record(conn, detail)
 
532
  record_event("info", "handover", f"the fly takes the selector's seat: {fly_pnl:+.3f} SOL vs {sel_pnl:+.3f} SOL over {n} trades"
533
  + (f" in {days} days" if days > 0 else ""), detail)
 
35
  import numpy as np
36
  import torch
37
 
38
+ from .. import config, markets
39
  from ..brain import activity, device as brain_device, plastic
40
  from ..brain.connectome import current_connectome_path
41
  from ..db.apilog import record_event
 
43
  from ..execution import ledger
44
  from ..execution.broker_paper import PaperBroker
45
  from ..ops.reset import activity_dir, fly_state_dir
46
+ from ..train import fly_calibrate, fly_governance as gov, fly_selector, selector, strategies
47
  from ..train.decisions import X_COLS
48
  from . import handover, paper_trading
49
 
50
  log = logging.getLogger(__name__)
51
+ BOOK, KIND = "paper_fly", "fly" # the Solana book's names (another chain's book takes its own from fly_trader/markets)
52
  BAR_KEEP = 270 # minutes of bars kept per watched mint (the longest strategy hold, 240, + next-open window + margin)
53
  NEXT_OPEN_S = 120.0 # entry at the next traded minute's open when it comes within two minutes (decisions.build)
54
  JUMP = 50.0
 
90
  return float(exit_px / entry * (1 - rows[0][3]) * (1 - rows[j][3]) - 1)
91
 
92
 
93
+ def chain_passed(v: dict, chain: str) -> tuple[bool, str]:
94
+ """Whether the replay lets this chain's book trade: its own per-chain verdict (a combined replay), else the one verdict
95
+ for Solana (a Solana-only replay says nothing about another chain)."""
96
+ if "chains" in v:
97
+ c = v["chains"].get(chain)
98
+ return (bool(c and c.get("passed")), (c or {}).get("reason") or f"the replay has no {chain} verdict")
99
+ return (bool(v.get("passed")) and chain == "sol", v.get("reason") if chain == "sol" else f"the replay judged no {chain} trades")
100
+
101
+
102
+ def try_start(live: bool = False, chain: str = "sol", brain: "FlyBook | None" = None) -> tuple["FlyBook | None", str]:
103
+ """A FlyBook for ``chain`` when the fly may trade there, else (None, why). ``brain``: the Solana book whose fly and
104
+ plastic memory another chain's book shares (one model learns from both chains)."""
105
  from ..train import fly_replay
106
  v = fly_replay.verdict()
107
  if not v:
108
  return None, "the fly's replay has not run on the current definitions (fly-trader fly-replay)"
109
+ ok, why = chain_passed(v, chain)
110
+ if not ok:
111
+ return None, f"the fly's replay did not pass on {markets.for_chain(chain).name}: {why}"
112
  with transaction() as conn:
113
  boot = fly_selector.latest_deployable(conn)
114
  if boot is None:
115
  return None, "no deployable fly bootstrap (the training pipeline bootstraps one)"
116
  try:
117
+ return FlyBook(boot, v, live=live, chain=chain, brain=brain), "ok"
118
  except Exception as e:
119
  log.exception("fly book failed to start")
120
  return None, f"fly book failed to start: {type(e).__name__}: {e}"
121
 
122
 
123
+ def no_teacher(n: int) -> dict:
124
+ """The decision when the teacher gate is on (FLY_TEACHER_GATE) but the fly's teacher could not be loaded: no row is
125
+ allowed, each recorded with why. A fresh install once lost the teacher's snapshot id (ops/release.remap_teacher)."""
126
+ return {"strategy": np.full(n, None, dtype=object), "score": np.full(n, -1.0), "hold_s": np.zeros(n), "threshold": np.full(n, np.inf),
127
+ "tables": [[] for _ in range(n)], "allow": np.zeros(n, bool),
128
+ "reason": np.full(n, "fail closed: the fly's teacher (the selector that taught it) is not loaded", dtype=object)}
129
+
130
 
131
+ class FlyBook:
132
+ def __init__(self, boot: dict, verdict: dict, live: bool = False, chain: str = "sol", brain: "FlyBook | None" = None):
133
  self.live = live; self.done = False; self.mirror = None
134
  self.verdict = verdict
135
+ self.chain = chain; self.market = markets.for_chain(chain); self.brain = brain
136
+ self.name = "fly" if chain == "sol" else f"fly_{chain}"
137
+ self.BOOK = self.market.fly_book; self.KIND = self.name; self.status_key = self.market.fly_status_key
138
+ self.state_path = fly_state_dir() / ("state.pt" if chain == "sol" else f"state_{chain}.pt")
139
  self._load_bootstrap(boot)
140
+ self.run_id = str(uuid.uuid4()); self.beat_no = 0; self.broker = PaperBroker(self.BOOK)
141
  with transaction() as conn:
142
  conn.execute("INSERT INTO runs (run_id, kind, config, brain_snapshot_id, status) VALUES (%s,%s,%s,%s,'running')",
143
+ (self.run_id, self.KIND, json.dumps({"channels": self.cfg, "book": self.BOOK, "chain": chain}, default=str), self.boot_id))
144
  self._restore_tags(conn)
145
  record_event("info", "fly", "fly session started", {"run_id": self.run_id, "bootstrap": self.boot_id, "channels": self.cfg, "lines": self.lines["plastic"],
146
  "pending": len(self.pending), "learning_frozen": self.learning_frozen})
 
148
  self.lines["plastic"], len(self.pending))
149
 
150
  # ---- model and state ----
151
+ def _load_bootstrap(self, boot: dict | None) -> None:
152
+ b = self.brain
153
+ if b is not None: # another chain's book: the Solana book's fly and plastic memory, this chain's own lines and tags
154
+ self.boot_id = b.boot_id; self.fly = b.fly; self.connectome_name = b.connectome_name; self.capture_ok = False
155
+ self.teacher_models = b.teacher_models; self.idx = b.idx; self.names = b.names; self.holds = b.holds; self.H = b.H
156
+ self.teacher_live, self.teacher_groups = b.teacher_live, b.teacher_groups
157
+ self.cfg = b.cfg; self.alpha = b.alpha; self.half_life = b.half_life; self.bank = b.bank
158
+ net = self.fly.net
159
+ else:
160
+ self.boot_id = int(boot["id"])
161
+ self.fly = fly_selector.load(boot["path"])
162
+ try:
163
+ self.connectome_name = current_connectome_path().name # stamped on the activity files the console draws
164
+ except Exception:
165
+ self.connectome_name = ""
166
+ self.capture_ok = True
167
+ self.teacher_models = self._teacher_models(boot)
168
+ self.idx = np.asarray([X_COLS.index(c) for c in self.fly.cols], dtype=int)
169
+ self.names = list(self.fly.strategies); self.holds = {k: float(self.fly.rules[k]["hold_min"]) * 60.0 for k in self.names}
170
+ self.H = max(self.holds.values())
171
+ per = self.verdict.get("per_strategy") or {self.names[0]: {"alpha": self.verdict.get("alpha", 0.0), "half_life_days": self.verdict.get("half_life_days")}}
172
+ self.cfg = {k: (float((per.get(k) or {}).get("alpha") or 0.0), (per.get(k) or {}).get("half_life_days")) for k in self.names}
173
+ self.alpha = self.cfg[self.names[0]][0]; self.half_life = float(self.cfg[self.names[0]][1]) if self.cfg[self.names[0]][1] is not None else math.inf
174
+ net = self.fly.net
175
+ self.bank = plastic.PlasticBank(net, [(self.alpha, self.half_life)], fly_selector.SCALE, learn=net.learn.cpu().numpy(), read=net.read.cpu().numpy())
176
+ for j, k in enumerate(self.names): # each strategy's channel learns and forgets at its own replay-chosen rate
177
+ a, h = self.cfg[k]; cols = self.bank.learn[j]
178
+ self.bank.alpha[0, cols] = a; self.bank.half_life_s[0, cols] = float(h) * 86400.0 if h is not None else float("inf")
179
  self.pending = plastic.PendingTags(net.n_kc, net.k_active)
180
  self.bars: dict[str, deque] = {}; self.watch: dict[str, int] = {}
181
+ own = {k: self.fly.line_for(k, self.chain) for k in self.names}; own_s = {k: list(self.fly.sizing_for(k, self.chain)) for k in self.names}
182
+ if config.FLY_SCORE == "teacher" and self.teacher_live: # it scores on its teacher's scale: it starts on its teacher's lines and bands
183
+ ts_ = self.teacher_live.get("strategies") or {}
184
+ own = {k: float(ts_[k]["line"]) if k in ts_ else v for k, v in own.items()}
185
+ own_s = {k: list(ts_[k].get("sizing") or v) if k in ts_ else v for k, v in own_s.items()}
186
+ self.lines = {"plastic": dict(own), "frozen": dict(own)}
187
+ self.sizing = {"plastic": {k: list(v) for k, v in own_s.items()}, "frozen": {k: list(v) for k, v in own_s.items()}}
188
  self.applied_through = -math.inf; self.learning_frozen = False; self.checks_paused_until = {k: 0.0 for k in self.names}
189
  self.race_started_at: float | None = None; self.last_day: int | None = None; self.last_hour: int | None = None
190
  self.rollbacks: dict = {k: [] for k in self.names}; self.nu_set = False; self.hour_stats = self._empty_stats(); self.last_checks: dict = {}
191
  s = self._read_state()
192
+ if b is not None:
193
+ self.nu_set = True # the shared memory is the Solana book's to load and save
194
+ if s and s.get("bootstrap_id") == self.boot_id and s.get("cfg") == self.cfg and s.get("score", "own") != config.FLY_SCORE:
195
+ self._archive_state(s) # learned under another scoring: it starts from its bootstrap
196
+ s = None
197
  if s and s.get("bootstrap_id") == self.boot_id and s.get("cfg") == self.cfg:
198
+ if b is None:
199
+ self.bank.load_state(s["bank"])
200
+ self.nu_set = True
201
  self.lines, self.sizing = s["lines"], s["sizing"]
202
  self.applied_through, self.learning_frozen, self.checks_paused_until = s["applied_through"], s["learning_frozen"], s["checks_paused_until"]
203
  self.race_started_at, self.last_day, self.last_hour, self.rollbacks = s["race_started_at"], s["last_day"], s["last_hour"], s["rollbacks"]
204
  elif s:
205
  self.race_started_at = s.get("race_started_at") # a new bootstrap keeps racing the same book
206
  with transaction() as conn:
207
+ conn.execute("UPDATE fly_scored SET state = 'dropped', x = NULL WHERE state = 'pending' AND chain = %s AND bootstrap_id IS DISTINCT FROM %s",
208
+ (self.chain, self.boot_id))
209
+
210
+ teacher_live: dict | None = None
211
+ teacher_groups: set = set()
212
+
213
+ def _teacher_models(self, boot: dict) -> dict | None:
214
+ """The stack that taught this fly (its snapshot's ``teacher_snapshot``: the holdout-blind copy when that is what
215
+ taught it), whose final decision on each scored row is stored as ``teacher_allow`` -- the match calibration
216
+ rule's budget. None (with a warning) when the fly was taught by a refit stack or the snapshot is gone."""
217
+ meta = boot.get("meta") or {}
218
+ sid = meta.get("teacher_snapshot")
219
+ if sid is None:
220
+ if fly_calibrate.RULE == "match":
221
+ log.warning("fly #%s names no teacher snapshot: the match rule keeps its bootstrap line", self.boot_id)
222
+ return None
223
+ try:
224
+ m = selector.load_snapshot(int(sid))
225
+ except Exception:
226
+ log.exception("fly #%s: teacher snapshot #%s could not be loaded", self.boot_id, sid)
227
+ return None
228
+ blind = getattr(m, "blind", None) or {}
229
+ models = blind["models"] if "blind to the calibration week" in str(meta.get("teacher") or "") and blind.get("models") else getattr(m, "stack", None)
230
+ live = getattr(m, "stack", None) # the gate: the teacher's full stack, as its selector book trades it
231
+ if live and live.get("strategies") and all(c in X_COLS for v in live["strategies"].values() for c in v["cols"]):
232
+ self.teacher_live = live
233
+ self.teacher_groups = set((getattr(m, "metrics", None) or {}).get("groups") or [])
234
+ if not models or not models.get("strategies") or any(c not in X_COLS for v in models["strategies"].values() for c in v["cols"]):
235
+ log.warning("fly #%s: teacher #%s has no usable strategy stack", self.boot_id, sid)
236
+ return None
237
+ return models
238
+
239
+ def _teacher_allow(self, X: np.ndarray, t: float) -> dict[str, np.ndarray]:
240
+ """Per strategy: would the teacher trade each row (trigger, line and every filter)? Empty when unknown."""
241
+ if self.teacher_models is None:
242
+ return {}
243
+ try:
244
+ per = strategies.decide_all(self.teacher_models, X, X_COLS, t)
245
+ return {k: np.asarray(v["allow"], bool) for k, v in per.items()}
246
+ except Exception:
247
+ log.exception("teacher decisions failed for this minute (the fly still trades)")
248
+ return {}
249
 
250
  def _read_state(self) -> dict | None:
251
  if not self.state_path.exists():
252
  return None
253
  try:
254
+ return torch.load(self.state_path, map_location="cpu", weights_only=True)
255
  except Exception:
256
  log.exception("fly state unreadable; starting from the bootstrap")
257
  return None
258
 
259
+ def _archive_state(self, s: dict) -> None:
260
+ """Keep a state learned under another scoring mode beside the new one (never loaded again)."""
261
+ try:
262
+ dst = self.state_path.with_name(f"{self.state_path.stem}_{s.get('score', 'own')}_{time.strftime('%Y%m%dT%H%M%S')}.pt")
263
+ os.replace(self.state_path, dst)
264
+ record_event("info", "fly", f"fly state learned with FLY_SCORE={s.get('score', 'own')} set aside: the fly starts from its bootstrap",
265
+ {"path": str(dst), "drift": None})
266
+ except OSError:
267
+ log.exception("could not set the old fly state aside")
268
+
269
  def _state(self) -> dict:
270
+ return {"bootstrap_id": self.boot_id, "cfg": self.cfg, "score": config.FLY_SCORE, "bank": self.bank.state() if self.brain is None else None, "lines": self.lines, "sizing": self.sizing,
271
  "applied_through": self.applied_through, "learning_frozen": self.learning_frozen, "checks_paused_until": self.checks_paused_until,
272
  "race_started_at": self.race_started_at, "last_day": self.last_day, "last_hour": self.last_hour, "rollbacks": self.rollbacks,
273
  "saved_at": time.time()}
 
278
  torch.save(self._state(), tmp); os.replace(tmp, self.state_path)
279
 
280
  def _restore_tags(self, conn) -> None:
281
+ rows = conn.execute("SELECT ts, mint, strategy, x, score, line FROM fly_scored WHERE state = 'pending' AND bootstrap_id = %s AND chain = %s ORDER BY ts",
282
+ (self.boot_id, self.chain)).fetchall()
283
  rows = [r for r in rows if r["strategy"] in self.holds]
284
  applied = [(r["ts"], r["mint"], r["strategy"]) for r in rows if r["ts"].timestamp() + self.holds[r["strategy"]] + LABEL_LAG_S <= self.applied_through]
285
  if applied:
 
291
  groups.setdefault((r["ts"].timestamp(), r["strategy"]), []).append(r)
292
  for (t, name), rs in sorted(groups.items()):
293
  j = self.names.index(name)
294
+ Xr = np.asarray([r["x"] for r in rs], dtype=np.float32)
295
+ Y, u0, k = self.fly.parts_all(Xr[:, self.idx])
296
+ tv = self._teacher_scores(Xr, t); loc = np.arange(len(rs))
297
+ if tv is not None:
298
+ loc = loc[np.isfinite(tv[:, j])]
299
+ if not len(loc):
300
+ continue
301
+ rs = [rs[i] for i in loc]; Xr = Xr[loc]; li = torch.as_tensor(loc, device=Y.device); Y, u0, k = Y[li], u0[li], k[li]
302
+ y = self._head(Y[:, j], tv, loc, j, u0)
303
+ if config.FLY_TEACHER_GATE and self.teacher_live is not None: # gated: it learns from its teacher's trades, as live
304
+ d = strategies.decide(self.teacher_live, Xr, X_COLS, t)
305
+ w = torch.tensor([[float(d["strategy"][i] == name and bool(d["allow"][i])) for i in range(len(rs))]])
306
+ elif config.FLY_TEACHER_GATE:
307
+ w = torch.zeros((1, len(rs)))
308
+ else:
309
+ w = plastic.row_weights(torch.tensor([[float(r["score"]) for r in rs]]), torch.tensor([float(rs[0]["line"])]))
310
+ self.pending.push([(t, r["mint"], name) for r in rs], np.full(len(rs), t), t + self.holds[name] + LABEL_LAG_S, y, u0, k, w, s=j)
311
  for r in rs:
312
  self.watch[r["mint"]] = self.watch.get(r["mint"], 0) + 1
313
 
314
  def maybe_reload(self) -> bool:
315
+ """A newer deployable bootstrap (a re-bootstrap) replaces this one at once: D = 0, pending tags dropped, the book carries on.
316
+ Another chain's book follows its brain (the Solana book reloads first)."""
317
+ if self.brain is not None:
318
+ if self.brain.boot_id == self.boot_id:
319
+ return False
320
+ self._write_state(); self._load_bootstrap(None)
321
+ return True
322
  from ..train import fly_replay
323
  with transaction() as conn:
324
  boot = fly_selector.latest_deployable(conn)
 
344
  q.append(b)
345
 
346
  def open_mints(self, conn) -> set[str]:
347
+ return {p["mint"] for p in ledger.open_positions(conn, self.BOOK)}
348
 
349
  def on_minute(self, ctx) -> dict:
350
  conn, t_now = ctx.conn, ctx.t_start
 
367
  self.race_started_at = ctx.m1_epoch
368
  self.beat_no += 1
369
  d = scored["decision"]
370
+ st = paper_trading.trade_minute(ctx, book=self.BOOK, run_id=self.run_id, beat_no=self.beat_no, broker=self.broker, kind=self.KIND, mints=ctx.mints, infos=ctx.infos,
371
  scores=d["score"], threshold=d["threshold"], table=None, horizon_s=self.holds[self.names[0]], holds=d["hold_s"],
372
+ strategies=d["strategy"], tables=d["tables"], flat=config.FLY_SIZING == "flat",
373
+ allow=d.get("allow"), reasons=d.get("reason"))
374
  out.update({k: v for k, v in st.items() if k != "entries"}); out["stage"] = "trading"
375
+ if self.live and handover.state(conn, self.chain) is not None:
376
  out["live"] = self._live(ctx, st)
377
  else:
378
  out["stage"] = "learning (not trading this minute)" if ctx.trade else "catching up"
 
387
  cur.executemany("UPDATE fly_scored SET label = %s, state = %s, resolved_at = now(), x = NULL WHERE ts = %s AND mint = %s AND strategy = %s", learned.pop("keys"))
388
  learned.pop("keys", None)
389
  out["checks"] = self.last_checks; out["updated_at"] = datetime.now(timezone.utc).isoformat()
390
+ conn.execute("INSERT INTO ui_settings (key, value) VALUES (%s, %s) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()",
391
+ (self.status_key, json.dumps({**out, "chain": self.chain}, default=str)))
392
  return out
393
 
394
  def _live(self, ctx, st: dict) -> dict:
395
  """The fly holds the seat: mirror this minute on the bot wallet (agent/fly_live.py) while signing is allowed."""
396
+ if self.chain != "sol":
397
+ from ..rh import live as rh_live # Robinhood Chain: the ETH wallet (rh/live.py)
398
+ return rh_live.minute(self, ctx, st)
399
  from ..chain.cluster_guard import signing_allowed
400
  if not signing_allowed():
401
  return {"stage": "live gated: signing is not allowed (LIVE_ENABLED=1 and the live prerequisites)"}
 
460
  self._capture(H, trig, ctx.t_start); del H
461
  if not self.nu_set:
462
  self.bank.estimate_nu(Y[:, 0], u0, k); self.nu_set = True
463
+ t = ctx.t_start; ts = datetime.fromtimestamp(t, timezone.utc); rows = []; ta = self._teacher_allow(ctx.X, t)
464
+ V = np.full((n, len(self.names)), -np.inf); F = np.full((n, len(self.names)), np.nan)
465
+ gated = config.FLY_TEACHER_GATE
466
+ if config.FLY_SCORE == "teacher" and self.teacher_live is None: # it starts from its teacher's score: none, no score
467
+ return {"decision": no_teacher(n), "picks": 0}
468
+ td = self._teacher_decision(ctx) if gated and self.teacher_live is not None else None
469
+ tv = self._teacher_scores(ctx.X, t)
470
  for j, name in enumerate(self.names):
471
  loc = np.flatnonzero(trig[:, j])
472
+ if tv is not None:
473
+ loc = loc[np.isfinite(tv[loc, j])] # a candidate its teacher scores
474
  if not len(loc):
475
  continue
476
  li = torch.as_tensor(loc, device=Y.device)
477
+ y = self._head(Y[li, j], tv, loc, j, u0[li])
478
  with torch.no_grad():
479
+ sc, _ = self.bank.predict(y, u0[li], k[li], s=np.full(len(loc), j)); sc = sc[0]
480
+ fr = self.bank.frozen(y, u0[li], s=np.full(len(loc), j))
481
+ v, f = sc.float().cpu().numpy(), fr.float().cpu().numpy(); V[loc, j] = v; F[loc, j] = f
482
  line, fline = self.lines["plastic"][name], self.lines["frozen"][name]
483
+ al = ta.get(name)
484
+ rows += [(ts, ctx.mints[i], name, int(self.holds[name] // 60), ctx.X[i].tolist(), float(v[q]), float(f[q]), line, fline, self.boot_id,
485
+ None if al is None else bool(al[i])) for q, i in enumerate(loc)]
486
+ if td is not None: # gated: its decisions are the teacher's trades of this strategy (before its own vetoes), so it learns from those
487
+ mine = np.array([td["strategy"][i] == name and bool(td["allow"][i]) for i in loc], dtype=np.float32)
488
+ w = torch.as_tensor(mine[None], dtype=sc.dtype, device=sc.device)
489
+ elif gated: # the gate is on but its teacher is not loaded: nothing to learn from until it is
490
+ w = torch.zeros((1, len(loc)), dtype=sc.dtype, device=sc.device)
491
+ else:
492
+ w = plastic.row_weights(sc[None], torch.tensor([line], dtype=sc.dtype, device=sc.device))
493
+ self.pending.push([(t, ctx.mints[i], name) for i in loc], np.full(len(loc), t), t + self.holds[name] + LABEL_LAG_S, y, u0[li], k[li], w, s=j)
494
  for i in loc:
495
  m = ctx.mints[i]; self.watch[m] = self.watch.get(m, 0) + 1
496
  q_ = self.bars.setdefault(m, deque(maxlen=BAR_KEEP)); b = ctx.bars.get(m)
 
498
  q_.append(b)
499
  if rows:
500
  with conn.cursor() as cur:
501
+ cur.executemany("INSERT INTO fly_scored (ts, mint, strategy, hold_min, x, score, frozen_score, line, frozen_line, bootstrap_id, teacher_allow, chain) "
502
+ "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT (ts, mint, strategy) DO NOTHING", [r + (self.chain,) for r in rows])
503
+ if self.chain == "sol":
504
+ self.fly.lines = dict(self.lines["plastic"]); self.fly.sizings = {k: list(v) for k, v in self.sizing["plastic"].items()}
505
+ else: # the shared model decides this chain's rows on this chain's lines
506
+ self.fly.chain_lines[self.chain] = dict(self.lines["plastic"]); self.fly.chain_sizings[self.chain] = {k: list(v) for k, v in self.sizing["plastic"].items()}
507
+ if td is not None:
508
+ d = self._gated(ctx, V, F, td)
509
+ return {"decision": d, "picks": int(d["allow"].sum())}
510
+ if gated: # never the ungated fly: it bought what the selector rejected and lost (2026-10-08)
511
+ return {"decision": no_teacher(n), "picks": 0}
512
  d = fly_selector.fly_decide(self.fly, ctx.X, X_COLS, np.where(np.isfinite(V), V, -np.inf))
513
  return {"decision": d, "picks": int(sum(1 for x in d["strategy"] if x is not None))}
514
 
515
+ def _teacher_scores(self, X: np.ndarray, t) -> np.ndarray | None:
516
+ """[B, S] the teacher's own score of each row per strategy (NaN off its trigger) when the fly scores as its teacher
517
+ plus its learned correction (config.FLY_SCORE = teacher), else None."""
518
+ if config.FLY_SCORE != "teacher" or self.teacher_live is None:
519
+ return None
520
+ return fly_selector.teacher_scores(self.teacher_live, X, t, self.names, X_COLS)
521
+
522
+ def _head(self, y_own: torch.Tensor, tv: np.ndarray | None, loc: np.ndarray, j: int, u0: torch.Tensor) -> torch.Tensor:
523
+ """The head output the bank scores with: the fly's own decoder output, or (FLY_SCORE = teacher) the one under which
524
+ its frozen score is the teacher's score exactly (PlasticBank.head_for), so only its learning moves it."""
525
+ if tv is None:
526
+ return y_own
527
+ return self.bank.head_for(torch.as_tensor(tv[loc, j], dtype=torch.float32, device=self.bank.dev), u0, s=np.full(len(loc), j))
528
+
529
+ def _teacher_decision(self, ctx) -> dict:
530
+ """The deployed selector's decision on this minute's rows, with its live fail-closed rules."""
531
+ from .selector_session import fail_closed
532
+ return fail_closed(self.teacher_groups, ctx.infos, strategies.decide(self.teacher_live, ctx.X, X_COLS, ctx.t_start))
533
+
534
+ def _gated(self, ctx, V: np.ndarray, F: np.ndarray, td: dict | None = None) -> dict:
535
+ """The fly's decision under its teacher: each row's strategy, trigger, line, win filter, gates, dump veto and
536
+ fail-closed rules are the deployed selector's (agent/selector_session.fail_closed), so a fresh bootstrap trades the
537
+ selector's set; the fly drops a teacher trade only where its learning has moved its score more than
538
+ ``FLY_LEARNED_VETO`` below its bootstrap score. A row the teacher blocks is recorded with the teacher's reason."""
539
+ n = len(ctx.mints)
540
+ td = td if td is not None else self._teacher_decision(ctx)
541
+ strat = np.full(n, None, dtype=object); score = np.full(n, -1.0); hold = np.zeros(n); thr = np.full(n, np.inf)
542
+ tables = [[] for _ in range(n)]; allow = np.zeros(n, bool); reason = np.full(n, "", dtype=object)
543
+ for i in range(n):
544
+ s_ = td["strategy"][i]
545
+ if s_ is None or s_ not in self.names:
546
+ continue
547
+ j = self.names.index(s_); v, f = V[i, j], F[i, j]
548
+ # the selector's own score, line and sizing table for the row: a gated fly buy is sized exactly as the selector
549
+ # sizes it (agent/sizing.size_position; FLY_SIZING=kelly: the same Kelly band, fraction and caps; flat: flat)
550
+ strat[i] = s_; score[i] = float(td["score"][i]); thr[i] = float(td["threshold"][i]); hold[i] = self.holds[s_]; tables[i] = td["tables"][i]
551
+ if not td["allow"][i]:
552
+ reason[i] = f"selector: {td['reason'][i]}"
553
+ elif np.isfinite(v) and np.isfinite(f) and v < f - config.FLY_LEARNED_VETO:
554
+ reason[i] = f"learned veto: score {v:.3f} < bootstrap {f:.3f} - {config.FLY_LEARNED_VETO:g}"
555
+ else:
556
+ allow[i] = True; reason[i] = "trade"
557
+ return {"strategy": strat, "score": score, "hold_s": hold, "threshold": thr, "tables": tables, "allow": allow, "reason": reason}
558
+
559
  def _capture(self, H: torch.Tensor, trig: np.ndarray, t: float) -> None:
560
  """The console's brain view: this minute's mean activity over the rows some strategy's trigger fired on (every
561
  row when none did), one file per minute (brain/activity.py). Never allowed to interrupt scoring."""
 
572
  # ---- daily and hourly ----
573
  def _resolved(self, conn, since_resolved: float, cols: str, name: str) -> list[dict]:
574
  """A strategy's resolved rows whose labels became known after ``since_resolved`` (epoch s)."""
575
+ return conn.execute(f"SELECT {cols} FROM fly_scored WHERE state = 'resolved' AND label IS NOT NULL AND bootstrap_id = %s AND strategy = %s AND chain = %s "
576
+ "AND ts >= %s ORDER BY ts", (self.boot_id, name, self.chain,
577
+ datetime.fromtimestamp(since_resolved - self.holds[name] - LABEL_LAG_S, timezone.utc))).fetchall()
578
 
579
  def _calibrate(self, conn, now: float) -> None:
580
  day = datetime.fromtimestamp(now, timezone.utc).date()
581
+ rule = fly_calibrate.RULE
582
  for name in self.names:
583
+ rows = self._resolved(conn, now - fly_calibrate.WINDOW_DAYS * 86400.0, "ts, mint, score, frozen_score, label, teacher_allow", name)
584
+ target = None
585
+ if rule == "match":
586
+ # the teacher's trades over the same rows are the budget; rows scored before it was recorded cannot count
587
+ rows = [r for r in rows if r["teacher_allow"] is not None]
588
+ if not rows:
589
+ log.info("fly %s: no scored rows with the teacher's decision yet; the line stays %.4f", name, self.lines["plastic"][name])
590
+ continue
591
  if not rows:
592
  continue
593
  ts = np.array([r["ts"].timestamp() for r in rows]); mint = np.array([r["mint"] for r in rows]); lab = np.array([r["label"] for r in rows], dtype=np.float64)
594
+ if rule == "match":
595
+ al = np.array([bool(r["teacher_allow"]) for r in rows]) & np.isfinite(lab)
596
+ target = len(fly_calibrate.taken_idx(ts, mint, self.holds[name], np.flatnonzero(al)))
597
  for arm, col in (("plastic", "score"), ("frozen", "frozen_score")):
598
  cal = fly_calibrate.calibrate(ts, mint, self.holds[name], np.array([r[col] for r in rows], dtype=np.float64), lab,
599
+ prev_line=self.lines[arm][name], prev_sizing=self.sizing[arm][name], rule=rule, target_trades=target)
600
  if cal.changed:
601
+ record_event("info", "fly", f"{name} {arm} line {self.lines[arm][name] * 100:+.2f}% → {cal.line * 100:+.2f}%",
602
+ {"trades": cal.trades, "mean": cal.mean, "total": cal.total, "rule": rule, "teacher_trades": target})
603
  self.lines[arm][name], self.sizing[arm][name] = cal.line, cal.sizing
604
  conn.execute("INSERT INTO fly_calibrations (day, arm, line, sizing, trades, total, mean, window_days) VALUES (%s,%s,%s,%s,%s,%s,%s,%s) "
605
  "ON CONFLICT (day, arm) DO UPDATE SET line = EXCLUDED.line, sizing = EXCLUDED.sizing, trades = EXCLUDED.trades, total = EXCLUDED.total, mean = EXCLUDED.mean",
606
+ (day, f"{arm}:{name}" if self.chain == "sol" else f"{arm}:{self.chain}:{name}", cal.line, json.dumps(cal.sizing), cal.trades,
607
+ cal.total, cal.mean, fly_calibrate.WINDOW_DAYS))
608
 
609
  def _drift(self, j: int) -> float:
610
  return float(self.bank.drift(j)[0]) if self.bank.learn.shape[0] > 1 else float(self.bank.drift()[0])
 
626
  return out
627
 
628
  def _hourly(self, conn, now: float) -> None:
629
+ if self.brain is not None:
630
+ # another chain's book: its own handover; the shared memory's checks, snapshots and rollbacks are the Solana book's
631
+ self._handover_check(conn, now)
632
+ return
633
  hour = datetime.fromtimestamp(math.floor(now / 3600) * 3600 - 3600, timezone.utc)
634
  checks = self._checks(conn, now); self.last_checks = checks
635
  h = self.hour_stats; first = checks.get(self.names[0]) or {}
 
650
  d = fly_state_dir(); d.mkdir(parents=True, exist_ok=True)
651
  path = d / f"snap_{datetime.fromtimestamp(now, timezone.utc).strftime('%Y%m%dT%H%M')}.pt"
652
  torch.save({"bank": self.bank.state(), "bootstrap_id": self.boot_id, "lines": self.lines, "sizing": self.sizing}, path)
653
+ note = {"bootstrap_id": self.boot_id, "data": fly_selector.FLY_VERSION, "score": config.FLY_SCORE, "good": good, "checks": checks,
654
  "drift": max((c["drift"] for c in checks.values()), default=0.0), "ic24": {k: c.get("ic") for k, c in checks.items()}}
655
  return int(conn.execute("INSERT INTO brain_snapshots (run_id, path, kind, note) VALUES (%s,%s,'fly_plastic',%s) RETURNING id",
656
  (self.run_id, str(path), json.dumps(note, default=str))).fetchone()["id"])
 
679
  (datetime.fromtimestamp(now - ROLLBACK_MIN_AGE_S, timezone.utc),)).fetchall():
680
  n = json.loads(r["note"] or "{}")
681
  ok = n.get("good") or not ((n.get("checks") or {}).get(name) or {}).get("triggers")
682
+ if ok and n.get("bootstrap_id") == self.boot_id and n.get("score", "own") == config.FLY_SCORE and Path(r["path"]).exists():
683
  target = r; break
684
  if target is not None:
685
  snap = plastic.PlasticBank(self.fly.net, [(0.0, math.inf)], fly_selector.SCALE, learn=self.bank.learn.cpu().numpy(), read=self.bank.read.cpu().numpy())
686
+ st = torch.load(target["path"], map_location="cpu", weights_only=True)["bank"]
687
  snap.D = torch.zeros_like(snap.D); snap.D[:, snap.mask.bool()] = st["D"].to(snap.dev).float()
688
  self.bank.D[0][:, cols] = snap.D[0][:, cols]
689
  else:
 
722
  whole race), closed ``config.HANDOVER_MIN_TRADES`` paper trades and, when ``config.HANDOVER_BEAT_SELECTOR``,
723
  realized at least the paper selector's P&L over that window."""
724
  days = config.HANDOVER_DAYS
725
+ if handover.state(conn, self.chain) is not None or self.race_started_at is None or now - self.race_started_at < days * 86400.0:
726
  return
727
  since = datetime.fromtimestamp(self.race_started_at if days <= 0 else now - days * 86400.0, timezone.utc)
728
+ fb, sb = self.BOOK, self.market.selector_book # this chain's race: its fly book against its selector book
729
  pnl = {b: conn.execute("SELECT COALESCE(sum(realized_sol), 0) AS s, count(*) AS n FROM positions WHERE book = %s AND status = 'closed' AND closed_at >= %s",
730
+ (b, since)).fetchone() for b in (fb, sb)}
731
+ fly_pnl, sel_pnl, n = float(pnl[fb]["s"]), float(pnl[sb]["s"]), int(pnl[fb]["n"])
732
  if n >= config.HANDOVER_MIN_TRADES and (fly_pnl >= sel_pnl or not config.HANDOVER_BEAT_SELECTOR):
733
  detail = {"at": datetime.fromtimestamp(now, timezone.utc).isoformat(), "fly_pnl_sol": fly_pnl, "selector_pnl_sol": sel_pnl, "fly_trades": n,
734
+ "selector_trades": int(pnl[sb]["n"]), "days": days, "beat_selector": config.HANDOVER_BEAT_SELECTOR, "bootstrap": self.boot_id,
735
+ "chain": self.chain, "unit": self.market.unit}
736
+ handover.record(conn, detail, self.chain)
737
  record_event("info", "handover", f"the fly takes the selector's seat: {fly_pnl:+.3f} SOL vs {sel_pnl:+.3f} SOL over {n} trades"
738
  + (f" in {days} days" if days > 0 else ""), detail)
fly_trader/agent/handover.py CHANGED
@@ -11,14 +11,22 @@ import json
11
  KEY = "handover"
12
 
13
 
14
- def state(conn) -> dict | None:
15
- r = conn.execute("SELECT value FROM ui_settings WHERE key = %s", (KEY,)).fetchone()
 
 
 
 
 
 
 
 
16
  if not r or r["value"] is None:
17
  return None
18
  v = r["value"] if isinstance(r["value"], dict) else json.loads(r["value"])
19
  return v or None
20
 
21
 
22
- def record(conn, detail: dict) -> None:
23
  conn.execute("INSERT INTO ui_settings (key, value) VALUES (%s, %s) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()",
24
- (KEY, json.dumps(detail, default=str)))
 
11
  KEY = "handover"
12
 
13
 
14
+ def _key(chain: str | None) -> str:
15
+ """Each chain's fly takes its own chain's seat (fly_trader/markets: 'handover', 'handover_rh')."""
16
+ if chain in (None, "sol"):
17
+ return KEY
18
+ from ..markets import for_chain
19
+ return for_chain(chain).handover_key
20
+
21
+
22
+ def state(conn, chain: str | None = "sol") -> dict | None:
23
+ r = conn.execute("SELECT value FROM ui_settings WHERE key = %s", (_key(chain),)).fetchone()
24
  if not r or r["value"] is None:
25
  return None
26
  v = r["value"] if isinstance(r["value"], dict) else json.loads(r["value"])
27
  return v or None
28
 
29
 
30
+ def record(conn, detail: dict, chain: str | None = "sol") -> None:
31
  conn.execute("INSERT INTO ui_settings (key, value) VALUES (%s, %s) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()",
32
+ (_key(chain), json.dumps(detail, default=str)))
fly_trader/agent/minute_engine.py CHANGED
@@ -30,6 +30,7 @@ from datetime import datetime, timezone
30
 
31
  import numpy as np
32
 
 
33
  from ..db.apilog import record_event
34
  from ..db.connection import transaction
35
  from ..market.exit_cost import PUMP_SUPPLY
@@ -78,6 +79,7 @@ class Minute:
78
  fresh: bool
79
  trade: bool
80
  engine: "MinuteEngine"
 
81
 
82
  @property
83
  def t_start(self) -> float:
@@ -91,7 +93,14 @@ class Minute:
91
 
92
 
93
  class MinuteEngine:
94
- def __init__(self, books: list | None = None, live: bool = False):
 
 
 
 
 
 
 
95
  self.books = list(books or []); self.live = live
96
  self.states: dict[str, MintState] = {}
97
  self.meta_cache: dict[str, dict] = {}; self.last_sweep = time.time(); self.last_minute: float | None = None
@@ -100,21 +109,34 @@ class MinuteEngine:
100
  def has_book(self, name: str) -> bool:
101
  return any(b.name == name for b in self.books)
102
 
 
 
 
 
 
 
 
103
  # ---- lookups ----
104
  def _meta(self, conn, mint: str) -> dict | None:
105
  """``corpus_meta`` row (graduation time + creation/creator features), cached for 10 minutes: rows appear at
106
  graduation and the archive rebuild later replaces stream values, so the cache is refreshed."""
107
  hit = self.meta_cache.get(mint)
108
  if hit is None or time.time() - hit["at"] > META_TTL_S:
109
- r = conn.execute("SELECT graduated_at, supply, " + ", ".join(META_COLS) + " FROM corpus_meta WHERE mint = %s", (mint,)).fetchone()
 
 
 
110
  hit = self.meta_cache[mint] = {"row": dict(r) if r else None, "at": time.time()}
111
  return hit["row"]
112
 
113
  def _state(self, conn, mint: str, pool: str | None, program_label: str | None) -> MintState:
114
  s = self.states.get(mint)
115
  if s is None:
116
- t = conn.execute("SELECT decimals FROM tokens WHERE mint = %s", (mint,)).fetchone()
117
- s = MintState(mint, int((t or {}).get("decimals") or 6), pool, program_label, None, None)
 
 
 
118
  self.states[mint] = s
119
  row = self._meta(conn, mint)
120
  s.meta_row = row
@@ -128,8 +150,11 @@ class MinuteEngine:
128
  s = self.states.get(mint)
129
  if s is not None and s.st.last_res_quote_sol:
130
  return s.st.last_res_quote_sol
131
- r = conn.execute("SELECT resq_sol FROM pump_minutes WHERE mint = %s AND resq_sol IS NOT NULL ORDER BY ts DESC LIMIT 1", (mint,)).fetchone()
132
- return float(r["resq_sol"]) if r else None
 
 
 
133
 
134
  def _mcap(self, mint: str, price: float | None) -> float | None:
135
  """Market cap in SOL (price × supply), which sets the pool fee tier; None without a price."""
@@ -138,7 +163,16 @@ class MinuteEngine:
138
 
139
  # ---- one minute ----
140
  def _aggregate(self, conn, m0: datetime, m1: datetime) -> dict[str, dict]:
141
- """PumpAPI minutes: every SOL-quoted PumpSwap pump.fun token, the same fields as the archive."""
 
 
 
 
 
 
 
 
 
142
  rows = conn.execute("SELECT mint, pool_id, open, high, low, close, buy_sol, sell_sol, n_buys, n_sells, n_traders, resq_sol, fee_rate, "
143
  "n_buyers, wash_sol, wash_buy_sol, top_sell_sol, insider_sell_sol, skill_buy FROM pump_minutes WHERE ts = %s", (m0,)).fetchall()
144
  return {r["mint"]: {"open": r["open"], "high": r["high"], "low": r["low"], "close": r["close"], "buy": r["buy_sol"] or 0.0, "sell": r["sell_sol"] or 0.0,
@@ -151,7 +185,9 @@ class MinuteEngine:
151
  """The full feature vector (``X_COLS``) of a mint for the minute ending ``t_end``, and what trading needs."""
152
  s = self._state(conn, mint, a["pool"], a["program_label"])
153
  st, price, resq = s.st, float(a["close"]), a["resq"]
154
- if (s.prev_close and not (1 / 50 <= price / s.prev_close <= 50)) or (resq is not None and resq > 1e5):
 
 
155
  s.broken = True # train/decisions.py: a scale break makes the mint ineligible from that minute on
156
  s.prev_close = price
157
  if a["buy"] > 0:
@@ -175,21 +211,27 @@ class MinuteEngine:
175
  "meta_known": 1.0 if meta.get("ttg_min") is not None else 0.0, **s.flow.features(t_end), "mkt_vol_1h": self.market.value(t_end)}
176
  by_name = {**{n: float(f[i]) for i, n in enumerate(FEATURES)}, **extra,
177
  **{c: float(meta.get(c)) if meta.get(c) is not None else 0.0 for c in META_COLS}}
 
 
 
178
  x = np.nan_to_num(np.asarray([by_name.get(c, 0.0) for c in X_COLS], dtype=np.float32), nan=0.0, posinf=0.0, neginf=0.0)
179
  info = {"price": price, "open": float(a["open"]) if a.get("open") else price, "resq": resq, "logvol_15m": f[FIDX["logvol_15m"]],
180
  # "ec" is the training label's own cost model (market/exit_cost.cost_at_size), at the size the book trades
181
- "ec": float(cost_at_size(f[FIDX["exit_cost_0p1"]], resq)), "age_h": (t_end - s.graduated_at) / 3600 if s.graduated_at else None,
 
182
  "mcap": price * s.meta.supply, "fee_rate": a.get("fee_rate"), "decimals": s.decimals, "pool": s.pool, "program_label": s.program_label, "broken": s.broken,
183
- "skill_missing": a.get("skill_buy") is None, "curve_known": bool(meta.get("curve_known"))}
 
184
  return x, info
185
 
186
- @staticmethod
187
- def eligible(info: dict) -> bool:
 
188
  return not info["broken"] and info["resq"] is not None and info["resq"] >= MIN_RESQ_SOL and info["logvol_15m"] >= math.log1p(MIN_VOL_15M_SOL)
189
 
190
  def _stream_through(self) -> float | None:
191
  with transaction() as conn:
192
- r = conn.execute("SELECT value->>'flushed_through' AS ft FROM ui_settings WHERE key = 'pumpstream_status'").fetchone()
193
  return datetime.fromisoformat(r["ft"]).timestamp() if r and r["ft"] else None
194
 
195
  def ready_through(self, m1_epoch: float) -> float:
@@ -224,7 +266,7 @@ class MinuteEngine:
224
  for mint, a in agg.items():
225
  _x, info = self._features(conn, mint, a, t1); n += 1
226
  bars[mint] = (t1 - 60.0, info["open"], info["price"], info["ec"])
227
- for b in self.books:
228
  b.on_bars(t1 - 60.0, bars)
229
  log.info("warm-up: %d token-minutes over the last %d minutes (%d tokens)", n, minutes, len(self.states))
230
  return n
@@ -247,9 +289,10 @@ class MinuteEngine:
247
  X, infos, mints, bars = self._minute_rows(conn, agg, m1_epoch)
248
  ctx = Minute(conn=conn, m0=m0, m1=m1, m1_epoch=m1_epoch, agg=agg, X=X, infos=infos, mints=mints, bars=bars,
249
  prices={m: float(a["close"]) for m, a in agg.items()}, resqs={m: a["resq"] for m, a in agg.items()},
250
- fees={m: a.get("fee_rate") for m, a in agg.items()}, fresh=trade and self.stream_fresh(m1_epoch - 60), trade=trade, engine=self)
 
251
  out.update(mints_traded=len(agg), eligible=len(mints)); held: set[str] = set()
252
- for b in list(self.books):
253
  try:
254
  with conn.transaction(): # a savepoint per book
255
  b.on_bars(m1_epoch - 60.0, bars)
@@ -259,7 +302,7 @@ class MinuteEngine:
259
  log.exception("%s: minute %s failed", b.name, m1.isoformat())
260
  record_event("error", b.name, f"minute failed: {m1.isoformat()}")
261
  self._sweep(m1_epoch, held)
262
- for b in [b for b in self.books if b.done]:
263
  b.finish(); self.books.remove(b)
264
  return out
265
 
@@ -288,35 +331,45 @@ def admit_books(engine: MinuteEngine, live: bool) -> list:
288
  from . import fly_session, handover
289
  from .selector_session import NoModel, SelectorBook
290
  new = []
291
- if not engine.has_book("fly"):
292
- fb, why = fly_session.try_start(live)
293
- if fb is not None:
294
- new.append(fb)
295
- else:
296
- _status("fly_status", {"stage": "not trading", "detail": why})
297
- fly_on = engine.has_book("fly") or bool(new)
298
- with transaction() as conn:
299
- handed = handover.state(conn) is not None
300
- if not engine.has_book("selector") and not (handed and fly_on):
301
- try:
302
- new.append(SelectorBook())
303
- except NoModel as e:
304
- _status("selector_status", {"stage": "waiting for a model", "detail": str(e)})
 
 
 
305
  return new
306
 
307
 
308
  def main(stop_event: threading.Event | None = None, live: bool = False) -> None:
309
- engine = MinuteEngine(live=live)
 
 
 
 
 
310
  while not (stop_event is not None and stop_event.is_set()):
311
- engine.books = admit_books(engine, live)
312
- if engine.books:
313
  break
314
  log.info("trading engine waiting: no book qualifies yet")
315
  (stop_event or threading.Event()).wait(MODEL_CHECK_S)
316
  else:
317
  return
318
  m_start = math.floor(time.time() / 60) * 60
319
- engine.warm_up(m_start); engine.last_minute = m_start - 60; last_check = time.time(); n_min = 0
 
 
320
  last_release, last_stamp_check = release_stamp(), 0.0
321
  try:
322
  while not (stop_event is not None and stop_event.is_set()):
@@ -327,31 +380,35 @@ def main(stop_event: threading.Event | None = None, live: bool = False) -> None:
327
  last_release, last_check = stamp, 0.0
328
  if now - last_check >= MODEL_CHECK_S:
329
  last_check = now
330
- for b in engine.books:
331
  try:
332
  b.maybe_reload()
333
  except Exception:
334
  log.exception("%s: model reload check failed", b.name)
335
  try:
336
- engine.books += admit_books(engine, live)
337
  except Exception:
338
  log.exception("book admission failed")
339
- if engine.books and m1 > engine.last_minute and now - m1 >= 4.0 and (upto := engine.ready_through(m1)) > engine.last_minute: # only minutes the stream has written
 
 
 
 
340
  for minute in range(int(engine.last_minute) + 60, int(upto) + 1, 60):
341
  try:
342
- st = engine.run_minute(float(minute), trade=(minute == int(m1))) # missed minutes update features (and the fly's learning) only
343
  n_min += 1
344
- if minute == int(m1) and n_min % 10 == 0:
345
- log.info("minute %s: traded %d eligible %d | %s", st["minute"], st.get("mints_traded", 0), st.get("eligible", 0),
346
  " | ".join(f"{b.name}: picks {(st.get(b.name) or {}).get('picks')} open {(st.get(b.name) or {}).get('open')} wealth {(st.get(b.name) or {}).get('wealth')}"
347
- for b in engine.books))
348
  except Exception:
349
- log.exception("minute %s failed", minute); record_event("error", "engine", f"minute failed: {minute}")
350
  engine.last_minute = upto
351
  time.sleep(0.5)
352
  finally:
353
- for b in engine.books:
354
  try:
355
  b.finish()
356
  except Exception:
357
- log.exception("%s: finish failed", b.name)
 
30
 
31
  import numpy as np
32
 
33
+ from .. import config, markets
34
  from ..db.apilog import record_event
35
  from ..db.connection import transaction
36
  from ..market.exit_cost import PUMP_SUPPLY
 
79
  fresh: bool
80
  trade: bool
81
  engine: "MinuteEngine"
82
+ market: object = None # fly_trader/markets.MarketSpec of these rows (None: Solana)
83
 
84
  @property
85
  def t_start(self) -> float:
 
93
 
94
 
95
  class MinuteEngine:
96
+ """One market's minutes (``market``: Solana — pump_minutes, corpus_meta — or Robinhood Chain — rh_minutes, rh_meta,
97
+ amounts in ETH). The engines of both markets share one list of books; each minute goes only to its market's books."""
98
+
99
+ def __init__(self, books: list | None = None, live: bool = False, market=None):
100
+ self.spec = market or markets.SOL; self.rh = self.spec.chain != "sol"
101
+ # how old a minute may be and still be traded: Solana's feed flushes within seconds; the RH stream's minutes land
102
+ # ~2–3 min late (confirmations, the index pass, the price marks), so on RH "the current minute" never arrives
103
+ self.trade_lag_s = float(config.RH_TRADE_LAG_S) if self.rh else 0.0
104
  self.books = list(books or []); self.live = live
105
  self.states: dict[str, MintState] = {}
106
  self.meta_cache: dict[str, dict] = {}; self.last_sweep = time.time(); self.last_minute: float | None = None
 
109
  def has_book(self, name: str) -> bool:
110
  return any(b.name == name for b in self.books)
111
 
112
+ def tradeable(self, minute: float, m1: float) -> bool:
113
+ """Whether minute ``minute`` (its end, epoch s) is recent enough to trade at wall-clock minute ``m1``."""
114
+ return int(m1) - self.trade_lag_s <= int(minute) <= int(m1)
115
+
116
+ def my_books(self) -> list:
117
+ return [b for b in self.books if getattr(b, "chain", "sol") == self.spec.chain]
118
+
119
  # ---- lookups ----
120
  def _meta(self, conn, mint: str) -> dict | None:
121
  """``corpus_meta`` row (graduation time + creation/creator features), cached for 10 minutes: rows appear at
122
  graduation and the archive rebuild later replaces stream values, so the cache is refreshed."""
123
  hit = self.meta_cache.get(mint)
124
  if hit is None or time.time() - hit["at"] > META_TTL_S:
125
+ if self.rh: # Robinhood Chain: rh_meta, the same columns (rh/meta.py), plus the quote class
126
+ r = conn.execute("SELECT graduated_at, supply, quote_class, " + ", ".join(META_COLS) + " FROM rh_meta WHERE mint = %s", (mint,)).fetchone()
127
+ else:
128
+ r = conn.execute("SELECT graduated_at, supply, " + ", ".join(META_COLS) + " FROM corpus_meta WHERE mint = %s", (mint,)).fetchone()
129
  hit = self.meta_cache[mint] = {"row": dict(r) if r else None, "at": time.time()}
130
  return hit["row"]
131
 
132
  def _state(self, conn, mint: str, pool: str | None, program_label: str | None) -> MintState:
133
  s = self.states.get(mint)
134
  if s is None:
135
+ if self.rh: # every Pons launch has 18 decimals
136
+ s = MintState(mint, 18, pool, program_label, None, None); s.meta.ec_size = 0.1 * self.spec.k()
137
+ else:
138
+ t = conn.execute("SELECT decimals FROM tokens WHERE mint = %s", (mint,)).fetchone()
139
+ s = MintState(mint, int((t or {}).get("decimals") or 6), pool, program_label, None, None)
140
  self.states[mint] = s
141
  row = self._meta(conn, mint)
142
  s.meta_row = row
 
150
  s = self.states.get(mint)
151
  if s is not None and s.st.last_res_quote_sol:
152
  return s.st.last_res_quote_sol
153
+ if self.rh:
154
+ r = conn.execute("SELECT resq_eth AS q FROM rh_minutes WHERE mint = %s AND resq_eth IS NOT NULL ORDER BY ts DESC LIMIT 1", (mint,)).fetchone()
155
+ else:
156
+ r = conn.execute("SELECT resq_sol AS q FROM pump_minutes WHERE mint = %s AND resq_sol IS NOT NULL ORDER BY ts DESC LIMIT 1", (mint,)).fetchone()
157
+ return float(r["q"]) if r else None
158
 
159
  def _mcap(self, mint: str, price: float | None) -> float | None:
160
  """Market cap in SOL (price × supply), which sets the pool fee tier; None without a price."""
 
163
 
164
  # ---- one minute ----
165
  def _aggregate(self, conn, m0: datetime, m1: datetime) -> dict[str, dict]:
166
+ """PumpAPI minutes: every SOL-quoted PumpSwap pump.fun token, the same fields as the archive. Robinhood Chain: the
167
+ rh_minutes of graduated Pons pools in ETH (rh/minutes.py), under the same keys."""
168
+ if self.rh:
169
+ rows = conn.execute("SELECT mint, pool_id, open, high, low, close, buy_eth, sell_eth, n_buys, n_sells, n_traders, resq_eth, fee_rate, "
170
+ "n_buyers, wash_eth, wash_buy_eth, top_sell_eth, insider_sell_eth, skill_buy FROM rh_minutes WHERE ts = %s", (m0,)).fetchall()
171
+ return {r["mint"]: {"open": r["open"], "high": r["high"], "low": r["low"], "close": r["close"], "buy": r["buy_eth"] or 0.0, "sell": r["sell_eth"] or 0.0,
172
+ "nb": r["n_buys"] or 0, "ns": r["n_sells"] or 0, "n_traders": int(r["n_traders"] or 0), "resq": r["resq_eth"],
173
+ "pool": r["pool_id"], "program_label": self.spec.program_label, "fee_rate": r["fee_rate"],
174
+ "n_buyers": r["n_buyers"], "wash_sol": r["wash_eth"], "wash_buy_sol": r["wash_buy_eth"], "top_sell_sol": r["top_sell_eth"],
175
+ "insider_sell_sol": r["insider_sell_eth"], "skill_buy": r["skill_buy"]} for r in rows if r["close"]}
176
  rows = conn.execute("SELECT mint, pool_id, open, high, low, close, buy_sol, sell_sol, n_buys, n_sells, n_traders, resq_sol, fee_rate, "
177
  "n_buyers, wash_sol, wash_buy_sol, top_sell_sol, insider_sell_sol, skill_buy FROM pump_minutes WHERE ts = %s", (m0,)).fetchall()
178
  return {r["mint"]: {"open": r["open"], "high": r["high"], "low": r["low"], "close": r["close"], "buy": r["buy_sol"] or 0.0, "sell": r["sell_sol"] or 0.0,
 
185
  """The full feature vector (``X_COLS``) of a mint for the minute ending ``t_end``, and what trading needs."""
186
  s = self._state(conn, mint, a["pool"], a["program_label"])
187
  st, price, resq = s.st, float(a["close"]), a["resq"]
188
+ if self.rh:
189
+ s.meta.pool_fee = a.get("fee_rate") # the Pons hook fee this minute charged (the cost model's fee)
190
+ if (s.prev_close and not (1 / 50 <= price / s.prev_close <= 50)) or (resq is not None and resq > 1e5 * self.spec.k()):
191
  s.broken = True # train/decisions.py: a scale break makes the mint ineligible from that minute on
192
  s.prev_close = price
193
  if a["buy"] > 0:
 
211
  "meta_known": 1.0 if meta.get("ttg_min") is not None else 0.0, **s.flow.features(t_end), "mkt_vol_1h": self.market.value(t_end)}
212
  by_name = {**{n: float(f[i]) for i, n in enumerate(FEATURES)}, **extra,
213
  **{c: float(meta.get(c)) if meta.get(c) is not None else 0.0 for c in META_COLS}}
214
+ if self.rh: # the chain inputs (train/decisions.CHAIN_COLS)
215
+ qc = meta.get("quote_class")
216
+ by_name.update(chain_rh=1.0, qc_stable=float(qc == "stable"), qc_btc=float(qc == "btc"), qc_stock=float(qc == "stock"))
217
  x = np.nan_to_num(np.asarray([by_name.get(c, 0.0) for c in X_COLS], dtype=np.float32), nan=0.0, posinf=0.0, neginf=0.0)
218
  info = {"price": price, "open": float(a["open"]) if a.get("open") else price, "resq": resq, "logvol_15m": f[FIDX["logvol_15m"]],
219
  # "ec" is the training label's own cost model (market/exit_cost.cost_at_size), at the size the book trades
220
+ "ec": float(cost_at_size(f[FIDX["exit_cost_0p1"]], resq, market=self.spec if self.rh else None)),
221
+ "age_h": (t_end - s.graduated_at) / 3600 if s.graduated_at else None,
222
  "mcap": price * s.meta.supply, "fee_rate": a.get("fee_rate"), "decimals": s.decimals, "pool": s.pool, "program_label": s.program_label, "broken": s.broken,
223
+ "skill_missing": a.get("skill_buy") is None, "curve_known": bool(meta.get("curve_known")), "chain": self.spec.chain,
224
+ "quote_class": meta.get("quote_class")}
225
  return x, info
226
 
227
+ def eligible(self, info: dict) -> bool:
228
+ if self.rh: # the same gates in the chain's unit (x K), as training (train/decisions)
229
+ return not info["broken"] and info["resq"] is not None and info["resq"] >= self.spec.min_resq() and info["logvol_15m"] >= math.log1p(self.spec.min_vol_15m())
230
  return not info["broken"] and info["resq"] is not None and info["resq"] >= MIN_RESQ_SOL and info["logvol_15m"] >= math.log1p(MIN_VOL_15M_SOL)
231
 
232
  def _stream_through(self) -> float | None:
233
  with transaction() as conn:
234
+ r = conn.execute("SELECT value->>'flushed_through' AS ft FROM ui_settings WHERE key = %s", (self.spec.stream_status_key,)).fetchone()
235
  return datetime.fromisoformat(r["ft"]).timestamp() if r and r["ft"] else None
236
 
237
  def ready_through(self, m1_epoch: float) -> float:
 
266
  for mint, a in agg.items():
267
  _x, info = self._features(conn, mint, a, t1); n += 1
268
  bars[mint] = (t1 - 60.0, info["open"], info["price"], info["ec"])
269
+ for b in self.my_books():
270
  b.on_bars(t1 - 60.0, bars)
271
  log.info("warm-up: %d token-minutes over the last %d minutes (%d tokens)", n, minutes, len(self.states))
272
  return n
 
289
  X, infos, mints, bars = self._minute_rows(conn, agg, m1_epoch)
290
  ctx = Minute(conn=conn, m0=m0, m1=m1, m1_epoch=m1_epoch, agg=agg, X=X, infos=infos, mints=mints, bars=bars,
291
  prices={m: float(a["close"]) for m, a in agg.items()}, resqs={m: a["resq"] for m, a in agg.items()},
292
+ fees={m: a.get("fee_rate") for m, a in agg.items()}, fresh=trade and self.stream_fresh(m1_epoch - 60), trade=trade, engine=self,
293
+ market=self.spec)
294
  out.update(mints_traded=len(agg), eligible=len(mints)); held: set[str] = set()
295
+ for b in list(self.my_books()):
296
  try:
297
  with conn.transaction(): # a savepoint per book
298
  b.on_bars(m1_epoch - 60.0, bars)
 
302
  log.exception("%s: minute %s failed", b.name, m1.isoformat())
303
  record_event("error", b.name, f"minute failed: {m1.isoformat()}")
304
  self._sweep(m1_epoch, held)
305
+ for b in [b for b in self.my_books() if b.done]:
306
  b.finish(); self.books.remove(b)
307
  return out
308
 
 
331
  from . import fly_session, handover
332
  from .selector_session import NoModel, SelectorBook
333
  new = []
334
+ for m in markets.enabled(): # Solana first: another chain's fly shares the Solana fly's brain
335
+ fly_name = "fly" if m.chain == "sol" else f"fly_{m.chain}"; sel_name = "selector" if m.chain == "sol" else f"selector_{m.chain}"
336
+ if not engine.has_book(fly_name) and not any(b.name == fly_name for b in new):
337
+ brain = next((b for b in engine.books + new if b.name == "fly"), None) if m.chain != "sol" else None
338
+ fb, why = fly_session.try_start(live, chain=m.chain, brain=brain)
339
+ if fb is not None:
340
+ new.append(fb)
341
+ else:
342
+ _status(m.fly_status_key, {"stage": "not trading", "detail": why, "chain": m.chain})
343
+ fly_on = engine.has_book(fly_name) or any(b.name == fly_name for b in new)
344
+ with transaction() as conn:
345
+ handed = handover.state(conn, m.chain) is not None
346
+ if not engine.has_book(sel_name) and not (handed and fly_on):
347
+ try:
348
+ new.append(SelectorBook(m.chain))
349
+ except NoModel as e:
350
+ _status(m.selector_status_key, {"stage": "waiting for a model", "detail": str(e), "chain": m.chain})
351
  return new
352
 
353
 
354
  def main(stop_event: threading.Event | None = None, live: bool = False) -> None:
355
+ """One engine per enabled chain (markets.enabled()), all sharing one list of books: each engine feeds its minutes, as
356
+ its own stream writes them, to its chain's books only — a lagging chain never holds another chain's minutes back."""
357
+ engines = [MinuteEngine(live=live, market=m) for m in markets.enabled()]
358
+ lead = engines[0]; books: list = []
359
+ for e in engines:
360
+ e.books = books # the same list object: admission and removal are seen by every engine
361
  while not (stop_event is not None and stop_event.is_set()):
362
+ books += admit_books(lead, live)
363
+ if books:
364
  break
365
  log.info("trading engine waiting: no book qualifies yet")
366
  (stop_event or threading.Event()).wait(MODEL_CHECK_S)
367
  else:
368
  return
369
  m_start = math.floor(time.time() / 60) * 60
370
+ for e in engines:
371
+ e.warm_up(m_start); e.last_minute = m_start - 60
372
+ last_check = time.time(); n_min = 0
373
  last_release, last_stamp_check = release_stamp(), 0.0
374
  try:
375
  while not (stop_event is not None and stop_event.is_set()):
 
380
  last_release, last_check = stamp, 0.0
381
  if now - last_check >= MODEL_CHECK_S:
382
  last_check = now
383
+ for b in books:
384
  try:
385
  b.maybe_reload()
386
  except Exception:
387
  log.exception("%s: model reload check failed", b.name)
388
  try:
389
+ books += admit_books(lead, live)
390
  except Exception:
391
  log.exception("book admission failed")
392
+ for engine in engines:
393
+ if not engine.my_books() or m1 <= engine.last_minute or now - m1 < 4.0:
394
+ continue
395
+ if (upto := engine.ready_through(m1)) <= engine.last_minute: # only minutes this chain's stream has written
396
+ continue
397
  for minute in range(int(engine.last_minute) + 60, int(upto) + 1, 60):
398
  try:
399
+ st = engine.run_minute(float(minute), trade=engine.tradeable(minute, m1)) # older minutes update features (and the fly's learning) only
400
  n_min += 1
401
+ if engine.tradeable(minute, m1) and n_min % 10 == 0:
402
+ log.info("%s minute %s: traded %d eligible %d | %s", engine.spec.chain, st["minute"], st.get("mints_traded", 0), st.get("eligible", 0),
403
  " | ".join(f"{b.name}: picks {(st.get(b.name) or {}).get('picks')} open {(st.get(b.name) or {}).get('open')} wealth {(st.get(b.name) or {}).get('wealth')}"
404
+ for b in engine.my_books()))
405
  except Exception:
406
+ log.exception("%s minute %s failed", engine.spec.chain, minute); record_event("error", "engine", f"{engine.spec.chain} minute failed: {minute}")
407
  engine.last_minute = upto
408
  time.sleep(0.5)
409
  finally:
410
+ for b in books:
411
  try:
412
  b.finish()
413
  except Exception:
414
+ log.exception("%s: finish failed", b.name)
fly_trader/agent/paper_trading.py CHANGED
@@ -20,10 +20,11 @@ from ..market.exit_cost import exit_cost_fraction
20
 
21
  def trade_minute(ctx, *, book: str, run_id: str, beat_no: int, broker, kind: str, mints: list, infos: list, scores, threshold,
22
  table: list | None, horizon_s: float, block: str | None = None, holds=None, strategies=None, allow=None, reasons=None,
23
- tables: list | None = None) -> dict:
24
  """Optional per-row arguments (the selector's strategy stack, train/strategies.decide): ``threshold`` may be per row,
25
  ``holds`` (s), ``strategies``, ``tables`` (sizing) and ``allow`` / ``reasons`` (a blocked row is recorded with its reason).
26
- Left out, the call behaves exactly as the single-strategy engine."""
 
27
  conn, m1 = ctx.conn, ctx.m1
28
  beat = conn.execute("INSERT INTO beats (run_id, ts, beat_no, n_slots_active, notes) VALUES (%s,%s,%s,%s,%s) RETURNING id",
29
  (run_id, m1, beat_no, len(mints), json.dumps({"minute": m1.isoformat(), "mints_traded": len(ctx.agg), "book": book}))).fetchone()["id"]
@@ -41,7 +42,9 @@ def trade_minute(ctx, *, book: str, run_id: str, beat_no: int, broker, kind: str
41
  opens_now = ledger.open_positions(conn, book); open_mints = {p["mint"] for p in opens_now}
42
  cash = ledger.paper_cash(conn, book); n_enter = 0; picks = []; entries = []
43
  bankroll = cash + sum(float(p["cost_sol"]) for p in opens_now) # wealth at cost: the sizing base (agent/sizing.py)
44
- circuit = conn.execute("SELECT kill_switch, entries_paused FROM circuit_state WHERE id = 1").fetchone()
 
 
45
  blocked = (block or ("kill switch" if circuit and circuit["kill_switch"] else None) or ("paused" if circuit and circuit["entries_paused"] else None)
46
  or ("book halted" if rails.book_halted(conn, book) else None))
47
  thr_v = np.broadcast_to(np.asarray(threshold, dtype=np.float64), (len(mints),)) if len(mints) else np.array([])
@@ -51,7 +54,7 @@ def trade_minute(ctx, *, book: str, run_id: str, beat_no: int, broker, kind: str
51
  continue
52
  picks.append((m, float(sc)))
53
  tab_k = tables[k] if tables is not None else table
54
- size, why = sizing.size_position(float(sc), thr_k, tab_k or [], bankroll, cash, i["resq"])
55
  denied = allow is not None and not bool(allow[k])
56
  if m in open_mints or blocked or denied or size <= 0:
57
  rail = blocked or ("held" if m in open_mints else (str(reasons[k]) if denied and reasons is not None else "filtered") if denied else "sizing")
@@ -84,6 +87,6 @@ def trade_minute(ctx, *, book: str, run_id: str, beat_no: int, broker, kind: str
84
  peak_row = conn.execute("SELECT max(wealth) AS pk FROM wealth_marks WHERE book = %s", (book,)).fetchone(); peak = max(float(peak_row["pk"] or 0.0), wealth)
85
  conn.execute("INSERT INTO wealth_marks (beat_id, book, ts, sol_free, positions_value, exit_cost, wealth, peak, drawdown, exposure, n_open) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)",
86
  (beat, book, m1, cash, positions_net, exit_cost, wealth, peak, (1.0 - wealth / peak) if peak > 0 else 0.0, exposure, len(opens)))
87
- rails.check_book_drawdown(conn, book, wealth, peak) # the book's own kill switch: blocks its next minutes' entries
88
  return {"beat_id": beat, "entries": entries, "entered": n_enter, "exited": n_exit, "open": len(opens), "cash": cash, "wealth": wealth, "picks": len(picks), "blocked": blocked,
89
  "top_scores": sorted(picks, key=lambda x: -x[1])[:5], "open_mints": sorted({p["mint"] for p in opens})}
 
20
 
21
  def trade_minute(ctx, *, book: str, run_id: str, beat_no: int, broker, kind: str, mints: list, infos: list, scores, threshold,
22
  table: list | None, horizon_s: float, block: str | None = None, holds=None, strategies=None, allow=None, reasons=None,
23
+ tables: list | None = None, flat: bool = False) -> dict:
24
  """Optional per-row arguments (the selector's strategy stack, train/strategies.decide): ``threshold`` may be per row,
25
  ``holds`` (s), ``strategies``, ``tables`` (sizing) and ``allow`` / ``reasons`` (a blocked row is recorded with its reason).
26
+ Left out, the call behaves exactly as the single-strategy engine. ``flat``: every buy the same share of the bankroll
27
+ (agent/sizing.size_position), the fly's rule (``config.FLY_SIZING``)."""
28
  conn, m1 = ctx.conn, ctx.m1
29
  beat = conn.execute("INSERT INTO beats (run_id, ts, beat_no, n_slots_active, notes) VALUES (%s,%s,%s,%s,%s) RETURNING id",
30
  (run_id, m1, beat_no, len(mints), json.dumps({"minute": m1.isoformat(), "mints_traded": len(ctx.agg), "book": book}))).fetchone()["id"]
 
42
  opens_now = ledger.open_positions(conn, book); open_mints = {p["mint"] for p in opens_now}
43
  cash = ledger.paper_cash(conn, book); n_enter = 0; picks = []; entries = []
44
  bankroll = cash + sum(float(p["cost_sol"]) for p in opens_now) # wealth at cost: the sizing base (agent/sizing.py)
45
+ from ..markets import for_book
46
+ market = for_book(book) # the book's market: its circuit, money settings and sizes
47
+ circuit = conn.execute("SELECT kill_switch, entries_paused FROM circuit_state WHERE id = %s", (market.circuit_id,)).fetchone()
48
  blocked = (block or ("kill switch" if circuit and circuit["kill_switch"] else None) or ("paused" if circuit and circuit["entries_paused"] else None)
49
  or ("book halted" if rails.book_halted(conn, book) else None))
50
  thr_v = np.broadcast_to(np.asarray(threshold, dtype=np.float64), (len(mints),)) if len(mints) else np.array([])
 
54
  continue
55
  picks.append((m, float(sc)))
56
  tab_k = tables[k] if tables is not None else table
57
+ size, why = sizing.size_position(float(sc), thr_k, tab_k or [], bankroll, cash, i["resq"], flat=flat, market=market)
58
  denied = allow is not None and not bool(allow[k])
59
  if m in open_mints or blocked or denied or size <= 0:
60
  rail = blocked or ("held" if m in open_mints else (str(reasons[k]) if denied and reasons is not None else "filtered") if denied else "sizing")
 
87
  peak_row = conn.execute("SELECT max(wealth) AS pk FROM wealth_marks WHERE book = %s", (book,)).fetchone(); peak = max(float(peak_row["pk"] or 0.0), wealth)
88
  conn.execute("INSERT INTO wealth_marks (beat_id, book, ts, sol_free, positions_value, exit_cost, wealth, peak, drawdown, exposure, n_open) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)",
89
  (beat, book, m1, cash, positions_net, exit_cost, wealth, peak, (1.0 - wealth / peak) if peak > 0 else 0.0, exposure, len(opens)))
90
+ rails.check_book_drawdown(conn, book, wealth, peak, drawdown=market.kill_drawdown(), unit=market.unit) # the book's own kill switch
91
  return {"beat_id": beat, "entries": entries, "entered": n_enter, "exited": n_exit, "open": len(opens), "cash": cash, "wealth": wealth, "picks": len(picks), "blocked": blocked,
92
  "top_scores": sorted(picks, key=lambda x: -x[1])[:5], "open_mints": sorted({p["mint"] for p in opens})}
fly_trader/agent/rails.py CHANGED
@@ -1,6 +1,9 @@
1
  """Risk rails outside the model. KillSwitch (−KILL_SWITCH_DRAWDOWN from peak wealth → no entries), CircuitBreaker
2
  (CIRCUIT_THRESHOLD consecutive execution failures), the rolling 24 h notional ledger, and PauseEntries (console
3
  toggle). State lives in circuit_state / circuit_events / notional_ledger; the trading engine reads it before entries.
 
 
 
4
  """
5
  from __future__ import annotations
6
 
@@ -13,6 +16,7 @@ from ..db.apilog import record_event
13
  from ..db.connection import transaction
14
 
15
  log = logging.getLogger(__name__)
 
16
 
17
 
18
  @dataclass
@@ -25,63 +29,80 @@ class CircuitState:
25
  entries_paused: bool = False
26
 
27
 
28
- def load_circuit(conn) -> CircuitState:
29
- r = conn.execute("SELECT * FROM circuit_state WHERE id = 1").fetchone()
30
  if not r:
31
  return CircuitState()
32
  return CircuitState(r["fail_count"], r["tripped"], r["kill_switch"], r["kill_reason"], r["peak_wealth"], r["entries_paused"])
33
 
34
 
35
- def _event(conn, kind: str, detail: dict | None = None) -> None:
36
- conn.execute("INSERT INTO circuit_events (kind, detail) VALUES (%s, %s)", (kind, json.dumps(detail or {}, default=str)))
37
 
38
 
39
- def record_failure(conn, reason: str) -> CircuitState:
40
  r = conn.execute(
41
- "UPDATE circuit_state SET fail_count = fail_count + 1, last_failure_ts = now(), updated_at = now() WHERE id = 1 RETURNING fail_count, tripped"
42
  ).fetchone()
43
  tripped = r["tripped"]
44
  if r["fail_count"] >= config.CIRCUIT_THRESHOLD and not tripped:
45
- conn.execute("UPDATE circuit_state SET tripped = true, updated_at = now() WHERE id = 1")
46
- _event(conn, "tripped", {"reason": reason, "fail_count": r["fail_count"]})
47
- record_event("error", "rails", "circuit breaker tripped", {"reason": reason})
48
- return load_circuit(conn)
49
 
50
 
51
- def record_success(conn) -> None:
52
- conn.execute("UPDATE circuit_state SET fail_count = 0, updated_at = now() WHERE id = 1")
53
 
54
 
55
- def check_drawdown(conn, wealth: float, peak: float) -> bool:
56
- """Trip the kill switch when ``wealth`` has fallen KILL_SWITCH_DRAWDOWN below ``peak`` (the book's own peak: the
57
- trading engine passes the highest wealth mark of the current run). Returns whether the kill switch is on."""
58
- st = load_circuit(conn)
59
  if st.kill_switch:
60
  return True
61
- if peak > 0 and wealth <= peak * (1.0 - config.KILL_SWITCH_DRAWDOWN):
62
- conn.execute("UPDATE circuit_state SET kill_switch = true, kill_reason = %s, peak_wealth = %s, updated_at = now() WHERE id = 1",
63
- (f"wealth {wealth:.4f} SOL <= {1 - config.KILL_SWITCH_DRAWDOWN:.2f} x peak {peak:.4f} SOL", peak))
64
- _event(conn, "kill_switch", {"wealth": wealth, "peak": peak})
65
- record_event("error", "rails", "kill switch tripped: entries blocked", {"wealth": wealth, "peak": peak})
 
 
 
 
 
 
 
 
 
66
  return True
67
  return False
68
 
69
 
70
- def kill_rebase_at(conn, circuit_id: int = 1):
71
- """When the kill switch was last cleared: peaks are measured from marks after it, so clearing really re-bases.
72
- (Before this, the live book recomputed its peak from every wealth mark ever and re-tripped the next minute.)"""
73
- r = conn.execute("SELECT max(ts) AS ts FROM circuit_events WHERE kind = 'reset' AND (detail->>'kill')::boolean "
 
74
  "AND COALESCE((detail->>'circuit')::int, 1) = %s", (circuit_id,)).fetchone()
75
  return r["ts"] if r else None
76
 
77
 
78
- def check_book_drawdown(conn, book: str, wealth: float, peak: float) -> bool:
79
- """A paper race book's own kill switch: KILL_SWITCH_DRAWDOWN below the book's peak halts that book's entries only,
80
- so one book's drawdown never stops the other during the race. Returns whether the book is halted."""
 
 
 
 
 
 
81
  if book_halted(conn, book):
82
  return True
83
- if peak > 0 and wealth <= peak * (1.0 - config.KILL_SWITCH_DRAWDOWN):
84
- reason = f"wealth {wealth:.4f} SOL <= {1 - config.KILL_SWITCH_DRAWDOWN:.2f} x peak {peak:.4f} SOL"
 
85
  conn.execute("INSERT INTO book_state (book, halted, reason, peak, updated_at) VALUES (%s, true, %s, %s, now()) "
86
  "ON CONFLICT (book) DO UPDATE SET halted = true, reason = EXCLUDED.reason, peak = EXCLUDED.peak, updated_at = now()",
87
  (book, reason, peak))
@@ -103,28 +124,29 @@ def clear_book_halt(book: str) -> None:
103
  record_event("info", "rails", f"{book}: drawdown halt cleared")
104
 
105
 
106
- def notional_24h(conn) -> float:
107
- r = conn.execute("SELECT COALESCE(sum(sol), 0) AS s FROM notional_ledger WHERE ts > now() - interval '24 hours'").fetchone()
 
108
  return float(r["s"])
109
 
110
 
111
- def record_notional(conn, sol: float, kind: str = "entry") -> None:
112
- conn.execute("INSERT INTO notional_ledger (sol, kind) VALUES (%s, %s)", (sol, kind))
113
 
114
 
115
- def reset_circuit(kill: bool = False) -> None:
116
  with transaction() as conn:
117
- conn.execute("UPDATE circuit_state SET fail_count = 0, tripped = false, updated_at = now() WHERE id = 1")
118
- _event(conn, "reset", {"kill": kill})
119
  if kill:
120
- conn.execute("UPDATE circuit_state SET kill_switch = false, kill_reason = NULL, peak_wealth = NULL, updated_at = now() WHERE id = 1")
121
- record_event("info", "rails", "circuit reset" + (" incl. kill switch (peak re-based)" if kill else ""))
122
  print("circuit reset" + (" + kill switch cleared" if kill else ""))
123
 
124
 
125
- def set_entries_paused(paused: bool) -> None:
126
  with transaction() as conn:
127
- conn.execute("UPDATE circuit_state SET entries_paused = %s, updated_at = now() WHERE id = 1", (paused,))
128
- _event(conn, "paused" if paused else "resumed", {})
129
- record_event("info", "rails", "entries paused" if paused else "entries resumed")
130
  print("entries paused" if paused else "entries resumed")
 
1
  """Risk rails outside the model. KillSwitch (−KILL_SWITCH_DRAWDOWN from peak wealth → no entries), CircuitBreaker
2
  (CIRCUIT_THRESHOLD consecutive execution failures), the rolling 24 h notional ledger, and PauseEntries (console
3
  toggle). State lives in circuit_state / circuit_events / notional_ledger; the trading engine reads it before entries.
4
+ One circuit per trading type: ``circuit_id`` 1 is the memecoin live book (the default everywhere), 2 the Kalshi live
5
+ books (``KALSHI_CIRCUIT``, fly_trader/kalshi), 3 the Robinhood Chain memecoin live book (``RH_CIRCUIT``,
6
+ fly_trader/rh); ``check_drawdown`` takes the drawdown limit of the caller's type.
7
  """
8
  from __future__ import annotations
9
 
 
16
  from ..db.connection import transaction
17
 
18
  log = logging.getLogger(__name__)
19
+ SOLANA_CIRCUIT, KALSHI_CIRCUIT, RH_CIRCUIT = 1, 2, 3
20
 
21
 
22
  @dataclass
 
29
  entries_paused: bool = False
30
 
31
 
32
+ def load_circuit(conn, circuit_id: int = SOLANA_CIRCUIT) -> CircuitState:
33
+ r = conn.execute("SELECT * FROM circuit_state WHERE id = %s", (circuit_id,)).fetchone()
34
  if not r:
35
  return CircuitState()
36
  return CircuitState(r["fail_count"], r["tripped"], r["kill_switch"], r["kill_reason"], r["peak_wealth"], r["entries_paused"])
37
 
38
 
39
+ def _event(conn, kind: str, detail: dict | None = None, circuit_id: int = SOLANA_CIRCUIT) -> None:
40
+ conn.execute("INSERT INTO circuit_events (kind, detail) VALUES (%s, %s)", (kind, json.dumps({**(detail or {}), "circuit": circuit_id}, default=str)))
41
 
42
 
43
+ def record_failure(conn, reason: str, circuit_id: int = SOLANA_CIRCUIT) -> CircuitState:
44
  r = conn.execute(
45
+ "UPDATE circuit_state SET fail_count = fail_count + 1, last_failure_ts = now(), updated_at = now() WHERE id = %s RETURNING fail_count, tripped", (circuit_id,)
46
  ).fetchone()
47
  tripped = r["tripped"]
48
  if r["fail_count"] >= config.CIRCUIT_THRESHOLD and not tripped:
49
+ conn.execute("UPDATE circuit_state SET tripped = true, updated_at = now() WHERE id = %s", (circuit_id,))
50
+ _event(conn, "tripped", {"reason": reason, "fail_count": r["fail_count"]}, circuit_id)
51
+ record_event("error", "rails", "circuit breaker tripped", {"reason": reason, "circuit": circuit_id})
52
+ return load_circuit(conn, circuit_id)
53
 
54
 
55
+ def record_success(conn, circuit_id: int = SOLANA_CIRCUIT) -> None:
56
+ conn.execute("UPDATE circuit_state SET fail_count = 0, updated_at = now() WHERE id = %s", (circuit_id,))
57
 
58
 
59
+ def check_drawdown(conn, wealth: float, peak: float, circuit_id: int = SOLANA_CIRCUIT, drawdown: float | None = None, unit: str = "SOL") -> bool:
60
+ """Trip the kill switch when ``wealth`` has fallen ``drawdown`` (default KILL_SWITCH_DRAWDOWN) below ``peak`` (the book's
61
+ own peak: the trading engine passes the highest wealth mark of the current run). Returns whether the kill switch is on."""
62
+ st = load_circuit(conn, circuit_id)
63
  if st.kill_switch:
64
  return True
65
+ dd = config.KILL_SWITCH_DRAWDOWN if drawdown is None else float(drawdown)
66
+ if peak > 0 and wealth <= peak * (1.0 - dd):
67
+ conn.execute("UPDATE circuit_state SET kill_switch = true, kill_reason = %s, peak_wealth = %s, updated_at = now() WHERE id = %s",
68
+ (f"wealth {wealth:.4f} {unit} <= {1 - dd:.2f} x peak {peak:.4f} {unit}", peak, circuit_id))
69
+ _event(conn, "kill_switch", {"wealth": wealth, "peak": peak}, circuit_id)
70
+ record_event("error", "rails", "kill switch tripped: entries blocked", {"wealth": wealth, "peak": peak, "circuit": circuit_id})
71
+ if config.VAULT_ENABLED: # the hosted fly: tell the operator at once
72
+ try:
73
+ from ..vault import alerts
74
+ alerts.send(f"KILL SWITCH tripped: {wealth:.4f} {unit} <= {1 - dd:.2f} x peak {peak:.4f} {unit}. Entries blocked"
75
+ + (", open positions are being liquidated" if config.KILL_SWITCH_LIQUIDATE else "")
76
+ + ". Clear it with `fly-trader reset-circuit --kill` once you know why.")
77
+ except Exception:
78
+ pass
79
  return True
80
  return False
81
 
82
 
83
+ def kill_rebase_at(conn, circuit_id: int = SOLANA_CIRCUIT):
84
+ """When the kill switch was last cleared, or money last withdrawn from the live wallet: peaks are measured from marks
85
+ after it, so clearing really re-bases and a withdrawal is never a drawdown. (Before this, the live book recomputed its
86
+ peak from every wealth mark ever and re-tripped the next minute.)"""
87
+ r = conn.execute("SELECT max(ts) AS ts FROM circuit_events WHERE ((kind = 'reset' AND (detail->>'kill')::boolean) OR kind = 'withdrawal') "
88
  "AND COALESCE((detail->>'circuit')::int, 1) = %s", (circuit_id,)).fetchone()
89
  return r["ts"] if r else None
90
 
91
 
92
+ def record_withdrawal(conn, detail: dict, circuit_id: int = SOLANA_CIRCUIT) -> None:
93
+ """The operator took money out of a live wallet (chain/withdraw.py, rh/withdraw.py): ``kill_rebase_at`` re-bases the
94
+ live book's peak at it."""
95
+ _event(conn, "withdrawal", detail, circuit_id)
96
+
97
+
98
+ def check_book_drawdown(conn, book: str, wealth: float, peak: float, drawdown: float | None = None, unit: str = "SOL") -> bool:
99
+ """A paper race book's own kill switch: ``drawdown`` (default KILL_SWITCH_DRAWDOWN) below the book's peak halts that
100
+ book's entries only, so one book's drawdown never stops the other during the race. Returns whether the book is halted."""
101
  if book_halted(conn, book):
102
  return True
103
+ dd = config.KILL_SWITCH_DRAWDOWN if drawdown is None else float(drawdown)
104
+ if peak > 0 and wealth <= peak * (1.0 - dd):
105
+ reason = f"wealth {wealth:.4f} {unit} <= {1 - dd:.2f} x peak {peak:.4f} {unit}"
106
  conn.execute("INSERT INTO book_state (book, halted, reason, peak, updated_at) VALUES (%s, true, %s, %s, now()) "
107
  "ON CONFLICT (book) DO UPDATE SET halted = true, reason = EXCLUDED.reason, peak = EXCLUDED.peak, updated_at = now()",
108
  (book, reason, peak))
 
124
  record_event("info", "rails", f"{book}: drawdown halt cleared")
125
 
126
 
127
+ def notional_24h(conn, market: str = "sol") -> float:
128
+ """The last 24 h of entries in ``market``'s unit (``sol`` holds ETH on the ``rh`` market's rows)."""
129
+ r = conn.execute("SELECT COALESCE(sum(sol), 0) AS s FROM notional_ledger WHERE ts > now() - interval '24 hours' AND market = %s", (market,)).fetchone()
130
  return float(r["s"])
131
 
132
 
133
+ def record_notional(conn, sol: float, kind: str = "entry", market: str = "sol") -> None:
134
+ conn.execute("INSERT INTO notional_ledger (sol, kind, market) VALUES (%s, %s, %s)", (sol, kind, market))
135
 
136
 
137
+ def reset_circuit(kill: bool = False, circuit_id: int = SOLANA_CIRCUIT) -> None:
138
  with transaction() as conn:
139
+ conn.execute("UPDATE circuit_state SET fail_count = 0, tripped = false, updated_at = now() WHERE id = %s", (circuit_id,))
140
+ _event(conn, "reset", {"kill": kill}, circuit_id)
141
  if kill:
142
+ conn.execute("UPDATE circuit_state SET kill_switch = false, kill_reason = NULL, peak_wealth = NULL, updated_at = now() WHERE id = %s", (circuit_id,))
143
+ record_event("info", "rails", "circuit reset" + (" incl. kill switch (peak re-based)" if kill else ""), {"circuit": circuit_id})
144
  print("circuit reset" + (" + kill switch cleared" if kill else ""))
145
 
146
 
147
+ def set_entries_paused(paused: bool, circuit_id: int = SOLANA_CIRCUIT) -> None:
148
  with transaction() as conn:
149
+ conn.execute("UPDATE circuit_state SET entries_paused = %s, updated_at = now() WHERE id = %s", (paused, circuit_id))
150
+ _event(conn, "paused" if paused else "resumed", {}, circuit_id)
151
+ record_event("info", "rails", "entries paused" if paused else "entries resumed", {"circuit": circuit_id})
152
  print("entries paused" if paused else "entries resumed")
fly_trader/agent/selector_session.py CHANGED
@@ -50,23 +50,53 @@ def pinned_snapshot() -> int | None:
50
  return int(v) if v is not None else None
51
 
52
 
53
- class SelectorBook:
54
- name = "selector"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
55
 
56
- def __init__(self):
 
 
57
  from ..train import selector as sel
 
 
 
58
  pin = pinned_snapshot()
59
  self.model = sel.load_snapshot(pin) if pin is not None else sel.load_latest()
60
  if self.model is None:
61
  raise NoModel("no model has qualified to trade yet (trained on the current data, with a backtest that made money after costs and beat random picks)")
 
 
 
62
  self.idx = _columns(self.model)
63
  self.horizon_s = self.model.horizon_min * 60
64
- self.broker = PaperBroker(BOOK); self.done = False
65
  self.run_id = str(uuid.uuid4()); self.beat_no = 0
66
  with transaction() as conn:
67
  self.snapshot_id = pin if pin is not None else sel.latest_current(conn)["id"] # the snapshot actually loaded
68
  conn.execute("INSERT INTO runs (run_id, kind, config, brain_snapshot_id, status) VALUES (%s,%s,%s,%s,'running')",
69
- (self.run_id, KIND, json.dumps({"threshold": self.model.threshold, "horizon_min": self.model.horizon_min, "book": BOOK}, default=str), self.snapshot_id))
 
70
  record_event("info", "selector", "selector session started", {"run_id": self.run_id, "snapshot": self.snapshot_id, "threshold": self.model.threshold, "horizon_min": self.model.horizon_min})
71
  log.info("selector session: snapshot %d threshold %.4f horizon %d min", self.snapshot_id, self.model.threshold, self.model.horizon_min)
72
 
@@ -74,8 +104,7 @@ class SelectorBook:
74
  """Switch to a newer deployable model without a restart (the paper book and open positions carry on). While a
75
  snapshot is pinned (``ui_settings['pinned_selector_snapshot']``) the book stays on it, so a forward test measures
76
  one model instead of a blend of every retrain that lands during it."""
77
- import joblib
78
- from ..train import selector as sel
79
  pin = pinned_snapshot()
80
  with transaction() as conn:
81
  if pin is not None: # a pin freezes the book, but a NEW pin (a release) switches to it
@@ -86,7 +115,7 @@ class SelectorBook:
86
  r = sel.latest_current(conn)
87
  if not r or r["id"] == self.snapshot_id:
88
  return False
89
- m = joblib.load(r["path"])
90
  try:
91
  idx = _columns(m)
92
  except RuntimeError as e:
@@ -95,7 +124,7 @@ class SelectorBook:
95
  old = self.snapshot_id; self.model, self.idx, self.snapshot_id, self.horizon_s = m, idx, r["id"], m.horizon_min * 60
96
  with transaction() as conn:
97
  conn.execute("UPDATE runs SET brain_snapshot_id = %s, config = %s WHERE run_id = %s",
98
- (r["id"], json.dumps({"threshold": m.threshold, "horizon_min": m.horizon_min, "book": BOOK}, default=str), self.run_id))
99
  record_event("info", "selector", f"switched to model #{r['id']}", {"from": old, "to": r["id"], "threshold": m.threshold})
100
  log.info("switched from model #%s to #%d (threshold %.4f)", old, r["id"], m.threshold)
101
  return True
@@ -104,7 +133,7 @@ class SelectorBook:
104
  pass
105
 
106
  def open_mints(self, conn) -> set[str]:
107
- return {p["mint"] for p in ledger.open_positions(conn, BOOK)}
108
 
109
  def _decide(self, ctx) -> dict | None:
110
  """The strategy stack's per-row decision (train/strategies.decide), with the live fail-closed rules: rows whose
@@ -113,15 +142,7 @@ class SelectorBook:
113
  if not (getattr(self.model, "stack", None) or {}).get("strategies") or not len(ctx.mints):
114
  return None
115
  d = self.model.decide(ctx.X, X_COLS, ctx.t_start)
116
- groups = set((getattr(self.model, "metrics", None) or {}).get("groups") or [])
117
- for k, i in enumerate(ctx.infos):
118
- if not d["allow"][k]:
119
- continue
120
- if "skill" in groups and i.get("skill_missing"):
121
- d["allow"][k] = False; d["reason"][k] = "fail closed: no wallet-skill table for today"
122
- elif "rug" in groups and i.get("age_h") is not None and not i.get("curve_known"):
123
- d["allow"][k] = False; d["reason"][k] = "fail closed: curve facts not known yet"
124
- return d
125
 
126
  def on_minute(self, ctx) -> dict:
127
  conn = ctx.conn
@@ -141,10 +162,10 @@ class SelectorBook:
141
  status = {**summary, "stage": "stream stale: holding (no entries or exits)", "updated_at": datetime.now(timezone.utc).isoformat()}
142
  self._write_status(conn, status)
143
  return status
144
- handed = handover.state(conn) is not None and ctx.engine.has_book("fly")
145
  self.beat_no += 1
146
  extra = {} if d is None else {"holds": d["hold_s"], "strategies": d["strategy"], "allow": d["allow"], "reasons": d["reason"], "tables": d["tables"]}
147
- st = paper_trading.trade_minute(ctx, book=BOOK, run_id=self.run_id, beat_no=self.beat_no, broker=self.broker, kind=KIND, mints=ctx.mints, infos=ctx.infos,
148
  scores=scores, threshold=thr, table=getattr(self.model, "sizing", None), horizon_s=self.horizon_s,
149
  block="handed over to the fly" if handed else None, **extra)
150
  if handed and st["open"] == 0:
@@ -155,10 +176,9 @@ class SelectorBook:
155
  self._write_status(conn, status)
156
  return status
157
 
158
- @staticmethod
159
- def _write_status(conn, status: dict) -> None:
160
- conn.execute("INSERT INTO ui_settings (key, value) VALUES ('selector_status', %s) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()",
161
- (json.dumps(status, default=str),))
162
 
163
  def finish(self) -> None:
164
  with transaction() as conn:
 
50
  return int(v) if v is not None else None
51
 
52
 
53
+ def chain_deployable(model, chain: str) -> tuple[bool, str]:
54
+ """Whether the selector may trade ``chain``: Solana by its own deployability (as before); another chain only when the
55
+ combined stack's holdout on that chain's rows showed an edge (train/strategies.score_holdout_by_chain)."""
56
+ if chain == "sol":
57
+ return True, "ok"
58
+ c = ((getattr(model, "metrics", None) or {}).get("chains") or {}).get(chain)
59
+ if not c:
60
+ return False, f"the selector was not judged on {chain} rows"
61
+ return bool(c.get("deployable")), f"{chain} holdout: {c.get('n', 0)} trades, mean {(c.get('mean') or 0) * 100:+.2f}% (random {(c.get('random_mean') or 0) * 100:+.2f}%)"
62
+
63
+
64
+ def fail_closed(groups: set, infos: list, d: dict) -> dict:
65
+ """The selector's live fail-closed rules on a ``strategies.decide`` result (in place): rows whose inputs training had
66
+ but live lacks are blocked. Shared by the fly's teacher gate (agent/fly_session.py), so both books obey the same rules."""
67
+ for k, i in enumerate(infos):
68
+ if not d["allow"][k]:
69
+ continue
70
+ if "skill" in groups and i.get("skill_missing"):
71
+ d["allow"][k] = False; d["reason"][k] = "fail closed: no wallet-skill table for today"
72
+ elif "rug" in groups and i.get("age_h") is not None and not i.get("curve_known"):
73
+ d["allow"][k] = False; d["reason"][k] = "fail closed: curve facts not known yet"
74
+ return d
75
+
76
 
77
+ class SelectorBook:
78
+ def __init__(self, chain: str = "sol"):
79
+ from .. import markets
80
  from ..train import selector as sel
81
+ self.chain = chain; self.market = markets.for_chain(chain)
82
+ self.name = "selector" if chain == "sol" else f"selector_{chain}"
83
+ self.BOOK = self.market.selector_book; self.KIND = self.name; self.status_key = self.market.selector_status_key
84
  pin = pinned_snapshot()
85
  self.model = sel.load_snapshot(pin) if pin is not None else sel.load_latest()
86
  if self.model is None:
87
  raise NoModel("no model has qualified to trade yet (trained on the current data, with a backtest that made money after costs and beat random picks)")
88
+ ok, why = chain_deployable(self.model, chain)
89
+ if not ok:
90
+ raise NoModel(f"the selector may not trade {self.market.name}: {why}")
91
  self.idx = _columns(self.model)
92
  self.horizon_s = self.model.horizon_min * 60
93
+ self.broker = PaperBroker(self.BOOK); self.done = False
94
  self.run_id = str(uuid.uuid4()); self.beat_no = 0
95
  with transaction() as conn:
96
  self.snapshot_id = pin if pin is not None else sel.latest_current(conn)["id"] # the snapshot actually loaded
97
  conn.execute("INSERT INTO runs (run_id, kind, config, brain_snapshot_id, status) VALUES (%s,%s,%s,%s,'running')",
98
+ (self.run_id, self.KIND, json.dumps({"threshold": self.model.threshold, "horizon_min": self.model.horizon_min, "book": self.BOOK, "chain": chain},
99
+ default=str), self.snapshot_id))
100
  record_event("info", "selector", "selector session started", {"run_id": self.run_id, "snapshot": self.snapshot_id, "threshold": self.model.threshold, "horizon_min": self.model.horizon_min})
101
  log.info("selector session: snapshot %d threshold %.4f horizon %d min", self.snapshot_id, self.model.threshold, self.model.horizon_min)
102
 
 
104
  """Switch to a newer deployable model without a restart (the paper book and open positions carry on). While a
105
  snapshot is pinned (``ui_settings['pinned_selector_snapshot']``) the book stays on it, so a forward test measures
106
  one model instead of a blend of every retrain that lands during it."""
107
+ from ..train import model_io, selector as sel
 
108
  pin = pinned_snapshot()
109
  with transaction() as conn:
110
  if pin is not None: # a pin freezes the book, but a NEW pin (a release) switches to it
 
115
  r = sel.latest_current(conn)
116
  if not r or r["id"] == self.snapshot_id:
117
  return False
118
+ m = model_io.load_selector(r["path"])
119
  try:
120
  idx = _columns(m)
121
  except RuntimeError as e:
 
124
  old = self.snapshot_id; self.model, self.idx, self.snapshot_id, self.horizon_s = m, idx, r["id"], m.horizon_min * 60
125
  with transaction() as conn:
126
  conn.execute("UPDATE runs SET brain_snapshot_id = %s, config = %s WHERE run_id = %s",
127
+ (r["id"], json.dumps({"threshold": m.threshold, "horizon_min": m.horizon_min, "book": self.BOOK}, default=str), self.run_id))
128
  record_event("info", "selector", f"switched to model #{r['id']}", {"from": old, "to": r["id"], "threshold": m.threshold})
129
  log.info("switched from model #%s to #%d (threshold %.4f)", old, r["id"], m.threshold)
130
  return True
 
133
  pass
134
 
135
  def open_mints(self, conn) -> set[str]:
136
+ return {p["mint"] for p in ledger.open_positions(conn, self.BOOK)}
137
 
138
  def _decide(self, ctx) -> dict | None:
139
  """The strategy stack's per-row decision (train/strategies.decide), with the live fail-closed rules: rows whose
 
142
  if not (getattr(self.model, "stack", None) or {}).get("strategies") or not len(ctx.mints):
143
  return None
144
  d = self.model.decide(ctx.X, X_COLS, ctx.t_start)
145
+ return fail_closed(set((getattr(self.model, "metrics", None) or {}).get("groups") or []), ctx.infos, d)
 
 
 
 
 
 
 
 
146
 
147
  def on_minute(self, ctx) -> dict:
148
  conn = ctx.conn
 
162
  status = {**summary, "stage": "stream stale: holding (no entries or exits)", "updated_at": datetime.now(timezone.utc).isoformat()}
163
  self._write_status(conn, status)
164
  return status
165
+ handed = handover.state(conn, self.chain) is not None and ctx.engine.has_book("fly" if self.chain == "sol" else f"fly_{self.chain}")
166
  self.beat_no += 1
167
  extra = {} if d is None else {"holds": d["hold_s"], "strategies": d["strategy"], "allow": d["allow"], "reasons": d["reason"], "tables": d["tables"]}
168
+ st = paper_trading.trade_minute(ctx, book=self.BOOK, run_id=self.run_id, beat_no=self.beat_no, broker=self.broker, kind=self.KIND, mints=ctx.mints, infos=ctx.infos,
169
  scores=scores, threshold=thr, table=getattr(self.model, "sizing", None), horizon_s=self.horizon_s,
170
  block="handed over to the fly" if handed else None, **extra)
171
  if handed and st["open"] == 0:
 
176
  self._write_status(conn, status)
177
  return status
178
 
179
+ def _write_status(self, conn, status: dict) -> None:
180
+ conn.execute("INSERT INTO ui_settings (key, value) VALUES (%s, %s) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()",
181
+ (self.status_key, json.dumps({**status, "chain": self.chain}, default=str)))
 
182
 
183
  def finish(self) -> None:
184
  with transaction() as conn:
fly_trader/agent/sizing.py CHANGED
@@ -4,16 +4,20 @@ Certainty is measured, not assumed. During training the backtest's out-of-sample
4
  score cleared the buy line (``build_table``: equal-count bands of ``score − threshold``). For each band the growth-optimal
5
  (Kelly) fraction is found on the band's actual net returns — the bet f that maximises the average of log(1 + f·r), so the
6
  rare −100 % rugs count in full; a band with no edge gets 0. Live (``size_position``): the current score's band →
7
- ``KELLY_FRACTION`` of that fraction of the deployable bankroll (wealth at cost minus the gas reserve), capped at
8
- ``MAX_POSITION_FRACTION`` of it, ``MAX_POOL_SHARE`` of the pool's quote reserve (bigger buys pay more price impact than
9
- the 0.1 SOL the backtest measured) and the free cash above the reserve; below ``MIN_POSITION_SOL`` the buy is skipped.
10
- A model without a table (trained before sizing) trades the fixed ``MAX_POSITION_SOL``.
 
 
 
11
  """
12
  from __future__ import annotations
13
 
14
  import numpy as np
15
 
16
  from .. import config
 
17
 
18
  N_BANDS = 4
19
  MIN_BAND_TRADES = 50
@@ -57,26 +61,31 @@ def band_for(table: list[dict], margin: float) -> dict | None:
57
 
58
 
59
  def size_position(score: float, threshold: float, table: list[dict] | None, bankroll_sol: float, cash_sol: float,
60
- res_quote_sol: float | None) -> tuple[float, str]:
61
- """(size in SOL, why). Size 0 means skip the buy."""
62
- reserve = config.GAS_RESERVE_SOL; free = cash_sol - reserve
63
- if not table:
64
- size = config.MAX_POSITION_SOL
65
- return (size, f"fixed {size:g} SOL (model has no sizing table)") if free >= size else (0.0, "not enough free cash")
66
- b = band_for(table, score - threshold)
67
- if b is None or b["kelly"] <= 0:
68
- return 0.0, "no edge in this score band"
69
  deployable = max(0.0, bankroll_sol - reserve)
70
- size = config.KELLY_FRACTION * b["kelly"] * deployable
71
- caps = {"bankroll cap": config.MAX_POSITION_FRACTION * deployable, "free cash": free}
72
- if res_quote_sol:
73
- caps["pool depth"] = config.MAX_POOL_SHARE * res_quote_sol
 
 
 
 
 
 
 
 
74
  limit = min(caps, key=caps.get)
75
- why = f"band from +{b['lo']:.3f} over the line: Kelly {b['kelly']:.2f} × {config.KELLY_FRACTION:g} of {deployable:.2f} SOL"
76
  if caps[limit] < size:
77
  size = caps[limit]; why += f", capped by {limit}"
78
- if size < config.MIN_POSITION_SOL:
79
- return 0.0, why + f" → {size:.3f} SOL is below the {config.MIN_POSITION_SOL:g} SOL minimum"
80
  return float(size), why + f" → {size:.3f} SOL"
81
 
82
 
 
4
  score cleared the buy line (``build_table``: equal-count bands of ``score − threshold``). For each band the growth-optimal
5
  (Kelly) fraction is found on the band's actual net returns — the bet f that maximises the average of log(1 + f·r), so the
6
  rare −100 % rugs count in full; a band with no edge gets 0. Live (``size_position``): the current score's band →
7
+ ``KELLY_FRACTION`` of that fraction of the deployable bankroll (wealth at cost minus the gas reserve); ``flat`` (the
8
+ fly, ``config.FLY_SIZING``): ``MAX_POSITION_FRACTION`` of it on every buy that cleared the line, no bands. Either way
9
+ capped at ``MAX_POSITION_FRACTION`` of the deployable bankroll, the free cash above the reserve, and the label size
10
+ (``market/exit_cost.label_size``: ``LABEL_SIZE_SOL`` or ``MAX_POOL_SHARE`` of the pool) -- the size every edge was
11
+ measured at, so the book behaves the same with 5 SOL or 500: a bigger bankroll buys no bigger than the backtest did,
12
+ it only runs out of cash later. Below ``MIN_POSITION_SOL`` the buy is skipped. A model without a table (trained
13
+ before sizing) trades the fixed ``MAX_POSITION_SOL``, under the same caps.
14
  """
15
  from __future__ import annotations
16
 
17
  import numpy as np
18
 
19
  from .. import config
20
+ from ..market.exit_cost import label_size
21
 
22
  N_BANDS = 4
23
  MIN_BAND_TRADES = 50
 
61
 
62
 
63
  def size_position(score: float, threshold: float, table: list[dict] | None, bankroll_sol: float, cash_sol: float,
64
+ res_quote_sol: float | None, flat: bool = False, market=None) -> tuple[float, str]:
65
+ """(size in the book's unit, why). Size 0 means skip the buy. ``flat``: ``MAX_POSITION_FRACTION`` of the deployable
66
+ bankroll on every buy that cleared the line (the fly); otherwise the Kelly band of ``table``. ``market``
67
+ (fly_trader/markets; default Solana): the gas reserve, minimum position and label size of the book's own market."""
68
+ rh = market is not None and market.chain != "sol"
69
+ reserve = market.gas_reserve() if rh else config.GAS_RESERVE_SOL; free = cash_sol - reserve
70
+ min_pos = market.min_position() if rh else config.MIN_POSITION_SOL
 
 
71
  deployable = max(0.0, bankroll_sol - reserve)
72
+ if flat:
73
+ size = config.MAX_POSITION_FRACTION * deployable; why = f"flat {config.MAX_POSITION_FRACTION:g} of {deployable:.2f} SOL"
74
+ elif not table:
75
+ size = config.MAX_POSITION_SOL * (market.k() if rh else 1.0); why = f"fixed {size:g} {'ETH' if rh else 'SOL'} (model has no sizing table)"
76
+ else:
77
+ b = band_for(table, score - threshold)
78
+ if b is None or b["kelly"] <= 0:
79
+ return 0.0, "no edge in this score band"
80
+ size = config.KELLY_FRACTION * b["kelly"] * deployable
81
+ why = f"band from +{b['lo']:.3f} over the line: Kelly {b['kelly']:.2f} × {config.KELLY_FRACTION:g} of {deployable:.2f} SOL"
82
+ # the label size is the largest buy any backtest measured: past it the edge is an extrapolation that impact eats
83
+ caps = {"bankroll cap": config.MAX_POSITION_FRACTION * deployable, "free cash": free, "label size": label_size(res_quote_sol, market if rh else None)}
84
  limit = min(caps, key=caps.get)
 
85
  if caps[limit] < size:
86
  size = caps[limit]; why += f", capped by {limit}"
87
+ if size < min_pos:
88
+ return 0.0, why + f" → {max(size, 0.0):.6f} is below the {min_pos:g} minimum"
89
  return float(size), why + f" → {size:.3f} SOL"
90
 
91
 
fly_trader/brain/connectome.py CHANGED
@@ -115,6 +115,9 @@ class Connectome:
115
  self.pop_ranges, self.pathway_gains)
116
  self.W_KM0 = torch.from_numpy(np.ascontiguousarray(z["W_KM0"])).to(torch.float32)
117
  self.M_KM = torch.from_numpy(np.ascontiguousarray(z["M_KM"])).to(torch.bool)
 
 
 
118
  self.spectral_radius_raw = float(z["spectral_radius_raw"])
119
  self.content_sha256 = str(z["content_sha256"])
120
  self.s = float(scale["s"]) if scale and "s" in scale else float(z["s0"])
@@ -159,3 +162,27 @@ class SubConnectome:
159
  self.pathway_gains = getattr(c, "pathway_gains", {})
160
  self.excluded = exclude
161
  self.keep = keep
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
115
  self.pop_ranges, self.pathway_gains)
116
  self.W_KM0 = torch.from_numpy(np.ascontiguousarray(z["W_KM0"])).to(torch.float32)
117
  self.M_KM = torch.from_numpy(np.ascontiguousarray(z["M_KM"])).to(torch.bool)
118
+ # FlyBrain's finer annotation per neuron (VIS_R1R6, VIS_ME, LPTC, ...): the Kalshi fly finds its photoreceptors by it
119
+ self.flybrain_group = np.asarray(z["flybrain_group"]) if "flybrain_group" in z.files else None
120
+ self.flybrain_group_names = [str(x) for x in z["flybrain_group_names"]] if "flybrain_group_names" in z.files else []
121
  self.spectral_radius_raw = float(z["spectral_radius_raw"])
122
  self.content_sha256 = str(z["content_sha256"])
123
  self.s = float(scale["s"]) if scale and "s" in scale else float(z["s0"])
 
162
  self.pathway_gains = getattr(c, "pathway_gains", {})
163
  self.excluded = exclude
164
  self.keep = keep
165
+ fg = getattr(c, "flybrain_group", None)
166
+ self.flybrain_group = fg[keep.numpy()] if fg is not None else None
167
+ self.flybrain_group_names = list(getattr(c, "flybrain_group_names", []))
168
+
169
+
170
+ # ---------------------------------------------------------------- the Kalshi fly's graph: the optic lobes kept, the memecoin senses dropped
171
+ VISUAL_SUB = tuple(AFFERENT_POPS) # exclude these and VISUAL stays: the second fly's sub-connectome
172
+ PHOTORECEPTOR_GROUPS = ("VIS_R1R6", "VIS_R7R8") # FlyBrain groups of the retina's photoreceptors (R7R8 is absent in the v783 join)
173
+
174
+
175
+ def photoreceptor_rows(graph) -> torch.Tensor:
176
+ """Rows of ``graph`` (a Connectome or SubConnectome) that are photoreceptors: VISUAL neurons whose FlyBrain group is
177
+ R1–R6 or R7–R8. The Kalshi fly's features enter here and travel the optic lobes into the central brain."""
178
+ if "VISUAL" not in graph.pop_ranges:
179
+ raise ValueError("this graph has no VISUAL population: the Kalshi fly needs the optic lobes")
180
+ fg = getattr(graph, "flybrain_group", None); names = list(getattr(graph, "flybrain_group_names", []) or [])
181
+ lo, hi = graph.pop_ranges["VISUAL"]
182
+ if fg is None or not names:
183
+ raise ValueError("the connectome artifact carries no flybrain_group annotation; rebuild it (brain/build.py)")
184
+ want = [i for i, n in enumerate(names) if n in PHOTORECEPTOR_GROUPS]
185
+ rows = np.flatnonzero(np.isin(fg[lo:hi], want)) + lo
186
+ if not len(rows):
187
+ raise ValueError(f"no photoreceptor rows: VISUAL groups present are {sorted(set(names[int(g)] for g in set(fg[lo:hi].tolist()) if int(g) < len(names)))}")
188
+ return torch.from_numpy(rows.astype(np.int64))
fly_trader/brain/geometry.py CHANGED
@@ -25,6 +25,8 @@ EXCLUDE = ("VISUAL",)
25
  POS_COLS = ["pos_x", "pos_y", "pos_z"]
26
  SOMA_COLS = ["soma_x", "soma_y", "soma_z"]
27
  BIN, META = "geometry.bin", "geometry.json"
 
 
28
 
29
 
30
  def keep_mask(pop_ranges: dict, n: int, exclude: tuple[str, ...] = EXCLUDE) -> np.ndarray:
@@ -60,10 +62,11 @@ def read_meta(out_dir: Path) -> dict | None:
60
 
61
 
62
  def build_geometry(out_dir: Path, connectome_path: str | Path | None = None, annotations: str | Path | None = None,
63
- exclude: tuple[str, ...] = EXCLUDE) -> dict:
64
  """Write (once per connectome) and return the geometry metadata. Soma centroid where known, else the neuron's
65
- annotation point, else its population's centroid (a handful of neurons); scaled to micrometres and centred."""
66
- out_dir = Path(out_dir); jpath, bpath = out_dir / META, out_dir / BIN
 
67
  path = Path(connectome_path) if connectome_path else current_connectome_path()
68
  with np.load(path, allow_pickle=False) as z:
69
  sha = str(z["content_sha256"]); n = int(z["N"]); pop_ranges = {k: tuple(v) for k, v in json.loads(str(z["pop_ranges"])).items()}
@@ -97,3 +100,39 @@ def build_geometry(out_dir: Path, connectome_path: str | Path | None = None, ann
97
  out_dir.mkdir(parents=True, exist_ok=True)
98
  _atomic(bpath, np.ascontiguousarray(xyz).tobytes()); _atomic(jpath, json.dumps(meta).encode())
99
  return meta
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
25
  POS_COLS = ["pos_x", "pos_y", "pos_z"]
26
  SOMA_COLS = ["soma_x", "soma_y", "soma_z"]
27
  BIN, META = "geometry.bin", "geometry.json"
28
+ FULL_BIN, FULL_META = "geometry_full.bin", "geometry_full.json" # every neuron: the scene both flies are drawn into (ui/brain3d)
29
+ FLY_EXCLUDES = {"memecoin": EXCLUDE, "kalshi": None} # None: brain/connectome.VISUAL_SUB (the memecoin afferents), resolved lazily
30
 
31
 
32
  def keep_mask(pop_ranges: dict, n: int, exclude: tuple[str, ...] = EXCLUDE) -> np.ndarray:
 
62
 
63
 
64
  def build_geometry(out_dir: Path, connectome_path: str | Path | None = None, annotations: str | Path | None = None,
65
+ exclude: tuple[str, ...] = EXCLUDE, names: tuple[str, str] = (BIN, META)) -> dict:
66
  """Write (once per connectome) and return the geometry metadata. Soma centroid where known, else the neuron's
67
+ annotation point, else its population's centroid (a handful of neurons); scaled to micrometres and centred.
68
+ ``exclude=()`` with ``names=(FULL_BIN, FULL_META)`` is the whole brain (``build_full_geometry``)."""
69
+ out_dir = Path(out_dir); bpath, jpath = out_dir / names[0], out_dir / names[1]
70
  path = Path(connectome_path) if connectome_path else current_connectome_path()
71
  with np.load(path, allow_pickle=False) as z:
72
  sha = str(z["content_sha256"]); n = int(z["N"]); pop_ranges = {k: tuple(v) for k, v in json.loads(str(z["pop_ranges"])).items()}
 
100
  out_dir.mkdir(parents=True, exist_ok=True)
101
  _atomic(bpath, np.ascontiguousarray(xyz).tobytes()); _atomic(jpath, json.dumps(meta).encode())
102
  return meta
103
+
104
+
105
+ def fly_index(pop_ranges: dict, n: int, exclude: tuple[str, ...]) -> np.ndarray:
106
+ """int32 map from a fly's sub-graph index to the full-graph index (the rows ``SubConnectome`` keeps, in order)."""
107
+ return np.flatnonzero(keep_mask(pop_ranges, n, exclude)).astype(np.int32)
108
+
109
+
110
+ def build_full_geometry(out_dir: Path, connectome_path: str | Path | None = None, annotations: str | Path | None = None) -> dict:
111
+ """The whole brain's geometry plus, per fly, the index map from its sub-graph into it (``index_<fly>.bin``): the
112
+ scene the console draws both flies into. ``geometry_full.json`` carries ``flies`` = {name: {"exclude", "n", "index"}}."""
113
+ from .connectome import VISUAL_SUB
114
+ out_dir = Path(out_dir); jpath = out_dir / FULL_META
115
+ meta = read_meta_named(out_dir, FULL_META)
116
+ path = Path(connectome_path) if connectome_path else current_connectome_path()
117
+ with np.load(path, allow_pickle=False) as z:
118
+ sha = str(z["content_sha256"]); n = int(z["N"]); pop_ranges = {k: tuple(v) for k, v in json.loads(str(z["pop_ranges"])).items()}
119
+ if meta and meta.get("connectome_sha256") == sha and meta.get("version") == VERSION and meta.get("flies") and (out_dir / FULL_BIN).exists() \
120
+ and all((out_dir / f["index"]).exists() for f in meta["flies"].values()):
121
+ return meta
122
+ meta = build_geometry(out_dir, path, annotations, exclude=(), names=(FULL_BIN, FULL_META))
123
+ flies = {}
124
+ for name, exc in FLY_EXCLUDES.items():
125
+ exc = tuple(exc) if exc is not None else tuple(VISUAL_SUB)
126
+ idx = fly_index(pop_ranges, n, exc); fname = f"index_{name}.bin"
127
+ _atomic(out_dir / fname, np.ascontiguousarray(idx).tobytes())
128
+ flies[name] = {"exclude": list(exc), "n": int(len(idx)), "index": fname}
129
+ meta = {**meta, "flies": flies}
130
+ _atomic(jpath, json.dumps(meta).encode())
131
+ return meta
132
+
133
+
134
+ def read_meta_named(out_dir: Path, name: str) -> dict | None:
135
+ try:
136
+ return json.loads((Path(out_dir) / name).read_text())
137
+ except (FileNotFoundError, ValueError):
138
+ return None
fly_trader/brain/pathways.py CHANGED
@@ -13,7 +13,8 @@ from datetime import datetime
13
  from pathlib import Path
14
 
15
  import numpy as np
16
- import torch
 
17
 
18
  TOP_K = 200
19
 
@@ -27,7 +28,7 @@ def km_pairs(connectome_path: str | Path) -> tuple[np.ndarray, np.ndarray, int,
27
 
28
  def bootstrap_weights(boot_path: str | Path) -> tuple[np.ndarray, np.ndarray, list[str]]:
29
  """(base weight per pair, learn [S, J], strategy names) from a fly bootstrap checkpoint (``fly_selector.save``)."""
30
- d = torch.load(boot_path, map_location="cpu", weights_only=False); sd = d["state_dict"]
31
  w0 = np.logaddexp(0.0, sd["theta_km"].detach().float().numpy()).astype(np.float32) # softplus
32
  n_mbon = int(sd["c_sign"].numel()) if "c_sign" in sd else int(sd["mask"].shape[1])
33
  learn = sd["learn"].numpy().astype(bool) if "learn" in sd else np.ones((1, n_mbon), dtype=bool)
@@ -37,7 +38,7 @@ def bootstrap_weights(boot_path: str | Path) -> tuple[np.ndarray, np.ndarray, li
37
 
38
  def bank_D(path: str | Path) -> tuple[np.ndarray, int | None]:
39
  """The learned change per pair from ``state.pt`` or a ``snap_*.pt`` (both carry ``bank`` and ``bootstrap_id``)."""
40
- s = torch.load(path, map_location="cpu", weights_only=False)
41
  D = s["bank"]["D"]; D = D[0] if D.ndim == 2 else D
42
  bid = s.get("bootstrap_id")
43
  return D.detach().float().numpy(), (int(bid) if bid is not None else None)
 
13
  from pathlib import Path
14
 
15
  import numpy as np
16
+
17
+ from ..train import model_io
18
 
19
  TOP_K = 200
20
 
 
28
 
29
  def bootstrap_weights(boot_path: str | Path) -> tuple[np.ndarray, np.ndarray, list[str]]:
30
  """(base weight per pair, learn [S, J], strategy names) from a fly bootstrap checkpoint (``fly_selector.save``)."""
31
+ d = model_io.torch_load(boot_path); sd = d["state_dict"]
32
  w0 = np.logaddexp(0.0, sd["theta_km"].detach().float().numpy()).astype(np.float32) # softplus
33
  n_mbon = int(sd["c_sign"].numel()) if "c_sign" in sd else int(sd["mask"].shape[1])
34
  learn = sd["learn"].numpy().astype(bool) if "learn" in sd else np.ones((1, n_mbon), dtype=bool)
 
38
 
39
  def bank_D(path: str | Path) -> tuple[np.ndarray, int | None]:
40
  """The learned change per pair from ``state.pt`` or a ``snap_*.pt`` (both carry ``bank`` and ``bootstrap_id``)."""
41
+ s = model_io.torch_load(path)
42
  D = s["bank"]["D"]; D = D[0] if D.ndim == 2 else D
43
  bid = s.get("bootstrap_id")
44
  return D.detach().float().numpy(), (int(bid) if bid is not None else None)
fly_trader/brain/plastic.py CHANGED
@@ -140,7 +140,7 @@ def row_weights(scores: torch.Tensor, lines: torch.Tensor) -> torch.Tensor:
140
  even at 10x the losing population outweighed the tradeable rows by more than 100:1 and dragged every prediction
141
  toward the candidate mean. Learning then made the fly worse than the same network frozen (PF 1.012 vs 1.098). A row
142
  below the line is not a decision, so it carries no weight."""
143
- return torch.where(scores >= lines[:, None], 1.0, 0.0)
144
 
145
 
146
  class PlasticBank:
@@ -187,6 +187,12 @@ class PlasticBank:
187
  m = torch.tanh(u)
188
  return (y_dn[None] + (((m - self.mu) / self.sigma) * self._read(s, len(y_dn))) @ self.c) / self.scale, m
189
 
 
 
 
 
 
 
190
  def frozen(self, y_dn: torch.Tensor, u0: torch.Tensor, s=None) -> torch.Tensor:
191
  """The shadow: the bootstrap fly's score (D = 0) [B]."""
192
  return (y_dn + (((torch.tanh(u0) - self.mu) / self.sigma) * self._read(s, len(y_dn))) @ self.c) / self.scale
 
140
  even at 10x the losing population outweighed the tradeable rows by more than 100:1 and dragged every prediction
141
  toward the candidate mean. Learning then made the fly worse than the same network frozen (PF 1.012 vs 1.098). A row
142
  below the line is not a decision, so it carries no weight."""
143
+ return torch.where(scores >= (lines if lines.dim() == 2 else lines[:, None]), 1.0, 0.0) # lines [C] or per row [C, B] (each row's chain)
144
 
145
 
146
  class PlasticBank:
 
187
  m = torch.tanh(u)
188
  return (y_dn[None] + (((m - self.mu) / self.sigma) * self._read(s, len(y_dn))) @ self.c) / self.scale, m
189
 
190
+ def head_for(self, target: torch.Tensor, u0: torch.Tensor, s=None) -> torch.Tensor:
191
+ """The head output under which the frozen score (D = 0) is exactly ``target`` (net-return units) [B]: with it,
192
+ ``predict`` is ``target`` plus the plastic change of the MBON readout, and ``update`` learns the error of that
193
+ sum -- the fly that scores as its teacher plus a learned correction (config.FLY_SCORE = teacher)."""
194
+ return target * self.scale - (((torch.tanh(u0) - self.mu) / self.sigma) * self._read(s, len(target))) @ self.c
195
+
196
  def frozen(self, y_dn: torch.Tensor, u0: torch.Tensor, s=None) -> torch.Tensor:
197
  """The shadow: the bootstrap fly's score (D = 0) [B]."""
198
  return (y_dn + (((torch.tanh(u0) - self.mu) / self.sigma) * self._read(s, len(y_dn))) @ self.c) / self.scale
fly_trader/chain/cluster_guard.py CHANGED
@@ -6,6 +6,9 @@ quotes, signature statuses) never do. Signing is allowed only when ALL of these
6
  exists), ``LIVE_ENABLED=1``, and ``config.live_prerequisites_missing()`` is empty (capital, position
7
  cap, gas reserve, bot key and API keys all set). ``config`` attributes are read at call time so a
8
  test can monkeypatch them.
 
 
 
9
  """
10
  from __future__ import annotations
11
 
@@ -35,6 +38,18 @@ def signing_allowed() -> bool:
35
  return False
36
 
37
 
 
 
 
 
 
 
 
 
 
 
 
 
38
  def assert_vault_signing_allowed() -> None:
39
  """The vault's own signing (claim payouts, principal withdrawals, token-account closes). On mainnet it is the same
40
  gate as trading. The devnet rehearsal (``VAULT_CLUSTER=devnet``) may sign with a throwaway key that is not a
 
6
  exists), ``LIVE_ENABLED=1``, and ``config.live_prerequisites_missing()`` is empty (capital, position
7
  cap, gas reserve, bot key and API keys all set). ``config`` attributes are read at call time so a
8
  test can monkeypatch them.
9
+
10
+ The one exception is a withdrawal to the operator's own wallet (``assert_withdraw_allowed``,
11
+ chain/withdraw.py): it takes money out of the fly's reach and never trades, so it needs no LIVE_ENABLED.
12
  """
13
  from __future__ import annotations
14
 
 
38
  return False
39
 
40
 
41
+ def assert_withdraw_allowed(dest: str) -> None:
42
+ """A withdrawal of the bot's SOL (chain/withdraw.py): mainnet, and only to a wallet listed in FUNDING_ADDRESSES — the
43
+ list lives in .env and is never typed into the console, so whoever reaches the console can at most send the bot's SOL
44
+ to its owner. Never on the hosted vault fly: its SOL leaves through the treasury (``fly-trader vault withdraw``)."""
45
+ if config.VAULT_ENABLED:
46
+ raise SigningRefused("refusing: the vault fly's SOL leaves through the treasury (fly-trader vault withdraw), not from the trading wallet")
47
+ if config.SOLANA_CLUSTER != "mainnet-beta":
48
+ raise SigningRefused(f"refusing to withdraw on cluster {config.SOLANA_CLUSTER!r}: the bot wallet is on mainnet-beta")
49
+ if dest not in config.FUNDING_ADDRESSES:
50
+ raise SigningRefused(f"refusing: {dest} is not one of your FUNDING_ADDRESSES (.env)")
51
+
52
+
53
  def assert_vault_signing_allowed() -> None:
54
  """The vault's own signing (claim payouts, principal withdrawals, token-account closes). On mainnet it is the same
55
  gate as trading. The devnet rehearsal (``VAULT_CLUSTER=devnet``) may sign with a throwaway key that is not a
fly_trader/chain/jupiter_swap.py CHANGED
@@ -58,7 +58,7 @@ JUPITER_EXECUTE_RPS = config.JUPITER_RPS # separate bucket, same rate
58
  JUPITER_LABELS_URL = "https://api.jup.ag/swap/v2/program-id-to-label"
59
  JUPITER_LABELS_CACHE = config.DATA_DIR / "jupiter_program_labels.json"
60
  JUPITER_LABELS_MAX_AGE_S = 86400.0
61
- DEFAULT_EXCLUDE_ROUTERS = "jupiterz"
62
 
63
 
64
  class JupiterError(RuntimeError):
 
58
  JUPITER_LABELS_URL = "https://api.jup.ag/swap/v2/program-id-to-label"
59
  JUPITER_LABELS_CACHE = config.DATA_DIR / "jupiter_program_labels.json"
60
  JUPITER_LABELS_MAX_AGE_S = 86400.0
61
+ DEFAULT_EXCLUDE_ROUTERS = "jupiterz,dflow,okx" # Metis only: the signer refuses other routers' programs (signer/policy.py)
62
 
63
 
64
  class JupiterError(RuntimeError):
fly_trader/chain/rpc.py CHANGED
@@ -224,3 +224,9 @@ class HttpSolanaRpc:
224
  sig = self.call("sendTransaction", [tx_b64, {"encoding": "base64", "skipPreflight": bool(skip_preflight),
225
  "preflightCommitment": COMMITMENT, "maxRetries": int(max_retries)}])
226
  return str(sig)
 
 
 
 
 
 
 
224
  sig = self.call("sendTransaction", [tx_b64, {"encoding": "base64", "skipPreflight": bool(skip_preflight),
225
  "preflightCommitment": COMMITMENT, "maxRetries": int(max_retries)}])
226
  return str(sig)
227
+
228
+
229
+ def check_rpc() -> "HttpSolanaRpc | None":
230
+ """A client for the second, independent provider (``SOLANA_CHECK_RPC_URL``) that money-moving reads must agree with,
231
+ or None when none is configured."""
232
+ return HttpSolanaRpc(config.SOLANA_CHECK_RPC_URL) if config.SOLANA_CHECK_RPC_URL else None
fly_trader/chain/withdraw.py ADDED
@@ -0,0 +1,129 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Withdraw SOL from the bot wallet to one of the operator's own wallets (System → Safety & wallet in the console).
2
+
3
+ - Only to a wallet listed in FUNDING_ADDRESSES and only on mainnet; never on the hosted vault fly, whose SOL leaves
4
+ through the treasury (cluster_guard.assert_withdraw_allowed). No LIVE_ENABLED: a withdrawal takes money out of the
5
+ fly's reach and never trades.
6
+ - What is left is 0 or at least the rent-exempt minimum, and while live positions are open at least the gas reserve
7
+ (their exits pay fees from it). An empty destination must receive at least the rent-exempt minimum.
8
+ - It holds the live broker's in-flight lock, so a swap's balance delta never contains a withdrawal.
9
+ - The signed transaction is recorded (wallet_events 'withdrawal', status 'signed') before it is broadcast; its outcome is
10
+ added to that row. A confirmed withdrawal re-bases the live book's kill-switch peak (agent/rails.record_withdrawal):
11
+ money taken out is not a drawdown.
12
+ """
13
+ from __future__ import annotations
14
+
15
+ import base64
16
+ import logging
17
+ from decimal import Decimal
18
+
19
+ from psycopg.types.json import Jsonb
20
+ from solders.hash import Hash
21
+ from solders.message import MessageV0
22
+ from solders.pubkey import Pubkey
23
+ from solders.system_program import TransferParams, transfer
24
+ from solders.transaction import VersionedTransaction
25
+
26
+ from .. import config
27
+ from ..agent import rails
28
+ from ..db.apilog import record_event
29
+ from ..db.connection import transaction
30
+ from ..logging_setup import scrub
31
+ from ..markets import SOL
32
+ from .cluster_guard import assert_withdraw_allowed
33
+ from .rpc import RpcError, RpcTransportError
34
+
35
+ log = logging.getLogger(__name__)
36
+ LAMPORTS = config.LAMPORTS_PER_SOL
37
+ FEE_LAMPORTS = 5_000 # one signature, no priority fee
38
+ RENT_EXEMPT_MIN = 890_880 # a system account without data: a balance below it (other than 0) is refused
39
+ LOCK_WAIT_S = 120.0 # a swap holds the broker's lock until it confirms or expires
40
+
41
+
42
+ class WithdrawRefused(RuntimeError):
43
+ """Nothing was signed; the message says why."""
44
+
45
+
46
+ def destinations() -> list[str]:
47
+ """The wallets SOL may go to: the valid Solana addresses in FUNDING_ADDRESSES."""
48
+ out = []
49
+ for a in config.FUNDING_ADDRESSES:
50
+ try:
51
+ Pubkey.from_string(a)
52
+ out.append(a)
53
+ except ValueError:
54
+ log.warning("FUNDING_ADDRESSES: %r is not a Solana address", a)
55
+ return out
56
+
57
+
58
+ def limits(rpc, me: str, to: str | None = None) -> dict:
59
+ """Fresh reads: the wallet's lamports, what must stay (the gas reserve while live positions are open), the most that
60
+ can go out, and the destination's lamports."""
61
+ balance = int(rpc.get_balance(me))
62
+ with transaction() as conn:
63
+ n_open = int(conn.execute("SELECT count(*) AS n FROM positions WHERE book = %s AND status = 'open'", (SOL.live_book,)).fetchone()["n"])
64
+ keep = max(int(config.GAS_RESERVE_SOL * LAMPORTS), RENT_EXEMPT_MIN) if n_open else 0
65
+ return {"balance": balance, "fee": FEE_LAMPORTS, "keep": keep, "n_open": n_open, "max": max(0, balance - FEE_LAMPORTS - keep),
66
+ "to_balance": int(rpc.get_balance(to)) if to else None}
67
+
68
+
69
+ def plan(lim: dict, sol: float | None) -> tuple[int, str | None]:
70
+ """(lamports to send, why not or None) for ``sol`` SOL, or everything withdrawable when ``sol`` is None."""
71
+ lam = lim["max"] if sol is None else int(Decimal(str(sol)) * LAMPORTS)
72
+ if lam <= 0:
73
+ return 0, "nothing to withdraw" if sol is None else "enter an amount"
74
+ if lam > lim["max"]:
75
+ return lam, (f"at most {lim['max'] / LAMPORTS:.6f} SOL can go out"
76
+ + (f": {lim['keep'] / LAMPORTS:g} SOL stays for the fees of {lim['n_open']} open live position(s)" if lim["keep"] else " after the fee"))
77
+ left = lim["balance"] - lam - lim["fee"]
78
+ if 0 < left < RENT_EXEMPT_MIN:
79
+ return lam, f"Solana refuses to leave {left / LAMPORTS:.6f} SOL behind: leave at least {RENT_EXEMPT_MIN / LAMPORTS:g} SOL or withdraw everything"
80
+ if lim.get("to_balance") == 0 and lam < RENT_EXEMPT_MIN:
81
+ return lam, f"an empty wallet must receive at least {RENT_EXEMPT_MIN / LAMPORTS:g} SOL"
82
+ return lam, None
83
+
84
+
85
+ def withdraw(sol: float | None, to: str, rpc=None, keypair=None) -> dict:
86
+ """Send ``sol`` SOL (None: everything withdrawable) from the bot wallet to ``to``. Returns {status, signature, lamports,
87
+ to, slot}; status 'confirmed', 'failed_on_chain' (only the fee moved) or 'expired' (nothing moved). Raises
88
+ WithdrawRefused, cluster_guard.SigningRefused or keys.WalletKeyError when nothing was signed."""
89
+ from ..execution.broker_live import _INFLIGHT, await_confirmation
90
+ assert_withdraw_allowed(to)
91
+ if keypair is None:
92
+ from .keys import load_keypair
93
+ keypair = load_keypair()
94
+ if rpc is None:
95
+ from .rpc import HttpSolanaRpc
96
+ rpc = HttpSolanaRpc()
97
+ me = str(keypair.pubkey())
98
+ if not _INFLIGHT.acquire(timeout=LOCK_WAIT_S):
99
+ raise WithdrawRefused("a swap is still in flight: try again in a minute")
100
+ try:
101
+ lam, why = plan(limits(rpc, me, to), sol)
102
+ if why:
103
+ raise WithdrawRefused(why)
104
+ bh = rpc.get_latest_blockhash()
105
+ ix = transfer(TransferParams(from_pubkey=keypair.pubkey(), to_pubkey=Pubkey.from_string(to), lamports=lam))
106
+ tx = VersionedTransaction(MessageV0.try_compile(keypair.pubkey(), [ix], [], Hash.from_string(bh["blockhash"])), [keypair])
107
+ sig = str(tx.signatures[0])
108
+ detail = {"to": to, "lamports": lam, "sol": lam / LAMPORTS, "signature": sig, "last_valid_block_height": bh["last_valid_block_height"], "status": "signed"}
109
+ with transaction() as conn: # recorded before the network ever sees it
110
+ eid = conn.execute("INSERT INTO wallet_events (kind, pubkey, detail) VALUES ('withdrawal', %s, %s) RETURNING id", (me, Jsonb(detail))).fetchone()["id"]
111
+ try:
112
+ rpc.send_transaction(base64.b64encode(bytes(tx)).decode())
113
+ except RpcError as e: # refused by the node (preflight): never broadcast
114
+ with transaction() as conn:
115
+ conn.execute("UPDATE wallet_events SET detail = detail || %s WHERE id = %s", (Jsonb({"status": "rejected", "error": scrub(str(e))[:300]}), eid))
116
+ raise WithdrawRefused(f"the network refused it: {scrub(str(e))[:200]}") from None
117
+ except RpcTransportError as e: # it may have gone out: its status decides
118
+ log.warning("withdrawal %s: send failed in transport (%s); polling its status", sig, e)
119
+ status, st = await_confirmation(rpc, sig, bh["last_valid_block_height"])
120
+ slot = int((st or {}).get("slot") or 0)
121
+ with transaction() as conn:
122
+ conn.execute("UPDATE wallet_events SET detail = detail || %s WHERE id = %s", (Jsonb({"status": status, "slot": slot}), eid))
123
+ if status == "confirmed":
124
+ rails.record_withdrawal(conn, {"sol": lam / LAMPORTS, "to": to, "signature": sig})
125
+ finally:
126
+ _INFLIGHT.release()
127
+ record_event("info" if status == "confirmed" else "warning", "wallet", f"withdrawal of {lam / LAMPORTS:.6f} SOL to {to}: {status}",
128
+ {"signature": sig, "slot": slot})
129
+ return {"status": status, "signature": sig, "lamports": lam, "to": to, "slot": slot}
fly_trader/config.py CHANGED
@@ -14,8 +14,10 @@ from dotenv import load_dotenv
14
  REPO_ROOT = Path(__file__).resolve().parent.parent
15
  load_dotenv(REPO_ROOT / ".env", override=False)
16
 
17
- SECRET_ENV_NAMES = ("HELIUS_API_KEY", "JUPITER_API_KEY", "BOT_PRIVATE_KEY",
18
- "RELAY_SECRET", "TELEGRAM_BOT_TOKEN", "VAULT_BACKUP_HF_TOKEN", "VAULT_SOLANA_RPC_URL")
 
 
19
 
20
 
21
  def utcnow() -> datetime:
@@ -76,9 +78,12 @@ KELLY_FRACTION = env_float("KELLY_FRACTION", 0.25) # quarter Kel
76
  MAX_POSITION_FRACTION = env_float("MAX_POSITION_FRACTION", 0.10) # of the deployable bankroll (wealth minus the gas reserve)
77
  MAX_POOL_SHARE = env_float("MAX_POOL_SHARE", 0.02) # of the pool's quote reserve (bounds price impact)
78
  # The training label charges what an exit really costs at the size the book takes. market/features.py prices
79
- # exit_cost_0p1 for a fixed 0.1 SOL, but positions are Kelly-sized up to MAX_POSITION_FRACTION of the bankroll, and
80
- # impact = value/(value + reserves) grows with size. 0 = the sizing rule's own ceiling (train/decisions.py).
81
- LABEL_SIZE_SOL = env_float("LABEL_SIZE_SOL", 0.0)
 
 
 
82
  MIN_POSITION_SOL = env_float("MIN_POSITION_SOL", 0.02) # smaller sizes are skipped
83
  KILL_SWITCH_DRAWDOWN = env_float("KILL_SWITCH_DRAWDOWN", 0.30)
84
  KILL_SWITCH_LIQUIDATE = env_bool("KILL_SWITCH_LIQUIDATE", False) # a tripped kill switch also sells every open live position (agent/fly_live.py)
@@ -103,6 +108,25 @@ LAMPORTS_PER_SOL = 1_000_000_000
103
  STATS_REFRESH_S = env_float("STATS_REFRESH_S", 600.0)
104
 
105
  # ---- connectome (the fly) ----
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
106
  DEVICE = env_str("DEVICE", "auto") # auto (CUDA, else MPS, else CPU) | cpu | cuda | cuda:N | mps — brain/device.py
107
  NT_SIGN_MODE = env_str("NT_SIGN_MODE", "shiu") # shiu | flybrain
108
  CELL_TYPES_SOURCE = env_str("CELL_TYPES_SOURCE", "annotations") # annotations | flybrain
@@ -158,8 +182,8 @@ def live_prerequisites_missing() -> list[str]:
158
  missing.append("MAX_POSITION_SOL")
159
  if GAS_RESERVE_SOL <= 0:
160
  missing.append("GAS_RESERVE_SOL")
161
- if not env_str("BOT_PRIVATE_KEY") and not env_str("BOT_PRIVATE_KEY_FILE"):
162
- missing.append("BOT_PRIVATE_KEY")
163
  if not JUPITER_API_KEY:
164
  missing.append("JUPITER_API_KEY")
165
  if not HELIUS_API_KEY:
@@ -168,9 +192,11 @@ def live_prerequisites_missing() -> list[str]:
168
 
169
  # ---- the $FLY vault (fly_trader/vault, docs/vault/SPEC.md): only the hosted vault fly sets VAULT_ENABLED ----
170
  VAULT_ENABLED = env_bool("VAULT_ENABLED", False)
 
 
171
  VAULT_CLUSTER = env_str("VAULT_CLUSTER", "mainnet-beta") # devnet only for the rehearsal (trading off, claims on devnet)
172
  VAULT_SOLANA_RPC_URL = env_str("VAULT_SOLANA_RPC_URL") # overrides Helius for the vault's reads/claims (the devnet rehearsal)
173
- FUNDING_ADDRESSES = env_list("FUNDING_ADDRESSES", []) # SOL from these is a deposit; from anyone else, profit
174
  RH_CHAIN_ID = env_int("RH_CHAIN_ID", 4663)
175
  RH_RPC_URL = env_str("RH_RPC_URL", "https://rpc.mainnet.chain.robinhood.com")
176
  VAULT_ADDRESS = env_str("VAULT_ADDRESS") # the FlyVault proxy
@@ -180,6 +206,8 @@ VAULT_IMPL_CODEHASHES = env_list("VAULT_IMPL_CODEHASHES", []) # implementa
180
  VAULT_PERIOD_S = env_int("VAULT_PERIOD_S", 7 * 86400) # one settlement period; the rehearsal uses 900
181
  VAULT_EPOCH = env_int("VAULT_EPOCH", 345_600) # boundaries at EPOCH + k*PERIOD; 345600 = Monday 1970-01-05 00:00 UTC
182
  CLAIM_MIN_LAMPORTS = env_int("CLAIM_MIN_LAMPORTS", 2_000_000) # 0.002 SOL: above the rent-exempt minimum of a new wallet
 
 
183
  RELAY_URL = env_str("RELAY_URL") # the site origin; the fly calls {RELAY_URL}/api/...
184
  RELAY_KEY_ID = env_str("RELAY_KEY_ID", "k1")
185
  RELAY_SECRET = env_str("RELAY_SECRET")
@@ -193,11 +221,161 @@ TELEGRAM_CHAT_ID = env_str("TELEGRAM_CHAT_ID")
193
  VAULT_BACKUP_RECIPIENT = env_str("VAULT_BACKUP_RECIPIENT") # age / ssh-ed25519 public key the backups are encrypted to
194
  VAULT_BACKUP_BUCKET = env_str("VAULT_BACKUP_BUCKET") # private HF storage bucket, e.g. bryceweiner/fly-vault-backups
195
  VAULT_BACKUP_HF_TOKEN = env_str("VAULT_BACKUP_HF_TOKEN") # scoped to that bucket only
 
 
 
 
 
 
 
 
 
 
 
196
 
197
 
198
  def vault_solana_rpc_url() -> str:
199
  return VAULT_SOLANA_RPC_URL or helius_http_url()
200
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
201
  # ---- historical corpus (every pump.fun graduation, pulled from the migration wallet + pump.fun's swap-api) ----
202
  CORPUS_DIR = DATA_DIR / "corpus"
203
  CORPUS_DAYS = env_int("CORPUS_DAYS", 240) # enumerate graduations this far back (must reach before CORPUS_PULL_BEFORE)
 
14
  REPO_ROOT = Path(__file__).resolve().parent.parent
15
  load_dotenv(REPO_ROOT / ".env", override=False)
16
 
17
+ SECRET_ENV_NAMES = ("HELIUS_API_KEY", "JUPITER_API_KEY", "BOT_PRIVATE_KEY", "KALSHI_API_KEY_ID", "KALSHI_PRIVATE_KEY_PATH",
18
+ "RELAY_SECRET", "TELEGRAM_BOT_TOKEN", "VAULT_BACKUP_HF_TOKEN", "VAULT_SOLANA_RPC_URL", "SIGNER_RPC_URL",
19
+ "SOLANA_CHECK_RPC_URL", "HEALTHCHECK_URL", "RH_BOT_PRIVATE_KEY", "RH_RPC_URL_LOGS", "ENVIO_API_TOKEN", "QUICKNODE_API_KEY",
20
+ "QUICKNODE_RH_API_KEY", "QUICKNODE_RH_API_URL", "RH_RPC_URL_TX")
21
 
22
 
23
  def utcnow() -> datetime:
 
78
  MAX_POSITION_FRACTION = env_float("MAX_POSITION_FRACTION", 0.10) # of the deployable bankroll (wealth minus the gas reserve)
79
  MAX_POOL_SHARE = env_float("MAX_POOL_SHARE", 0.02) # of the pool's quote reserve (bounds price impact)
80
  # The training label charges what an exit really costs at the size the book takes. market/features.py prices
81
+ # exit_cost_0p1 for a fixed 0.1 SOL, but impact = value/(value + reserves) grows with size. Labels are priced at
82
+ # min(LABEL_SIZE_SOL, MAX_POOL_SHARE of the pool) -- the "label size" -- and no position is ever bigger (agent/sizing.py):
83
+ # every edge the models show was measured at that size. A fixed reference, never a share of the bankroll: derived from
84
+ # CAPITAL_SOL it made the labels, and the sizes, grow with the bankroll, and in the 2026-09-26 replay a 500 SOL book
85
+ # sized past it lost 426 SOL over 60 days where one capped at it made the same +13 SOL as a 5 or 50 SOL book.
86
+ LABEL_SIZE_SOL = env_float("LABEL_SIZE_SOL", 0.5)
87
  MIN_POSITION_SOL = env_float("MIN_POSITION_SOL", 0.02) # smaller sizes are skipped
88
  KILL_SWITCH_DRAWDOWN = env_float("KILL_SWITCH_DRAWDOWN", 0.30)
89
  KILL_SWITCH_LIQUIDATE = env_bool("KILL_SWITCH_LIQUIDATE", False) # a tripped kill switch also sells every open live position (agent/fly_live.py)
 
108
  STATS_REFRESH_S = env_float("STATS_REFRESH_S", 600.0)
109
 
110
  # ---- connectome (the fly) ----
111
+ # How the fly picks its buy line (train/fly_calibrate.py): total | selector | match. match = its teacher's selectivity:
112
+ # in the 2026-09-26 replays it took 92 trades at +10.1 % (58 % winners, PF 1.92) where total took 1,607 at +2.0 %.
113
+ FLY_CALIB_RULE = env_str("FLY_CALIB_RULE", "match")
114
+ # How the fly sizes a buy (agent/sizing.py): kelly = the selector's own Kelly band for the row (its score, line and sizing
115
+ # table; with FLY_TEACHER_GATE every fly buy is a selector buy, sized exactly as the selector sizes it) | flat =
116
+ # MAX_POSITION_FRACTION of the deployable bankroll on every buy. (Flat was the default while the fly sized from its own
117
+ # calibrated bands, which flipped between Kelly 0.97, 0 and 0.15 on its few trades a week; the selector's bands are fitted
118
+ # on its walk-forward and do not.)
119
+ FLY_SIZING = env_str("FLY_SIZING", "kelly")
120
+ # The fly buys only what its teacher (the deployed selector stack) would buy: the same trigger, line, win filter, gates,
121
+ # dump veto and fail-closed rules. Its own learning (plastic score below its bootstrap score by FLY_LEARNED_VETO) can
122
+ # only drop a teacher trade, never add one (2026-10-08: ungated, the fly bought rows the selector rejected and lost).
123
+ FLY_TEACHER_GATE = env_bool("FLY_TEACHER_GATE", True)
124
+ FLY_LEARNED_VETO = env_float("FLY_LEARNED_VETO", 0.02)
125
+ # How the fly scores a row. teacher: its teacher's own score plus what its mushroom body has learned (the plastic change of
126
+ # its KC->MBON synapses, zero at bootstrap), so it starts exactly as the selector and its learning only corrects the
127
+ # selector. own: its distilled copy of the selector (until 2026-10-11): that copy ranked the selector's top 1 % of
128
+ # candidates only ~0.6-correlated, so its own judgement at the line was half noise (fly #157 and #251 alike).
129
+ FLY_SCORE = env_str("FLY_SCORE", "teacher")
130
  DEVICE = env_str("DEVICE", "auto") # auto (CUDA, else MPS, else CPU) | cpu | cuda | cuda:N | mps — brain/device.py
131
  NT_SIGN_MODE = env_str("NT_SIGN_MODE", "shiu") # shiu | flybrain
132
  CELL_TYPES_SOURCE = env_str("CELL_TYPES_SOURCE", "annotations") # annotations | flybrain
 
182
  missing.append("MAX_POSITION_SOL")
183
  if GAS_RESERVE_SOL <= 0:
184
  missing.append("GAS_RESERVE_SOL")
185
+ if not SIGNER_SOCKET and not env_str("BOT_PRIVATE_KEY") and not env_str("BOT_PRIVATE_KEY_FILE"):
186
+ missing.append("BOT_PRIVATE_KEY") # the vault server's brain has no key: its signer does
187
  if not JUPITER_API_KEY:
188
  missing.append("JUPITER_API_KEY")
189
  if not HELIUS_API_KEY:
 
192
 
193
  # ---- the $FLY vault (fly_trader/vault, docs/vault/SPEC.md): only the hosted vault fly sets VAULT_ENABLED ----
194
  VAULT_ENABLED = env_bool("VAULT_ENABLED", False)
195
+ PAYOUT_PRIVATE_KEY_FILE = env_str("PAYOUT_PRIVATE_KEY_FILE") # the key that pays claims from the treasury (spending limit L2); read by the signer only
196
+ VAULT_BOOK = env_str("VAULT_BOOK", "live") # the book the vault shares: live; a paper book (e.g. paper_fly) for a dry run on real decisions without real SOL
197
  VAULT_CLUSTER = env_str("VAULT_CLUSTER", "mainnet-beta") # devnet only for the rehearsal (trading off, claims on devnet)
198
  VAULT_SOLANA_RPC_URL = env_str("VAULT_SOLANA_RPC_URL") # overrides Helius for the vault's reads/claims (the devnet rehearsal)
199
+ FUNDING_ADDRESSES = env_list("FUNDING_ADDRESSES", []) # your own Solana wallets: SOL from these is a deposit (vault); the console withdraws only to these
200
  RH_CHAIN_ID = env_int("RH_CHAIN_ID", 4663)
201
  RH_RPC_URL = env_str("RH_RPC_URL", "https://rpc.mainnet.chain.robinhood.com")
202
  VAULT_ADDRESS = env_str("VAULT_ADDRESS") # the FlyVault proxy
 
206
  VAULT_PERIOD_S = env_int("VAULT_PERIOD_S", 7 * 86400) # one settlement period; the rehearsal uses 900
207
  VAULT_EPOCH = env_int("VAULT_EPOCH", 345_600) # boundaries at EPOCH + k*PERIOD; 345600 = Monday 1970-01-05 00:00 UTC
208
  CLAIM_MIN_LAMPORTS = env_int("CLAIM_MIN_LAMPORTS", 2_000_000) # 0.002 SOL: above the rent-exempt minimum of a new wallet
209
+ CLAIM_MIN_WEI = env_int("CLAIM_MIN_WEI", 100_000_000_000_000) # 0.0001 ETH: smaller ETH owed waits for the next claim
210
+ VAULT_RH_GAS_RESERVE_WEI = env_int("VAULT_RH_GAS_RESERVE_WEI", 200_000_000_000_000) # ETH the settlement never allocates (the RH wallet pays payout gas)
211
  RELAY_URL = env_str("RELAY_URL") # the site origin; the fly calls {RELAY_URL}/api/...
212
  RELAY_KEY_ID = env_str("RELAY_KEY_ID", "k1")
213
  RELAY_SECRET = env_str("RELAY_SECRET")
 
221
  VAULT_BACKUP_RECIPIENT = env_str("VAULT_BACKUP_RECIPIENT") # age / ssh-ed25519 public key the backups are encrypted to
222
  VAULT_BACKUP_BUCKET = env_str("VAULT_BACKUP_BUCKET") # private HF storage bucket, e.g. bryceweiner/fly-vault-backups
223
  VAULT_BACKUP_HF_TOKEN = env_str("VAULT_BACKUP_HF_TOKEN") # scoped to that bucket only
224
+ # custody (docs/vault/SPEC.md "Custody"): a Squads v4 treasury the hot keys draw on only through its spending limits
225
+ VAULT_MULTISIG = env_str("VAULT_MULTISIG") # the Squads multisig; the treasury is its vault 0
226
+ VAULT_LIMIT_TRADING = env_str("VAULT_LIMIT_TRADING") # L1: trading key -> the trading wallet only, per day
227
+ VAULT_LIMIT_PAYOUT = env_str("VAULT_LIMIT_PAYOUT") # L2: payout key -> any holder, per week
228
+ TREASURY_FLOAT_SOL = env_float("TREASURY_FLOAT_SOL", 2.0) # the most the trading wallet keeps idle; the rest goes back
229
+ TOPUP_MIN_SOL = env_float("TOPUP_MIN_SOL", 0.1) # smaller refills and returns are skipped
230
+ SIGNER_SOCKET = env_str("SIGNER_SOCKET") # the signer container's socket; unset: an in-process signer
231
+ SIGNER_RPC_URL = env_str("SIGNER_RPC_URL") # the signer's own RPC (a second provider)
232
+ SOLANA_CHECK_RPC_URL = env_str("SOLANA_CHECK_RPC_URL") # a second provider money-moving reads must agree with
233
+ TELEGRAM_ADMIN_USER_ID = env_str("TELEGRAM_ADMIN_USER_ID") # the only Telegram user whose /panic and /status count
234
+ HEALTHCHECK_URL = env_str("HEALTHCHECK_URL") # dead-man ping (healthchecks.io) every 5 min
235
 
236
 
237
  def vault_solana_rpc_url() -> str:
238
  return VAULT_SOLANA_RPC_URL or helius_http_url()
239
 
240
+
241
+ def signer_env() -> dict:
242
+ """What an in-process signer reads (fly_trader/signer; the signer container has its own environment)."""
243
+ return {k: str(v) for k, v in {"VAULT_MULTISIG": VAULT_MULTISIG, "VAULT_LIMIT_TRADING": VAULT_LIMIT_TRADING,
244
+ "VAULT_LIMIT_PAYOUT": VAULT_LIMIT_PAYOUT,
245
+ "TREASURY_FLOAT_SOL": TREASURY_FLOAT_SOL, "VAULT_BACKUP_RECIPIENT": VAULT_BACKUP_RECIPIENT}.items() if v}
246
+
247
+ # ---- Robinhood Chain memecoins (fly_trader/rh; Pons launches graduated into Uniswap v4; branch rh-memecoins) ----
248
+ # The same selector and fly as Solana, trained on both chains' rows (a chain input tells them apart); RH books keep their
249
+ # money in native ETH. Amounts on RH rows are ETH (non-ETH-quoted pools converted at the minute's base/ETH price).
250
+ RH_ENABLED = env_bool("RH_ENABLED", True) # index RH and paper-trade it
251
+ RH_LIVE_ENABLED = env_bool("RH_LIVE_ENABLED", False) # sign and send real RH transactions (live_rh)
252
+ RH_TESTNET = env_bool("RH_TESTNET", False) # chain 46630 (the rehearsal); the guard refuses a mismatch
253
+ RH_EXPECTED_CHAIN_ID = 46630 if RH_TESTNET else RH_CHAIN_ID
254
+ RH_RPC_URL_LOGS = env_str("RH_RPC_URL_LOGS") # optional better endpoint for eth_getLogs / tx lookups (else RH_RPC_URL)
255
+
256
+
257
+ def quicknode_rh_url() -> str | None:
258
+ """The QuickNode RH endpoint: QUICKNODE_RH_API_URL, with QUICKNODE_RH_API_KEY appended when the URL does not carry it."""
259
+ url, key = env_str("QUICKNODE_RH_API_URL"), env_str("QUICKNODE_RH_API_KEY")
260
+ if not url:
261
+ return None
262
+ return url if not key or key in url else url.rstrip("/") + "/" + key
263
+
264
+
265
+ # sending and reading transactions (sends, receipts, nonces, balances, eth_call, gas): every RhRpc() without an explicit URL.
266
+ # Never logs: QuickNode's free Discover plan caps eth_getLogs at 5 blocks (measured 2026-10-09), so logs stay on HyperSync /
267
+ # RH_RPC_URL_LOGS / RH_RPC_URL; its 10M credits a month cover the trading path, not a log tail. QuickNode serves mainnet
268
+ # only while RH_RPC_URL is the default: a run that points RH_RPC_URL elsewhere (anvil, the testnet rehearsal) sends there.
269
+ RH_RPC_URL_TX = env_str("RH_RPC_URL_TX") or (quicknode_rh_url() if env_str("RH_RPC_URL") is None and not RH_TESTNET else None)
270
+ # block headers for timing the index (rh/blocktime.py): PublicNode serves header batches fast but refuses eth_getLogs (403),
271
+ # the official RPC serves logs but rate-limits at ~20 requests/min (measured 2026-09-28) — so each does what it does well
272
+ RH_RPC_URL_BLOCKS = env_str("RH_RPC_URL_BLOCKS", "https://robinhood-rpc.publicnode.com")
273
+ RH_HYPERSYNC_URL = env_str("RH_HYPERSYNC_URL", "https://robinhood.hypersync.xyz") # log history in bulk when ENVIO_API_TOKEN is set (rh/hypersync.py)
274
+ RH_HYPERSYNC_RPM = env_float("RH_HYPERSYNC_RPM", 40) # HyperSync queries per minute (free tier: fair use, refused ~6 % at 60; Starter plan: 100)
275
+ RH_BOT_ADDRESS = env_str("RH_BOT_ADDRESS") # the address RH_BOT_PRIVATE_KEY must derive to (a guard, not a secret)
276
+ RH_FUNDING_ADDRESSES = env_list("RH_FUNDING_ADDRESSES", []) # your own EVM wallets: the console withdraws the RH bot's ETH only to these
277
+ RH_CONFIRMATIONS = env_int("RH_CONFIRMATIONS", 3) # blocks before a live receipt or a live log is applied
278
+ RH_START_BLOCK = env_int("RH_START_BLOCK", 0) # the indexer's first block (0: the Pons V2 factory's deployment block)
279
+ RH_START_DAY = env_str("RH_START_DAY", "2026-08-04") # Pons V2 went live: the first RH corpus day
280
+ # Fixed at build time by tools/rh_constants.py (2026-09-28 13:12 UTC: SOL/USD 119.51, ETH/USD 2682.05): the Solana sizes in ETH.
281
+ # Never derived live: they set the size every RH label is priced at (see LABEL_SIZE_SOL).
282
+ RH_ETH_PER_SOL = env_float("RH_ETH_PER_SOL", 0.0445583) # K: converts fixed-unit thresholds (eligibility, triggers) for RH rows
283
+ RH_CAPITAL_ETH = env_float("RH_CAPITAL_ETH", 0.222791) # the RH paper books' bankroll (5 SOL at build time)
284
+ RH_LABEL_SIZE_ETH = env_float("RH_LABEL_SIZE_ETH", 0.0222791) # 0.5 SOL at build time: RH labels are priced at it; no RH buy is bigger
285
+ RH_MIN_POSITION_ETH = env_float("RH_MIN_POSITION_ETH", 0.000891166) # 0.02 SOL at build time
286
+ RH_GAS_RESERVE_ETH = env_float("RH_GAS_RESERVE_ETH", 0.0029) # 200 round trips at 2x the gas rh-probe measured (356k gas, 0.02 gwei, 2026-09-28)
287
+ RH_MAX_FEE_GWEI = env_float("RH_MAX_FEE_GWEI", 5.0) # a transaction never offers more per gas
288
+ RH_TX_FEE_ETH = env_float("RH_TX_FEE_ETH", 7.2e-6) # one swap's gas in ETH (rh-probe 2026-09-28: 356k gas × 0.02 gwei); fixed like the SOL fee
289
+ RH_HOOK_FEE = env_float("RH_HOOK_FEE", 0.0099) # Pons hook fee + creator tax per side when a minute has none: the median of 52k indexed swaps (1 % of the net = 0.99 % of gross; high-tax launches reach 5.7 %)
290
+ RH_KILL_SWITCH_DRAWDOWN = env_float("RH_KILL_SWITCH_DRAWDOWN", 0.30)
291
+ RH_MIN_SWEEP_ETH = env_float("RH_MIN_SWEEP_ETH", 0.0002) # base-asset or dead-bag balances worth less are dust
292
+ RH_STREAM_POLL_S = env_float("RH_STREAM_POLL_S", 2.0)
293
+ RH_TRADE_LAG_S = env_float("RH_TRADE_LAG_S", 240.0) # an RH minute this old (s) is still traded: its stream lands ~2–3 min late
294
+ RH_MINUTES_KEEP_DAYS = env_int("RH_MINUTES_KEEP_DAYS", 14)
295
+ RH_DIR_NAME = "rh" # data/corpus/rh/... (the RH corpus beside the Solana one)
296
+ # contracts on Robinhood Chain (mainnet; verified to carry code 2026-09-28)
297
+ PONS_FACTORY = env_str("PONS_FACTORY", "0x7ed598bcef8bd9edd8c97a195c6d13f40801ec7e")
298
+ PONS_ROUTER = env_str("PONS_ROUTER", "0xe33e9e479df8802cb0866d5d05258bec4cf62948")
299
+ PONS_HOOK = env_str("PONS_HOOK", "0xe5e702641ea86f4ae6cc3cdaed2b886f976be044")
300
+ PONS_LOCKER = env_str("PONS_LOCKER", "0x267444d099b10fb5ed7c3cc7b7c767adca574952")
301
+ PONS_GRAD_EXECUTOR = env_str("PONS_GRAD_EXECUTOR", "0xc7819b64a1daecd7ec19856d026cb14efbd89046")
302
+ V4_POOL_MANAGER = env_str("V4_POOL_MANAGER", "0x8366a39cc670b4001a1121b8f6a443a643e40951")
303
+ V4_QUOTER = env_str("V4_QUOTER", "0x8dc178efb8111bb0973dd9d722ebeff267c98f94")
304
+ V4_STATE_VIEW = env_str("V4_STATE_VIEW", "0xf3334192d15450cdd385c8b70e03f9a6bd9e673b")
305
+ UNIVERSAL_ROUTER = env_str("UNIVERSAL_ROUTER", "0x8876789976decbfcbbbe364623c63652db8c0904")
306
+ PERMIT2 = env_str("PERMIT2", "0x000000000022D473030F116dDEE9F6B43aC78BA3")
307
+ V3_FACTORY = env_str("V3_FACTORY", "0x1f7d7550b1b028f7571e69a784071f0205fd2efa")
308
+ V3_QUOTER_V2 = env_str("V3_QUOTER_V2", "0x33e885ed0ec9bf04ecfb19341582aadcb4c8a9e7")
309
+ RH_WETH = env_str("RH_WETH", "0x0bd7d308f8e1639fab988df18a8011f41eacad73") # SwapRouter02.WETH9()
310
+ RH_USDG = env_str("RH_USDG", "0x5fc5360D0400a0Fd4f2af552ADD042D716F1d168")
311
+ KYBER_API = env_str("KYBER_API", "https://aggregator-api.kyberswap.com/robinhood/api/v1")
312
+ KYBER_ROUTER = env_str("KYBER_ROUTER", "0x6131B5fae19EA4f9D964eAc0408E4408b66337b5") # MetaAggregationRouterV2: the only router a built Kyber tx may call
313
+ KYBER_CLIENT_ID = env_str("KYBER_CLIENT_ID", "fly-trader")
314
+
315
+
316
+ def rh_live_prerequisites_missing() -> list[str]:
317
+ missing = []
318
+ if not RH_LIVE_ENABLED:
319
+ missing.append("RH_LIVE_ENABLED=1")
320
+ if not env_str("RH_BOT_PRIVATE_KEY"):
321
+ missing.append("RH_BOT_PRIVATE_KEY")
322
+ if not RH_BOT_ADDRESS:
323
+ missing.append("RH_BOT_ADDRESS")
324
+ for name in ("RH_CAPITAL_ETH", "RH_LABEL_SIZE_ETH", "RH_MIN_POSITION_ETH", "RH_ETH_PER_SOL", "RH_GAS_RESERVE_ETH"):
325
+ if globals()[name] <= 0:
326
+ missing.append(name)
327
+ return missing
328
+
329
+
330
+ # ---- Kalshi prediction markets (fly_trader/kalshi; the same key names as better_bot, whose client is vendored) ----
331
+ KALSHI_API_KEY_ID = env_str("KALSHI_API_KEY_ID")
332
+ KALSHI_PRIVATE_KEY_PATH = env_str("KALSHI_PRIVATE_KEY_PATH")
333
+ KALSHI_API_BASE_URL = env_str("KALSHI_API_BASE_URL", "https://api.elections.kalshi.com/trade-api/v2")
334
+ KALSHI_WS_URL = env_str("KALSHI_WS_URL", "wss://api.elections.kalshi.com/trade-api/ws/v2")
335
+ KALSHI_SUBACCOUNT = env_int("KALSHI_SUBACCOUNT", 0) # 0 = the primary account: only a dedicated subaccount may trade live
336
+ KALSHI_FEE_FACTOR = env_float("KALSHI_FEE_FACTOR", 0.07) # taker fee per contract = factor x multiplier x P(1-P)
337
+ KALSHI_MAKER_FEE_FACTOR = env_float("KALSHI_MAKER_FEE_FACTOR", 0.0175)
338
+ KALSHI_CAPITAL_USD = env_float("KALSHI_CAPITAL_USD", 100.0) # the cap both Kalshi arms share (paper start; live never sizes above it)
339
+ KALSHI_CASH_FLOOR_USD = env_float("KALSHI_CASH_FLOOR_USD", 5.0) # the gas reserve's analogue: never spent
340
+ KALSHI_MIN_PAYOUT_CENTS = env_int("KALSHI_MIN_PAYOUT_CENTS", 25) # a fill whose winning profit is below this is skipped
341
+ KALSHI_LIVE_ENABLED = env_bool("KALSHI_LIVE_ENABLED", False)
342
+ KALSHI_TAKER_LIVE = env_bool("KALSHI_TAKER_LIVE", True)
343
+ KALSHI_MAKER_LIVE = env_bool("KALSHI_MAKER_LIVE", True)
344
+ KALSHI_KILL_SWITCH_DRAWDOWN = env_float("KALSHI_KILL_SWITCH_DRAWDOWN", 0.30)
345
+ KALSHI_MAX_SLATE_FRACTION = env_float("KALSHI_MAX_SLATE_FRACTION", 0.50) # of the cap deployed at once across every open position
346
+ KALSHI_MAX_DAYS_TO_CLOSE = env_float("KALSHI_MAX_DAYS_TO_CLOSE", 10.0) # the evidence base (Whelan 2026) covers the last 10 days
347
+ KALSHI_MIN_MINUTES_TO_CLOSE = env_float("KALSHI_MIN_MINUTES_TO_CLOSE", 5.0)
348
+ KALSHI_MAX_SPREAD_CENTS = env_int("KALSHI_MAX_SPREAD_CENTS", 10)
349
+ KALSHI_MIN_OPEN_INTEREST = env_float("KALSHI_MIN_OPEN_INTEREST", 100.0)
350
+ KALSHI_MIN_VOLUME_24H = env_float("KALSHI_MIN_VOLUME_24H", 50.0)
351
+ KALSHI_MAKER_QUIET_MIN = env_float("KALSHI_MAKER_QUIET_MIN", 30.0) # no resting orders this close to the market's close
352
+ KALSHI_MAKER_TTL_H = env_float("KALSHI_MAKER_TTL_H", 6.0)
353
+ KALSHI_MAKER_MAX_RESTING = env_int("KALSHI_MAKER_MAX_RESTING", 40)
354
+ KALSHI_HISTORY_START = env_str("KALSHI_HISTORY_START", "2025-01-01")
355
+ KALSHI_DATASET_DIR = env_str("KALSHI_DATASET_DIR") # jon-becker/prediction-market-analysis checkout (data/kalshi/{markets,trades})
356
+ KALSHI_MINUTES_KEEP_DAYS = env_int("KALSHI_MINUTES_KEEP_DAYS", 7)
357
+ KALSHI_MARKETS_PER_DAY = env_int("KALSHI_MARKETS_PER_DAY", 150) # the event sample's target: about this many markets per category per day (kalshi/history.sample_rates);
358
+ # events are kept by md5(event ticker) < the category's rate, blind to outcome and volume
359
+ KALSHI_DIR = DATA_DIR / "kalshi"
360
+ KALSHI_TRAIN_MAX_ROWS = env_int("KALSHI_TRAIN_MAX_ROWS", 8_000_000) # training rows per walk-forward block / final fit (uniform random subset)
361
+ KALSHI_FILL_THREADS = env_int("KALSHI_FILL_THREADS", 8) # markets filled at once by the history worker, all under the one KALSHI_RPS bucket
362
+ KALSHI_RPS = env_float("KALSHI_RPS", 16.0) # per process; the account's Advanced grant reads 200 tokens/s at 10 per call = 20 calls/s, the feed uses ~3
363
+
364
+
365
+ def kalshi_live_prerequisites_missing() -> list[str]:
366
+ missing = []
367
+ if not KALSHI_LIVE_ENABLED:
368
+ missing.append("KALSHI_LIVE_ENABLED=1")
369
+ if not KALSHI_API_KEY_ID:
370
+ missing.append("KALSHI_API_KEY_ID")
371
+ if not KALSHI_PRIVATE_KEY_PATH or not Path(KALSHI_PRIVATE_KEY_PATH).expanduser().exists():
372
+ missing.append("KALSHI_PRIVATE_KEY_PATH (readable PEM file)")
373
+ if KALSHI_SUBACCOUNT <= 0:
374
+ missing.append("KALSHI_SUBACCOUNT (a dedicated subaccount, never the primary)")
375
+ if KALSHI_CAPITAL_USD <= 0:
376
+ missing.append("KALSHI_CAPITAL_USD")
377
+ return missing
378
+
379
  # ---- historical corpus (every pump.fun graduation, pulled from the migration wallet + pump.fun's swap-api) ----
380
  CORPUS_DIR = DATA_DIR / "corpus"
381
  CORPUS_DAYS = env_int("CORPUS_DAYS", 240) # enumerate graduations this far back (must reach before CORPUS_PULL_BEFORE)
fly_trader/db/schema.py CHANGED
@@ -6,7 +6,9 @@ appending a version-gated block AND bumping SCHEMA_VERSION, or it never runs.
6
  """
7
  from __future__ import annotations
8
 
 
9
  import logging
 
10
  from datetime import datetime, timedelta, timezone
11
 
12
  import psycopg
@@ -17,8 +19,15 @@ from .connection import connect, database_name, database_url
17
 
18
  log = logging.getLogger(__name__)
19
 
20
- SCHEMA_VERSION = 6 # 5: the plastic fly's tables and per-book halts (BASE_DDL); 6: the selector's strategy stack (MIGRATIONS[6])
21
  _LOCK_KEY = 0x666C795F6D6967 # "fly_mig"
 
 
 
 
 
 
 
22
 
23
  BASE_DDL: list[str] = [
24
  """CREATE TABLE IF NOT EXISTS schema_version (
@@ -150,11 +159,11 @@ BASE_DDL: list[str] = [
150
  running_mean real[], running_var real[], n bigint NOT NULL DEFAULT 0,
151
  updated_at timestamptz NOT NULL DEFAULT now())""",
152
  """CREATE TABLE IF NOT EXISTS circuit_state (
153
- id int PRIMARY KEY CHECK (id = 1), fail_count int NOT NULL DEFAULT 0, last_failure_ts timestamptz,
154
  tripped boolean NOT NULL DEFAULT false, kill_switch boolean NOT NULL DEFAULT false, kill_reason text,
155
  peak_wealth double precision, entries_paused boolean NOT NULL DEFAULT false,
156
  updated_at timestamptz NOT NULL DEFAULT now())""",
157
- "INSERT INTO circuit_state (id) VALUES (1) ON CONFLICT DO NOTHING",
158
  """CREATE TABLE IF NOT EXISTS circuit_events (
159
  id bigserial PRIMARY KEY, ts timestamptz NOT NULL DEFAULT now(), kind text NOT NULL, detail jsonb)""",
160
  """CREATE TABLE IF NOT EXISTS notional_ledger (
@@ -223,6 +232,8 @@ BASE_DDL: list[str] = [
223
  label real, state text NOT NULL DEFAULT 'pending', resolved_at timestamptz, bootstrap_id bigint,
224
  PRIMARY KEY (ts, mint))""",
225
  "CREATE INDEX IF NOT EXISTS fly_scored_state_ts_idx ON fly_scored (state, ts)",
 
 
226
  """CREATE TABLE IF NOT EXISTS fly_calibrations (
227
  day date NOT NULL, arm text NOT NULL, line real, sizing jsonb, trades int, total real, mean real, window_days int,
228
  created_at timestamptz NOT NULL DEFAULT now(), PRIMARY KEY (day, arm))""",
@@ -261,6 +272,105 @@ MIGRATIONS: dict[int, list[str]] = {
261
  "ALTER TABLE fly_scored ADD COLUMN IF NOT EXISTS hold_min int",
262
  "ALTER TABLE fly_scored DROP CONSTRAINT IF EXISTS fly_scored_pkey",
263
  "ALTER TABLE fly_scored ADD PRIMARY KEY (ts, mint, strategy)"],
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
264
  }
265
 
266
 
@@ -278,41 +388,93 @@ def ensure_database(url: str | None = None) -> bool:
278
  return True
279
 
280
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
281
  def apply_schema(url: str | None = None) -> int:
282
- """Apply the ladder under an advisory lock. Returns the resulting version."""
 
 
283
  with connect(url) as conn:
284
- with conn.cursor() as cur:
285
- cur.execute("SELECT pg_advisory_lock(%s)", (_LOCK_KEY,))
286
- try:
287
- for stmt in BASE_DDL:
288
- cur.execute(stmt)
289
- cur.execute("SELECT version FROM schema_version WHERE singleton")
290
- row = cur.fetchone()
291
- current = int(row["version"]) if row else 0
292
- for v in sorted(MIGRATIONS):
293
- if current < v:
294
- for stmt in MIGRATIONS[v]:
295
- cur.execute(stmt)
296
- current = v
297
- from ..vault.schema import VAULT_DDL # idempotent, outside the ladder: same DDL on master and distribution
298
- for stmt in VAULT_DDL:
299
- cur.execute(stmt)
300
- target = max(current, SCHEMA_VERSION)
301
- cur.execute(
302
- "INSERT INTO schema_version (singleton, version) VALUES (true, %s) "
303
- "ON CONFLICT (singleton) DO UPDATE SET version = EXCLUDED.version, updated_at = now()",
304
- (target,),
305
- )
306
- conn.commit()
307
- ensure_partitions(conn)
308
- conn.commit()
309
- return target
310
- except Exception:
311
- conn.rollback()
312
- raise
313
- finally:
314
- cur.execute("SELECT pg_advisory_unlock(%s)", (_LOCK_KEY,))
315
- conn.commit()
316
 
317
 
318
  def _partition_name(base: str, start: datetime, granularity: str) -> str:
 
6
  """
7
  from __future__ import annotations
8
 
9
+ import hashlib
10
  import logging
11
+ import time
12
  from datetime import datetime, timedelta, timezone
13
 
14
  import psycopg
 
19
 
20
  log = logging.getLogger(__name__)
21
 
22
+ SCHEMA_VERSION = 12 # 5: the plastic fly's tables and per-book halts (BASE_DDL); 6: the selector's strategy stack (MIGRATIONS[6]); 7: Kalshi (MIGRATIONS[7]); 8: Kalshi minute extremes
23
  _LOCK_KEY = 0x666C795F6D6967 # "fly_mig"
24
+ # Applying the DDL takes ACCESS EXCLUSIVE on the tables it touches (Postgres takes it for ALTER TABLE ... ADD COLUMN IF NOT
25
+ # EXISTS even when the column exists) and holds it to the end of the transaction, while live workers write those tables.
26
+ # 2026-10-07: the RH stream's start-up apply deadlocked with the trading engine on positions/decisions and crashed. So:
27
+ # unchanged DDL is not re-run at all (its fingerprint is stored), and a run never waits more than LOCK_TIMEOUT for a
28
+ # lock: it rolls back, releases everything it holds, and retries.
29
+ LOCK_TIMEOUT = "5s"
30
+ APPLY_TRIES = 30
31
 
32
  BASE_DDL: list[str] = [
33
  """CREATE TABLE IF NOT EXISTS schema_version (
 
159
  running_mean real[], running_var real[], n bigint NOT NULL DEFAULT 0,
160
  updated_at timestamptz NOT NULL DEFAULT now())""",
161
  """CREATE TABLE IF NOT EXISTS circuit_state (
162
+ id int PRIMARY KEY, fail_count int NOT NULL DEFAULT 0, last_failure_ts timestamptz,
163
  tripped boolean NOT NULL DEFAULT false, kill_switch boolean NOT NULL DEFAULT false, kill_reason text,
164
  peak_wealth double precision, entries_paused boolean NOT NULL DEFAULT false,
165
  updated_at timestamptz NOT NULL DEFAULT now())""",
166
+ "INSERT INTO circuit_state (id) VALUES (1) ON CONFLICT DO NOTHING", # id 2 (the Kalshi rails) is inserted by MIGRATIONS[7], after the CHECK is dropped
167
  """CREATE TABLE IF NOT EXISTS circuit_events (
168
  id bigserial PRIMARY KEY, ts timestamptz NOT NULL DEFAULT now(), kind text NOT NULL, detail jsonb)""",
169
  """CREATE TABLE IF NOT EXISTS notional_ledger (
 
232
  label real, state text NOT NULL DEFAULT 'pending', resolved_at timestamptz, bootstrap_id bigint,
233
  PRIMARY KEY (ts, mint))""",
234
  "CREATE INDEX IF NOT EXISTS fly_scored_state_ts_idx ON fly_scored (state, ts)",
235
+ # would the fly's teacher have traded this row (its final decision)? the match calibration rule's budget (train/fly_calibrate.py)
236
+ "ALTER TABLE fly_scored ADD COLUMN IF NOT EXISTS teacher_allow boolean",
237
  """CREATE TABLE IF NOT EXISTS fly_calibrations (
238
  day date NOT NULL, arm text NOT NULL, line real, sizing jsonb, trades int, total real, mean real, window_days int,
239
  created_at timestamptz NOT NULL DEFAULT now(), PRIMARY KEY (day, arm))""",
 
272
  "ALTER TABLE fly_scored ADD COLUMN IF NOT EXISTS hold_min int",
273
  "ALTER TABLE fly_scored DROP CONSTRAINT IF EXISTS fly_scored_pkey",
274
  "ALTER TABLE fly_scored ADD PRIMARY KEY (ts, mint, strategy)"],
275
+ # 7: Kalshi prediction markets (fly_trader/kalshi): the exchange catalogue, live minutes and quotes, the corpus registry,
276
+ # the four Kalshi books' positions/orders/fills, the visual fly's tables, and a second circuit (id 2) for the Kalshi rails
277
+ 7: ["ALTER TABLE circuit_state DROP CONSTRAINT IF EXISTS circuit_state_id_check",
278
+ "INSERT INTO circuit_state (id) VALUES (2) ON CONFLICT DO NOTHING",
279
+ """CREATE TABLE IF NOT EXISTS kalshi_series (
280
+ ticker text PRIMARY KEY, title text, category text, categories text[], tags text[], frequency text, fee_type text,
281
+ fee_multiplier double precision, settlement_sources jsonb, updated_at timestamptz NOT NULL DEFAULT now())""",
282
+ """CREATE TABLE IF NOT EXISTS kalshi_events (
283
+ event_ticker text PRIMARY KEY, series_ticker text, title text, sub_title text, category text, mutually_exclusive boolean,
284
+ strike_date timestamptz, strike_period text, collateral_return_type text, exchange_index int, raw jsonb,
285
+ updated_at timestamptz NOT NULL DEFAULT now())""",
286
+ "CREATE INDEX IF NOT EXISTS kalshi_events_series_idx ON kalshi_events (series_ticker)",
287
+ """CREATE TABLE IF NOT EXISTS kalshi_markets (
288
+ ticker text PRIMARY KEY, event_ticker text, market_type text, title text, yes_sub_title text, no_sub_title text, status text,
289
+ open_time timestamptz, close_time timestamptz, expected_expiration_time timestamptz, latest_expiration_time timestamptz,
290
+ settlement_ts timestamptz, result text, settlement_value double precision, strike_type text, floor_strike double precision,
291
+ cap_strike double precision, exchange_index int, price_level_structure text, is_provisional boolean, can_close_early boolean,
292
+ created_time timestamptz, source text, raw jsonb, updated_at timestamptz NOT NULL DEFAULT now())""",
293
+ "CREATE INDEX IF NOT EXISTS kalshi_markets_status_close_idx ON kalshi_markets (status, close_time)",
294
+ "CREATE INDEX IF NOT EXISTS kalshi_markets_event_idx ON kalshi_markets (event_ticker)",
295
+ "CREATE INDEX IF NOT EXISTS kalshi_markets_settled_idx ON kalshi_markets (settlement_ts)",
296
+ # the pump_minutes analogue: one row per (market, UTC minute) from the ticker and trade channels
297
+ """CREATE TABLE IF NOT EXISTS kalshi_minutes (
298
+ ticker text NOT NULL, ts timestamptz NOT NULL, yes_bid real, yes_ask real, last real, bid_size real, ask_size real,
299
+ volume_fp double precision, open_interest_fp double precision, dollar_volume double precision,
300
+ taker_buy_yes double precision, taker_buy_no double precision, n_trades int, max_trade double precision, block_contracts double precision,
301
+ PRIMARY KEY (ticker, ts))""",
302
+ "CREATE INDEX IF NOT EXISTS kalshi_minutes_ts_idx ON kalshi_minutes (ts)",
303
+ # the live top of book (every ticker message), read by the maker arm and the vendored venue's spread tiers
304
+ """CREATE TABLE IF NOT EXISTS kalshi_quotes (
305
+ ticker text PRIMARY KEY, ts timestamptz, yes_bid real, yes_ask real, mid real, bid_size real, ask_size real, last real,
306
+ updated_at timestamptz NOT NULL DEFAULT now())""",
307
+ """CREATE OR REPLACE VIEW venue_market_cache AS
308
+ SELECT 'kalshi'::text AS venue, ticker AS market_id, yes_ask AS yes_ask_cents, mid AS market_prob FROM kalshi_quotes""",
309
+ # the corpus registry (kalshi/history.py) and the daily feature builds (kalshi/mature.py)
310
+ """CREATE TABLE IF NOT EXISTS kalshi_corpus (
311
+ ticker text PRIMARY KEY, status text NOT NULL DEFAULT 'pending', settled_ts timestamptz, result text, open_time timestamptz,
312
+ close_time timestamptz, candles_1m int, trades int, candle_path text, trade_path text, seeded_from text, last_error text,
313
+ updated_at timestamptz NOT NULL DEFAULT now())""",
314
+ "CREATE INDEX IF NOT EXISTS kalshi_corpus_status_idx ON kalshi_corpus (status, settled_ts DESC)",
315
+ """CREATE TABLE IF NOT EXISTS kalshi_days (
316
+ day date PRIMARY KEY, markets int, rows int, took_s real, built_at timestamptz NOT NULL DEFAULT now())""",
317
+ # the four Kalshi books (paper_kalshi_taker, paper_kalshi_maker, live_kalshi_taker, live_kalshi_maker); money in cents
318
+ """CREATE TABLE IF NOT EXISTS kalshi_positions (
319
+ id bigserial PRIMARY KEY, book text NOT NULL, ticker text NOT NULL, side text NOT NULL, contracts double precision NOT NULL,
320
+ cost_cents double precision NOT NULL, fee_cents double precision NOT NULL DEFAULT 0, avg_price_cents double precision,
321
+ opened_at timestamptz NOT NULL DEFAULT now(), closed_at timestamptz, entry_decision_id bigint, exit_decision_id bigint,
322
+ order_id text, status text NOT NULL DEFAULT 'open', result text, payout_cents double precision, realized_cents double precision,
323
+ strategy text, arm text, score real, line real, last_mark_cents real, last_mark_ts timestamptz)""",
324
+ "CREATE INDEX IF NOT EXISTS kalshi_positions_book_status_idx ON kalshi_positions (book, status)",
325
+ "CREATE INDEX IF NOT EXISTS kalshi_positions_ticker_idx ON kalshi_positions (ticker)",
326
+ """CREATE TABLE IF NOT EXISTS kalshi_orders (
327
+ id bigserial PRIMARY KEY, book text NOT NULL, order_id text, client_order_id text, decision_id bigint, ticker text NOT NULL,
328
+ side text NOT NULL, action text NOT NULL DEFAULT 'buy', price_cents int NOT NULL, count double precision NOT NULL, tif text NOT NULL,
329
+ post_only boolean NOT NULL DEFAULT false, expiration_ts timestamptz, status text NOT NULL, fill_count double precision NOT NULL DEFAULT 0,
330
+ remaining double precision, avg_fill_cents double precision, fee_cents double precision, exchange_index int, subaccount int,
331
+ fair_cents real, margin_cents real, strategy text, request jsonb, response jsonb, error text,
332
+ ts timestamptz NOT NULL DEFAULT now(), updated_at timestamptz NOT NULL DEFAULT now())""",
333
+ "CREATE INDEX IF NOT EXISTS kalshi_orders_book_status_idx ON kalshi_orders (book, status)",
334
+ "CREATE UNIQUE INDEX IF NOT EXISTS kalshi_orders_order_id_uniq ON kalshi_orders (order_id) WHERE order_id IS NOT NULL",
335
+ """CREATE TABLE IF NOT EXISTS kalshi_fills (
336
+ trade_id text PRIMARY KEY, order_id text, book text, ticker text NOT NULL, side text NOT NULL, price_cents double precision NOT NULL,
337
+ count double precision NOT NULL, fee_cents double precision, is_taker boolean, action text, ts timestamptz NOT NULL,
338
+ post_position double precision, raw jsonb)""",
339
+ "CREATE INDEX IF NOT EXISTS kalshi_fills_order_idx ON kalshi_fills (order_id)",
340
+ # the visual fly (kalshi/fly_session.py): scored (market, minute, side) rows waiting for settlement, calibrations, learning, rollbacks
341
+ """CREATE TABLE IF NOT EXISTS kalshi_fly_scored (
342
+ ts timestamptz NOT NULL, ticker text NOT NULL, side text NOT NULL, strategy text NOT NULL, x real[], score real, frozen_score real,
343
+ line real, frozen_line real, label real, state text NOT NULL DEFAULT 'pending', due_ts timestamptz, resolved_at timestamptz,
344
+ bootstrap_id bigint, PRIMARY KEY (ts, ticker, side, strategy))""",
345
+ "CREATE INDEX IF NOT EXISTS kalshi_fly_scored_state_idx ON kalshi_fly_scored (state, due_ts)",
346
+ """CREATE TABLE IF NOT EXISTS kalshi_fly_calibrations (
347
+ day date NOT NULL, arm text NOT NULL, line real, sizing jsonb, trades int, total real, mean real, window_days int,
348
+ created_at timestamptz NOT NULL DEFAULT now(), PRIMARY KEY (day, arm))""",
349
+ """CREATE TABLE IF NOT EXISTS kalshi_fly_updates (
350
+ hour timestamptz PRIMARY KEY, n int, mean_delta real, mean_abs_delta real, step real, capped int, drift real, ic real, detail jsonb)""",
351
+ """CREATE TABLE IF NOT EXISTS kalshi_fly_rollbacks (
352
+ id bigserial PRIMARY KEY, ts timestamptz NOT NULL DEFAULT now(), reason text, checks jsonb, from_snapshot bigint, to_snapshot bigint)"""],
353
+ # the minute's lowest YES ask and highest YES bid: whether a resting order would have filled (kalshi/maker.py) — the
354
+ # candle archive's yes_ask_low / yes_bid_high, which the label rule reads (kalshi/mature.py)
355
+ 8: ["ALTER TABLE kalshi_minutes ADD COLUMN IF NOT EXISTS yes_ask_low real",
356
+ "ALTER TABLE kalshi_minutes ADD COLUMN IF NOT EXISTS yes_bid_high real"],
357
+ # 9: a market's lifetime volume on its corpus row, so the corpus can keep the most traded markets per day (kalshi/history.prune_pending)
358
+ 9: ["ALTER TABLE kalshi_corpus ADD COLUMN IF NOT EXISTS volume double precision"],
359
+ # 10: every settled single market Kalshi listed (the dataset's and the exchange walk's), from which the corpus samples whole
360
+ # events outcome-blind (kalshi/history.select_corpus)
361
+ 10: ["""CREATE TABLE IF NOT EXISTS kalshi_universe (
362
+ ticker text PRIMARY KEY, event_ticker text, series_ticker text, title text, open_time timestamptz, close_time timestamptz,
363
+ settled_ts timestamptz, result text, volume double precision, source text, selected boolean NOT NULL DEFAULT false,
364
+ updated_at timestamptz NOT NULL DEFAULT now())""",
365
+ "CREATE INDEX IF NOT EXISTS kalshi_universe_close_idx ON kalshi_universe (close_time)",
366
+ "CREATE INDEX IF NOT EXISTS kalshi_universe_selected_idx ON kalshi_universe (selected) WHERE selected"],
367
+ # 11: a market's early-close condition: such a market may close before its listed close when its event happens, so its
368
+ # realized close carries the outcome; its time anchor is the scheduled expected expiration (kalshi/mature.load_meta)
369
+ 11: ["ALTER TABLE kalshi_markets ADD COLUMN IF NOT EXISTS early_close_condition text"],
370
+ # 12: the minute's taker sales at their extremes: the lowest YES price a taker sold YES at (bought NO) and the highest a
371
+ # taker bought YES at (sold NO at 100 − it): a resting bid fills when a taker sells into it (kalshi/decisions.maker_filled)
372
+ 12: ["ALTER TABLE kalshi_minutes ADD COLUMN IF NOT EXISTS yes_sold_low real",
373
+ "ALTER TABLE kalshi_minutes ADD COLUMN IF NOT EXISTS yes_bought_high real"],
374
  }
375
 
376
 
 
388
  return True
389
 
390
 
391
+ def ddl_fingerprint() -> str:
392
+ """sha256 of every statement apply_schema runs (the ladder, the idempotent vault/RH/ETH DDL) and SCHEMA_VERSION."""
393
+ from ..rh.schema import RH_DDL
394
+ from ..vault.eth_schema import ETH_DDL
395
+ from ..vault.schema import VAULT_DDL
396
+ h = hashlib.sha256()
397
+ for stmt in [str(SCHEMA_VERSION), *BASE_DDL, *(x for v in sorted(MIGRATIONS) for x in MIGRATIONS[v]), *VAULT_DDL, *RH_DDL, *ETH_DDL]:
398
+ h.update(stmt.encode()); h.update(b"\0")
399
+ return h.hexdigest()
400
+
401
+
402
+ def _applied(conn, fp: str) -> int | None:
403
+ """The schema version when exactly this DDL is already applied (nothing to run), else None."""
404
+ try:
405
+ row = conn.execute("SELECT version, ddl_sha256 FROM schema_version WHERE singleton").fetchone()
406
+ except (psycopg.errors.UndefinedTable, psycopg.errors.UndefinedColumn):
407
+ conn.rollback()
408
+ return None
409
+ conn.commit()
410
+ if row and row["ddl_sha256"] == fp and int(row["version"]) >= SCHEMA_VERSION:
411
+ return int(row["version"])
412
+ return None
413
+
414
+
415
+ def _apply_all(conn, fp: str) -> int:
416
+ with conn.cursor() as cur:
417
+ cur.execute(f"SET LOCAL lock_timeout = '{LOCK_TIMEOUT}'")
418
+ for stmt in BASE_DDL:
419
+ cur.execute(stmt)
420
+ cur.execute("ALTER TABLE schema_version ADD COLUMN IF NOT EXISTS ddl_sha256 text")
421
+ cur.execute("SELECT version FROM schema_version WHERE singleton")
422
+ row = cur.fetchone()
423
+ current = int(row["version"]) if row else 0
424
+ for v in sorted(MIGRATIONS):
425
+ if current < v:
426
+ for stmt in MIGRATIONS[v]:
427
+ cur.execute(stmt)
428
+ current = v
429
+ from ..vault.schema import VAULT_DDL # idempotent, outside the ladder: same DDL on master and distribution
430
+ for stmt in VAULT_DDL:
431
+ cur.execute(stmt)
432
+ from ..rh.schema import RH_DDL # idempotent, outside the ladder: the Robinhood Chain market (fly_trader/rh)
433
+ for stmt in RH_DDL:
434
+ cur.execute(stmt)
435
+ from ..vault.eth_schema import ETH_DDL # the vault's ETH pot (Robinhood Chain profits paid in ETH)
436
+ for stmt in ETH_DDL:
437
+ cur.execute(stmt)
438
+ target = max(current, SCHEMA_VERSION)
439
+ cur.execute(
440
+ "INSERT INTO schema_version (singleton, version, ddl_sha256) VALUES (true, %s, %s) "
441
+ "ON CONFLICT (singleton) DO UPDATE SET version = EXCLUDED.version, ddl_sha256 = EXCLUDED.ddl_sha256, updated_at = now()",
442
+ (target, fp),
443
+ )
444
+ return target
445
+
446
+
447
  def apply_schema(url: str | None = None) -> int:
448
+ """Apply the ladder under an advisory lock. Returns the resulting version. A no-op (no table locks) when this exact
449
+ DDL was applied before; otherwise every lock wait is bounded by LOCK_TIMEOUT and the whole run is retried."""
450
+ fp = ddl_fingerprint()
451
  with connect(url) as conn:
452
+ conn.execute("SELECT pg_advisory_lock(%s)", (_LOCK_KEY,))
453
+ conn.commit() # the advisory lock is the session's; no transaction stays open
454
+ try:
455
+ target = _applied(conn, fp)
456
+ if target is None:
457
+ for attempt in range(APPLY_TRIES):
458
+ try:
459
+ target = _apply_all(conn, fp)
460
+ conn.commit()
461
+ break
462
+ except (psycopg.errors.LockNotAvailable, psycopg.errors.DeadlockDetected) as e:
463
+ conn.rollback() # releases every table lock this run took
464
+ if attempt == APPLY_TRIES - 1:
465
+ raise
466
+ wait = min(30.0, 1.0 + attempt)
467
+ log.warning("schema: %s while applying the DDL (live workers hold the tables); retrying in %.0f s", type(e).__name__, wait)
468
+ time.sleep(wait)
469
+ ensure_partitions(conn)
470
+ conn.commit()
471
+ return target
472
+ except Exception:
473
+ conn.rollback()
474
+ raise
475
+ finally:
476
+ conn.execute("SELECT pg_advisory_unlock(%s)", (_LOCK_KEY,))
477
+ conn.commit()
 
 
 
 
 
 
478
 
479
 
480
  def _partition_name(base: str, start: datetime, granularity: str) -> str:
fly_trader/execution/broker_live.py CHANGED
@@ -23,7 +23,6 @@ Every write is committed immediately so a crash mid-poll leaves an auditable row
23
  """
24
  from __future__ import annotations
25
 
26
- import base64
27
  import logging
28
  import re
29
  import threading
@@ -31,20 +30,16 @@ import time
31
  from dataclasses import dataclass
32
 
33
  from psycopg.types.json import Jsonb
34
- from solders.hash import Hash
35
- from solders.instruction import AccountMeta, Instruction
36
- from solders.message import MessageV0
37
- from solders.pubkey import Pubkey
38
- from solders.transaction import VersionedTransaction
39
-
40
  from .. import config
41
  from ..chain.balances import tx_deltas, Snapshot, compute_delta, snapshot_balances
42
  from ..chain.cluster_guard import assert_signing_allowed
43
  from ..chain.jupiter_swap import JupiterError, JupiterSwap
44
  from ..db.connection import transaction
45
- from ..chain.keys import load_keypair
46
  from ..chain.rpc import TOKEN_2022_PROGRAM_ID, TOKEN_PROGRAM_ID, HttpSolanaRpc
47
- from ..chain.signing import NotASigner, n_lookup_programs, sign_transaction_b64, static_program_ids, transaction_id
 
 
 
48
  from ..db.apilog import record_event
49
  from ..db.connection import connect, transaction
50
  from ..logging_setup import scrub
@@ -110,10 +105,14 @@ def await_confirmation(rpc, signature: str | None, last_valid_block_height: int
110
 
111
 
112
  class LiveBroker:
113
- def __init__(self, rpc=None, jup=None, keypair=None):
 
 
 
114
  self._rpc = rpc
115
  self._jup = jup
116
- self._keypair = keypair
 
117
  self._labels: dict[str, str] | None = None
118
 
119
  @property
@@ -129,14 +128,16 @@ class LiveBroker:
129
  return self._jup
130
 
131
  @property
132
- def keypair(self):
133
- if self._keypair is None:
134
- self._keypair = load_keypair()
135
- return self._keypair
136
 
137
  @property
138
  def pubkey(self) -> str:
139
- return str(self.keypair.pubkey())
 
 
140
 
141
  @property
142
  def labels(self) -> dict[str, str]:
@@ -185,7 +186,7 @@ class LiveBroker:
185
  taker = self.pubkey
186
  pre = snapshot_balances(self.rpc, taker)
187
  request = {"inputMint": input_mint, "outputMint": output_mint, "amount": str(amount_in), "taker": taker,
188
- "slippageBps": slippage_bps, "excludeRouters": "jupiterz"}
189
  base = dict(decision_id=decision_id, book=book, attempt=attempt, side=side, input_mint=input_mint,
190
  output_mint=output_mint, amount_in=amount_in, slippage_bps=slippage_bps, request=request)
191
 
@@ -200,7 +201,7 @@ class LiveBroker:
200
 
201
  # 1. order
202
  try:
203
- order = self.jup.order(input_mint, output_mint, amount_in, taker=taker, slippage_bps=slippage_bps)
204
  except JupiterError as e:
205
  return fail("order_error", scrub(str(e)))
206
  err_code = _int_or_none(order.get("errorCode"))
@@ -223,14 +224,18 @@ class LiveBroker:
223
  conn, **base, order_response=order, request_id=order.get("requestId"), router=order.get("router"),
224
  fee_bps=_int_or_none(order.get("feeBps")), program_ids=program_ids, lvbh=lvbh, status="ordered")
225
 
226
- # 2. sign (our slot only)
227
  try:
228
- signed_b64, _idx = sign_transaction_b64(tx_b64, self.keypair)
229
- except (NotASigner, ValueError) as e:
230
- return fail("sign_error", f"{type(e).__name__}: {scrub(str(e))}", None, order_id=order_id)
231
- if _idx != 0 and not order.get("gasless"):
232
- return fail("sign_error", f"our key is signer {_idx}, but a non-gasless order must have the taker as fee payer", None, order_id=order_id)
233
- tx_sig = transaction_id(signed_b64)
 
 
 
 
234
 
235
  # 3. execute
236
  exec_req = {"requestId": order.get("requestId"), "lastValidBlockHeight": lvbh, "signedTransaction": signed_b64}
@@ -419,14 +424,13 @@ def swap_smoke(sol: float) -> None:
419
  print(f"net lamports delta: {net} ({net / config.LAMPORTS_PER_SOL:.9f} SOL)")
420
 
421
 
422
- def close_empty_atas(rpc=None, keypair=None, url: str | None = None, batch: int = CLOSE_ATA_BATCH, skip_mints: set | None = None) -> dict:
423
- """Close every zero-balance token account owned by the bot (rent back to the wallet).
424
- SPL Token / Token-2022 CloseAccount = instruction index 9, accounts [account, destination, owner].
425
- One v0 transaction per ``batch`` accounts; each is recorded as wallet_events(kind='ata_closed')."""
426
  assert_signing_allowed()
427
- kp = keypair or load_keypair()
428
- owner = kp.pubkey()
429
- owner_s = str(owner)
430
  rpc = rpc or HttpSolanaRpc()
431
  skip = set(skip_mints or ())
432
  empties = [a for a in rpc.get_token_accounts_by_owner(owner_s) if int(a["amount"]) == 0 and a["mint"] not in skip and not a.get("is_native")]
@@ -436,16 +440,17 @@ def close_empty_atas(rpc=None, keypair=None, url: str | None = None, batch: int
436
  for i in range(0, len(empties), batch):
437
  chunk = empties[i:i + batch]
438
  accounts = [a["address"] for a in chunk]
439
- ixs = [Instruction(Pubkey.from_string(a["program"]), bytes([9]),
440
- [AccountMeta(Pubkey.from_string(a["address"]), False, True),
441
- AccountMeta(owner, False, True),
442
- AccountMeta(owner, True, False)]) for a in chunk]
443
- bh = rpc.get_latest_blockhash()
444
- msg = MessageV0.try_compile(owner, ixs, [], Hash.from_string(bh["blockhash"]))
445
- tx = VersionedTransaction(msg, [kp])
446
- tx_b64 = base64.b64encode(bytes(tx)).decode()
447
  try:
448
- sig = rpc.send_transaction(tx_b64)
 
 
 
 
 
 
 
 
 
449
  except Exception as e:
450
  err = f"{type(e).__name__}: {scrub(str(e))}"
451
  result["failed"].append({"accounts": accounts, "error": err})
 
23
  """
24
  from __future__ import annotations
25
 
 
26
  import logging
27
  import re
28
  import threading
 
30
  from dataclasses import dataclass
31
 
32
  from psycopg.types.json import Jsonb
 
 
 
 
 
 
33
  from .. import config
34
  from ..chain.balances import tx_deltas, Snapshot, compute_delta, snapshot_balances
35
  from ..chain.cluster_guard import assert_signing_allowed
36
  from ..chain.jupiter_swap import JupiterError, JupiterSwap
37
  from ..db.connection import transaction
 
38
  from ..chain.rpc import TOKEN_2022_PROGRAM_ID, TOKEN_PROGRAM_ID, HttpSolanaRpc
39
+ from ..chain.signing import n_lookup_programs, static_program_ids
40
+ from ..signer import client as signer_client
41
+ from ..signer.client import SignerUnavailable
42
+ from ..signer.policy import EXCLUDE_ROUTERS, PolicyError
43
  from ..db.apilog import record_event
44
  from ..db.connection import connect, transaction
45
  from ..logging_setup import scrub
 
105
 
106
 
107
  class LiveBroker:
108
+ """Swaps through Jupiter; the trading key never enters this process: ``signer`` (fly_trader/signer, the signer
109
+ container's socket on the vault server) checks and signs each transaction."""
110
+
111
+ def __init__(self, rpc=None, jup=None, signer=None):
112
  self._rpc = rpc
113
  self._jup = jup
114
+ self._signer = signer
115
+ self._pubkey: str | None = None
116
  self._labels: dict[str, str] | None = None
117
 
118
  @property
 
128
  return self._jup
129
 
130
  @property
131
+ def signer(self):
132
+ if self._signer is None:
133
+ self._signer = signer_client.get()
134
+ return self._signer
135
 
136
  @property
137
  def pubkey(self) -> str:
138
+ if self._pubkey is None:
139
+ self._pubkey = self.signer.call("pubkeys")["trading"]
140
+ return self._pubkey
141
 
142
  @property
143
  def labels(self) -> dict[str, str]:
 
186
  taker = self.pubkey
187
  pre = snapshot_balances(self.rpc, taker)
188
  request = {"inputMint": input_mint, "outputMint": output_mint, "amount": str(amount_in), "taker": taker,
189
+ "slippageBps": slippage_bps, "excludeRouters": EXCLUDE_ROUTERS}
190
  base = dict(decision_id=decision_id, book=book, attempt=attempt, side=side, input_mint=input_mint,
191
  output_mint=output_mint, amount_in=amount_in, slippage_bps=slippage_bps, request=request)
192
 
 
201
 
202
  # 1. order
203
  try:
204
+ order = self.jup.order(input_mint, output_mint, amount_in, taker=taker, slippage_bps=slippage_bps, exclude_routers=EXCLUDE_ROUTERS)
205
  except JupiterError as e:
206
  return fail("order_error", scrub(str(e)))
207
  err_code = _int_or_none(order.get("errorCode"))
 
224
  conn, **base, order_response=order, request_id=order.get("requestId"), router=order.get("router"),
225
  fee_bps=_int_or_none(order.get("feeBps")), program_ids=program_ids, lvbh=lvbh, status="ordered")
226
 
227
+ # 2. sign: the signer checks the transaction (programs, roles, a simulation of our balances) and signs our slot
228
  try:
229
+ signed = self.signer.call("sign_swap", tx=tx_b64, in_mint=input_mint, out_mint=output_mint, in_amount=amount_in,
230
+ min_out=max(1, int(_int_or_none(order.get("otherAmountThreshold")) or 1)))
231
+ except PolicyError as e:
232
+ r = fail("sign_refused", f"{e.code}: {scrub(str(e))}", None, order_id=order_id)
233
+ if e.code == "policy":
234
+ record_event("error", "signer", f"swap refused: {e}", {"order_id": order_id, "side": side, "mint": mint})
235
+ return r[0], e.code == "in_flight" # a simulation that failed (slippage, stale route): retry with a new order
236
+ except SignerUnavailable as e:
237
+ return fail("sign_error", scrub(str(e)), None, order_id=order_id)
238
+ signed_b64, tx_sig = signed["tx"], signed["signature"]
239
 
240
  # 3. execute
241
  exec_req = {"requestId": order.get("requestId"), "lastValidBlockHeight": lvbh, "signedTransaction": signed_b64}
 
424
  print(f"net lamports delta: {net} ({net / config.LAMPORTS_PER_SOL:.9f} SOL)")
425
 
426
 
427
+ def close_empty_atas(rpc=None, signer=None, url: str | None = None, batch: int = CLOSE_ATA_BATCH, skip_mints: set | None = None) -> dict:
428
+ """Close every zero-balance token account owned by the bot (rent back to the wallet). The signer builds and signs
429
+ the CloseAccount transactions (it re-checks that each account is ours and empty); each is recorded as
430
+ wallet_events(kind='ata_closed')."""
431
  assert_signing_allowed()
432
+ signer = signer or signer_client.get()
433
+ owner_s = signer.call("pubkeys")["trading"]
 
434
  rpc = rpc or HttpSolanaRpc()
435
  skip = set(skip_mints or ())
436
  empties = [a for a in rpc.get_token_accounts_by_owner(owner_s) if int(a["amount"]) == 0 and a["mint"] not in skip and not a.get("is_native")]
 
440
  for i in range(0, len(empties), batch):
441
  chunk = empties[i:i + batch]
442
  accounts = [a["address"] for a in chunk]
 
 
 
 
 
 
 
 
443
  try:
444
+ built = signer.call("close_atas", accounts=accounts)
445
+ except (PolicyError, SignerUnavailable) as e:
446
+ result["failed"].append({"accounts": accounts, "error": str(e)})
447
+ record_event("error", "close_empty_atas", "the signer refused", {"accounts": accounts, "error": str(e)})
448
+ continue
449
+ if not built:
450
+ continue
451
+ bh = {"last_valid_block_height": built[0]["last_valid_block_height"]}
452
+ try:
453
+ sig = rpc.send_transaction(built[0]["tx"])
454
  except Exception as e:
455
  err = f"{type(e).__name__}: {scrub(str(e))}"
456
  result["failed"].append({"accounts": accounts, "error": err})
fly_trader/execution/broker_paper.py CHANGED
@@ -32,7 +32,9 @@ class PaperFill:
32
 
33
  class PaperBroker:
34
  def __init__(self, book: str):
35
- assert book in ("paper_selector", "paper_fly") or book.startswith("replay_"), book # the race books; replay_* for tests
 
 
36
  self.book = book
37
 
38
  def buy(self, conn, *, decision_id: int | None, mint: str, pool: str | None, size_sol: float, price: float | None,
@@ -45,7 +47,7 @@ class PaperBroker:
45
  return PaperFill(False, None, None, 0.0, 0.0, 0, 0.0, "no price")
46
  if res_quote_sol is None or res_quote_sol <= 0:
47
  return PaperFill(False, None, None, price, 0.0, 0, 0.0, "no liquidity estimate")
48
- fee_frac = fee_fraction(size_sol, mcap_sol, pool_fee)
49
  impact = impact_fraction(size_sol, res_quote_sol, program_label)
50
  if impact > config.EXECUTABILITY_MAX_IMPACT:
51
  # a router would not fill this either; refuse like the live executability check
@@ -74,7 +76,7 @@ class PaperBroker:
74
  gross = qty_raw / (10 ** decimals) * price
75
  c = exit_cost_fraction(gross, res_quote_sol, mcap_sol, program_label, pool_fee)
76
  proceeds = gross * (1.0 - c)
77
- fee_sol = gross * fee_fraction(gross, mcap_sol, pool_fee)
78
  if fraction < 0.999: # partial: shrink the position, realise the sold part
79
  realized = ledger.realize_partial(conn, position_id=int(position["id"]), qty_raw=qty_raw,
80
  cost_part=float(position["cost_sol"]) * fraction, proceeds_sol=proceeds,
 
32
 
33
  class PaperBroker:
34
  def __init__(self, book: str):
35
+ from ..markets import MARKETS
36
+ paper = {b for m in MARKETS.values() for b in m.paper_books}
37
+ assert book in paper or book.startswith("replay_"), book # the race books of every market; replay_* for tests
38
  self.book = book
39
 
40
  def buy(self, conn, *, decision_id: int | None, mint: str, pool: str | None, size_sol: float, price: float | None,
 
47
  return PaperFill(False, None, None, 0.0, 0.0, 0, 0.0, "no price")
48
  if res_quote_sol is None or res_quote_sol <= 0:
49
  return PaperFill(False, None, None, price, 0.0, 0, 0.0, "no liquidity estimate")
50
+ fee_frac = fee_fraction(size_sol, mcap_sol, pool_fee, program_label)
51
  impact = impact_fraction(size_sol, res_quote_sol, program_label)
52
  if impact > config.EXECUTABILITY_MAX_IMPACT:
53
  # a router would not fill this either; refuse like the live executability check
 
76
  gross = qty_raw / (10 ** decimals) * price
77
  c = exit_cost_fraction(gross, res_quote_sol, mcap_sol, program_label, pool_fee)
78
  proceeds = gross * (1.0 - c)
79
+ fee_sol = gross * fee_fraction(gross, mcap_sol, pool_fee, program_label)
80
  if fraction < 0.999: # partial: shrink the position, realise the sold part
81
  realized = ledger.realize_partial(conn, position_id=int(position["id"]), qty_raw=qty_raw,
82
  cost_part=float(position["cost_sol"]) * fraction, proceeds_sol=proceeds,
fly_trader/execution/ledger.py CHANGED
@@ -20,7 +20,8 @@ log = logging.getLogger(__name__)
20
 
21
  def open_positions(conn, book: str) -> list[dict]:
22
  rows = conn.execute(
23
- """SELECT p.*, t.decimals, t.graduated_at, wp.program_label,
 
24
  COALESCE(p.hold_s, (d.detail->>'hold_s')::float) AS hold_s, COALESCE(p.strategy, d.detail->>'strategy') AS strategy
25
  FROM positions p LEFT JOIN tokens t ON t.mint = p.mint
26
  LEFT JOIN watch_pools wp ON wp.pool = p.pool
@@ -38,7 +39,8 @@ def paper_cash(conn, book: str) -> float:
38
  FROM positions WHERE book = %s""",
39
  (book,),
40
  ).fetchone()
41
- return config.CAPITAL_SOL + float(r["realized"]) - float(r["open_cost"])
 
42
 
43
 
44
  def record_fill(conn, *, order_id: int | None, book: str, mint: str, side: str, token_delta: int,
@@ -55,6 +57,11 @@ def record_fill(conn, *, order_id: int | None, book: str, mint: str, side: str,
55
  return int(row["id"])
56
 
57
 
 
 
 
 
 
58
  def open_position(conn, *, book: str, mint: str, pool: str | None, qty_raw: int, cost_sol: float,
59
  entry_price: float, decision_id: int | None, fees_sol: float, ts: datetime | None = None,
60
  hold_s: float | None = None, strategy: str | None = None) -> int:
@@ -70,9 +77,9 @@ def open_position(conn, *, book: str, mint: str, pool: str | None, qty_raw: int,
70
  return int(existing["id"])
71
  row = conn.execute(
72
  """INSERT INTO positions (book, mint, pool, opened_at, entry_decision_id, qty, cost_sol, entry_price, peak_price,
73
- last_mark_price, last_mark_ts, fees_sol, status, last_swap_ts, hold_s, strategy)
74
- VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,'open',%s,%s,%s) RETURNING id""",
75
- (book, mint, pool, ts, decision_id, qty_raw, cost_sol, entry_price, entry_price, entry_price, ts, fees_sol, ts, hold_s, strategy),
76
  ).fetchone()
77
  return int(row["id"])
78
 
@@ -127,11 +134,11 @@ def mark_positions(conn, book: str, prices: dict[str, float], last_swaps: dict[s
127
 
128
  def verify_fills() -> None:
129
  """Reconcile live positions against on-chain token balances; prints mismatches, writes wallet_events."""
130
- from ..chain import keys
131
  from ..chain.rpc import HttpSolanaRpc
132
  from ..db.connection import transaction
 
133
  rpc = HttpSolanaRpc()
134
- pub = keys.bot_pubkey()
135
  accounts = rpc.get_token_accounts_by_owner(pub)
136
  onchain = {a["mint"]: int(a["amount"]) for a in accounts}
137
  with transaction() as conn:
 
20
 
21
  def open_positions(conn, book: str) -> list[dict]:
22
  rows = conn.execute(
23
+ """SELECT p.*, COALESCE(t.decimals, CASE WHEN p.chain = 'rh' THEN 18 END) AS decimals, t.graduated_at,
24
+ COALESCE(wp.program_label, CASE WHEN p.chain = 'rh' THEN 'Pons v4' END) AS program_label,
25
  COALESCE(p.hold_s, (d.detail->>'hold_s')::float) AS hold_s, COALESCE(p.strategy, d.detail->>'strategy') AS strategy
26
  FROM positions p LEFT JOIN tokens t ON t.mint = p.mint
27
  LEFT JOIN watch_pools wp ON wp.pool = p.pool
 
39
  FROM positions WHERE book = %s""",
40
  (book,),
41
  ).fetchone()
42
+ from ..markets import for_book
43
+ return for_book(book).capital() + float(r["realized"]) - float(r["open_cost"]) # the book's own market's capital (SOL / ETH)
44
 
45
 
46
  def record_fill(conn, *, order_id: int | None, book: str, mint: str, side: str, token_delta: int,
 
57
  return int(row["id"])
58
 
59
 
60
+ def _chain_of(book: str) -> str:
61
+ from ..markets import for_book
62
+ return for_book(book).chain
63
+
64
+
65
  def open_position(conn, *, book: str, mint: str, pool: str | None, qty_raw: int, cost_sol: float,
66
  entry_price: float, decision_id: int | None, fees_sol: float, ts: datetime | None = None,
67
  hold_s: float | None = None, strategy: str | None = None) -> int:
 
77
  return int(existing["id"])
78
  row = conn.execute(
79
  """INSERT INTO positions (book, mint, pool, opened_at, entry_decision_id, qty, cost_sol, entry_price, peak_price,
80
+ last_mark_price, last_mark_ts, fees_sol, status, last_swap_ts, hold_s, strategy, chain)
81
+ VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,'open',%s,%s,%s,%s) RETURNING id""",
82
+ (book, mint, pool, ts, decision_id, qty_raw, cost_sol, entry_price, entry_price, entry_price, ts, fees_sol, ts, hold_s, strategy, _chain_of(book)),
83
  ).fetchone()
84
  return int(row["id"])
85
 
 
134
 
135
  def verify_fills() -> None:
136
  """Reconcile live positions against on-chain token balances; prints mismatches, writes wallet_events."""
 
137
  from ..chain.rpc import HttpSolanaRpc
138
  from ..db.connection import transaction
139
+ from ..signer import client
140
  rpc = HttpSolanaRpc()
141
+ pub = client.get().call("pubkeys")["trading"]
142
  accounts = rpc.get_token_accounts_by_owner(pub)
143
  onchain = {a["mint"]: int(a["amount"]) for a in accounts}
144
  with transaction() as conn:
fly_trader/kalshi/__init__.py ADDED
@@ -0,0 +1 @@
 
 
1
+ """Kalshi prediction markets: the fly's second trading type, on the optic lobes (see the plan of 2026-09-22)."""
fly_trader/kalshi/client.py ADDED
@@ -0,0 +1,342 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """fly-trader's Kalshi REST client: one persistent, authenticated SDK client on a private event-loop thread, paced under
2
+ the account's rate tier, every call logged to ``api_calls`` (service 'kalshi'; the key never appears).
3
+
4
+ The vendored better_bot client (``kalshi/vendor/kalshi_client.py``) opens and closes an SDK client per call, which is
5
+ right for a daily batch and wrong for a history pull of tens of thousands of candlestick requests or a trading loop.
6
+ This wrapper keeps the same authentication (``Configuration.api_key_id`` / ``private_key_pem``, RSA-PSS by the SDK) and
7
+ the same ``call_api`` transport, adds the endpoints better_bot never needed — subaccounts, resting (GTC, post-only)
8
+ orders, cancel/amend, the historical tier, series and events — and passes ``KALSHI_SUBACCOUNT`` on every portfolio and
9
+ order call. Prices cross this boundary in cents (ints) and contracts as floats; the exchange's fixed-point dollar
10
+ strings are converted here and nowhere else. Paths and body fields verified against kalshi-python-async 3.10 on
11
+ 2026-09-22 (``docs.kalshi.com``): V2 orders are ``POST /portfolio/orders`` with ``yes_price``/``no_price`` in cents.
12
+ """
13
+ from __future__ import annotations
14
+
15
+ import asyncio
16
+ import json
17
+ import logging
18
+ import threading
19
+ import time
20
+ import uuid
21
+ from urllib.parse import urlencode
22
+
23
+ from .. import config
24
+ from ..chain.jupiter_tokens import TokenBucket
25
+ from ..db.apilog import record_api_call
26
+ from ..logging_setup import scrub
27
+
28
+ log = logging.getLogger(__name__)
29
+ RETRIES_429 = 5
30
+
31
+
32
+ class KalshiApiError(RuntimeError):
33
+ def __init__(self, method: str, path: str, status: int | None, body):
34
+ self.method, self.path, self.status, self.body = method, path, status, body
35
+ err = (body or {}).get("error") if isinstance(body, dict) else None
36
+ msg = (err.get("message") if isinstance(err, dict) else None) or (body.get("message") if isinstance(body, dict) else None) or str(body)[:200]
37
+ super().__init__(f"kalshi {method} {path}: HTTP {status}: {scrub(str(msg))}")
38
+
39
+
40
+ def _f(x) -> float | None:
41
+ try:
42
+ return float(x) if x not in (None, "") else None
43
+ except (TypeError, ValueError):
44
+ return None
45
+
46
+
47
+ def cents(dollars) -> int | None:
48
+ """A fixed-point dollar string ('0.5600') as whole cents; None when absent."""
49
+ v = _f(dollars)
50
+ return int(round(v * 100)) if v is not None else None
51
+
52
+
53
+ def _summary(params: dict | None, body: dict | None) -> dict | None:
54
+ out = {}
55
+ if params:
56
+ out["params"] = {k: v for k, v in params.items()}
57
+ if body:
58
+ out["body"] = {k: v for k, v in body.items() if k not in ("signedTransaction",)}
59
+ return out or None
60
+
61
+
62
+ class KalshiRest:
63
+ """Synchronous facade over the async SDK client (one per process is enough: MPS and the minute loop are single-threaded)."""
64
+
65
+ def __init__(self, rps: float | None = None, subaccount: int | None = None, base_url: str | None = None):
66
+ self.base = (base_url or config.KALSHI_API_BASE_URL).rstrip("/")
67
+ self.subaccount = config.KALSHI_SUBACCOUNT if subaccount is None else int(subaccount)
68
+ self.bucket = TokenBucket(rps or config.KALSHI_RPS)
69
+ self._loop: asyncio.AbstractEventLoop | None = None
70
+ self._thread: threading.Thread | None = None
71
+ self._client = None
72
+ self._lock = threading.Lock()
73
+
74
+ # ---- lifecycle ----
75
+ def _ensure(self):
76
+ with self._lock:
77
+ if self._client is not None:
78
+ return self._client
79
+ if not config.KALSHI_API_KEY_ID or not config.KALSHI_PRIVATE_KEY_PATH:
80
+ raise RuntimeError("KALSHI_API_KEY_ID and KALSHI_PRIVATE_KEY_PATH are not set")
81
+ from kalshi_python_async import Configuration, KalshiClient
82
+ cfg = Configuration(); cfg.host = self.base
83
+ cfg.api_key_id = config.KALSHI_API_KEY_ID
84
+ with open(config.KALSHI_PRIVATE_KEY_PATH) as f:
85
+ cfg.private_key_pem = f.read()
86
+ try:
87
+ import certifi
88
+ cfg.ssl_ca_cert = certifi.where()
89
+ except ImportError:
90
+ pass
91
+ self._loop = asyncio.new_event_loop()
92
+ self._thread = threading.Thread(target=self._loop.run_forever, name=f"{threading.current_thread().name}-kalshi-io", daemon=True)
93
+ self._thread.start()
94
+ self._client = asyncio.run_coroutine_threadsafe(self._make(cfg, KalshiClient), self._loop).result(30)
95
+ return self._client
96
+
97
+ @staticmethod
98
+ async def _make(cfg, KalshiClient):
99
+ return KalshiClient(configuration=cfg)
100
+
101
+ def close(self) -> None:
102
+ with self._lock:
103
+ if self._client is not None and self._loop is not None:
104
+ try:
105
+ asyncio.run_coroutine_threadsafe(self._client.close(), self._loop).result(10)
106
+ except Exception:
107
+ pass
108
+ self._loop.call_soon_threadsafe(self._loop.stop)
109
+ self._client = None
110
+
111
+ # ---- transport ----
112
+ async def _call(self, method: str, url: str, body: dict | None):
113
+ raw = await self._client.call_api(method, url, body=body, header_params={"Content-Type": "application/json"} if body is not None else None)
114
+ data = await raw.read()
115
+ text = data.decode("utf-8", "replace") if isinstance(data, (bytes, bytearray)) else str(data)
116
+ start = text.find("{")
117
+ try:
118
+ parsed = json.loads(text[start:]) if start >= 0 else {}
119
+ except ValueError:
120
+ parsed = {"raw": text[:300]}
121
+ return int(raw.status), parsed
122
+
123
+ def request(self, method: str, path: str, params: dict | None = None, body: dict | None = None, timeout: float = 60.0) -> dict:
124
+ """One paced, logged call. Raises ``KalshiApiError`` on HTTP >= 400 (429 is retried with backoff first)."""
125
+ client = self._ensure()
126
+ q = {k: v for k, v in (params or {}).items() if v is not None}
127
+ url = self.base + path + (("?" + urlencode(q)) if q else "")
128
+ t0 = time.monotonic(); status = None; err = None; parsed = None
129
+ try:
130
+ for attempt in range(RETRIES_429 + 1):
131
+ self.bucket.acquire()
132
+ status, parsed = asyncio.run_coroutine_threadsafe(self._call(method, url, body), self._loop).result(timeout)
133
+ if status == 429 and attempt < RETRIES_429:
134
+ time.sleep(1.0 + attempt)
135
+ continue
136
+ break
137
+ if status >= 400:
138
+ raise KalshiApiError(method, path, status, parsed)
139
+ return parsed
140
+ except KalshiApiError as e:
141
+ err = str(e); raise
142
+ except Exception as e:
143
+ err = f"{type(e).__name__}: {scrub(str(e))}"; raise
144
+ finally:
145
+ record_api_call("kalshi", path, method, status, int((time.monotonic() - t0) * 1000), err is None, err,
146
+ request=_summary(q, body), response_bytes=len(json.dumps(parsed)) if parsed is not None else None)
147
+
148
+ def get(self, path: str, **params) -> dict:
149
+ return self.request("GET", path, params)
150
+
151
+ def pages(self, path: str, key: str, params: dict | None = None, limit: int = 1000, max_pages: int = 10_000, cursor: str | None = None, with_cursor: bool = False):
152
+ """Cursor walk: yields each page's ``key`` list until the cursor ends or a page is empty. ``cursor`` resumes a walk
153
+ from a page cursor seen before; ``with_cursor`` yields ``(items, next_cursor)`` so the caller can persist its place."""
154
+ for _ in range(max_pages):
155
+ data = self.request("GET", path, {**(params or {}), "limit": limit, "cursor": cursor})
156
+ items = data.get(key) or []; nxt = data.get("cursor") or None
157
+ if items:
158
+ yield (items, nxt) if with_cursor else items
159
+ cursor = nxt
160
+ if not cursor or not items:
161
+ break
162
+
163
+ # ---- exchange / catalogue ----
164
+ def exchange_status(self) -> dict:
165
+ return self.get("/exchange/status")
166
+
167
+ def markets(self, cursor: str | None = None, with_cursor: bool = False, **params):
168
+ """Pages of market rows (``status`` one of unopened|open|closed|settled|...; ``min_settled_ts`` etc.)."""
169
+ yield from self.pages("/markets", "markets", params, cursor=cursor, with_cursor=with_cursor)
170
+
171
+ def historical_markets(self, cursor: str | None = None, with_cursor: bool = False, **params):
172
+ yield from self.pages("/historical/markets", "markets", params, cursor=cursor, with_cursor=with_cursor)
173
+
174
+ def historical_cutoff(self) -> dict:
175
+ return self.get("/historical/cutoff")
176
+
177
+ def market(self, ticker: str) -> dict:
178
+ return (self.get(f"/markets/{ticker}") or {}).get("market") or {}
179
+
180
+ def historical_market(self, ticker: str) -> dict:
181
+ return (self.get(f"/historical/markets/{ticker}") or {}).get("market") or {}
182
+
183
+ def events(self, **params):
184
+ """Pages of events (``with_nested_markets=true`` nests their markets); limit is capped at 200 by the exchange."""
185
+ yield from self.pages("/events", "events", params, limit=200)
186
+
187
+ def event(self, event_ticker: str, with_nested_markets: bool = True) -> dict:
188
+ return self.get(f"/events/{event_ticker}", with_nested_markets=str(with_nested_markets).lower())
189
+
190
+ def series_list(self, category: str | None = None, **params) -> list[dict]:
191
+ return (self.get("/series", category=category, **params) or {}).get("series") or []
192
+
193
+ def series(self, series_ticker: str) -> dict:
194
+ return (self.get(f"/series/{series_ticker}") or {}).get("series") or {}
195
+
196
+ def orderbook(self, ticker: str, depth: int | None = None) -> dict:
197
+ return self.get(f"/markets/{ticker}/orderbook", depth=depth)
198
+
199
+ def candlesticks(self, series_ticker: str, ticker: str, start_ts: int, end_ts: int, period_interval: int = 1) -> list[dict]:
200
+ """1/60/1440-minute candles; a market archived past the historical cutoff answers on the historical path."""
201
+ params = {"start_ts": int(start_ts), "end_ts": int(end_ts), "period_interval": int(period_interval)}
202
+ try:
203
+ return (self.request("GET", f"/series/{series_ticker}/markets/{ticker}/candlesticks", params) or {}).get("candlesticks") or []
204
+ except KalshiApiError as e:
205
+ if e.status != 404:
206
+ raise
207
+ cs = (self.request("GET", f"/historical/markets/{ticker}/candlesticks", params) or {}).get("candlesticks") or []
208
+ return [{**c, "_units": "dollars"} for c in cs] # the archive sends prices as dollar strings under plain keys ("close": "0.0100")
209
+
210
+ def trades(self, ticker: str, min_ts: int | None = None, max_ts: int | None = None, historical: bool = False):
211
+ yield from self.pages("/historical/trades" if historical else "/markets/trades", "trades", {"ticker": ticker, "min_ts": min_ts, "max_ts": max_ts})
212
+
213
+ # ---- portfolio (always on this process's subaccount unless told otherwise) ----
214
+ def _sub(self, subaccount) -> int:
215
+ return self.subaccount if subaccount is None else int(subaccount)
216
+
217
+ def balance(self, subaccount: int | None = None) -> dict:
218
+ """``{"balance_cents", "portfolio_value_cents"}`` of one subaccount (0 = primary)."""
219
+ d = self.get("/portfolio/balance", subaccount=self._sub(subaccount))
220
+ return {"balance_cents": int(d.get("balance") or 0), "portfolio_value_cents": int(d.get("portfolio_value") or 0), "updated_ts": d.get("updated_ts")}
221
+
222
+ def subaccount_balances(self) -> list[dict]:
223
+ return (self.get("/portfolio/subaccounts/balances") or {}).get("subaccount_balances") or []
224
+
225
+ def create_subaccount(self) -> int:
226
+ d = self.request("POST", "/portfolio/subaccounts", body={})
227
+ return int(d["subaccount_number"])
228
+
229
+ def transfer_between_subaccounts(self, from_subaccount: int, to_subaccount: int, amount_cents: int) -> dict:
230
+ body = {"client_transfer_id": str(uuid.uuid4()), "from_subaccount": int(from_subaccount), "to_subaccount": int(to_subaccount), "amount_cents": int(amount_cents)}
231
+ return self.request("POST", "/portfolio/subaccounts/transfer", body=body)
232
+
233
+ def positions(self, subaccount: int | None = None, **params) -> list[dict]:
234
+ out = []
235
+ for page in self.pages("/portfolio/positions", "market_positions", {**params, "subaccount": self._sub(subaccount)}, limit=1000):
236
+ out.extend(page)
237
+ return out
238
+
239
+ def orders(self, status: str | None = None, subaccount: int | None = None, **params) -> list[dict]:
240
+ out = []
241
+ for page in self.pages("/portfolio/orders", "orders", {**params, "status": status, "subaccount": self._sub(subaccount)}, limit=1000):
242
+ out.extend(page)
243
+ return out
244
+
245
+ def order(self, order_id: str) -> dict:
246
+ return (self.get(f"/portfolio/orders/{order_id}") or {}).get("order") or {}
247
+
248
+ def fills(self, subaccount: int | None = None, **params) -> list[dict]:
249
+ out = []
250
+ for page in self.pages("/portfolio/fills", "fills", {**params, "subaccount": self._sub(subaccount)}, limit=1000):
251
+ out.extend(page)
252
+ return out
253
+
254
+ def settlements(self, subaccount: int | None = None, **params) -> list[dict]:
255
+ out = []
256
+ for page in self.pages("/portfolio/settlements", "settlements", {**params, "subaccount": self._sub(subaccount)}, limit=1000):
257
+ out.extend(page)
258
+ return out
259
+
260
+ # ---- orders ----
261
+ def create_order(self, ticker: str, side: str, price_cents: int, count: float, *, tif: str = "immediate_or_cancel", post_only: bool = False,
262
+ expiration_ts: int | None = None, client_order_id: str | None = None, subaccount: int | None = None, action: str = "buy") -> dict:
263
+ """Buy (or sell) ``count`` contracts of ``side`` ('yes'|'no') at ``price_cents`` each. ``tif``: immediate_or_cancel |
264
+ good_till_canceled | fill_or_kill; a resting order is GTC (+ ``post_only`` never crosses, ``expiration_ts`` ends it).
265
+ Returns the exchange's order object (``order_id``, ``status``, ``fill_count``/``remaining_count`` ...)."""
266
+ if side not in ("yes", "no"):
267
+ raise ValueError(f"side must be yes|no, got {side!r}")
268
+ if not 1 <= int(price_cents) <= 99:
269
+ raise ValueError(f"price {price_cents} outside 1..99 cents")
270
+ body = {"ticker": ticker, "client_order_id": client_order_id or f"fly-{uuid.uuid4().hex[:16]}", "side": side, "action": action,
271
+ "count_fp": f"{float(count):.2f}", ("yes_price" if side == "yes" else "no_price"): int(price_cents), "time_in_force": tif,
272
+ "post_only": bool(post_only), "self_trade_prevention_type": "maker" if post_only else "taker_at_cross",
273
+ "subaccount": self._sub(subaccount)}
274
+ if expiration_ts is not None:
275
+ body["expiration_ts"] = int(expiration_ts)
276
+ d = self.request("POST", "/portfolio/orders", body=body)
277
+ return d.get("order") or d
278
+
279
+ def cancel_order(self, order_id: str, subaccount: int | None = None) -> dict:
280
+ d = self.request("DELETE", f"/portfolio/orders/{order_id}", {"subaccount": self._sub(subaccount)})
281
+ return d.get("order") or d
282
+
283
+ def amend_order(self, order_id: str, ticker: str, side: str, price_cents: int, count: float, client_order_id: str, subaccount: int | None = None,
284
+ action: str = "buy") -> dict:
285
+ body = {"ticker": ticker, "side": side, "action": action, "client_order_id": client_order_id, "updated_client_order_id": f"fly-{uuid.uuid4().hex[:16]}",
286
+ ("yes_price" if side == "yes" else "no_price"): int(price_cents), "count_fp": f"{float(count):.2f}", "subaccount": self._sub(subaccount)}
287
+ d = self.request("POST", f"/portfolio/orders/{order_id}/amend", body=body)
288
+ return d.get("order") or d
289
+
290
+
291
+ _REST: KalshiRest | None = None
292
+ _REST_LOCK = threading.Lock()
293
+
294
+
295
+ def rest() -> KalshiRest:
296
+ """The process-wide client (built on first use)."""
297
+ global _REST
298
+ with _REST_LOCK:
299
+ if _REST is None:
300
+ _REST = KalshiRest()
301
+ return _REST
302
+
303
+
304
+ # ---- operator commands (fly-trader kalshi-subaccount / kalshi-fund / kalshi-status) ----
305
+ def subaccount_create(write_env: bool = True) -> int:
306
+ """Create the dedicated subaccount and record it in .env (``KALSHI_SUBACCOUNT``); refuses when one is already set."""
307
+ import re
308
+ from ..db.apilog import record_event
309
+ if config.KALSHI_SUBACCOUNT > 0:
310
+ raise RuntimeError(f"KALSHI_SUBACCOUNT={config.KALSHI_SUBACCOUNT} is already set; refusing to create another")
311
+ n = KalshiRest(subaccount=0).create_subaccount()
312
+ if write_env:
313
+ path = config.REPO_ROOT / ".env"; text = path.read_text() if path.exists() else ""
314
+ text = re.sub(r"^KALSHI_SUBACCOUNT=.*$", f"KALSHI_SUBACCOUNT={n}", text, flags=re.M) if re.search(r"^KALSHI_SUBACCOUNT=", text, re.M) else text.rstrip("\n") + f"\nKALSHI_SUBACCOUNT={n}\n"
315
+ path.write_text(text)
316
+ record_event("info", "kalshi", f"subaccount {n} created", {"subaccount": n})
317
+ return n
318
+
319
+
320
+ def fund(usd: float, to_subaccount: int | None = None) -> dict:
321
+ """Move ``usd`` from the primary account into the fly's subaccount."""
322
+ from ..db.apilog import record_event
323
+ to = config.KALSHI_SUBACCOUNT if to_subaccount is None else int(to_subaccount)
324
+ if to <= 0:
325
+ raise RuntimeError("no subaccount: run fly-trader kalshi-subaccount create first")
326
+ out = KalshiRest(subaccount=0).transfer_between_subaccounts(0, to, int(round(usd * 100)))
327
+ record_event("info", "kalshi", f"funded subaccount {to} with ${usd:.2f}", {"transfer": out})
328
+ return out
329
+
330
+
331
+ def status() -> dict:
332
+ r = rest()
333
+ out = {"exchange": r.exchange_status(), "subaccount": r.subaccount, "balances": r.subaccount_balances(),
334
+ "prerequisites_missing": config.kalshi_live_prerequisites_missing()}
335
+ try:
336
+ out["balance"] = r.balance()
337
+ out["resting_orders"] = r.orders(status="resting")
338
+ out["positions"] = [p for p in r.positions() if _f(p.get("position_fp") or p.get("position")) not in (None, 0.0)]
339
+ out["settlements_24h"] = r.settlements(min_ts=int(time.time()) - 86400)
340
+ except KalshiApiError as e:
341
+ out["error"] = str(e)
342
+ return out
fly_trader/kalshi/data.py ADDED
@@ -0,0 +1,164 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """The Kalshi corpus on disk and the catalogue rows both the history puller and the live stream write.
2
+
3
+ Layout (``config.KALSHI_DIR``):
4
+ - ``candles/<ticker>.parquet`` — 1-minute candles (``CANDLE_SCHEMA``: period end, YES bid/ask OHLC in cents, trade price
5
+ OHLC when traded, contracts traded in the period, open interest at its end);
6
+ - ``trades/<ticker>.parquet`` — public trades (``TRADE_SCHEMA``: time, YES price in cents, contracts, the taker's side);
7
+ - ``features/<day>/part.parquet`` — the daily feature rows (kalshi/mature.py).
8
+ Rows of ``kalshi_markets`` / ``kalshi_events`` / ``kalshi_series`` are upserted from the exchange's JSON by
9
+ ``upsert_market`` and friends; prices are kept in cents, contracts as floats, times as UTC timestamps.
10
+ """
11
+ from __future__ import annotations
12
+
13
+ import json
14
+ from datetime import datetime, timezone
15
+ from pathlib import Path
16
+
17
+ import pyarrow as pa
18
+
19
+ from .. import config
20
+
21
+ CANDLES_DIR = config.KALSHI_DIR / "candles"
22
+ TRADES_DIR = config.KALSHI_DIR / "trades"
23
+ FEATURES_DIR = config.KALSHI_DIR / "features"
24
+ CANDLE_SCHEMA = pa.schema([("end_ts", pa.int64()), ("yes_bid_open", pa.float32()), ("yes_bid_high", pa.float32()), ("yes_bid_low", pa.float32()),
25
+ ("yes_bid_close", pa.float32()), ("yes_ask_open", pa.float32()), ("yes_ask_high", pa.float32()), ("yes_ask_low", pa.float32()),
26
+ ("yes_ask_close", pa.float32()), ("price_open", pa.float32()), ("price_high", pa.float32()), ("price_low", pa.float32()),
27
+ ("price_close", pa.float32()), ("price_mean", pa.float32()), ("volume", pa.float64()), ("open_interest", pa.float64())])
28
+ TRADE_SCHEMA = pa.schema([("ts", pa.timestamp("ms", tz="UTC")), ("yes_price", pa.int16()), ("count", pa.float64()), ("taker_side", pa.string()), ("is_block", pa.bool_())])
29
+ # discovery categories the exchange uses, folded into the fixed set the features one-hot (kalshi/features.CATEGORIES)
30
+ CATEGORY_KEYS = (("politic", "politics"), ("election", "politics"), ("sport", "sports"), ("crypto", "crypto"), ("bitcoin", "crypto"), ("econom", "economics"),
31
+ ("financ", "economics"), ("weather", "weather"), ("climate", "weather"), ("entertain", "entertainment"), ("culture", "entertainment"),
32
+ ("science", "science"), ("tech", "science"), ("compan", "companies"), ("world", "world"), ("health", "health"))
33
+
34
+
35
+ def category_key(raw: str | None) -> str:
36
+ s = raw.lower() if isinstance(raw, str) else "" # None or NaN (a market whose series is not catalogued yet): 'other'
37
+ for needle, key in CATEGORY_KEYS:
38
+ if needle in s:
39
+ return key
40
+ return "other"
41
+
42
+
43
+ def ts(v) -> datetime | None:
44
+ """An ISO-8601 string (or epoch seconds) from the exchange as an aware UTC datetime."""
45
+ if v in (None, ""):
46
+ return None
47
+ if isinstance(v, (int, float)):
48
+ return datetime.fromtimestamp(float(v), timezone.utc)
49
+ s = str(v).replace("Z", "+00:00")
50
+ try:
51
+ d = datetime.fromisoformat(s)
52
+ except ValueError:
53
+ return None
54
+ return d if d.tzinfo else d.replace(tzinfo=timezone.utc)
55
+
56
+
57
+ def _f(v) -> float | None:
58
+ try:
59
+ return float(v) if v not in (None, "") else None
60
+ except (TypeError, ValueError):
61
+ return None
62
+
63
+
64
+ def price_cents(row: dict, key: str) -> float | None:
65
+ """``<key>_dollars`` (fixed point) or ``<key>`` (cents) of a market/ticker row, as cents."""
66
+ d = _f(row.get(f"{key}_dollars"))
67
+ if d is not None:
68
+ return d * 100.0
69
+ return _f(row.get(key))
70
+
71
+
72
+ def series_ticker_of(ticker: str | None) -> str | None:
73
+ return str(ticker).split("-")[0] if ticker else None
74
+
75
+
76
+ MARKET_COLS = ("ticker", "event_ticker", "market_type", "title", "yes_sub_title", "no_sub_title", "status", "open_time", "close_time",
77
+ "expected_expiration_time", "latest_expiration_time", "settlement_ts", "result", "settlement_value", "strike_type", "floor_strike",
78
+ "cap_strike", "exchange_index", "price_level_structure", "is_provisional", "can_close_early", "early_close_condition", "created_time", "source", "raw")
79
+
80
+
81
+ def market_row(m: dict, source: str) -> dict:
82
+ return {"ticker": m.get("ticker"), "event_ticker": m.get("event_ticker"), "market_type": m.get("market_type"), "title": m.get("title"),
83
+ "yes_sub_title": m.get("yes_sub_title"), "no_sub_title": m.get("no_sub_title"), "status": m.get("status"), "open_time": ts(m.get("open_time")),
84
+ "close_time": ts(m.get("close_time")), "expected_expiration_time": ts(m.get("expected_expiration_time")),
85
+ "latest_expiration_time": ts(m.get("latest_expiration_time")), "settlement_ts": ts(m.get("settlement_ts") or m.get("settled_time")),
86
+ "result": m.get("result") or None, "settlement_value": _f(m.get("settlement_value_dollars")) if m.get("settlement_value_dollars") is not None else _f(m.get("settlement_value")),
87
+ "strike_type": m.get("strike_type"), "floor_strike": _f(m.get("floor_strike")), "cap_strike": _f(m.get("cap_strike")),
88
+ "exchange_index": m.get("exchange_index"), "price_level_structure": m.get("price_level_structure"), "is_provisional": m.get("is_provisional"),
89
+ "can_close_early": m.get("can_close_early"), "early_close_condition": m.get("early_close_condition") or None, "created_time": ts(m.get("created_time")), "source": source,
90
+ "raw": json.dumps({k: v for k, v in m.items() if k not in ("rules_primary", "rules_secondary")}, default=str)}
91
+
92
+
93
+ def upsert_markets(conn, markets: list[dict], source: str) -> int:
94
+ rows = [market_row(m, source) for m in markets if m.get("ticker")]
95
+ if not rows:
96
+ return 0
97
+ cols = MARKET_COLS
98
+ sets = ", ".join(f"{c} = COALESCE(EXCLUDED.{c}, kalshi_markets.{c})" for c in cols if c not in ("ticker", "source", "raw"))
99
+ conn.cursor().executemany(
100
+ f"INSERT INTO kalshi_markets ({', '.join(cols)}) VALUES ({', '.join(['%s'] * len(cols))}) ON CONFLICT (ticker) DO UPDATE SET {sets}, "
101
+ "source = EXCLUDED.source, raw = EXCLUDED.raw, updated_at = now()", [tuple(r[c] for c in cols) for r in rows])
102
+ return len(rows)
103
+
104
+
105
+ def upsert_event(conn, e: dict) -> None:
106
+ if not e.get("event_ticker"):
107
+ return
108
+ conn.execute("INSERT INTO kalshi_events (event_ticker, series_ticker, title, sub_title, category, mutually_exclusive, strike_date, strike_period, "
109
+ "collateral_return_type, exchange_index, raw) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT (event_ticker) DO UPDATE SET "
110
+ "series_ticker = COALESCE(EXCLUDED.series_ticker, kalshi_events.series_ticker), title = COALESCE(EXCLUDED.title, kalshi_events.title), "
111
+ "sub_title = COALESCE(EXCLUDED.sub_title, kalshi_events.sub_title), category = COALESCE(EXCLUDED.category, kalshi_events.category), "
112
+ "mutually_exclusive = COALESCE(EXCLUDED.mutually_exclusive, kalshi_events.mutually_exclusive), strike_date = COALESCE(EXCLUDED.strike_date, kalshi_events.strike_date), "
113
+ "strike_period = COALESCE(EXCLUDED.strike_period, kalshi_events.strike_period), collateral_return_type = COALESCE(EXCLUDED.collateral_return_type, kalshi_events.collateral_return_type), "
114
+ "exchange_index = COALESCE(EXCLUDED.exchange_index, kalshi_events.exchange_index), raw = EXCLUDED.raw, updated_at = now()",
115
+ (e["event_ticker"], e.get("series_ticker") or series_ticker_of(e["event_ticker"]), e.get("title"), e.get("sub_title"), e.get("category"),
116
+ e.get("mutually_exclusive"), ts(e.get("strike_date")), e.get("strike_period"), e.get("collateral_return_type"), e.get("exchange_index"),
117
+ json.dumps({k: v for k, v in e.items() if k != "markets"}, default=str)))
118
+
119
+
120
+ def upsert_series(conn, s: dict) -> None:
121
+ if not s.get("ticker"):
122
+ return
123
+ conn.execute("INSERT INTO kalshi_series (ticker, title, category, categories, tags, frequency, fee_type, fee_multiplier, settlement_sources) "
124
+ "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT (ticker) DO UPDATE SET title = EXCLUDED.title, category = EXCLUDED.category, "
125
+ "categories = EXCLUDED.categories, tags = EXCLUDED.tags, frequency = EXCLUDED.frequency, fee_type = EXCLUDED.fee_type, "
126
+ "fee_multiplier = EXCLUDED.fee_multiplier, settlement_sources = EXCLUDED.settlement_sources, updated_at = now()",
127
+ (s["ticker"], s.get("title"), s.get("category"), list(s.get("categories") or []), list(s.get("tags") or []), s.get("frequency"),
128
+ s.get("fee_type"), _f(s.get("fee_multiplier")), json.dumps(s.get("settlement_sources") or [], default=str)))
129
+
130
+
131
+ def candle_rows(candles: list[dict]) -> list[dict]:
132
+ """Exchange candlesticks → ``CANDLE_SCHEMA`` rows (cents; missing trade prices stay NaN)."""
133
+ out = []
134
+ for c in candles:
135
+ yb, ya, pr = c.get("yes_bid") or {}, c.get("yes_ask") or {}, c.get("price") or {}
136
+ plain = 100.0 if c.get("_units") == "dollars" else 1.0 # live: <k>_dollars or <k> in cents; archive: <k> in dollars
137
+
138
+ def q(d, k, plain=plain):
139
+ v = _f(d.get(f"{k}_dollars"))
140
+ return v * 100.0 if v is not None else (_f(d.get(k)) * plain if d.get(k) is not None and _f(d.get(k)) is not None else float("nan"))
141
+ out.append({"end_ts": int(c.get("end_period_ts") or 0),
142
+ "yes_bid_open": q(yb, "open"), "yes_bid_high": q(yb, "high"), "yes_bid_low": q(yb, "low"), "yes_bid_close": q(yb, "close"),
143
+ "yes_ask_open": q(ya, "open"), "yes_ask_high": q(ya, "high"), "yes_ask_low": q(ya, "low"), "yes_ask_close": q(ya, "close"),
144
+ "price_open": q(pr, "open"), "price_high": q(pr, "high"), "price_low": q(pr, "low"), "price_close": q(pr, "close"), "price_mean": q(pr, "mean"),
145
+ "volume": _f(c.get("volume_fp") or c.get("volume")) or 0.0, "open_interest": _f(c.get("open_interest_fp") or c.get("open_interest")) or 0.0})
146
+ return out
147
+
148
+
149
+ def trade_rows(trades: list[dict]) -> list[dict]:
150
+ out = []
151
+ for t in trades:
152
+ p = price_cents(t, "yes_price")
153
+ if p is None:
154
+ continue
155
+ out.append({"ts": ts(t.get("created_time")), "yes_price": int(round(p)), "count": _f(t.get("count_fp") or t.get("count")) or 0.0,
156
+ "taker_side": t.get("taker_outcome_side") or t.get("taker_side") or "", "is_block": bool(t.get("is_block_trade"))})
157
+ return [r for r in out if r["ts"] is not None]
158
+
159
+
160
+ def write_parquet(rows: list[dict], schema: pa.Schema, path: Path) -> None:
161
+ import pyarrow.parquet as pq
162
+ path.parent.mkdir(parents=True, exist_ok=True)
163
+ tmp = path.with_name(path.name + ".tmp")
164
+ pq.write_table(pa.Table.from_pylist(rows, schema=schema), tmp, compression="zstd"); tmp.replace(path)
fly_trader/kalshi/decisions.py ADDED
@@ -0,0 +1,149 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Decision points for the Kalshi selector and fly: one row per (market, minute, side) of the feature parts
2
+ (kalshi/mature.py), with the settlement labels both arms trade on.
3
+
4
+ Reuses ``train.decisions.DecisionSet`` unchanged: ``mint`` = ``"<ticker>:<side>"``, ``ts`` = minute start, ``day`` =
5
+ the minute's date, and — because every position is held to settlement — a per-row hold ``hold_s`` = settlement − ts
6
+ (``taken_idx`` / ``evaluate`` already take per-row holds). Labels, per dollar staked at the fee-inclusive price:
7
+ - ``fwd_pess`` (taker): ``(100·y − eff) / eff`` with ``eff = effective_price_cents(side ask)`` and ``y`` the settlement
8
+ of the side (1 when it paid);
9
+ - ``fwd`` (reference): the same at the side's mid;
10
+ - ``fwd_h["maker"]``: the maker arm's order (kalshi/maker.py): a bid resting one tick above the side's bid, never at or
11
+ through its ask (``maker_rest``), for the order's lifetime (``KALSHI_MAKER_TTL_H``, ending ``KALSHI_MAKER_QUIET_MIN``
12
+ before close); filled (``maker_filled``) when the side's ask came down to it — every bid at that price, ours included,
13
+ was taken — or a taker sold the side below it, or at it when our bid set a new best price and so stood first in its
14
+ queue; the maker fee where the series charges one; NaN when unfilled. Until feature version 5 the order rested one tick
15
+ inside the ask, for the market's whole life, and filled only when the ask later fell strictly below it: it paid away
16
+ the spread makers earn and counted only the fills that came with an adverse move (2026-10-09).
17
+ Eligibility (the live engine applies the same, ``kalshi/engine.KalshiMinuteEngine.eligible``): both quotes present,
18
+ spread ≤ KALSHI_MAX_SPREAD_CENTS, side ask in 1..99, open interest ≥ KALSHI_MIN_OPEN_INTEREST, 24 h volume ≥
19
+ KALSHI_MIN_VOLUME_24H, settlement yes/no, time to close inside the entry window.
20
+ """
21
+ from __future__ import annotations
22
+
23
+ import glob
24
+ import math
25
+ from datetime import date
26
+
27
+ import numpy as np
28
+ import pandas as pd
29
+ import pyarrow.parquet as pq
30
+
31
+ from .. import config
32
+ from ..train.corpus_features import _epoch_s
33
+ from ..train.decisions import DecisionSet
34
+ from . import data as D
35
+ from .features import K_COLS, KALSHI_FEATURE_VERSION, KIDX
36
+ from .mature import part_current
37
+ from .vendor import kalshi_client as kc
38
+
39
+ X_COLS = list(K_COLS)
40
+
41
+
42
+ def maker_rest(side_bid, side_ask) -> np.ndarray:
43
+ """The maker arm's bid (cents): one tick above the side's best bid, at most one tick below its ask (``post_only``
44
+ never crosses); with a one-tick spread it joins the bid's queue."""
45
+ return np.clip(np.minimum(np.round(np.asarray(side_bid, dtype=float)) + 1.0, np.round(np.asarray(side_ask, dtype=float)) - 1.0), 1.0, 99.0)
46
+
47
+
48
+ def maker_filled(rest, side_bid, ask_low, sold_low) -> np.ndarray:
49
+ """Would a bid at ``rest``, posted when the side's bid was ``side_bid``, have filled, given the lowest ask the side
50
+ showed (``ask_low``) and the lowest price a taker sold the side at (``sold_low``) while it rested (NaN: none). The ask
51
+ reaching ``rest`` means every bid at ``rest`` was taken; a taker sale below ``rest`` means the same; a sale at
52
+ ``rest`` reaches us only when we were first there — our bid improved on the best bid, so nobody was ahead of us."""
53
+ rest = np.asarray(rest, dtype=float); ask_low = np.asarray(ask_low, dtype=float); sold_low = np.asarray(sold_low, dtype=float)
54
+ first = rest > np.round(np.asarray(side_bid, dtype=float))
55
+ with np.errstate(invalid="ignore"):
56
+ return (np.isfinite(ask_low) & (ask_low <= rest)) | (np.isfinite(sold_low) & ((sold_low < rest) | (first & (sold_low <= rest))))
57
+
58
+
59
+ def maker_fee_cents(price_cents, fee_mult, charged) -> np.ndarray:
60
+ """Per-contract maker fee in cents where the series charges makers (KALSHI_MAKER_FEE_FACTOR·mult·p(1−p)·100), else 0."""
61
+ p = np.asarray(price_cents, dtype=float) / 100.0
62
+ return np.where(np.asarray(charged, dtype=float) > 0, config.KALSHI_MAKER_FEE_FACTOR * np.asarray(fee_mult, dtype=float) * p * (1 - p) * 100.0, 0.0)
63
+
64
+
65
+ def eligible_mask(X: np.ndarray, cols: list[str], to_close_s: np.ndarray) -> np.ndarray:
66
+ c = {n: i for i, n in enumerate(cols)}
67
+ return ((X[:, c["spread"]] <= config.KALSHI_MAX_SPREAD_CENTS) & (X[:, c["side_ask"]] >= 1.0) & (X[:, c["side_ask"]] <= 99.0)
68
+ & (X[:, c["side_bid"]] >= 1.0) & (np.expm1(X[:, c["log_oi"]]) >= config.KALSHI_MIN_OPEN_INTEREST)
69
+ & (np.expm1(X[:, c["log_vol_24h"]]) >= config.KALSHI_MIN_VOLUME_24H)
70
+ & (to_close_s >= config.KALSHI_MIN_MINUTES_TO_CLOSE * 60.0) & (to_close_s <= config.KALSHI_MAX_DAYS_TO_CLOSE * 86400.0))
71
+
72
+
73
+ def build(days: int | None = None, feature_dir=None, label_thr: float = 0.0) -> DecisionSet:
74
+ """The decision rows of the most recent ``days`` feature days (all of them when None). Each part is reduced to its
75
+ eligible rows as it is read — float32 inputs, labels, integer market ids — so the whole corpus (tens of millions of
76
+ rows) fits: reading every part into one pandas frame with its string columns peaked near three times the final size."""
77
+ root = feature_dir or D.FEATURES_DIR
78
+ dirs = sorted(glob.glob(str(root / "*")))
79
+ if days:
80
+ dirs = dirs[-days:]
81
+ files = [f for d in dirs for f in sorted(glob.glob(str(root / d.split("/")[-1] / "part-*.parquet")))]
82
+ if not files:
83
+ raise RuntimeError("no Kalshi feature parts; run kalshi-build")
84
+ stale = [f for f in files if not part_current(f)]
85
+ if stale:
86
+ raise RuntimeError(f"{len(stale)} Kalshi feature part(s) from another version (e.g. {stale[0]}); run kalshi-build")
87
+ need = list(dict.fromkeys([*X_COLS, "ticker", "side", "ts", "category", "close_ts", "end_ts", "settled_ts", "result", "fill_ask_low", "fill_sold_low"]))
88
+ tick_id: dict[str, int] = {}; tick_names: list[str] = []; cat_id: dict[str, int] = {}; cat_names: list[str] = []
89
+ acc: dict[str, list] = {k: [] for k in ("X", "ts", "settled", "y", "fwd", "fwd_pess", "maker", "tick", "side", "cat", "ord")}
90
+ for f in files:
91
+ t = pq.read_table(f, columns=need)
92
+ if not t.num_rows:
93
+ continue
94
+ res = np.asarray(t["result"].to_numpy(zero_copy_only=False), dtype=object); side = np.asarray(t["side"].to_numpy(zero_copy_only=False), dtype=object)
95
+ ok = (res == "yes") | (res == "no")
96
+ if not ok.any():
97
+ continue
98
+ X = np.column_stack([t[c].to_numpy(zero_copy_only=False).astype(np.float32) for c in X_COLS])
99
+ X[~np.isfinite(X)] = 0.0
100
+ ts = t["ts"].cast("int64").to_numpy().astype(np.float64) / 1000.0
101
+ close = t["close_ts"].to_numpy(zero_copy_only=False).astype(np.float64); st_ = t["settled_ts"].to_numpy(zero_copy_only=False).astype(np.float64)
102
+ end = t["end_ts"].to_numpy(zero_copy_only=False).astype(np.float64)
103
+ settled = np.where(np.isfinite(st_), st_, end); to_close = close - ts # window on the anchor; a label's timing on the realized end
104
+ y = (res == side).astype(np.float64)
105
+ eff = X[:, KIDX["eff_price"]].astype(np.float64); mid = np.clip(X[:, KIDX["side_mid"]].astype(np.float64), 1.0, 99.0)
106
+ fwd_pess = (100.0 * y - eff) / eff; fwd = (100.0 * y - mid) / mid
107
+ rest = maker_rest(X[:, KIDX["side_bid"]], X[:, KIDX["side_ask"]])
108
+ filled = maker_filled(rest, X[:, KIDX["side_bid"]], t["fill_ask_low"].to_numpy(zero_copy_only=False).astype(np.float64),
109
+ t["fill_sold_low"].to_numpy(zero_copy_only=False).astype(np.float64))
110
+ m_eff = rest + maker_fee_cents(rest, X[:, KIDX["fee_mult"]], X[:, KIDX["maker_fee"]])
111
+ maker = np.where(filled, (100.0 * y - m_eff) / m_eff, np.nan)
112
+ elig = ok & eligible_mask(X, X_COLS, to_close) & np.isfinite(fwd_pess) & (settled > ts)
113
+ if not elig.any():
114
+ continue
115
+ tk = t["ticker"].to_numpy(zero_copy_only=False)[elig]; cats = t["category"].to_numpy(zero_copy_only=False)[elig]
116
+ acc["X"].append(X[elig]); acc["ts"].append(ts[elig]); acc["settled"].append(settled[elig]); acc["y"].append(y[elig])
117
+ acc["fwd"].append(fwd[elig].astype(np.float32)); acc["fwd_pess"].append(fwd_pess[elig].astype(np.float32)); acc["maker"].append(maker[elig].astype(np.float32))
118
+ acc["tick"].append(np.fromiter((tick_id.setdefault(x, len(tick_id)) for x in tk), dtype=np.int64, count=len(tk)))
119
+ acc["side"].append((side[elig] == "yes").astype(np.int8))
120
+ acc["cat"].append(np.fromiter((cat_id.setdefault(x, len(cat_id)) for x in cats), dtype=np.int16, count=len(cats)))
121
+ acc["ord"].append((np.floor(ts[elig] / 86400.0) + 719163).astype(np.int32)) # date.fromordinal(719163) is 1970-01-01
122
+ del X, t
123
+ if not acc["X"]:
124
+ raise RuntimeError("no eligible Kalshi decision rows")
125
+ cat = lambda k: np.concatenate(acc.pop(k))
126
+ X = cat("X"); ts = cat("ts"); settled = cat("settled"); y = cat("y"); fwd = cat("fwd"); fwd_pess = cat("fwd_pess"); maker = cat("maker")
127
+ tick = cat("tick"); side = cat("side"); catc = cat("cat"); ords = cat("ord")
128
+ tick_names = np.array([None] * len(tick_id), dtype=object)
129
+ for k_, v in tick_id.items():
130
+ tick_names[v] = k_
131
+ cat_names = np.array([None] * len(cat_id), dtype=object)
132
+ for k_, v in cat_id.items():
133
+ cat_names[v] = k_
134
+ day_objs = {o: date.fromordinal(int(o)) for o in np.unique(ords)}
135
+ day_arr = np.array([day_objs[o] for o in sorted(day_objs)], dtype=object)[np.searchsorted(np.array(sorted(day_objs)), ords)]
136
+ ds = DecisionSet(X=X, y=(fwd_pess > label_thr).astype(np.int8), fwd=fwd, fwd_pess=fwd_pess, day=day_arr, ts=ts, mint=tick * 2 + side.astype(np.int64),
137
+ cols=list(X_COLS), horizon_s=float(np.median(settled - ts)), fwd_h={"maker": maker})
138
+ ds._day_ord = ords
139
+ ds.train_max_rows = int(config.KALSHI_TRAIN_MAX_ROWS) or None # training rows per fit: a random subset (train/strategies.cap_rows)
140
+ ds.hold_s = (settled - ts).astype(np.float64) # per-row: every position is held to its own settlement
141
+ ds.data_version = f"kalshi-features-{KALSHI_FEATURE_VERSION}" # in the walk-forward cache key: another feature version never reuses a fit
142
+ ds.label_ts = (settled + 60.0).astype(np.float64) # when the outcome is known: training for a period may use only rows known before it
143
+ ds.outcome = y.astype(np.float32)
144
+ ds.side = np.array(["no", "yes"], dtype=object)[side]; ds.ticker = tick_names[tick]; ds.category = cat_names[catc]
145
+ # the stack's bars (train/strategies.Score): a positive mean with a weekly sign test, not the memecoin PF ≥ 1.3 and 65 %
146
+ # winners — a contract bought at 95c that pays 5c when it wins needs a 1.3 % edge for PF 1.3, about all the edge the
147
+ # literature finds left on Kalshi, and its win rate is set by its price (operator decision, 2026-10-09)
148
+ ds.gate = "significance"
149
+ return ds