release 1 (7ab2355a0c)
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- .env.example +40 -5
- .gitattributes +1 -0
- CHANGELOG.md +131 -0
- Dockerfile +40 -12
- README.md +96 -33
- deploy/bootstrap-host.sh +433 -38
- deploy/check-host.sh +106 -0
- deploy/custody.env.example +26 -0
- deploy/docker-compose.vault.yml +55 -11
- deploy/fly +9 -0
- deploy/fly-aide.service +8 -0
- deploy/fly-aide.timer +9 -0
- deploy/fly-audit-watch.service +8 -0
- deploy/fly-audit-watch.timer +10 -0
- deploy/fly_audit_watch.py +109 -0
- deploy/fly_update.py +151 -51
- deploy/signer.env.example +4 -0
- deploy/update.json.example +6 -1
- deploy/vault.env.example +22 -12
- docker-compose.yml +7 -0
- docker/bootstrap.sh +42 -8
- docker/signer-requirements.txt +51 -0
- docs/vault/RUNBOOK.md +424 -109
- docs/vault/SPEC.md +233 -57
- fly_trader/__main__.py +159 -1
- fly_trader/agent/fly_live.py +16 -8
- fly_trader/agent/fly_session.py +280 -75
- fly_trader/agent/handover.py +12 -4
- fly_trader/agent/minute_engine.py +102 -45
- fly_trader/agent/paper_trading.py +8 -5
- fly_trader/agent/rails.py +65 -43
- fly_trader/agent/selector_session.py +45 -25
- fly_trader/agent/sizing.py +29 -20
- fly_trader/brain/connectome.py +27 -0
- fly_trader/brain/geometry.py +42 -3
- fly_trader/brain/pathways.py +4 -3
- fly_trader/brain/plastic.py +7 -1
- fly_trader/chain/cluster_guard.py +15 -0
- fly_trader/chain/jupiter_swap.py +1 -1
- fly_trader/chain/rpc.py +6 -0
- fly_trader/chain/withdraw.py +129 -0
- fly_trader/config.py +186 -8
- fly_trader/db/schema.py +198 -36
- fly_trader/execution/broker_live.py +46 -41
- fly_trader/execution/broker_paper.py +5 -3
- fly_trader/execution/ledger.py +14 -7
- fly_trader/kalshi/__init__.py +1 -0
- fly_trader/kalshi/client.py +342 -0
- fly_trader/kalshi/data.py +164 -0
- fly_trader/kalshi/decisions.py +149 -0
.env.example
CHANGED
|
@@ -2,10 +2,14 @@
|
|
| 2 |
# Keep values bare: Docker Compose keeps a trailing "# comment" as part of an empty value, so comments live on their own lines.
|
| 3 |
|
| 4 |
# --- keys (never logged) ---
|
| 5 |
-
# Helius: https://dashboard.helius.dev (RPC for balances and transactions)
|
| 6 |
HELIUS_API_KEY=
|
| 7 |
-
# Jupiter: https://portal.jup.ag (swaps and token stats)
|
| 8 |
JUPITER_API_KEY=
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9 |
|
| 10 |
# --- database (docker-compose.yml sets the container's URL; this one is for running outside Docker) ---
|
| 11 |
DATABASE_URL=postgresql:///fly_trader
|
|
@@ -15,7 +19,7 @@ DATABASE_URL=postgresql:///fly_trader
|
|
| 15 |
LIVE_ENABLED=1
|
| 16 |
# What you fund the wallet with; also the paper warm-up bankroll.
|
| 17 |
CAPITAL_SOL=0.5
|
| 18 |
-
#
|
| 19 |
MAX_POSITION_SOL=0.05
|
| 20 |
# Per position, of the bankroll minus the gas reserve.
|
| 21 |
MAX_POSITION_FRACTION=0.10
|
|
@@ -26,6 +30,13 @@ MIN_POSITION_SOL=0.02
|
|
| 26 |
# Never spent (fees and token-account rent).
|
| 27 |
GAS_RESERVE_SOL=0.05
|
| 28 |
KELLY_FRACTION=0.25
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 29 |
|
| 30 |
# --- rails ---
|
| 31 |
# 20 % below the wallet's peak: stop entering ...
|
|
@@ -41,18 +52,42 @@ HANDOVER_BEAT_SELECTOR=0
|
|
| 41 |
# A restart keeps the paper book, so the warm-up count survives it.
|
| 42 |
RESET_ON_START=0
|
| 43 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 44 |
# --- runtime ---
|
| 45 |
SOLANA_CLUSTER=mainnet-beta
|
| 46 |
# auto: an NVIDIA GPU when the container has one (docker-compose.cuda.yml), else the CPU. Or cpu | cuda | cuda:1.
|
| 47 |
DEVICE=auto
|
| 48 |
# Batch sizes are derived from the device's free memory; on a shared GPU cap what the fly may use, in GB.
|
| 49 |
# DEVICE_MEMORY_GB=
|
|
|
|
| 50 |
DATA_DIR=data
|
| 51 |
LOG_DIR=logs
|
| 52 |
|
| 53 |
# --- staying current ---
|
| 54 |
-
# Every UPDATE_INTERVAL_S the container asks Hugging Face whether HF_REPO has a new release
|
| 55 |
-
#
|
|
|
|
| 56 |
# rollback if the new one fails to start. AUTO_UPDATE=0 pins this install to what it was built from.
|
| 57 |
AUTO_UPDATE=1
|
| 58 |
HF_REPO=bryceweiner/fly-trader
|
|
|
|
| 2 |
# Keep values bare: Docker Compose keeps a trailing "# comment" as part of an empty value, so comments live on their own lines.
|
| 3 |
|
| 4 |
# --- keys (never logged) ---
|
| 5 |
+
# Helius: https://dashboard.helius.dev (RPC for balances and transactions; also the signer's, unless SIGNER_RPC_URL is set)
|
| 6 |
HELIUS_API_KEY=
|
| 7 |
+
# Jupiter: https://portal.jup.ag (swaps, through Jupiter's own Metis routes only, and token stats)
|
| 8 |
JUPITER_API_KEY=
|
| 9 |
+
# The bot key: `docker compose exec fly fly-trader wallet new` appends BOT_PRIVATE_KEY=... to this file. Do not add an
|
| 10 |
+
# empty BOT_PRIVATE_KEY= line yourself: wallet new refuses when the name is already here.
|
| 11 |
+
# Optional: a second Solana RPC from another provider for the signer, which simulates every swap before signing it.
|
| 12 |
+
# SIGNER_RPC_URL=
|
| 13 |
|
| 14 |
# --- database (docker-compose.yml sets the container's URL; this one is for running outside Docker) ---
|
| 15 |
DATABASE_URL=postgresql:///fly_trader
|
|
|
|
| 19 |
LIVE_ENABLED=1
|
| 20 |
# What you fund the wallet with; also the paper warm-up bankroll.
|
| 21 |
CAPITAL_SOL=0.5
|
| 22 |
+
# The fixed size for a model without a sizing table (the shipped fly has one); must be above 0 to trade live.
|
| 23 |
MAX_POSITION_SOL=0.05
|
| 24 |
# Per position, of the bankroll minus the gas reserve.
|
| 25 |
MAX_POSITION_FRACTION=0.10
|
|
|
|
| 30 |
# Never spent (fees and token-account rent).
|
| 31 |
GAS_RESERVE_SOL=0.05
|
| 32 |
KELLY_FRACTION=0.25
|
| 33 |
+
# The signer's own limits (defaults shown): SOL per buy, SOL of buys per day, swaps per minute (buys and sells), priority
|
| 34 |
+
# fee per swap, and the least value a swap may return for what it costs, at the signer's own Jupiter prices.
|
| 35 |
+
# SIGNER_MAX_BUY_SOL=1
|
| 36 |
+
# SIGNER_DAILY_BUY_SOL=10
|
| 37 |
+
# SIGNER_SWAPS_PER_MIN=6
|
| 38 |
+
# SIGNER_MAX_PRIORITY_SOL=0.005
|
| 39 |
+
# SIGNER_MIN_VALUE_RATIO=0.6
|
| 40 |
|
| 41 |
# --- rails ---
|
| 42 |
# 20 % below the wallet's peak: stop entering ...
|
|
|
|
| 52 |
# A restart keeps the paper book, so the warm-up count survives it.
|
| 53 |
RESET_ON_START=0
|
| 54 |
|
| 55 |
+
# --- Robinhood Chain memecoins (Pons launches graduated into Uniswap v4; money in ETH) ---
|
| 56 |
+
# Off by default. 1 = index Robinhood Chain (the rh_stream worker) and paper-trade it in ETH beside Solana. Indexing needs
|
| 57 |
+
# an Envio HyperSync token (ENVIO_API_TOKEN below; the free tier works, slower) -- the public RPC is far too slow for
|
| 58 |
+
# the backfill -- and disk for the history: ~30 GB from 2026-08-04. RH_START_DAY a ~40 days back indexes much less
|
| 59 |
+
# (the RH models need about a month of history per wallet); the RH books trade once the stream reaches the present.
|
| 60 |
+
RH_ENABLED=0
|
| 61 |
+
# ENVIO_API_TOKEN=
|
| 62 |
+
# RH_START_DAY=2026-08-04
|
| 63 |
+
# Real RH transactions stay off: the RH books trade paper until you set 1 and add the key and address below.
|
| 64 |
+
RH_LIVE_ENABLED=0
|
| 65 |
+
# 1 = the rehearsal chain (46630); the guard refuses any other chain id than the one selected here.
|
| 66 |
+
RH_TESTNET=0
|
| 67 |
+
# RPCs (defaults shown): logs from HyperSync (the official RPC when HyperSync fails), transactions from the official RPC,
|
| 68 |
+
# block headers from PublicNode. A QuickNode Robinhood Chain endpoint, if you have one, serves the transactions.
|
| 69 |
+
# RH_RPC_URL=https://rpc.mainnet.chain.robinhood.com
|
| 70 |
+
# RH_RPC_URL_BLOCKS=https://robinhood-rpc.publicnode.com
|
| 71 |
+
# QUICKNODE_RH_API_URL=
|
| 72 |
+
# The address RH_BOT_PRIVATE_KEY must derive to (a guard, not a secret). Needed only for live.
|
| 73 |
+
RH_BOT_ADDRESS=
|
| 74 |
+
# RH_BOT_PRIVATE_KEY (hex) goes in this .env only, on its own line, when you enable live. It is read from here at run
|
| 75 |
+
# time and never baked into the image. Do not leave an empty RH_BOT_PRIVATE_KEY= line.
|
| 76 |
+
|
| 77 |
# --- runtime ---
|
| 78 |
SOLANA_CLUSTER=mainnet-beta
|
| 79 |
# auto: an NVIDIA GPU when the container has one (docker-compose.cuda.yml), else the CPU. Or cpu | cuda | cuda:1.
|
| 80 |
DEVICE=auto
|
| 81 |
# Batch sizes are derived from the device's free memory; on a shared GPU cap what the fly may use, in GB.
|
| 82 |
# DEVICE_MEMORY_GB=
|
| 83 |
+
# docker-compose.yml sets these to /app/data and /app/logs, your ./data and ./logs
|
| 84 |
DATA_DIR=data
|
| 85 |
LOG_DIR=logs
|
| 86 |
|
| 87 |
# --- staying current ---
|
| 88 |
+
# Every UPDATE_INTERVAL_S the container asks Hugging Face whether HF_REPO has a new release. It installs one only if it is
|
| 89 |
+
# signed by a key in release/allowed_signers and newer than the installed one, and every file matches the signed
|
| 90 |
+
# manifest; then it restarts itself (about a minute; open positions are untouched) and keeps the previous release for a
|
| 91 |
# rollback if the new one fails to start. AUTO_UPDATE=0 pins this install to what it was built from.
|
| 92 |
AUTO_UPDATE=1
|
| 93 |
HF_REPO=bryceweiner/fly-trader
|
.gitattributes
CHANGED
|
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
models/selectors/selector_20261003T201413Z.skops filter=lfs diff=lfs merge=lfs -text
|
CHANGELOG.md
ADDED
|
@@ -0,0 +1,131 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Changelog
|
| 2 |
+
|
| 3 |
+
Releases are identified by date. The Docker distribution (branch `distribution`, Hugging Face `bryceweiner/fly-trader`)
|
| 4 |
+
ships signed releases built from this development line.
|
| 5 |
+
|
| 6 |
+
## 2026-10-11: Robinhood Chain, and a fly that starts as its selector
|
| 7 |
+
|
| 8 |
+
Not yet published to Hugging Face. Ships fly #157 (network) with selector #154, both trained on Solana and Robinhood
|
| 9 |
+
Chain rows together.
|
| 10 |
+
|
| 11 |
+
### Upgrading
|
| 12 |
+
|
| 13 |
+
- **Docker installs from before October 2026 must rebuild once:** `git pull && docker compose up -d --build`. The old
|
| 14 |
+
image cannot verify signed releases, so it stops updating after this release. Data, the fly's learned state and
|
| 15 |
+
open positions are kept.
|
| 16 |
+
- **Robinhood Chain is off by default** in the distribution. To turn it on, set `RH_ENABLED=1` and `ENVIO_API_TOKEN` (an
|
| 17 |
+
Envio HyperSync token; the free tier works) in `.env`.
|
| 18 |
+
- Indexing the full history from 2026-08-04 takes about 30 GB.
|
| 19 |
+
- Setting `RH_START_DAY` about 40 days back is enough to trade.
|
| 20 |
+
- **New settings:**
|
| 21 |
+
- `FLY_SCORE` (default `teacher`), `FLY_TEACHER_GATE` (default on), `FLY_LEARNED_VETO` (default 0.02) and `FLY_SIZING`
|
| 22 |
+
(default `kelly`).
|
| 23 |
+
- Robinhood Chain: `ENVIO_API_TOKEN`, `RH_*`, and optionally `QUICKNODE_RH_API_URL` for transactions.
|
| 24 |
+
|
| 25 |
+
### The fly
|
| 26 |
+
|
| 27 |
+
- **It scores as its selector plus what it has learned** (`FLY_SCORE=teacher`).
|
| 28 |
+
- On the first day its score equals the selector's exactly, verified live to 3e-8 on every row.
|
| 29 |
+
- Its learnable mushroom-body pathway (the plastic KC→MBON synapses) adds a correction, learned from realized returns,
|
| 30 |
+
that starts at zero.
|
| 31 |
+
- It therefore learns only where the selector is wrong.
|
| 32 |
+
- The earlier fly was a distilled copy of the selector, accurate overall but not at the buy line: it ranked the
|
| 33 |
+
selector's top 1 % of candidates only about 0.6-correlated.
|
| 34 |
+
- **It buys only what the selector buys** (`FLY_TEACHER_GATE`).
|
| 35 |
+
- The selector's trigger, buy line, win filter, dump veto and fail-closed rules decide every row.
|
| 36 |
+
- The fly's learning can drop a selector trade (a learned veto: its score more than 0.02 below its starting score) but
|
| 37 |
+
can never add one.
|
| 38 |
+
- Ungated, the fly had bought rows the selector rejected; those rows averaged −25.9 % per trade.
|
| 39 |
+
- **It sizes each buy exactly as the selector sizes it** (`FLY_SIZING=kelly`): the selector's score, buy line and Kelly
|
| 40 |
+
table for that row.
|
| 41 |
+
- **It learns from the selector's trades,** live and in the replay, after a restart too. It no longer learns from rows
|
| 42 |
+
above its own buy line.
|
| 43 |
+
- **It never trades without its teacher.** A fly whose selector cannot be loaded enters nothing. A release install now
|
| 44 |
+
points the fly at the shipped selector's new database id. Before that fix, the Docker boot test found a fresh install
|
| 45 |
+
running the fly ungated.
|
| 46 |
+
- **Replay verdict** (2026-09-24 to 10-09, the fly against the selector itself):
|
| 47 |
+
- Solana +3.78 % per trade over 308 trades.
|
| 48 |
+
- Robinhood Chain +2.66 % over 1,477 trades (selector +2.63 %).
|
| 49 |
+
- Random picks lost 4.4–4.6 %.
|
| 50 |
+
- **Smaller changes:**
|
| 51 |
+
- The fly calibrates on days its teacher never saw.
|
| 52 |
+
- A NaN mean can no longer pass the deploy gate.
|
| 53 |
+
- A second chain's teacher gets 28 days of its own data before the replay starts.
|
| 54 |
+
- `fly-replay --chains` re-judges one chain and keeps the others' verdicts.
|
| 55 |
+
|
| 56 |
+
### Robinhood Chain (new)
|
| 57 |
+
|
| 58 |
+
- **Pons memecoins traded beside pump.fun.** It trades tokens launched on Pons V2 after they graduate into Uniswap v4 pools
|
| 59 |
+
with the Pons hook.
|
| 60 |
+
- Each chain has its own books in its own unit (Robinhood Chain in ETH), with its own bankroll, kill switch, handover
|
| 61 |
+
and verdict.
|
| 62 |
+
- One selector and one fly cover both chains. A chain trades only if its own replay verdict passed.
|
| 63 |
+
- **Its own indexer.** It reads launches, graduations, swaps, the base assets' prices and 1-minute rows.
|
| 64 |
+
- Logs come from Envio HyperSync, in one query per pass.
|
| 65 |
+
- Minutes land about 15–25 s after they close (they took 35–95 s at first).
|
| 66 |
+
- If HyperSync fails, the official RPC carries the feed until it recovers.
|
| 67 |
+
- RH minutes up to 240 s old are still traded (`RH_TRADE_LAG_S`).
|
| 68 |
+
- **Live trading** (`RH_LIVE_ENABLED=0` by default):
|
| 69 |
+
- Swaps route through KyberSwap, with a Uniswap v4 fallback.
|
| 70 |
+
- One transaction is in flight at a time, booked from measured balances.
|
| 71 |
+
- It recovers after a crash and is reconciled against the chain.
|
| 72 |
+
- It has its own circuit breaker (circuit 3).
|
| 73 |
+
- `fly-trader rh-smoke` runs a tiny mainnet round trip.
|
| 74 |
+
- A QuickNode endpoint (`QUICKNODE_RH_API_URL`), if set, carries transactions. Its free plan caps log queries at 5
|
| 75 |
+
blocks, so logs stay on HyperSync.
|
| 76 |
+
- **Costs:** the Pons hook fee and creator tax (0.99 % per side at the median), price impact and gas.
|
| 77 |
+
- **Console:**
|
| 78 |
+
- A chain picker switches the Overview and Trades pages.
|
| 79 |
+
- A Robinhood Chain page shows the indexer, quote-asset prices and the ETH wallet.
|
| 80 |
+
- Safety & wallet shows both chains' bot wallets in USD and withdraws only to your own wallets.
|
| 81 |
+
|
| 82 |
+
### $FLY vault
|
| 83 |
+
|
| 84 |
+
- Robinhood Chain profit is paid to lockers in ETH, beside the SOL pot.
|
| 85 |
+
- **Custody:** a Squads treasury with spending limits and a key-isolated signer container. Claims are paid from a
|
| 86 |
+
separate payout wallet, and code releases are delayed.
|
| 87 |
+
- **Nothing published helps trade ahead of the fly.** The trading wallet's address, open positions and settlement
|
| 88 |
+
timing are not shown.
|
| 89 |
+
- **Audits:** red-team tests, mutation testing, Echidna and Halmos proofs.
|
| 90 |
+
- Fixed: a claim double-pay on resume.
|
| 91 |
+
- Added: Kyber calldata and fee checks, and relay griefing limits.
|
| 92 |
+
|
| 93 |
+
### Docker distribution
|
| 94 |
+
|
| 95 |
+
- Runs the development line's code again, merged instead of hand-ported (it had fallen about 35 commits behind).
|
| 96 |
+
- **Signed releases only.**
|
| 97 |
+
- The container installs a release only if its manifest is signed by a key in `release/allowed_signers`, its number
|
| 98 |
+
is higher than the installed one, and every file matches its hash.
|
| 99 |
+
- Releases carry no `.git*` file and no model they do not name.
|
| 100 |
+
- Startup removes model files an earlier release left behind.
|
| 101 |
+
- **Models are accepted whether or not Robinhood Chain is on.** Before release, an install with RH off would have
|
| 102 |
+
rejected the combined models and crash-looped at boot.
|
| 103 |
+
- The Robinhood Chain reference table ships in the seed.
|
| 104 |
+
- Kalshi ships dormant: no worker starts, and its pages appear only once `KALSHI_API_KEY_ID` is set.
|
| 105 |
+
|
| 106 |
+
### Kalshi prediction markets (development line)
|
| 107 |
+
|
| 108 |
+
- Under development, not traded in the distribution. Covers:
|
| 109 |
+
- an outcome-blind event corpus
|
| 110 |
+
- a selector judged by a weekly sign test
|
| 111 |
+
- maker and taker arms
|
| 112 |
+
- a visual fly
|
| 113 |
+
|
| 114 |
+
### Fixes
|
| 115 |
+
|
| 116 |
+
- **Schema changes no longer deadlock live workers.** An unchanged schema is skipped, and a changed one waits at most
|
| 117 |
+
5 s for a lock, then retries.
|
| 118 |
+
- **The RH paper books had made no trades.** A day's wallet-skill table did not exist yet when its minutes were scored.
|
| 119 |
+
Tables are now written ahead of their day, and the stream falls back to the newest one.
|
| 120 |
+
- **HyperSync could stall for 76 s.** A dead server address hung new connections; connections now time out in 5 s and
|
| 121 |
+
are kept alive.
|
| 122 |
+
- **The RH price scan ran about 270 queries per pass.** It now reads every price cursor at once.
|
| 123 |
+
|
| 124 |
+
## 2026-09-24: Fly #107 taught by selector #100
|
| 125 |
+
|
| 126 |
+
- The selector was retrained on real fees, and the fly was bootstrapped from it.
|
| 127 |
+
- The container keeps itself current from Hugging Face.
|
| 128 |
+
|
| 129 |
+
## 2026-09-22: First distribution
|
| 130 |
+
|
| 131 |
+
- Fire-and-forget trading in Docker, with the fly, its teacher selector and their connectome.
|
Dockerfile
CHANGED
|
@@ -1,28 +1,56 @@
|
|
| 1 |
# fly-trader: the trading engine and its console, on the CPU (a Linux container has no Metal GPU; the fly infers fine on CPU).
|
| 2 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4 |
ENV PYTHONUNBUFFERED=1 \
|
| 5 |
UV_PROJECT_ENVIRONMENT=/app/.venv \
|
|
|
|
|
|
|
| 6 |
PATH="/app/.venv/bin:${PATH}"
|
| 7 |
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
|
|
|
|
|
|
|
| 11 |
|
| 12 |
WORKDIR /app
|
| 13 |
COPY . .
|
| 14 |
# Which torch: TORCH_INDEX=cpu (default: any machine, ~200 MB) or cu126 / cu130 for an NVIDIA GPU (the CUDA runtime is
|
| 15 |
# inside the wheel; the host needs only the driver and the NVIDIA Container Toolkit -- see docker-compose.cuda.yml).
|
| 16 |
-
#
|
| 17 |
-
#
|
| 18 |
-
# to, the console's static directory, config/. A plain install ran a copy under site-packages and looked there instead.
|
| 19 |
ARG TORCH_INDEX=cpu
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
&&
|
| 23 |
-
&&
|
|
|
|
|
|
|
|
|
|
| 24 |
|
| 25 |
EXPOSE 8501
|
| 26 |
-
# healthy = the database answers
|
| 27 |
HEALTHCHECK --interval=30s --timeout=20s --start-period=300s --retries=5 CMD fly-trader health || exit 1
|
| 28 |
ENTRYPOINT ["/app/docker/entrypoint.sh"]
|
|
|
|
| 1 |
# fly-trader: the trading engine and its console, on the CPU (a Linux container has no Metal GPU; the fly infers fine on CPU).
|
| 2 |
+
# Two images from one file:
|
| 3 |
+
# default (last stage, "brain"): the engine, console and vault worker. It holds NO wallet key.
|
| 4 |
+
# --target signer: the only holder of the hot keys (fly_trader/signer): standard library + solders + httpx, nothing
|
| 5 |
+
# else of fly_trader, so none of the brain's dependencies ever run next to a key.
|
| 6 |
+
# Base images are pinned by digest and every Python dependency by hash (uv.lock, docker/signer-requirements.txt): a
|
| 7 |
+
# release builds exactly what was reviewed, whatever the registries serve later.
|
| 8 |
+
ARG PYTHON_IMAGE=python:3.13-slim@sha256:7c61056e61ac89e852de05f3dc6fa51a6dd2181797bceed46aa725dd7cb2cd3b
|
| 9 |
|
| 10 |
+
# ---------------------------------------------------------------- the signer
|
| 11 |
+
FROM ${PYTHON_IMAGE} AS signer
|
| 12 |
+
ENV PYTHONUNBUFFERED=1 PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=/app
|
| 13 |
+
RUN apt-get update && apt-get install -y --no-install-recommends age && rm -rf /var/lib/apt/lists/* \
|
| 14 |
+
&& groupadd -g 10000 fly && useradd -u 10002 -g 10000 -M -s /usr/sbin/nologin signer
|
| 15 |
+
COPY docker/signer-requirements.txt /tmp/req.txt
|
| 16 |
+
RUN pip install --no-cache-dir --require-hashes --only-binary=:all: -r /tmp/req.txt && rm /tmp/req.txt
|
| 17 |
+
WORKDIR /app
|
| 18 |
+
COPY fly_trader/__init__.py fly_trader/__init__.py
|
| 19 |
+
COPY fly_trader/signer fly_trader/signer
|
| 20 |
+
USER 10002:10000
|
| 21 |
+
HEALTHCHECK --interval=30s --timeout=10s --start-period=20s --retries=3 CMD python -m fly_trader.signer ping || exit 1
|
| 22 |
+
CMD ["python", "-m", "fly_trader.signer", "serve"]
|
| 23 |
+
|
| 24 |
+
# ---------------------------------------------------------------- the brain (default)
|
| 25 |
+
FROM ${PYTHON_IMAGE} AS brain
|
| 26 |
ENV PYTHONUNBUFFERED=1 \
|
| 27 |
UV_PROJECT_ENVIRONMENT=/app/.venv \
|
| 28 |
+
UV_COMPILE_BYTECODE=1 \
|
| 29 |
+
HOME=/tmp \
|
| 30 |
PATH="/app/.venv/bin:${PATH}"
|
| 31 |
|
| 32 |
+
# openssh-client: ssh-keygen verifies signed releases (the self-updater, fly_trader/ops/autoupdate.py)
|
| 33 |
+
RUN apt-get update && apt-get install -y --no-install-recommends postgresql-client ca-certificates curl age openssh-client \
|
| 34 |
+
&& rm -rf /var/lib/apt/lists/* \
|
| 35 |
+
&& groupadd -g 10000 fly && useradd -u 10001 -g 10000 -M -s /usr/sbin/nologin fly
|
| 36 |
+
COPY --from=ghcr.io/astral-sh/uv:0.12.19@sha256:04d046b13e60d6bcec73cbc5e1cad25d680dea90c8573340950a0ac2d1aef424 /uv /usr/local/bin/uv
|
| 37 |
|
| 38 |
WORKDIR /app
|
| 39 |
COPY . .
|
| 40 |
# Which torch: TORCH_INDEX=cpu (default: any machine, ~200 MB) or cu126 / cu130 for an NVIDIA GPU (the CUDA runtime is
|
| 41 |
# inside the wheel; the host needs only the driver and the NVIDIA Container Toolkit -- see docker-compose.cuda.yml).
|
| 42 |
+
# It names the project extra that pins torch to that index. --frozen: exactly uv.lock with its hashes, never a fresh
|
| 43 |
+
# resolution. The project installs editable: the package IS /app/fly_trader, so config.REPO_ROOT is /app.
|
|
|
|
| 44 |
ARG TORCH_INDEX=cpu
|
| 45 |
+
ENV TORCH_INDEX=${TORCH_INDEX}
|
| 46 |
+
RUN uv sync --frozen --no-dev --extra "${TORCH_INDEX}" \
|
| 47 |
+
&& chmod +x /app/docker/entrypoint.sh \
|
| 48 |
+
&& mkdir -p /app/fly_trader/ui/static/brain && chown -R 10001:10000 /app/fly_trader/ui/static/brain
|
| 49 |
+
# The vault server runs this unprivileged user and never updates itself (AUTO_UPDATE=0). The self-hosted compose file
|
| 50 |
+
# runs the container as root instead: its self-updater replaces /app, and it writes the bind-mounted data/, logs/, .env.
|
| 51 |
+
USER 10001:10000
|
| 52 |
|
| 53 |
EXPOSE 8501
|
| 54 |
+
# healthy = the database answers, the console's worker threads run and (on the vault server) the signer answers
|
| 55 |
HEALTHCHECK --interval=30s --timeout=20s --start-period=300s --retries=5 CMD fly-trader health || exit 1
|
| 56 |
ENTRYPOINT ["/app/docker/entrypoint.sh"]
|
README.md
CHANGED
|
@@ -6,6 +6,7 @@ tags:
|
|
| 6 |
- solana
|
| 7 |
- memecoin
|
| 8 |
- pump.fun
|
|
|
|
| 9 |
- trading
|
| 10 |
- connectome
|
| 11 |
- drosophila
|
|
@@ -14,12 +15,12 @@ tags:
|
|
| 14 |
|
| 15 |
# fly-trader — distribution
|
| 16 |
|
| 17 |
-
**A fruit-fly connectome that trades
|
| 18 |
|
| 19 |
> ## ⚠️ Read this first
|
| 20 |
>
|
| 21 |
-
> This software signs real transactions from a wallet you fund. It buys tokens that graduated from pump.fun,
|
| 22 |
-
>
|
| 23 |
> months of data; the market it trades is adversarial and changes. Nothing here is a claim of profit, and nothing here is
|
| 24 |
> financial advice. Run it only with money you can lose entirely, on a wallet used for nothing else. MIT licence — no
|
| 25 |
> warranty of any kind. See [LICENSE](LICENSE).
|
|
@@ -32,10 +33,16 @@ connectome they run on. The training pipeline lives on [`master`](https://github
|
|
| 32 |
- Every minute it scores every PumpSwap token that graduated from pump.fun at least six hours ago, from live 1-minute
|
| 33 |
candles (a token needs that much history before its features mean anything).
|
| 34 |
- The fly — the FlyWire *Drosophila* central brain (41,756 neurons) as a rate network on its real signed synapses —
|
| 35 |
-
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 39 |
- **It starts on paper.** After 20 closed paper trades it switches itself to the wallet and stays there. Before that,
|
| 40 |
and whenever the wallet is unfunded or a key is missing, it trades paper and says so on the console.
|
| 41 |
- **Rails:** 20 % below the wallet's peak it stops entering and sells every open position (the kill switch); three
|
|
@@ -46,13 +53,23 @@ connectome they run on. The training pipeline lives on [`master`](https://github
|
|
| 46 |
|
| 47 |
## Requirements
|
| 48 |
|
| 49 |
-
- Docker (Desktop on macOS/Windows, or Engine + Compose v2 on Linux). ~2 GB of disk for the image and database
|
|
|
|
| 50 |
- A [Helius](https://dashboard.helius.dev) API key (free tier is enough) and a [Jupiter](https://portal.jup.ag) API key.
|
|
|
|
|
|
|
| 51 |
- 0.5 SOL you are prepared to lose. Not more: the caps below are sized for 0.5.
|
| 52 |
- Any 64-bit CPU: the fly scores about as fast on 16 CPU threads as on a GPU (its forward pass is a memory-bound
|
| 53 |
scatter over a million synapses). An NVIDIA GPU is used when you start it with the CUDA override (see below). Apple
|
| 54 |
GPUs are not reachable from inside Docker.
|
| 55 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 56 |
## Start trading
|
| 57 |
|
| 58 |
1. Download this branch and open a terminal in it:
|
|
@@ -67,13 +84,15 @@ connectome they run on. The training pipeline lives on [`master`](https://github
|
|
| 67 |
```bash
|
| 68 |
docker compose up -d --build
|
| 69 |
```
|
| 70 |
-
The first start builds the image (several minutes), creates the database,
|
| 71 |
-
feed and the trading engine.
|
|
|
|
| 72 |
4. Create the bot wallet — it prints the public key and writes the secret into your `.env`:
|
| 73 |
```bash
|
| 74 |
docker compose exec fly fly-trader wallet new
|
| 75 |
```
|
| 76 |
-
**Back up `.env` now.** It is the only copy of that key.
|
|
|
|
| 77 |
5. Send **0.5 SOL** to the printed address, then restart the engine so it picks up the key:
|
| 78 |
```bash
|
| 79 |
docker compose restart fly
|
|
@@ -92,7 +111,7 @@ scores anything); then it trades paper until 20 trades have closed; then it trad
|
|
| 92 |
| Clear a tripped kill switch | Safety & wallet → **Clear kill switch**, or `docker compose exec fly fly-trader reset-circuit --kill` (re-bases the peak) |
|
| 93 |
| Stop everything | `docker compose down` — **open positions stay open in the wallet**; start again and it sells them on schedule, or sell them yourself |
|
| 94 |
| See the logs | `docker compose logs -f fly` |
|
| 95 |
-
| Start again from nothing | `docker compose down -v` (deletes the database) and
|
| 96 |
|
| 97 |
The console listens on `127.0.0.1` only. Do not expose it: it can start and stop the engine and clear the rails.
|
| 98 |
|
|
@@ -100,15 +119,17 @@ The console listens on `127.0.0.1` only. Do not expose it: it can start and stop
|
|
| 100 |
|
| 101 |
| Path | What |
|
| 102 |
|---|---|
|
| 103 |
-
| `models/policies/fly_*.pt` | the fly (
|
| 104 |
-
| `models/selectors/selector_*.
|
| 105 |
| `models/connectome/` | the FlyWire FAFB v783 central brain as the fly runs on it (14 MB), with its calibrated scale |
|
| 106 |
-
| `models/wallet_skill/` | the fitted wallet-skill inputs: `config.json` (the horizon and lookback the model was trained with)
|
| 107 |
-
| `seed/seed.sql` | the database rows that admit the fly to trade: the model records, the passed replay verdict, autostart |
|
|
|
|
| 108 |
| `.env.example` | every knob, set for 0.5 SOL |
|
| 109 |
-
| `docker-compose.yml`, `Dockerfile`, `docker/
|
|
|
|
| 110 |
|
| 111 |
-
The
|
| 112 |
factor 1.13) against −5.27 % for random picks of the same tokens. That is a backtest on one corpus, not a forecast.
|
| 113 |
|
| 114 |
## Tuning
|
|
@@ -118,14 +139,40 @@ Edit `.env`, then `docker compose restart fly`. The ones that matter:
|
|
| 118 |
| Variable | Shipped | Meaning |
|
| 119 |
|---|---|---|
|
| 120 |
| `CAPITAL_SOL` | 0.5 | what you funded; also the paper warm-up bankroll |
|
| 121 |
-
| `
|
|
|
|
| 122 |
| `GAS_RESERVE_SOL` | 0.05 | never spent |
|
| 123 |
| `KILL_SWITCH_DRAWDOWN` / `KILL_SWITCH_LIQUIDATE` | 0.20 / 1 | drawdown from peak that halts and sells everything |
|
| 124 |
| `HANDOVER_MIN_TRADES` | 20 | closed paper trades before it goes live |
|
| 125 |
| `LIVE_ENABLED` | 1 | set 0 to run paper only |
|
| 126 |
|
| 127 |
-
Scaling the wallet up scales the
|
| 128 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 129 |
|
| 130 |
## NVIDIA GPU
|
| 131 |
|
|
@@ -139,31 +186,47 @@ docker compose -f docker-compose.yml -f docker-compose.cuda.yml up -d --build
|
|
| 139 |
|
| 140 |
The image is built with the CUDA 12.6 torch wheel (`TORCH_INDEX: cu126` in `docker-compose.cuda.yml`; `cu130` for the
|
| 141 |
newest drivers) and the container is given every GPU. `DEVICE=auto` in `.env` then resolves to the GPU; the console's
|
| 142 |
-
Model page shows which device the fly is on. Batch sizes are derived from the card's free memory, so an 8 GB card works;
|
| 143 |
on a shared card set `DEVICE_MEMORY_GB` in `.env` to what the fly may use.
|
| 144 |
|
| 145 |
## Updates
|
| 146 |
|
| 147 |
-
The container keeps itself current. Every hour it asks Hugging Face whether
|
| 148 |
-
[`bryceweiner/fly-trader`](https://huggingface.co/bryceweiner/fly-trader) has a new release
|
| 149 |
-
|
| 150 |
-
|
| 151 |
-
|
| 152 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 153 |
"Refreshing the model" steps below are the manual way.
|
| 154 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 155 |
## Refreshing the model
|
| 156 |
|
| 157 |
The model is a snapshot of one training run. To ship a newer one, on a `master` checkout with its database, after a
|
| 158 |
passed replay:
|
| 159 |
|
| 160 |
```bash
|
| 161 |
-
.venv/bin/python tools/make_seed.py --out dist
|
| 162 |
-
cp -R dist/models/* models/ && cp -R dist/seed/* seed/
|
| 163 |
```
|
| 164 |
|
| 165 |
-
|
| 166 |
-
|
|
|
|
|
|
|
| 167 |
|
| 168 |
## Also on Hugging Face
|
| 169 |
|
|
@@ -174,6 +237,6 @@ changed by it, so pull the branch, `docker compose up -d --build`, and the new m
|
|
| 174 |
|
| 175 |
FlyWire consortium, Dorkenwald et al., *Neuronal wiring diagram of an adult brain*, Nature 2024; annotations Schlegel
|
| 176 |
et al. 2024; synapse signs Shiu et al. 2024. Connectome export: [snedea/flybrain](https://github.com/snedea/flybrain)
|
| 177 |
-
(MIT). Market data: [PumpAPI](https://pumpapi.io); routing: [Jupiter](https://jup.ag).
|
| 178 |
|
| 179 |
[MIT](LICENSE).
|
|
|
|
| 6 |
- solana
|
| 7 |
- memecoin
|
| 8 |
- pump.fun
|
| 9 |
+
- robinhood-chain
|
| 10 |
- trading
|
| 11 |
- connectome
|
| 12 |
- drosophila
|
|
|
|
| 15 |
|
| 16 |
# fly-trader — distribution
|
| 17 |
|
| 18 |
+
**A fruit-fly connectome that trades memecoins with your money: on Solana, and on Robinhood Chain if you turn it on.**
|
| 19 |
|
| 20 |
> ## ⚠️ Read this first
|
| 21 |
>
|
| 22 |
+
> This software signs real transactions from a wallet you fund. It buys tokens that graduated from pump.fun (and, if
|
| 23 |
+
> you turn Robinhood Chain on, from Pons), holds them for two to four hours, and sells. **It can lose everything in that wallet, quickly.** The model was trained on a few
|
| 24 |
> months of data; the market it trades is adversarial and changes. Nothing here is a claim of profit, and nothing here is
|
| 25 |
> financial advice. Run it only with money you can lose entirely, on a wallet used for nothing else. MIT licence — no
|
| 26 |
> warranty of any kind. See [LICENSE](LICENSE).
|
|
|
|
| 33 |
- Every minute it scores every PumpSwap token that graduated from pump.fun at least six hours ago, from live 1-minute
|
| 34 |
candles (a token needs that much history before its features mean anything).
|
| 35 |
- The fly — the FlyWire *Drosophila* central brain (41,756 neurons) as a rate network on its real signed synapses —
|
| 36 |
+
starts as the gradient-boosted selector that taught it: it scores every token with the selector's own prediction of
|
| 37 |
+
its net return. Its mushroom body's KC→MBON synapses then learn, from what the trades earn, a correction to that
|
| 38 |
+
prediction, and can veto a selector buy they have learned to distrust.
|
| 39 |
+
- It buys only what the selector buys (the selector's strategy, buy line and filters), sizes the position exactly as the
|
| 40 |
+
selector does (a quarter of the growth-optimal bet for its score band, capped), holds two to four hours (the
|
| 41 |
+
strategy's hold), sells.
|
| 42 |
+
- **Every transaction is checked before it is signed** (`fly_trader/signer`): swaps go only through Jupiter's own
|
| 43 |
+
routes, each one is simulated first and refused if it would spend more than ordered, deliver the token anywhere but
|
| 44 |
+
your wallet, touch another holding, or return far less value than it costs; buys are capped per trade, per day and per
|
| 45 |
+
minute.
|
| 46 |
- **It starts on paper.** After 20 closed paper trades it switches itself to the wallet and stays there. Before that,
|
| 47 |
and whenever the wallet is unfunded or a key is missing, it trades paper and says so on the console.
|
| 48 |
- **Rails:** 20 % below the wallet's peak it stops entering and sells every open position (the kill switch); three
|
|
|
|
| 53 |
|
| 54 |
## Requirements
|
| 55 |
|
| 56 |
+
- Docker (Desktop on macOS/Windows, or Engine + Compose v2 on Linux). ~2 GB of disk for the image and database
|
| 57 |
+
(Robinhood Chain, if you turn it on, adds ~30 GB for its full history; see below).
|
| 58 |
- A [Helius](https://dashboard.helius.dev) API key (free tier is enough) and a [Jupiter](https://portal.jup.ag) API key.
|
| 59 |
+
Optional: a second Solana RPC from another provider as `SIGNER_RPC_URL`, so the checks above do not rely on Helius
|
| 60 |
+
alone.
|
| 61 |
- 0.5 SOL you are prepared to lose. Not more: the caps below are sized for 0.5.
|
| 62 |
- Any 64-bit CPU: the fly scores about as fast on 16 CPU threads as on a GPU (its forward pass is a memory-bound
|
| 63 |
scatter over a million synapses). An NVIDIA GPU is used when you start it with the CUDA override (see below). Apple
|
| 64 |
GPUs are not reachable from inside Docker.
|
| 65 |
|
| 66 |
+
> ### ⚠️ Robinhood Chain needs an Envio HyperSync API token
|
| 67 |
+
>
|
| 68 |
+
> Robinhood Chain is off by default. To trade it, create a token in the [Envio dashboard](https://envio.dev/app/api-tokens)
|
| 69 |
+
> (personal accounts get one free), put it in `.env` as `ENVIO_API_TOKEN`, and set `RH_ENABLED=1`. Its feed reads the
|
| 70 |
+
> chain's logs from [Envio HyperSync](https://envio.dev/pricing/hypersync); the public Robinhood Chain RPC is far too slow
|
| 71 |
+
> to index its history. Without a token, leave Robinhood Chain off: Solana trades either way.
|
| 72 |
+
|
| 73 |
## Start trading
|
| 74 |
|
| 75 |
1. Download this branch and open a terminal in it:
|
|
|
|
| 84 |
```bash
|
| 85 |
docker compose up -d --build
|
| 86 |
```
|
| 87 |
+
The first start builds the image (several minutes), creates the database, downloads the wallet-skill table (~300 MB,
|
| 88 |
+
checked against its pinned sha256), installs the models and starts the market feed and the trading engine. The
|
| 89 |
+
Processes page also lists a `vault` worker that stops at once: it runs only on the hosted $FLY vault server.
|
| 90 |
4. Create the bot wallet — it prints the public key and writes the secret into your `.env`:
|
| 91 |
```bash
|
| 92 |
docker compose exec fly fly-trader wallet new
|
| 93 |
```
|
| 94 |
+
**Back up `.env` now.** It is the only copy of that key. (`wallet new` refuses if `.env` already has a
|
| 95 |
+
`BOT_PRIVATE_KEY=` line, even an empty one.)
|
| 96 |
5. Send **0.5 SOL** to the printed address, then restart the engine so it picks up the key:
|
| 97 |
```bash
|
| 98 |
docker compose restart fly
|
|
|
|
| 111 |
| Clear a tripped kill switch | Safety & wallet → **Clear kill switch**, or `docker compose exec fly fly-trader reset-circuit --kill` (re-bases the peak) |
|
| 112 |
| Stop everything | `docker compose down` — **open positions stay open in the wallet**; start again and it sells them on schedule, or sell them yourself |
|
| 113 |
| See the logs | `docker compose logs -f fly` |
|
| 114 |
+
| Start again from nothing | `docker compose down -v` (deletes the database) and `sudo rm -rf data/` (the container writes it as root; it also holds the signer's ledger) |
|
| 115 |
|
| 116 |
The console listens on `127.0.0.1` only. Do not expose it: it can start and stop the engine and clear the rails.
|
| 117 |
|
|
|
|
| 119 |
|
| 120 |
| Path | What |
|
| 121 |
|---|---|
|
| 122 |
+
| `models/policies/fly_*.pt` | the fly (#107 on the training machine; a fresh install gives it its own id): taught by selector #100, one strategy (`capitulation`), 240-minute hold, its own buy line. Loaded weights-only: the file cannot run code |
|
| 123 |
+
| `models/selectors/selector_*.skops` | the selector that taught it (#100), kept so the console can show the paper selector alongside. Saved with skops, not pickle |
|
| 124 |
| `models/connectome/` | the FlyWire FAFB v783 central brain as the fly runs on it (14 MB), with its calibrated scale |
|
| 125 |
+
| `models/wallet_skill/` | the fitted wallet-skill inputs: `config.json` (the horizon and lookback the model was trained with), the name of the newest per-day table (`TABLE`) and its sha256 (`TABLE.sha256`). The table itself (~300 MB) is on Hugging Face; the container downloads it once and uses it only if the hash matches |
|
| 126 |
+
| `seed/release_seed.json` (`seed/seed.sql` is the fallback) | the database rows that admit the fly to trade: the model records (by file hash), the passed replay verdict, autostart |
|
| 127 |
+
| `release/allowed_signers` | the release keys this install trusts for automatic updates (see Updates) |
|
| 128 |
| `.env.example` | every knob, set for 0.5 SOL |
|
| 129 |
+
| `docker-compose.yml`, `Dockerfile`, `docker/` | one Postgres, one engine + console (the Dockerfile's default stage; its `signer` stage is for the vault server) |
|
| 130 |
+
| `deploy/` | the hosted $FLY vault server: a separate signer container, host hardening, the signed-release updater ([docs/vault/RUNBOOK.md](docs/vault/RUNBOOK.md)) |
|
| 131 |
|
| 132 |
+
The replay verdict shipped here: over the 29 evaluation days it made +1.52 % per trade after real costs (profit
|
| 133 |
factor 1.13) against −5.27 % for random picks of the same tokens. That is a backtest on one corpus, not a forecast.
|
| 134 |
|
| 135 |
## Tuning
|
|
|
|
| 139 |
| Variable | Shipped | Meaning |
|
| 140 |
|---|---|---|
|
| 141 |
| `CAPITAL_SOL` | 0.5 | what you funded; also the paper warm-up bankroll |
|
| 142 |
+
| `MAX_POSITION_FRACTION` | 0.10 | the most of the bankroll one position may take |
|
| 143 |
+
| `MAX_POSITION_SOL` | 0.05 | the fixed size for a model without a sizing table (the shipped fly has one); must be above 0 to trade live |
|
| 144 |
| `GAS_RESERVE_SOL` | 0.05 | never spent |
|
| 145 |
| `KILL_SWITCH_DRAWDOWN` / `KILL_SWITCH_LIQUIDATE` | 0.20 / 1 | drawdown from peak that halts and sells everything |
|
| 146 |
| `HANDOVER_MIN_TRADES` | 20 | closed paper trades before it goes live |
|
| 147 |
| `LIVE_ENABLED` | 1 | set 0 to run paper only |
|
| 148 |
|
| 149 |
+
Scaling the wallet up scales the positions with it (`MAX_POSITION_FRACTION`). The signer has its own limits that do
|
| 150 |
+
not scale: at most `SIGNER_MAX_BUY_SOL` (1) per buy, `SIGNER_DAILY_BUY_SOL` (10) of buys a day and
|
| 151 |
+
`SIGNER_SWAPS_PER_MIN` (6) swaps a minute; raise them in `.env` deliberately.
|
| 152 |
+
|
| 153 |
+
## Robinhood Chain (Pons) memecoins
|
| 154 |
+
|
| 155 |
+
The same engine also trades Robinhood Chain: tokens launched on Pons that graduated into Uniswap v4 pools.
|
| 156 |
+
|
| 157 |
+
- **Off by default; turn it on with `RH_ENABLED=1` and `ENVIO_API_TOKEN` in `.env`** (an Envio HyperSync token; the
|
| 158 |
+
free tier works). The `rh_stream` worker then starts with the others and indexes Pons launches, graduations and swaps
|
| 159 |
+
into 1-minute rows: a backfill from `RH_START_DAY` (default 2026-08-04, ~30 GB; ~40 days back is enough to trade),
|
| 160 |
+
then live, with RH minutes landing ~15-25 s after they close. If HyperSync fails, the official RPC carries the feed
|
| 161 |
+
until it is back.
|
| 162 |
+
- **Paper, in ETH.** The Robinhood Chain books
|
| 163 |
+
(`paper_rh_selector`, `paper_rh_fly`) keep their money in native ETH (`RH_CAPITAL_ETH`, 5 SOL's worth at build time)
|
| 164 |
+
and pay Pons' hook fee, price impact and gas. They have their own kill switch and circuit (Safety & wallet →
|
| 165 |
+
Robinhood Chain trading rails) and their own handover. The console's chain picker switches Overview and Trades
|
| 166 |
+
between the two chains, and the **Robinhood Chain** page shows the indexer, the quote-asset prices and the ETH wallet.
|
| 167 |
+
- **One model for both chains.** The shipped selector and fly were trained on both chains' rows together, and the
|
| 168 |
+
replay judged each chain on its own trades; a chain's books trade only where its verdict passed. With RH off the same
|
| 169 |
+
models trade Solana alone.
|
| 170 |
+
- **Live** needs `RH_LIVE_ENABLED=1`, `RH_BOT_PRIVATE_KEY` (in `.env` only, never in the image) and the address it
|
| 171 |
+
derives to as `RH_BOT_ADDRESS`, plus a funded wallet (ETH on Robinhood Chain, chain 4663). Then
|
| 172 |
+
`docker compose restart fly`. As on Solana, the fly trades the wallet only after it has won the RH paper race.
|
| 173 |
+
- `RH_ENABLED=0` turns all of it off.
|
| 174 |
+
|
| 175 |
+
RH training data does not ship: an install that turns RH on builds its own from the stream, as Solana does.
|
| 176 |
|
| 177 |
## NVIDIA GPU
|
| 178 |
|
|
|
|
| 186 |
|
| 187 |
The image is built with the CUDA 12.6 torch wheel (`TORCH_INDEX: cu126` in `docker-compose.cuda.yml`; `cu130` for the
|
| 188 |
newest drivers) and the container is given every GPU. `DEVICE=auto` in `.env` then resolves to the GPU; the console's
|
| 189 |
+
Model & training page shows which device the fly is on. Batch sizes are derived from the card's free memory, so an 8 GB card works;
|
| 190 |
on a shared card set `DEVICE_MEMORY_GB` in `.env` to what the fly may use.
|
| 191 |
|
| 192 |
## Updates
|
| 193 |
|
| 194 |
+
The container keeps itself current, with **signed releases only**. Every hour it asks Hugging Face whether
|
| 195 |
+
[`bryceweiner/fly-trader`](https://huggingface.co/bryceweiner/fly-trader) has a new release. It installs one only if:
|
| 196 |
+
- its `release.json` is signed by a key in this install's `release/allowed_signers`
|
| 197 |
+
- its release number is higher than the installed one
|
| 198 |
+
- every file matches the size and sha256 the signed manifest lists
|
| 199 |
+
|
| 200 |
+
Then it restarts (about a minute; the paper books, the fly's learned state and any open positions are untouched --
|
| 201 |
+
open positions are sold on schedule by the restarted engine) and keeps the previous release beside the new one. If the
|
| 202 |
+
new release does not come up within five minutes, it goes back to the previous one and never tries that release again.
|
| 203 |
+
An unsigned or wrongly signed commit is refused and logged. While `release/allowed_signers` lists no key, nothing is
|
| 204 |
+
installed. `docker compose logs -f fly` shows what it did. `AUTO_UPDATE=0` in `.env` turns this off; then the
|
| 205 |
"Refreshing the model" steps below are the manual way.
|
| 206 |
|
| 207 |
+
**Installed before October 2026? Rebuild once.** The updater in those images installed whatever Hugging Face held,
|
| 208 |
+
unverified; it may install this release by itself, but only a rebuilt image can check signatures (it needs `ssh-keygen`),
|
| 209 |
+
so it stops at this release. From your install directory:
|
| 210 |
+
|
| 211 |
+
```bash
|
| 212 |
+
git pull && docker compose up -d --build
|
| 213 |
+
```
|
| 214 |
+
|
| 215 |
+
Your data, the fly's learned state and open positions are kept. Robinhood Chain stays off until you turn it on.
|
| 216 |
+
|
| 217 |
## Refreshing the model
|
| 218 |
|
| 219 |
The model is a snapshot of one training run. To ship a newer one, on a `master` checkout with its database, after a
|
| 220 |
passed replay:
|
| 221 |
|
| 222 |
```bash
|
| 223 |
+
.venv/bin/python tools/make_seed.py --out ../fly-trader-dist
|
|
|
|
| 224 |
```
|
| 225 |
|
| 226 |
+
Then publish it as a signed release with `tools/publish_release.py` (docs/vault/RUNBOOK.md, section 4.1): installs
|
| 227 |
+
with automatic updates pick it up within the hour. For a manual install, commit `models/` and `seed/` on this branch;
|
| 228 |
+
every start runs `fly-trader apply-release`, which reuses rows it already has (same file hash) and registers new
|
| 229 |
+
models, so pull the branch, `docker compose up -d --build`, and the new model is in place.
|
| 230 |
|
| 231 |
## Also on Hugging Face
|
| 232 |
|
|
|
|
| 237 |
|
| 238 |
FlyWire consortium, Dorkenwald et al., *Neuronal wiring diagram of an adult brain*, Nature 2024; annotations Schlegel
|
| 239 |
et al. 2024; synapse signs Shiu et al. 2024. Connectome export: [snedea/flybrain](https://github.com/snedea/flybrain)
|
| 240 |
+
(MIT). Market data: [PumpAPI](https://pumpapi.io); routing: [Jupiter](https://jup.ag) (Metis routes only).
|
| 241 |
|
| 242 |
[MIT](LICENSE).
|
deploy/bootstrap-host.sh
CHANGED
|
@@ -1,64 +1,459 @@
|
|
| 1 |
#!/usr/bin/env bash
|
| 2 |
-
# One-time setup of the vault fly's server
|
| 3 |
-
#
|
| 4 |
-
#
|
| 5 |
-
#
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6 |
set -euo pipefail
|
| 7 |
-
RELEASE_PUBKEY="${1:?usage: bootstrap-host.sh \"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8 |
|
| 9 |
-
|
| 10 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 11 |
install -m 0755 -d /etc/apt/keyrings
|
| 12 |
curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
|
| 13 |
echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu $(. /etc/os-release && echo "$VERSION_CODENAME") stable" > /etc/apt/sources.list.d/docker.list
|
| 14 |
-
apt-get update
|
| 15 |
-
apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
|
| 16 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 17 |
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 21 |
ufw default deny incoming
|
| 22 |
-
ufw default
|
| 23 |
-
ufw
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 24 |
ufw --force enable
|
| 25 |
|
| 26 |
-
#
|
| 27 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 28 |
install -d -m 0700 /srv/fly/secrets
|
| 29 |
install -d -m 0755 /etc/fly /usr/local/lib/fly
|
| 30 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 31 |
chmod 0644 /etc/fly/allowed_signers
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
|
| 36 |
-
openssl
|
| 37 |
-
|
| 38 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 39 |
A = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"
|
| 40 |
def b58(b):
|
| 41 |
n = int.from_bytes(b, "big"); s = ""
|
| 42 |
while n: n, r = divmod(n, 58); s = A[r] + s
|
| 43 |
return "1" * (len(b) - len(b.lstrip(b"\0"))) + s
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
|
|
|
|
| 48 |
PY
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 53 |
install -m 0755 "$HERE/fly_update.py" /usr/local/lib/fly/fly_update.py
|
|
|
|
| 54 |
[ -f /etc/fly/update.json ] || install -m 0600 "$HERE/update.json.example" /etc/fly/update.json
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 55 |
[ -f /srv/fly/vault.env ] || install -m 0600 "$HERE/vault.env.example" /srv/fly/vault.env
|
|
|
|
|
|
|
| 56 |
install -m 0644 "$HERE/docker-compose.vault.yml" /srv/fly/docker-compose.yml
|
| 57 |
install -m 0644 "$HERE/fly-update.service" /etc/systemd/system/fly-update.service
|
| 58 |
install -m 0644 "$HERE/fly-update.timer" /etc/systemd/system/fly-update.timer
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 59 |
systemctl daemon-reload
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
#!/usr/bin/env bash
|
| 2 |
+
# One-time setup of the vault fly's server: a dedicated machine (e.g. a mini PC at home) on a fresh Ubuntu Server 24.04
|
| 3 |
+
# with full-disk encryption, hardened along the lines of the CIS Ubuntu 24.04 benchmark for a single-purpose host that
|
| 4 |
+
# holds a hot wallet. Run with sudo, in two passes:
|
| 5 |
+
#
|
| 6 |
+
# 1. sudo TS_AUTHKEY=tskey-auth-... RECOVERY_PUBKEY="ssh-ed25519 AAAA... recovery" EXPECT_MANIFEST=<sha256 of release.json> \
|
| 7 |
+
# TPM_UNLOCK=1 ADMIN_USER=fly-admin bash bootstrap-host.sh "<release public key line: Secretive's ecdsa-sha2-nistp256 ...>"
|
| 8 |
+
# then, from your Mac, log in over Tailscale: ssh fly-admin@fly-vault (MagicDNS name or the 100.x address)
|
| 9 |
+
# 2. sudo CLOSE_PUBLIC_SSH=1 bash bootstrap-host.sh "<same release key>"
|
| 10 |
+
# removes the SSH rule for the local network / internet: after it the machine accepts SSH only over Tailscale and
|
| 11 |
+
# has no other open port.
|
| 12 |
+
#
|
| 13 |
+
# TPM_UNLOCK=1 binds the root disk's LUKS encryption to the TPM (clevis, PCR 7 = the Secure Boot state), so an
|
| 14 |
+
# unattended security reboot comes back up without a passphrase; it asks once for the disk passphrase. Without it the
|
| 15 |
+
# machine waits at every boot for the passphrase on its console, and automatic reboots are turned off.
|
| 16 |
+
#
|
| 17 |
+
# ADMIN_USER (default fly-admin) becomes the only account that can log in over SSH (key only, sudo without a
|
| 18 |
+
# password; root login is switched off). If it has no ~/.ssh/authorized_keys yet, root's are copied to it first, and
|
| 19 |
+
# the script stops before touching SSH if neither exists. Keep this root session open until you have logged in as
|
| 20 |
+
# ADMIN_USER from a second terminal. deploy/check-host.sh verifies every setting afterwards.
|
| 21 |
+
#
|
| 22 |
+
# RELEASE_PUBKEY / RECOVERY_PUBKEY: the two keys whose signatures the updater accepts (principals bryce and
|
| 23 |
+
# bryce-recovery), valid for the release namespace only. EXPECT_MANIFEST pins the first release the updater may
|
| 24 |
+
# install (tools/publish_release.py prints it as the manifest sha256); until it is set the updater installs nothing.
|
| 25 |
+
# TS_AUTHKEY: a one-off, pre-authorised key tagged tag:fly from the Tailscale admin console; used once, not stored.
|
| 26 |
+
#
|
| 27 |
+
# Idempotent: safe to run again (keys and edited config files are never overwritten).
|
| 28 |
set -euo pipefail
|
| 29 |
+
RELEASE_PUBKEY="${1:?usage: ADMIN_USER=fly-admin bootstrap-host.sh \"ecdsa-sha2-nistp256 AAAA... release-key\"}"
|
| 30 |
+
RECOVERY_PUBKEY="${RECOVERY_PUBKEY:-}"
|
| 31 |
+
EXPECT_MANIFEST="${EXPECT_MANIFEST:-}"
|
| 32 |
+
TS_AUTHKEY="${TS_AUTHKEY:-}"
|
| 33 |
+
CLOSE_PUBLIC_SSH="${CLOSE_PUBLIC_SSH:-0}"
|
| 34 |
+
TPM_UNLOCK="${TPM_UNLOCK:-0}"
|
| 35 |
+
ADMIN_USER="${ADMIN_USER:-fly-admin}"
|
| 36 |
+
# container users (the images' USER lines); with userns-remap each is REMAP_UID + id on the host
|
| 37 |
+
BRAIN_UID=10001; SIGNER_UID=10002; FLY_GID=10000
|
| 38 |
+
HERE="$(cd "$(dirname "$0")" && pwd)"
|
| 39 |
+
export DEBIAN_FRONTEND=noninteractive
|
| 40 |
+
step() { printf '\n== %s\n' "$*"; }
|
| 41 |
|
| 42 |
+
# ---------------------------------------------------------------- 1. packages
|
| 43 |
+
step "packages"
|
| 44 |
+
apt-get update -q
|
| 45 |
+
apt-get install -y -q --no-install-recommends ca-certificates curl ufw unattended-upgrades python3 openssh-client openssl \
|
| 46 |
+
fail2ban python3-systemd auditd audispd-plugins aide aide-common lynis apparmor-utils systemd-timesyncd needrestart \
|
| 47 |
+
libpam-tmpdir debsums apt-show-versions apt-listchanges acct sysstat mokutil
|
| 48 |
install -m 0755 -d /etc/apt/keyrings
|
| 49 |
curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
|
| 50 |
echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu $(. /etc/os-release && echo "$VERSION_CODENAME") stable" > /etc/apt/sources.list.d/docker.list
|
| 51 |
+
apt-get update -q
|
| 52 |
+
apt-get install -y -q docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
|
| 53 |
+
# Tailscale: SSH reaches this box only over your tailnet (WireGuard); its repo key is pinned like Docker's
|
| 54 |
+
curl -fsSL "https://pkgs.tailscale.com/stable/ubuntu/$(. /etc/os-release && echo "$VERSION_CODENAME").noarmor.gpg" -o /etc/apt/keyrings/tailscale.gpg
|
| 55 |
+
echo "deb [signed-by=/etc/apt/keyrings/tailscale.gpg] https://pkgs.tailscale.com/stable/ubuntu $(. /etc/os-release && echo "$VERSION_CODENAME") main" > /etc/apt/sources.list.d/tailscale.list
|
| 56 |
+
apt-get update -q
|
| 57 |
+
apt-get install -y -q tailscale
|
| 58 |
+
# nothing on this box needs snaps; fewer daemons, fewer holes
|
| 59 |
+
if dpkg -l snapd >/dev/null 2>&1; then apt-get purge -y -q snapd; fi
|
| 60 |
+
# no mail server: nothing here sends mail (alerts go to Telegram), and an MTA is one more listening daemon
|
| 61 |
+
if dpkg -l postfix 2>/dev/null | grep -q '^ii'; then apt-get purge -y -q postfix; fi
|
| 62 |
+
|
| 63 |
+
# ---------------------------------------------------------------- 1b. the machine itself
|
| 64 |
+
step "machine: wireless off, DMA protection, disk encryption"
|
| 65 |
+
# wired Ethernet only: Wi-Fi and Bluetooth drivers can never load (a radio is a way in that bypasses the router)
|
| 66 |
+
printf 'install cfg80211 /bin/false\ninstall mac80211 /bin/false\ninstall bluetooth /bin/false\ninstall btusb /bin/false\n' \
|
| 67 |
+
> /etc/modprobe.d/61-fly-no-wireless.conf
|
| 68 |
+
command -v rfkill >/dev/null && rfkill block all 2>/dev/null || true
|
| 69 |
+
systemctl disable --now bluetooth.service wpa_supplicant.service 2>/dev/null || true
|
| 70 |
+
# USB4 / Thunderbolt / OCuLink devices can read memory directly; the IOMMU confines them
|
| 71 |
+
if [ -z "$(ls -A /sys/kernel/iommu_groups 2>/dev/null)" ]; then
|
| 72 |
+
echo "WARNING: the IOMMU is off: enable it (AMD-Vi / IOMMU) in the firmware settings"
|
| 73 |
+
fi
|
| 74 |
+
# the root filesystem must sit on LUKS: the wallet keys are files on this disk
|
| 75 |
+
ROOT_LUKS="$(lsblk -rno NAME,TYPE -s "$(findmnt -no SOURCE /)" 2>/dev/null | awk '$2=="crypt"{found=1} found && $2=="part"{print "/dev/"$1; exit}')"
|
| 76 |
+
if [ -z "$ROOT_LUKS" ]; then
|
| 77 |
+
echo "WARNING: the root filesystem is not on an encrypted (LUKS) disk: reinstall with 'Encrypt the LVM group with LUKS'"
|
| 78 |
+
elif [ "$TPM_UNLOCK" = 1 ]; then
|
| 79 |
+
apt-get install -y -q --no-install-recommends clevis clevis-luks clevis-tpm2 clevis-initramfs
|
| 80 |
+
if ! clevis luks list -d "$ROOT_LUKS" 2>/dev/null | grep -q tpm2; then
|
| 81 |
+
echo "binding $ROOT_LUKS to the TPM (PCR 7); enter the disk passphrase when asked"
|
| 82 |
+
clevis luks bind -d "$ROOT_LUKS" tpm2 '{"pcr_ids":"7"}'
|
| 83 |
+
update-initramfs -u -k all
|
| 84 |
+
fi
|
| 85 |
+
fi
|
| 86 |
+
|
| 87 |
+
# ---------------------------------------------------------------- 2. patching
|
| 88 |
+
step "automatic security updates (reboot 04:00 UTC when a kernel needs it)"
|
| 89 |
+
cat > /etc/apt/apt.conf.d/20auto-upgrades <<'EOF'
|
| 90 |
+
APT::Periodic::Update-Package-Lists "1";
|
| 91 |
+
APT::Periodic::Unattended-Upgrade "1";
|
| 92 |
+
APT::Periodic::AutocleanInterval "7";
|
| 93 |
+
EOF
|
| 94 |
+
REBOOT=true # a disk that needs a typed passphrase at boot: no unattended reboots
|
| 95 |
+
if [ -n "${ROOT_LUKS:-}" ] && ! { command -v clevis >/dev/null && clevis luks list -d "$ROOT_LUKS" 2>/dev/null | grep -q tpm2; }; then
|
| 96 |
+
REBOOT=false
|
| 97 |
+
fi
|
| 98 |
+
cat > /etc/apt/apt.conf.d/52fly-unattended <<EOF
|
| 99 |
+
Unattended-Upgrade::Automatic-Reboot "$REBOOT";
|
| 100 |
+
Unattended-Upgrade::Automatic-Reboot-Time "04:00";
|
| 101 |
+
Unattended-Upgrade::Remove-Unused-Dependencies "true";
|
| 102 |
+
EOF
|
| 103 |
+
systemctl enable --now unattended-upgrades.service
|
| 104 |
+
timedatectl set-timezone UTC
|
| 105 |
+
systemctl enable --now systemd-timesyncd.service # the relay refuses the fly's requests if the clock is > 5 min off
|
| 106 |
+
|
| 107 |
+
# ---------------------------------------------------------------- 3. admin account
|
| 108 |
+
step "admin account: $ADMIN_USER"
|
| 109 |
+
id "$ADMIN_USER" >/dev/null 2>&1 || useradd -m -s /bin/bash "$ADMIN_USER"
|
| 110 |
+
passwd -l "$ADMIN_USER" >/dev/null # no password at all: SSH key only
|
| 111 |
+
home="$(getent passwd "$ADMIN_USER" | cut -d: -f6)"
|
| 112 |
+
install -d -m 0700 -o "$ADMIN_USER" -g "$ADMIN_USER" "$home/.ssh"
|
| 113 |
+
if [ ! -s "$home/.ssh/authorized_keys" ]; then
|
| 114 |
+
if [ -s /root/.ssh/authorized_keys ]; then
|
| 115 |
+
install -m 0600 -o "$ADMIN_USER" -g "$ADMIN_USER" /root/.ssh/authorized_keys "$home/.ssh/authorized_keys"
|
| 116 |
+
else
|
| 117 |
+
echo "STOP: neither $ADMIN_USER nor root has ~/.ssh/authorized_keys; add your key first" >&2; exit 1
|
| 118 |
+
fi
|
| 119 |
+
fi
|
| 120 |
+
echo "$ADMIN_USER ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/90-fly-admin
|
| 121 |
+
chmod 0440 /etc/sudoers.d/90-fly-admin
|
| 122 |
+
visudo -cf /etc/sudoers.d/90-fly-admin >/dev/null
|
| 123 |
+
# NOT in the docker group: that group is root without sudo's log line; docker runs through sudo
|
| 124 |
+
gpasswd -d "$ADMIN_USER" docker >/dev/null 2>&1 || true
|
| 125 |
+
|
| 126 |
+
# ---------------------------------------------------------------- 4. SSH
|
| 127 |
+
step "SSH: $ADMIN_USER only, key only, modern crypto"
|
| 128 |
+
cat > /etc/ssh/sshd_config.d/10-fly-hardening.conf <<EOF
|
| 129 |
+
PermitRootLogin no
|
| 130 |
+
PasswordAuthentication no
|
| 131 |
+
KbdInteractiveAuthentication no
|
| 132 |
+
PubkeyAuthentication yes
|
| 133 |
+
AuthenticationMethods publickey
|
| 134 |
+
AllowUsers $ADMIN_USER
|
| 135 |
+
MaxAuthTries 3
|
| 136 |
+
MaxSessions 4
|
| 137 |
+
LoginGraceTime 20
|
| 138 |
+
ClientAliveInterval 300
|
| 139 |
+
ClientAliveCountMax 2
|
| 140 |
+
PermitEmptyPasswords no
|
| 141 |
+
PermitUserEnvironment no
|
| 142 |
+
X11Forwarding no
|
| 143 |
+
AllowAgentForwarding no
|
| 144 |
+
# local forwarding stays on: it is how you reach the console (ssh -L 8501:127.0.0.1:8501)
|
| 145 |
+
AllowTcpForwarding local
|
| 146 |
+
GatewayPorts no
|
| 147 |
+
PermitTunnel no
|
| 148 |
+
Compression no
|
| 149 |
+
TCPKeepAlive no
|
| 150 |
+
MaxStartups 10:30:60
|
| 151 |
+
LogLevel VERBOSE
|
| 152 |
+
KexAlgorithms sntrup761x25519-sha512@openssh.com,curve25519-sha256,curve25519-sha256@libssh.org
|
| 153 |
+
Ciphers chacha20-poly1305@openssh.com,aes256-gcm@openssh.com,aes128-gcm@openssh.com
|
| 154 |
+
MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com
|
| 155 |
+
HostKeyAlgorithms ssh-ed25519,rsa-sha2-512,rsa-sha2-256
|
| 156 |
+
# client keys: ed25519, FIDO/hardware-backed keys, and P-256 (Secure Enclave keys from Secretive; the Ledger later)
|
| 157 |
+
PubkeyAcceptedAlgorithms sk-ssh-ed25519@openssh.com,ssh-ed25519,sk-ecdsa-sha2-nistp256@openssh.com,ecdsa-sha2-nistp256
|
| 158 |
+
EOF
|
| 159 |
+
# older lines in the main file must not win: Include is at the top, and first match wins in sshd
|
| 160 |
+
sed -i 's/^\(PermitRootLogin\|PasswordAuthentication\|KbdInteractiveAuthentication\|X11Forwarding\)/# fly: see sshd_config.d\/10-fly-hardening.conf\n#&/' /etc/ssh/sshd_config
|
| 161 |
+
install -d -m 0755 /run/sshd # 24.04 starts sshd on demand; the check needs this dir
|
| 162 |
+
sshd -t
|
| 163 |
+
systemctl reload ssh 2>/dev/null || systemctl restart ssh
|
| 164 |
|
| 165 |
+
step "fail2ban: SSH brute force"
|
| 166 |
+
cat > /etc/fail2ban/jail.d/fly-sshd.local <<'EOF'
|
| 167 |
+
[sshd]
|
| 168 |
+
enabled = true
|
| 169 |
+
backend = systemd
|
| 170 |
+
maxretry = 4
|
| 171 |
+
findtime = 10m
|
| 172 |
+
bantime = 1h
|
| 173 |
+
bantime.increment = true
|
| 174 |
+
EOF
|
| 175 |
+
systemctl enable --now fail2ban
|
| 176 |
+
systemctl restart fail2ban
|
| 177 |
+
|
| 178 |
+
# ---------------------------------------------------------------- 5. Tailscale + firewall
|
| 179 |
+
step "Tailscale"
|
| 180 |
+
systemctl enable --now tailscaled
|
| 181 |
+
if [ -n "$TS_AUTHKEY" ] && ! tailscale status >/dev/null 2>&1; then
|
| 182 |
+
tailscale up --auth-key="$TS_AUTHKEY" --hostname=fly-vault --advertise-tags=tag:fly --ssh=false --accept-dns=false
|
| 183 |
+
fi
|
| 184 |
+
unset TS_AUTHKEY
|
| 185 |
+
TS_UP=0; tailscale status >/dev/null 2>&1 && TS_UP=1
|
| 186 |
+
|
| 187 |
+
step "firewall: SSH over Tailscale only (plus LAN/public SSH until CLOSE_PUBLIC_SSH=1); out: HTTPS, HTTP (apt), DNS, NTP, WireGuard"
|
| 188 |
+
ufw --force reset >/dev/null
|
| 189 |
ufw default deny incoming
|
| 190 |
+
ufw default deny outgoing
|
| 191 |
+
ufw default deny routed
|
| 192 |
+
ufw allow in on tailscale0 to any port 22 proto tcp comment 'ssh over the tailnet'
|
| 193 |
+
if [ "$CLOSE_PUBLIC_SSH" = 1 ]; then
|
| 194 |
+
[ "$TS_UP" = 1 ] || { echo "STOP: Tailscale is not up; closing public SSH now would lock you out" >&2; exit 1; }
|
| 195 |
+
echo "public SSH closed: log in with ssh $ADMIN_USER@fly-vault (Tailscale)"
|
| 196 |
+
else
|
| 197 |
+
ufw limit in 22/tcp comment 'ssh (rate limited) -- removed by CLOSE_PUBLIC_SSH=1'
|
| 198 |
+
fi
|
| 199 |
+
ufw allow out 443/tcp comment 'https: RPCs, relay, HF, Telegram, apt, Tailscale control'
|
| 200 |
+
ufw allow out 80/tcp comment 'http: apt mirrors'
|
| 201 |
+
ufw allow out 53 comment 'dns'
|
| 202 |
+
ufw allow out 123/udp comment 'ntp'
|
| 203 |
+
ufw allow out 41641/udp comment 'tailscale wireguard'
|
| 204 |
+
ufw allow out 3478/udp comment 'tailscale stun'
|
| 205 |
+
ufw logging low
|
| 206 |
ufw --force enable
|
| 207 |
|
| 208 |
+
# Docker writes its own iptables rules and bypasses ufw; the DOCKER-USER chain is where the host gets a say. Containers
|
| 209 |
+
# may only talk to each other, reach DNS/HTTPS out, and nothing on the internet may open a connection to them.
|
| 210 |
+
cat > /usr/local/sbin/fly-docker-firewall <<'EOF'
|
| 211 |
+
#!/bin/sh
|
| 212 |
+
set -e
|
| 213 |
+
iptables -N DOCKER-USER 2>/dev/null || true
|
| 214 |
+
iptables -F DOCKER-USER
|
| 215 |
+
iptables -A DOCKER-USER -m conntrack --ctstate ESTABLISHED,RELATED -j RETURN
|
| 216 |
+
iptables -A DOCKER-USER -i docker0 -o docker0 -j RETURN
|
| 217 |
+
iptables -A DOCKER-USER -i br-+ -o br-+ -j RETURN
|
| 218 |
+
for p in 443 80 53; do # 80: apt mirrors during image builds (apt checks its own signatures)
|
| 219 |
+
iptables -A DOCKER-USER -i br-+ -p tcp --dport $p -j RETURN
|
| 220 |
+
iptables -A DOCKER-USER -i docker0 -p tcp --dport $p -j RETURN
|
| 221 |
+
done
|
| 222 |
+
iptables -A DOCKER-USER -i br-+ -p udp --dport 53 -j RETURN
|
| 223 |
+
iptables -A DOCKER-USER -i docker0 -p udp --dport 53 -j RETURN
|
| 224 |
+
iptables -A DOCKER-USER -i br-+ -j DROP
|
| 225 |
+
iptables -A DOCKER-USER -i docker0 -j DROP
|
| 226 |
+
iptables -A DOCKER-USER -m conntrack --ctstate NEW -j DROP
|
| 227 |
+
EOF
|
| 228 |
+
chmod 0750 /usr/local/sbin/fly-docker-firewall
|
| 229 |
+
cat > /etc/systemd/system/fly-docker-firewall.service <<'EOF'
|
| 230 |
+
[Unit]
|
| 231 |
+
Description=fly-trader: restrict container traffic (DOCKER-USER chain)
|
| 232 |
+
After=docker.service
|
| 233 |
+
Requires=docker.service
|
| 234 |
+
PartOf=docker.service
|
| 235 |
+
[Service]
|
| 236 |
+
Type=oneshot
|
| 237 |
+
RemainAfterExit=yes
|
| 238 |
+
ExecStart=/usr/local/sbin/fly-docker-firewall
|
| 239 |
+
[Install]
|
| 240 |
+
WantedBy=docker.service
|
| 241 |
+
EOF
|
| 242 |
+
|
| 243 |
+
# ---------------------------------------------------------------- 6. kernel
|
| 244 |
+
step "kernel and network settings"
|
| 245 |
+
cat > /etc/sysctl.d/60-fly-hardening.conf <<'EOF'
|
| 246 |
+
net.ipv4.conf.all.accept_redirects = 0
|
| 247 |
+
net.ipv4.conf.default.accept_redirects = 0
|
| 248 |
+
net.ipv4.conf.all.secure_redirects = 0
|
| 249 |
+
net.ipv4.conf.default.secure_redirects = 0
|
| 250 |
+
net.ipv4.conf.all.send_redirects = 0
|
| 251 |
+
net.ipv4.conf.default.send_redirects = 0
|
| 252 |
+
net.ipv4.conf.all.accept_source_route = 0
|
| 253 |
+
net.ipv4.conf.default.accept_source_route = 0
|
| 254 |
+
net.ipv4.conf.all.rp_filter = 1
|
| 255 |
+
net.ipv4.conf.default.rp_filter = 1
|
| 256 |
+
net.ipv4.conf.all.log_martians = 1
|
| 257 |
+
net.ipv4.conf.default.log_martians = 1
|
| 258 |
+
net.ipv4.icmp_echo_ignore_broadcasts = 1
|
| 259 |
+
net.ipv4.icmp_ignore_bogus_error_responses = 1
|
| 260 |
+
net.ipv4.tcp_syncookies = 1
|
| 261 |
+
net.ipv6.conf.all.accept_redirects = 0
|
| 262 |
+
net.ipv6.conf.default.accept_redirects = 0
|
| 263 |
+
net.ipv6.conf.all.accept_source_route = 0
|
| 264 |
+
net.ipv6.conf.default.accept_source_route = 0
|
| 265 |
+
kernel.kptr_restrict = 2
|
| 266 |
+
kernel.dmesg_restrict = 1
|
| 267 |
+
kernel.unprivileged_bpf_disabled = 1
|
| 268 |
+
net.core.bpf_jit_harden = 2
|
| 269 |
+
kernel.yama.ptrace_scope = 2
|
| 270 |
+
kernel.randomize_va_space = 2
|
| 271 |
+
kernel.sysrq = 0
|
| 272 |
+
fs.protected_hardlinks = 1
|
| 273 |
+
fs.protected_symlinks = 1
|
| 274 |
+
fs.protected_fifos = 2
|
| 275 |
+
fs.protected_regular = 2
|
| 276 |
+
fs.suid_dumpable = 0
|
| 277 |
+
EOF
|
| 278 |
+
sysctl -q --system
|
| 279 |
+
# core dumps could write a key to disk
|
| 280 |
+
cat > /etc/security/limits.d/60-fly-nocore.conf <<'EOF'
|
| 281 |
+
* hard core 0
|
| 282 |
+
EOF
|
| 283 |
+
mkdir -p /etc/systemd/coredump.conf.d
|
| 284 |
+
printf '[Coredump]\nStorage=none\nProcessSizeMax=0\n' > /etc/systemd/coredump.conf.d/60-fly.conf
|
| 285 |
+
aa-status --enabled # AppArmor must be on (Docker's default profile uses it)
|
| 286 |
+
# protocols, filesystems and devices this server never uses: not loadable at all
|
| 287 |
+
for m in dccp sctp rds tipc usb-storage firewire-core cramfs freevxfs hfs hfsplus jffs2 udf; do
|
| 288 |
+
echo "install $m /bin/false"
|
| 289 |
+
done > /etc/modprobe.d/60-fly-blacklist.conf
|
| 290 |
+
# new files readable by their group at most, never by everyone
|
| 291 |
+
sed -i 's/^UMASK.*/UMASK\t\t027/' /etc/login.defs
|
| 292 |
+
printf 'Authorized access only. Activity on this system is logged.\n' > /etc/issue
|
| 293 |
+
cp /etc/issue /etc/issue.net
|
| 294 |
+
sed -i 's/^ENABLED=.*/ENABLED="true"/' /etc/default/sysstat 2>/dev/null || true
|
| 295 |
+
systemctl enable --now acct sysstat >/dev/null 2>&1 || true
|
| 296 |
+
|
| 297 |
+
# ---------------------------------------------------------------- 7. Docker
|
| 298 |
+
step "Docker: user namespaces, no new privileges, bounded logs"
|
| 299 |
+
install -d -m 0755 /etc/docker
|
| 300 |
+
[ -f /etc/docker/daemon.json ] || cat > /etc/docker/daemon.json <<'EOF'
|
| 301 |
+
{
|
| 302 |
+
"userns-remap": "default",
|
| 303 |
+
"no-new-privileges": true,
|
| 304 |
+
"icc": false,
|
| 305 |
+
"live-restore": true,
|
| 306 |
+
"userland-proxy": false,
|
| 307 |
+
"log-driver": "json-file",
|
| 308 |
+
"log-opts": {"max-size": "50m", "max-file": "5"}
|
| 309 |
+
}
|
| 310 |
+
EOF
|
| 311 |
+
systemctl restart docker
|
| 312 |
+
systemctl daemon-reload
|
| 313 |
+
systemctl enable --now fly-docker-firewall.service
|
| 314 |
+
# with user namespaces, root inside a container is this unprivileged uid on the host
|
| 315 |
+
REMAP_UID="$(awk -F: '$1=="dockremap"{print $2; exit}' /etc/subuid)"
|
| 316 |
+
|
| 317 |
+
# ---------------------------------------------------------------- 8. layout, keys, updater
|
| 318 |
+
step "fly layout and wallets"
|
| 319 |
+
install -d -m 0750 /srv/fly
|
| 320 |
+
install -d -m 0755 /srv/fly/releases # public release files (from HF); the remapped container root reads them
|
| 321 |
+
H_BRAIN=$((REMAP_UID + BRAIN_UID)); H_SIGNER=$((REMAP_UID + SIGNER_UID)); H_FLY=$((REMAP_UID + FLY_GID))
|
| 322 |
+
install -d -m 0750 -o "$H_BRAIN" -g "$H_FLY" /srv/fly/data /srv/fly/logs
|
| 323 |
+
install -d -m 0700 -o "$H_SIGNER" -g "$H_SIGNER" /srv/fly/signer # the signer's own ledger (policy state)
|
| 324 |
+
install -d -m 0750 -o "$H_SIGNER" -g "$H_FLY" /srv/fly/run # its socket: the brain may connect, nothing else
|
| 325 |
install -d -m 0700 /srv/fly/secrets
|
| 326 |
install -d -m 0755 /etc/fly /usr/local/lib/fly
|
| 327 |
+
# release signers: valid for the release namespace only, so these keys sign nothing else this box would accept. A re-run
|
| 328 |
+
# without RECOVERY_PUBKEY keeps the recovery signer already installed (so closing SSH or updating the scripts never drops it).
|
| 329 |
+
if [ -z "$RECOVERY_PUBKEY" ] && [ -f /etc/fly/allowed_signers ]; then
|
| 330 |
+
RECOVERY_PUBKEY="$(awk '$1=="bryce-recovery"{ $1=""; $2=""; sub(/^ */, ""); print; exit }' /etc/fly/allowed_signers)"
|
| 331 |
+
fi
|
| 332 |
+
{
|
| 333 |
+
echo "bryce namespaces=\"fly-trader-release\" ${RELEASE_PUBKEY}"
|
| 334 |
+
if [ -n "$RECOVERY_PUBKEY" ]; then echo "bryce-recovery namespaces=\"fly-trader-release\" ${RECOVERY_PUBKEY}"; fi
|
| 335 |
+
} > /etc/fly/allowed_signers
|
| 336 |
chmod 0644 /etc/fly/allowed_signers
|
| 337 |
+
[ -n "$RECOVERY_PUBKEY" ] || echo "WARNING: no RECOVERY_PUBKEY: losing the release key would mean re-bootstrapping the updater"
|
| 338 |
+
install -m 0755 "$HERE/fly" /usr/local/bin/fly # sudo fly <command>: fly-trader inside the running container
|
| 339 |
+
# the database password: made here, root-only, handed to both database and brain by compose
|
| 340 |
+
if [ ! -s /srv/fly/db.env ]; then
|
| 341 |
+
pw="$(openssl rand -hex 24)"
|
| 342 |
+
( umask 077; printf 'POSTGRES_PASSWORD=%s\nDATABASE_URL=postgresql://fly:%s@db:5432/fly_trader\n' "$pw" "$pw" > /srv/fly/db.env )
|
| 343 |
+
unset pw
|
| 344 |
+
fi
|
| 345 |
+
chmod 0600 /srv/fly/db.env
|
| 346 |
+
# two hot keys, each an ed25519 seed made HERE and never copied off this box in plaintext (the signer writes
|
| 347 |
+
# age-encrypted backups for you): the trading key (bot_key: the trading wallet; refills itself from the treasury under
|
| 348 |
+
# L1) and the payout key (payout_key: holds nothing; pays claims from the treasury under L2). Only the signer container
|
| 349 |
+
# mounts them. Their public keys are printed below, once; later: sudo fly signer status.
|
| 350 |
+
make_key() {
|
| 351 |
+
local name="$1"
|
| 352 |
+
if [ ! -e "/srv/fly/secrets/$name" ]; then
|
| 353 |
+
umask 077
|
| 354 |
+
openssl genpkey -algorithm ed25519 -out "/srv/fly/secrets/$name.pem"
|
| 355 |
+
python3 - "$name" <<'PY'
|
| 356 |
+
import subprocess, sys
|
| 357 |
+
name = sys.argv[1]
|
| 358 |
A = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"
|
| 359 |
def b58(b):
|
| 360 |
n = int.from_bytes(b, "big"); s = ""
|
| 361 |
while n: n, r = divmod(n, 58); s = A[r] + s
|
| 362 |
return "1" * (len(b) - len(b.lstrip(b"\0"))) + s
|
| 363 |
+
pem = f"/srv/fly/secrets/{name}.pem"
|
| 364 |
+
priv = subprocess.run(["openssl", "pkey", "-in", pem, "-outform", "DER"], check=True, capture_output=True).stdout[-32:]
|
| 365 |
+
pub = subprocess.run(["openssl", "pkey", "-in", pem, "-pubout", "-outform", "DER"], check=True, capture_output=True).stdout[-32:]
|
| 366 |
+
open(f"/srv/fly/secrets/{name}", "x").write(b58(priv) + "\n")
|
| 367 |
+
print(f"{name}: {b58(pub)}")
|
| 368 |
PY
|
| 369 |
+
shred -u "/srv/fly/secrets/$name.pem"
|
| 370 |
+
fi
|
| 371 |
+
# readable only by the signer container's user (remapped), not by the brain container or any host account
|
| 372 |
+
chown "$H_SIGNER:$H_SIGNER" "/srv/fly/secrets/$name"
|
| 373 |
+
chmod 0400 "/srv/fly/secrets/$name"
|
| 374 |
+
}
|
| 375 |
+
make_key bot_key # the trading hot key: holds at most the float; refills itself from the Squads treasury (L1)
|
| 376 |
+
make_key payout_key # the payout key: holds nothing; pays claims from the treasury within its weekly cap (L2)
|
| 377 |
install -m 0755 "$HERE/fly_update.py" /usr/local/lib/fly/fly_update.py
|
| 378 |
+
install -m 0755 "$HERE/check-host.sh" /usr/local/lib/fly/check-host.sh
|
| 379 |
[ -f /etc/fly/update.json ] || install -m 0600 "$HERE/update.json.example" /etc/fly/update.json
|
| 380 |
+
if [ -n "$EXPECT_MANIFEST" ]; then # pin the first release: the updater refuses anything else until one is installed
|
| 381 |
+
python3 - "$EXPECT_MANIFEST" <<'PY'
|
| 382 |
+
import json, re, sys
|
| 383 |
+
pin = sys.argv[1].strip().lower()
|
| 384 |
+
if not re.fullmatch(r"[0-9a-f]{64}", pin):
|
| 385 |
+
raise SystemExit("EXPECT_MANIFEST must be a sha256 hex digest")
|
| 386 |
+
p = "/etc/fly/update.json"; c = json.load(open(p)); c["expect_manifest_sha256"] = pin
|
| 387 |
+
open(p, "w").write(json.dumps(c, indent=1) + "\n")
|
| 388 |
+
PY
|
| 389 |
+
fi
|
| 390 |
[ -f /srv/fly/vault.env ] || install -m 0600 "$HERE/vault.env.example" /srv/fly/vault.env
|
| 391 |
+
[ -f /srv/fly/custody.env ] || install -m 0600 "$HERE/custody.env.example" /srv/fly/custody.env
|
| 392 |
+
[ -f /srv/fly/signer.env ] || install -m 0600 "$HERE/signer.env.example" /srv/fly/signer.env
|
| 393 |
install -m 0644 "$HERE/docker-compose.vault.yml" /srv/fly/docker-compose.yml
|
| 394 |
install -m 0644 "$HERE/fly-update.service" /etc/systemd/system/fly-update.service
|
| 395 |
install -m 0644 "$HERE/fly-update.timer" /etc/systemd/system/fly-update.timer
|
| 396 |
+
install -m 0755 "$HERE/fly_audit_watch.py" /usr/local/lib/fly/fly_audit_watch.py
|
| 397 |
+
install -m 0644 "$HERE/fly-audit-watch.service" /etc/systemd/system/fly-audit-watch.service
|
| 398 |
+
install -m 0644 "$HERE/fly-audit-watch.timer" /etc/systemd/system/fly-audit-watch.timer
|
| 399 |
+
install -m 0644 "$HERE/fly-aide.service" /etc/systemd/system/fly-aide.service
|
| 400 |
+
install -m 0644 "$HERE/fly-aide.timer" /etc/systemd/system/fly-aide.timer
|
| 401 |
systemctl daemon-reload
|
| 402 |
+
systemctl enable --now fly-audit-watch.timer # a key read by anything but the signer goes to Telegram within a minute
|
| 403 |
+
systemctl enable --now fly-aide.timer # the daily file-integrity check reports to Telegram (there is no mail here)
|
| 404 |
+
|
| 405 |
+
# ---------------------------------------------------------------- 9. audit and integrity
|
| 406 |
+
step "audit log: key reads and config changes"
|
| 407 |
+
cat > /etc/audit/rules.d/60-fly.rules <<'EOF'
|
| 408 |
+
-w /srv/fly/secrets -p rwa -k fly_keys
|
| 409 |
+
-w /etc/fly -p wa -k fly_conf
|
| 410 |
+
-w /srv/fly/vault.env -p rwa -k fly_conf
|
| 411 |
+
-w /srv/fly/db.env -p rwa -k fly_conf
|
| 412 |
+
-w /srv/fly/custody.env -p wa -k fly_conf
|
| 413 |
+
-w /srv/fly/signer.env -p rwa -k fly_conf
|
| 414 |
+
-w /srv/fly/signer -p wa -k fly_signer
|
| 415 |
+
-w /usr/local/lib/fly -p wa -k fly_updater
|
| 416 |
+
-w /etc/ssh/sshd_config -p wa -k sshd
|
| 417 |
+
-w /etc/ssh/sshd_config.d -p wa -k sshd
|
| 418 |
+
-w /etc/sudoers -p wa -k sudoers
|
| 419 |
+
-w /etc/sudoers.d -p wa -k sudoers
|
| 420 |
+
-w /etc/docker -p wa -k docker_conf
|
| 421 |
+
-w /etc/systemd/system -p wa -k systemd_units
|
| 422 |
+
-w /etc/passwd -p wa -k identity
|
| 423 |
+
-w /etc/shadow -p wa -k identity
|
| 424 |
+
-w /etc/group -p wa -k identity
|
| 425 |
+
EOF
|
| 426 |
+
augenrules --load >/dev/null
|
| 427 |
+
systemctl enable --now auditd
|
| 428 |
+
|
| 429 |
+
step "file integrity baseline (AIDE, checked daily; takes a few minutes)"
|
| 430 |
+
echo 'Checksums = sha512' > /etc/aide/aide.conf.d/10_fly_checksums
|
| 431 |
+
cat > /etc/aide/aide.conf.d/90_fly_ignore <<'EOF'
|
| 432 |
+
!/srv/fly/data
|
| 433 |
+
!/srv/fly/logs
|
| 434 |
+
!/srv/fly/releases
|
| 435 |
+
!/var/lib/docker
|
| 436 |
+
!/var/lib/containerd
|
| 437 |
+
EOF
|
| 438 |
+
[ -f /var/lib/aide/aide.db ] || { aideinit -y -f >/dev/null 2>&1 && cp /var/lib/aide/aide.db.new /var/lib/aide/aide.db; }
|
| 439 |
+
|
| 440 |
+
# ---------------------------------------------------------------- 10. logs
|
| 441 |
+
step "persistent journal"
|
| 442 |
+
mkdir -p /etc/systemd/journald.conf.d
|
| 443 |
+
printf '[Journal]\nStorage=persistent\nSystemMaxUse=2G\n' > /etc/systemd/journald.conf.d/60-fly.conf
|
| 444 |
+
systemctl restart systemd-journald
|
| 445 |
+
|
| 446 |
+
step "done"
|
| 447 |
+
if [ "$CLOSE_PUBLIC_SSH" = 1 ]; then
|
| 448 |
+
next="Verify: sudo /usr/local/lib/fly/check-host.sh (every control should pass now)"
|
| 449 |
+
else
|
| 450 |
+
next="From your Mac, BEFORE closing this session: ssh $ADMIN_USER@fly-vault (over Tailscale; root login is now off)
|
| 451 |
+
Then run this script again with CLOSE_PUBLIC_SSH=1 to remove public SSH."
|
| 452 |
+
fi
|
| 453 |
+
cat <<EOF
|
| 454 |
+
$next
|
| 455 |
+
Next (docs/vault/RUNBOOK.md, sections 4.3 and 4.4):
|
| 456 |
+
1. the treasury on https://fly-trader.app/owner.html with your owner wallet (the two keys printed above)
|
| 457 |
+
2. sudoedit /srv/fly/{custody,signer,vault}.env and /etc/fly/update.json (Telegram)
|
| 458 |
+
3. sudo systemctl enable --now fly-update.timer # pulls, verifies and starts the pinned first release
|
| 459 |
+
EOF
|
deploy/check-host.sh
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
# Verify the vault server's hardening (bootstrap-host.sh). Run as root: sudo /usr/local/lib/fly/check-host.sh
|
| 3 |
+
# Prints PASS/FAIL per control and a Lynis hardening index; exits 1 if any control fails.
|
| 4 |
+
set -uo pipefail
|
| 5 |
+
fail=0
|
| 6 |
+
pass() { printf ' PASS %s\n' "$1"; }
|
| 7 |
+
bad() { printf ' FAIL %s\n' "$1"; fail=1; }
|
| 8 |
+
check() { local what="$1"; shift; if "$@" >/dev/null 2>&1; then pass "$what"; else bad "$what"; fi; }
|
| 9 |
+
sshd_is() { sshd -T 2>/dev/null | grep -qix "$1 $2"; }
|
| 10 |
+
sysctl_is() { [ "$(sysctl -n "$1" 2>/dev/null)" = "$2" ]; }
|
| 11 |
+
|
| 12 |
+
echo "SSH"
|
| 13 |
+
check "root login off" sshd_is permitrootlogin no
|
| 14 |
+
check "passwords off" sshd_is passwordauthentication no
|
| 15 |
+
check "keyboard-interactive off" sshd_is kbdinteractiveauthentication no
|
| 16 |
+
check "only the admin account" sh -c "sshd -T | grep -qi '^allowusers '"
|
| 17 |
+
check "max 3 auth tries" sshd_is maxauthtries 3
|
| 18 |
+
check "X11 / agent forwarding off" sh -c "sshd -T | grep -qix 'x11forwarding no' && sshd -T | grep -qix 'allowagentforwarding no'"
|
| 19 |
+
check "only local TCP forwarding" sshd_is allowtcpforwarding local
|
| 20 |
+
check "no weak ciphers" sh -c "! sshd -T | grep -i '^ciphers' | grep -Eqi 'cbc|3des|arcfour'"
|
| 21 |
+
check "fail2ban guards sshd" sh -c "fail2ban-client status sshd"
|
| 22 |
+
check "client keys: ed25519/P-256/FIDO only" sh -c "sshd -T | grep -i '^pubkeyacceptedalgorithms' | grep -vqi rsa"
|
| 23 |
+
check "admin not in the docker group" sh -c "! id -nG \"\$(sshd -T | awk 'tolower(\$1)==\"allowusers\"{print \$2}')\" | grep -qw docker"
|
| 24 |
+
|
| 25 |
+
echo "Machine"
|
| 26 |
+
root_luks() { lsblk -rno NAME,TYPE -s "$(findmnt -no SOURCE /)" | awk '$2=="crypt"{f=1} f && $2=="part"{print "/dev/"$1; exit}'; }
|
| 27 |
+
check "root filesystem encrypted (LUKS)" sh -c "[ -n \"\$(lsblk -rno TYPE -s \$(findmnt -no SOURCE /) | grep -x crypt)\" ]"
|
| 28 |
+
check "unattended reboots unlock (TPM) or are off" sh -c "grep -q 'Automatic-Reboot \"false\"' /etc/apt/apt.conf.d/52fly-unattended || clevis luks list -d $(root_luks) 2>/dev/null | grep -q tpm2"
|
| 29 |
+
check "IOMMU on (USB4/OCuLink DMA)" sh -c "[ -n \"\$(ls -A /sys/kernel/iommu_groups)\" ]"
|
| 30 |
+
check "Secure Boot on" sh -c "mokutil --sb-state 2>/dev/null | grep -qi 'SecureBoot enabled'"
|
| 31 |
+
check "no Wi-Fi or Bluetooth" sh -c "! ls /sys/class/net/*/wireless >/dev/null 2>&1 && [ -z \"\$(ls -A /sys/class/bluetooth 2>/dev/null)\" ]"
|
| 32 |
+
|
| 33 |
+
echo "Tailscale"
|
| 34 |
+
check "tailscaled up" sh -c "tailscale status >/dev/null"
|
| 35 |
+
check "Tailnet Lock enabled" sh -c "tailscale lock status 2>/dev/null | grep -qi 'lock is enabled'"
|
| 36 |
+
check "no public SSH (tailnet only)" sh -c "! ufw status | grep -E '^22/tcp' | grep -v tailscale0 | grep -q ."
|
| 37 |
+
|
| 38 |
+
echo "Firewall"
|
| 39 |
+
check "ufw active" sh -c "ufw status | grep -q 'Status: active'"
|
| 40 |
+
check "incoming denied by default" sh -c "ufw status verbose | grep -q 'deny (incoming)'"
|
| 41 |
+
check "outgoing denied by default" sh -c "ufw status verbose | grep -q 'deny (outgoing)'"
|
| 42 |
+
check "only SSH listens publicly" sh -c "! ss -Hltn | awk '{print \$4}' | grep -vE '^(127\\.|\\[::1\\]|\\[::ffff:127|100\\.|\\[fd7a:)' | grep -vE ':22\$' | grep -q ."
|
| 43 |
+
check "container traffic restricted" sh -c "iptables -S DOCKER-USER | grep -q -- '-j DROP'"
|
| 44 |
+
check "console bound to localhost" sh -c "! ss -Hltn | grep -E ':8501 ' | grep -vqE '127\\.0\\.0\\.1:8501'"
|
| 45 |
+
|
| 46 |
+
echo "Kernel"
|
| 47 |
+
for kv in net.ipv4.conf.all.accept_redirects=0 net.ipv4.conf.all.send_redirects=0 net.ipv4.conf.all.accept_source_route=0 \
|
| 48 |
+
net.ipv4.conf.all.rp_filter=1 net.ipv4.tcp_syncookies=1 kernel.kptr_restrict=2 kernel.dmesg_restrict=1 \
|
| 49 |
+
kernel.unprivileged_bpf_disabled=1 kernel.yama.ptrace_scope=2 kernel.randomize_va_space=2 fs.suid_dumpable=0 \
|
| 50 |
+
fs.protected_symlinks=1 fs.protected_hardlinks=1; do
|
| 51 |
+
check "${kv%%=*} = ${kv#*=}" sysctl_is "${kv%%=*}" "${kv#*=}"
|
| 52 |
+
done
|
| 53 |
+
check "core dumps off" sh -c "grep -qs 'Storage=none' /etc/systemd/coredump.conf.d/60-fly.conf"
|
| 54 |
+
check "AppArmor enforcing" aa-status --enabled
|
| 55 |
+
|
| 56 |
+
echo "Docker"
|
| 57 |
+
check "user namespace remapping" sh -c "docker info 2>/dev/null | grep -qi 'userns'"
|
| 58 |
+
check "no-new-privileges by default" sh -c "grep -q '\"no-new-privileges\": true' /etc/docker/daemon.json"
|
| 59 |
+
check "log size bounded" sh -c "grep -q 'max-size' /etc/docker/daemon.json"
|
| 60 |
+
check "live restore" sh -c "docker info 2>/dev/null | grep -qi 'Live Restore Enabled: true'"
|
| 61 |
+
|
| 62 |
+
echo "Secrets"
|
| 63 |
+
for k in bot_key payout_key; do
|
| 64 |
+
f=/srv/fly/secrets/$k
|
| 65 |
+
check "$k exists, mode 400" sh -c "[ -f $f ] && [ \"\$(stat -c %a $f)\" = 400 ]"
|
| 66 |
+
check "$k not owned by a login user" sh -c "[ \"\$(stat -c %U $f)\" != root ] || true; ! getent passwd \"\$(stat -c %U $f)\" | grep -qE ':/bin/(ba)?sh\$'"
|
| 67 |
+
done
|
| 68 |
+
check "secrets dir only root" sh -c "[ \"\$(stat -c %a /srv/fly/secrets)\" = 700 ]"
|
| 69 |
+
check "signer up (it refuses equal keys)" sh -c "docker inspect fly-fly-signer-1 --format '{{.State.Health.Status}}' | grep -q healthy"
|
| 70 |
+
check "release signer pinned" sh -c "[ -s /etc/fly/allowed_signers ] && [ \"\$(stat -c %U /etc/fly/allowed_signers)\" = root ]"
|
| 71 |
+
check "updater owned by root" sh -c "[ \"\$(stat -c %U /usr/local/lib/fly/fly_update.py)\" = root ] && ! [ -w /usr/local/lib/fly/fly_update.py -a \"\$(stat -c %a /usr/local/lib/fly/fly_update.py)\" != 755 ]"
|
| 72 |
+
for e in vault custody signer db; do
|
| 73 |
+
check "$e.env mode 600" sh -c "[ \"\$(stat -c %a /srv/fly/$e.env)\" = 600 ]"
|
| 74 |
+
done
|
| 75 |
+
check "no default database password" sh -c "! grep -qE 'POSTGRES_PASSWORD(:|=) *fly\$' /srv/fly/docker-compose.yml /srv/fly/db.env"
|
| 76 |
+
check "release signers namespaced" sh -c "! grep -v 'namespaces=\"fly-trader-release\"' /etc/fly/allowed_signers | grep -q ."
|
| 77 |
+
check "first release pinned (no TOFU)" sh -c "grep -Eq '\"expect_manifest_sha256\": *\"[0-9a-f]{64}\"' /etc/fly/update.json || [ -s /srv/fly/state.json ]"
|
| 78 |
+
|
| 79 |
+
echo "Key isolation"
|
| 80 |
+
check "keys mounted into the signer only" sh -c "! docker inspect fly-fly-1 --format '{{range .Mounts}}{{.Source}} {{end}}' | grep -q /srv/fly/secrets"
|
| 81 |
+
check "signer has the keys" sh -c "docker inspect fly-fly-signer-1 --format '{{range .Mounts}}{{.Source}} {{end}}' | grep -q /srv/fly/secrets/bot_key"
|
| 82 |
+
check "signer filesystem read-only" sh -c "docker inspect fly-fly-signer-1 --format '{{.HostConfig.ReadonlyRootfs}}' | grep -q true"
|
| 83 |
+
check "brain runs as uid 10001" sh -c "docker inspect fly-fly-1 --format '{{.Config.User}}' | grep -q '^10001'"
|
| 84 |
+
check "database has no route out" sh -c "docker network inspect fly_internal --format '{{.Internal}}' | grep -q true"
|
| 85 |
+
|
| 86 |
+
echo "Monitoring"
|
| 87 |
+
check "auditd running" systemctl is-active auditd
|
| 88 |
+
check "key reads audited" sh -c "auditctl -l | grep -q fly_keys"
|
| 89 |
+
check "key-read watcher running" systemctl is-active fly-audit-watch.timer
|
| 90 |
+
check "daily AIDE report to Telegram" systemctl is-active fly-aide.timer
|
| 91 |
+
check "AIDE baseline present" test -f /var/lib/aide/aide.db
|
| 92 |
+
check "journal persistent" test -d /var/log/journal
|
| 93 |
+
check "automatic security updates" sh -c "systemctl is-enabled unattended-upgrades && grep -q 'Automatic-Reboot' /etc/apt/apt.conf.d/52fly-unattended"
|
| 94 |
+
check "clock synchronised" sh -c "timedatectl show -p NTPSynchronized --value | grep -q yes"
|
| 95 |
+
check "no snapd" sh -c "! dpkg -l snapd 2>/dev/null | grep -q '^ii'"
|
| 96 |
+
|
| 97 |
+
echo "Lynis"
|
| 98 |
+
if command -v lynis >/dev/null; then
|
| 99 |
+
lynis audit system --quick --no-colors >/tmp/lynis.out 2>&1
|
| 100 |
+
printf ' hardening index: %s\n' "$(grep -o 'Hardening index : [0-9]*' /tmp/lynis.out | grep -o '[0-9]*$' || echo '?')"
|
| 101 |
+
grep -E '^\s+\* ' /var/log/lynis-report.dat >/dev/null 2>&1 || true
|
| 102 |
+
fi
|
| 103 |
+
|
| 104 |
+
echo
|
| 105 |
+
[ "$fail" = 0 ] && echo "all controls pass" || echo "SOME CONTROLS FAILED"
|
| 106 |
+
exit $fail
|
deploy/custody.env.example
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# /srv/fly/custody.env (mode 600): the treasury setup and the caps, read by BOTH the signer and the brain.
|
| 2 |
+
# Nothing secret: all of it is visible on chain or harmless. Comments on their own lines only.
|
| 3 |
+
# Create the multisig and its limits on https://fly-trader.app/owner.html with your owner wallet (docs/vault/RUNBOOK.md
|
| 4 |
+
# section 4.3); the page prints the three lines below. After changing any value: sudo python3 /usr/local/lib/fly/fly_update.py restart
|
| 5 |
+
|
| 6 |
+
# the Squads v4 multisig whose vault 0 is the treasury (members: your owner wallets only; never a server key)
|
| 7 |
+
VAULT_MULTISIG=
|
| 8 |
+
# L1: SOL, daily, member = the trading key, the ONLY destination = the trading wallet. A new limit has a new address.
|
| 9 |
+
VAULT_LIMIT_TRADING=
|
| 10 |
+
# L2: SOL, weekly, member = the payout key, any destination (claims)
|
| 11 |
+
VAULT_LIMIT_PAYOUT=
|
| 12 |
+
# the most SOL the trading wallet keeps (positions are sized on the whole book, spent from this float); also the largest
|
| 13 |
+
# single top-up the signer signs. Moves smaller than TOPUP_MIN_SOL are skipped.
|
| 14 |
+
TREASURY_FLOAT_SOL=2.0
|
| 15 |
+
TOPUP_MIN_SOL=0.1
|
| 16 |
+
# the fixed size for a model without a sizing table (sizing is otherwise MAX_POSITION_FRACTION of the book); must be > 0
|
| 17 |
+
MAX_POSITION_SOL=0.5
|
| 18 |
+
# the signer refuses any buy larger than this (a bound on a compromised brain, not sizing)
|
| 19 |
+
SIGNER_MAX_BUY_SOL=0.6
|
| 20 |
+
# the signer's own limits: SOL bought per day, swaps per minute, priority fee per swap, the worst value a swap may return
|
| 21 |
+
SIGNER_DAILY_BUY_SOL=10
|
| 22 |
+
SIGNER_SWAPS_PER_MIN=6
|
| 23 |
+
SIGNER_MAX_PRIORITY_SOL=0.005
|
| 24 |
+
SIGNER_MIN_VALUE_RATIO=0.6
|
| 25 |
+
# your age1... public key (NOT the release key): the signer encrypts the two hot keys to it, the brain the nightly ledger
|
| 26 |
+
VAULT_BACKUP_RECIPIENT=
|
deploy/docker-compose.vault.yml
CHANGED
|
@@ -1,47 +1,91 @@
|
|
| 1 |
# The hosted $FLY vault fly (docs/vault/SPEC.md). Installed by bootstrap-host.sh as /srv/fly/docker-compose.yml and
|
| 2 |
-
# driven by the updater (fly_update.py): FLY_RELEASE picks the image
|
| 3 |
-
# Nothing listens on a public interface: the console is on 127.0.0.1 (reach it with an SSH tunnel
|
| 4 |
-
#
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5 |
services:
|
| 6 |
db:
|
| 7 |
-
image: postgres:17
|
|
|
|
| 8 |
environment:
|
| 9 |
POSTGRES_USER: fly
|
| 10 |
-
POSTGRES_PASSWORD: fly
|
| 11 |
POSTGRES_DB: fly_trader
|
| 12 |
volumes:
|
| 13 |
- pgdata:/var/lib/postgresql/data
|
|
|
|
| 14 |
healthcheck:
|
| 15 |
test: ["CMD-SHELL", "pg_isready -U fly -d fly_trader"]
|
| 16 |
interval: 5s
|
| 17 |
timeout: 3s
|
| 18 |
retries: 30
|
| 19 |
restart: unless-stopped
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 20 |
logging: {driver: json-file, options: {max-size: "20m", max-file: "5"}}
|
|
|
|
| 21 |
fly:
|
| 22 |
-
image: fly-trader:${FLY_RELEASE:
|
| 23 |
-
env_file: /srv/fly/vault.env
|
| 24 |
environment:
|
| 25 |
-
DATABASE_URL: postgresql://fly:fly@db:5432/fly_trader
|
| 26 |
DATA_DIR: /app/data
|
| 27 |
LOG_DIR: /app/logs
|
| 28 |
-
|
| 29 |
VAULT_ENABLED: "1"
|
|
|
|
| 30 |
volumes:
|
| 31 |
- /srv/fly/data:/app/data
|
| 32 |
- /srv/fly/logs:/app/logs
|
| 33 |
-
- /srv/fly/
|
| 34 |
- ${FLY_RELEASES:-/srv/fly/releases}:/releases:ro
|
|
|
|
| 35 |
ports:
|
| 36 |
- "127.0.0.1:8501:8501"
|
| 37 |
depends_on:
|
| 38 |
db:
|
| 39 |
condition: service_healthy
|
|
|
|
|
|
|
| 40 |
restart: unless-stopped
|
| 41 |
stop_grace_period: 150s
|
| 42 |
security_opt: ["no-new-privileges:true"]
|
| 43 |
cap_drop: [ALL]
|
| 44 |
-
mem_limit: 14g
|
| 45 |
logging: {driver: json-file, options: {max-size: "50m", max-file: "5"}}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 46 |
volumes:
|
| 47 |
pgdata:
|
|
|
|
| 1 |
# The hosted $FLY vault fly (docs/vault/SPEC.md). Installed by bootstrap-host.sh as /srv/fly/docker-compose.yml and
|
| 2 |
+
# driven by the updater (fly_update.py): FLY_RELEASE picks the image tags it built from a verified release.
|
| 3 |
+
# Nothing listens on a public interface: the console is on 127.0.0.1 (reach it with an SSH tunnel over Tailscale) and
|
| 4 |
+
# the containers only make outbound connections.
|
| 5 |
+
#
|
| 6 |
+
# fly-signer the ONLY container that can read the two hot keys; it checks and signs, over a Unix socket in
|
| 7 |
+
# /srv/fly/run (a directory only it and the brain's group can enter). Read-only filesystem, no capabilities.
|
| 8 |
+
# fly the brain: engine, console, vault worker, data feeds. No key is mounted into it.
|
| 9 |
+
# db Postgres on an internal network (no route out), its password generated by bootstrap-host.sh.
|
| 10 |
+
#
|
| 11 |
+
# Settings: /srv/fly/custody.env (public: the treasury multisig, its limits, the float and caps; read by both),
|
| 12 |
+
# /srv/fly/signer.env (the signer's own RPC URL), /srv/fly/vault.env (the brain's API keys, relay, Telegram, backups),
|
| 13 |
+
# /srv/fly/db.env (the database password). All root-only (0600) on the host.
|
| 14 |
services:
|
| 15 |
db:
|
| 16 |
+
image: postgres:17@sha256:d74eeac9a635390a49bc21bd49fccd973de707e2a53a76ac49b552b8712ec46f
|
| 17 |
+
env_file: /srv/fly/db.env
|
| 18 |
environment:
|
| 19 |
POSTGRES_USER: fly
|
|
|
|
| 20 |
POSTGRES_DB: fly_trader
|
| 21 |
volumes:
|
| 22 |
- pgdata:/var/lib/postgresql/data
|
| 23 |
+
networks: [internal]
|
| 24 |
healthcheck:
|
| 25 |
test: ["CMD-SHELL", "pg_isready -U fly -d fly_trader"]
|
| 26 |
interval: 5s
|
| 27 |
timeout: 3s
|
| 28 |
retries: 30
|
| 29 |
restart: unless-stopped
|
| 30 |
+
security_opt: ["no-new-privileges:true"]
|
| 31 |
+
logging: {driver: json-file, options: {max-size: "20m", max-file: "5"}}
|
| 32 |
+
|
| 33 |
+
fly-signer:
|
| 34 |
+
image: fly-signer:${FLY_RELEASE:-none}
|
| 35 |
+
env_file: [/srv/fly/custody.env, /srv/fly/signer.env]
|
| 36 |
+
environment:
|
| 37 |
+
BOT_PRIVATE_KEY_FILE: /run/secrets/bot_key
|
| 38 |
+
PAYOUT_PRIVATE_KEY_FILE: /run/secrets/payout_key
|
| 39 |
+
SIGNER_SOCKET: /run/fly-signer/sock
|
| 40 |
+
SIGNER_DB: /srv/signer/ledger.sqlite
|
| 41 |
+
volumes:
|
| 42 |
+
- /srv/fly/secrets/bot_key:/run/secrets/bot_key:ro
|
| 43 |
+
- /srv/fly/secrets/payout_key:/run/secrets/payout_key:ro
|
| 44 |
+
- /srv/fly/signer:/srv/signer
|
| 45 |
+
- /srv/fly/run:/run/fly-signer
|
| 46 |
+
networks: [egress] # its own RPC and Jupiter's price API; nothing reaches in
|
| 47 |
+
read_only: true
|
| 48 |
+
tmpfs: [/tmp]
|
| 49 |
+
restart: unless-stopped
|
| 50 |
+
security_opt: ["no-new-privileges:true"]
|
| 51 |
+
cap_drop: [ALL]
|
| 52 |
+
mem_limit: 512m
|
| 53 |
+
pids_limit: 64
|
| 54 |
logging: {driver: json-file, options: {max-size: "20m", max-file: "5"}}
|
| 55 |
+
|
| 56 |
fly:
|
| 57 |
+
image: fly-trader:${FLY_RELEASE:-none}
|
| 58 |
+
env_file: [/srv/fly/custody.env, /srv/fly/vault.env, /srv/fly/db.env]
|
| 59 |
environment:
|
|
|
|
| 60 |
DATA_DIR: /app/data
|
| 61 |
LOG_DIR: /app/logs
|
| 62 |
+
SIGNER_SOCKET: /run/fly-signer/sock
|
| 63 |
VAULT_ENABLED: "1"
|
| 64 |
+
AUTO_UPDATE: "0" # releases come through the host updater (signed, 24 h hold on code), never from inside
|
| 65 |
volumes:
|
| 66 |
- /srv/fly/data:/app/data
|
| 67 |
- /srv/fly/logs:/app/logs
|
| 68 |
+
- /srv/fly/run:/run/fly-signer
|
| 69 |
- ${FLY_RELEASES:-/srv/fly/releases}:/releases:ro
|
| 70 |
+
networks: [internal, egress]
|
| 71 |
ports:
|
| 72 |
- "127.0.0.1:8501:8501"
|
| 73 |
depends_on:
|
| 74 |
db:
|
| 75 |
condition: service_healthy
|
| 76 |
+
fly-signer:
|
| 77 |
+
condition: service_healthy
|
| 78 |
restart: unless-stopped
|
| 79 |
stop_grace_period: 150s
|
| 80 |
security_opt: ["no-new-privileges:true"]
|
| 81 |
cap_drop: [ALL]
|
| 82 |
+
mem_limit: 48g # a dedicated 128 GB box; lower it on a smaller one (16 GB: 14g)
|
| 83 |
logging: {driver: json-file, options: {max-size: "50m", max-file: "5"}}
|
| 84 |
+
|
| 85 |
+
networks:
|
| 86 |
+
internal:
|
| 87 |
+
internal: true # the database has no route to anything but its two clients
|
| 88 |
+
egress: {}
|
| 89 |
+
|
| 90 |
volumes:
|
| 91 |
pgdata:
|
deploy/fly
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/bin/sh
|
| 2 |
+
# Run a fly-trader command inside the running brain container, e.g. sudo fly vault status
|
| 3 |
+
# sudo fly signer <cmd> runs python -m fly_trader.signer <cmd> in the signer container (status, clear-panic)
|
| 4 |
+
set -e
|
| 5 |
+
if [ "$1" = "signer" ]; then
|
| 6 |
+
shift
|
| 7 |
+
exec docker exec -i fly-fly-signer-1 python -m fly_trader.signer "$@"
|
| 8 |
+
fi
|
| 9 |
+
exec docker exec -i fly-fly-1 fly-trader "$@"
|
deploy/fly-aide.service
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[Unit]
|
| 2 |
+
Description=fly-trader vault: daily file-integrity check (AIDE), findings to Telegram
|
| 3 |
+
|
| 4 |
+
[Service]
|
| 5 |
+
Type=oneshot
|
| 6 |
+
Nice=10
|
| 7 |
+
ExecStart=/usr/bin/python3 /usr/local/lib/fly/fly_audit_watch.py aide
|
| 8 |
+
TimeoutStartSec=3600
|
deploy/fly-aide.timer
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[Unit]
|
| 2 |
+
Description=Run the AIDE check daily
|
| 3 |
+
|
| 4 |
+
[Timer]
|
| 5 |
+
OnCalendar=*-*-* 05:30:00 UTC
|
| 6 |
+
Persistent=true
|
| 7 |
+
|
| 8 |
+
[Install]
|
| 9 |
+
WantedBy=timers.target
|
deploy/fly-audit-watch.service
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[Unit]
|
| 2 |
+
Description=fly-trader vault: alert when anything but the signer reads a wallet key
|
| 3 |
+
After=auditd.service
|
| 4 |
+
|
| 5 |
+
[Service]
|
| 6 |
+
Type=oneshot
|
| 7 |
+
ExecStart=/usr/bin/python3 /usr/local/lib/fly/fly_audit_watch.py run
|
| 8 |
+
TimeoutStartSec=120
|
deploy/fly-audit-watch.timer
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[Unit]
|
| 2 |
+
Description=Check the key-read audit log every minute
|
| 3 |
+
|
| 4 |
+
[Timer]
|
| 5 |
+
OnBootSec=1min
|
| 6 |
+
OnUnitActiveSec=1min
|
| 7 |
+
AccuracySec=5s
|
| 8 |
+
|
| 9 |
+
[Install]
|
| 10 |
+
WantedBy=timers.target
|
deploy/fly_audit_watch.py
ADDED
|
@@ -0,0 +1,109 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Tell the operator when anything but the signer reads a wallet key (runs on the HOST, root, every minute).
|
| 3 |
+
|
| 4 |
+
auditd logs every access to /srv/fly/secrets (key ``fly_keys``, bootstrap-host.sh). The signer container's Python is
|
| 5 |
+
the only reader there should ever be (plus AIDE's daily integrity scan); a read by anything else -- a shell, cat, a
|
| 6 |
+
process in the brain container -- is the earliest sign of a compromise, so it goes to Telegram at once, with the
|
| 7 |
+
program, user and time. ``ausearch --checkpoint`` makes each event reported once.
|
| 8 |
+
|
| 9 |
+
fly_audit_watch.py run (the systemd timer, every minute)
|
| 10 |
+
fly_audit_watch.py aide (fly-aide.timer, daily: AIDE's integrity check; changes go to Telegram, since
|
| 11 |
+
this host has no mail. After reviewing expected changes: sudo aide --update and
|
| 12 |
+
sudo cp /var/lib/aide/aide.db.new /var/lib/aide/aide.db)
|
| 13 |
+
fly_audit_watch.py test (send a test message)
|
| 14 |
+
"""
|
| 15 |
+
from __future__ import annotations
|
| 16 |
+
|
| 17 |
+
import json
|
| 18 |
+
import re
|
| 19 |
+
import subprocess
|
| 20 |
+
import sys
|
| 21 |
+
import urllib.request
|
| 22 |
+
from pathlib import Path
|
| 23 |
+
|
| 24 |
+
CONF = Path("/etc/fly/update.json") # the updater's Telegram settings
|
| 25 |
+
CHECKPOINT = Path("/var/lib/fly/audit.checkpoint")
|
| 26 |
+
SIGNER_EXE = {"/usr/local/bin/python3.13", "/usr/local/bin/python3"} # the signer image's interpreter
|
| 27 |
+
ALLOWED_EXE = {"/usr/bin/aide", "/usr/sbin/aide"} # AIDE hashes every file daily
|
| 28 |
+
SIGNER_UID_OFFSET = 10002 # the signer container's user; on the host: dockremap base + this
|
| 29 |
+
|
| 30 |
+
FIELD = re.compile(r'(\w+)=("[^"]*"|\S+)')
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def remap_base() -> int:
|
| 34 |
+
for line in Path("/etc/subuid").read_text().splitlines():
|
| 35 |
+
name, start, _ = line.split(":")
|
| 36 |
+
if name == "dockremap":
|
| 37 |
+
return int(start)
|
| 38 |
+
return -1
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def events(raw: str) -> list[dict]:
|
| 42 |
+
"""SYSCALL records of ``ausearch --raw`` output as dicts (pure)."""
|
| 43 |
+
out = []
|
| 44 |
+
for line in raw.splitlines():
|
| 45 |
+
if not line.startswith("type=SYSCALL"):
|
| 46 |
+
continue
|
| 47 |
+
out.append({k: v.strip('"') for k, v in FIELD.findall(line)})
|
| 48 |
+
return out
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def suspicious(evs: list[dict], signer_uid: int) -> list[dict]:
|
| 52 |
+
"""Reads that are not the signer's interpreter as the signer's user, nor AIDE (pure)."""
|
| 53 |
+
bad = []
|
| 54 |
+
for e in evs:
|
| 55 |
+
exe, uid = e.get("exe", ""), int(e.get("uid", "-1")) if e.get("uid", "").lstrip("-").isdigit() else -1
|
| 56 |
+
if exe in SIGNER_EXE and uid == signer_uid:
|
| 57 |
+
continue
|
| 58 |
+
if exe in ALLOWED_EXE and uid == 0:
|
| 59 |
+
continue
|
| 60 |
+
bad.append(e)
|
| 61 |
+
return bad
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def alert(text: str) -> None:
|
| 65 |
+
c = json.loads(CONF.read_text()) if CONF.exists() else {}
|
| 66 |
+
tok, chat = c.get("telegram_bot_token"), c.get("telegram_chat_id")
|
| 67 |
+
print("ALERT", text, flush=True)
|
| 68 |
+
if tok and chat:
|
| 69 |
+
body = json.dumps({"chat_id": chat, "text": "[fly host] " + text}).encode()
|
| 70 |
+
urllib.request.urlopen(urllib.request.Request(f"https://api.telegram.org/bot{tok}/sendMessage", data=body,
|
| 71 |
+
headers={"Content-Type": "application/json"}), timeout=10)
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
def run() -> int:
|
| 75 |
+
CHECKPOINT.parent.mkdir(parents=True, exist_ok=True)
|
| 76 |
+
r = subprocess.run(["ausearch", "-k", "fly_keys", "--raw", "--checkpoint", str(CHECKPOINT), "--start", "checkpoint"],
|
| 77 |
+
capture_output=True, text=True)
|
| 78 |
+
if r.returncode not in (0, 1): # 1: no matches; 10/11/12: checkpoint problems -> start fresh next run
|
| 79 |
+
CHECKPOINT.unlink(missing_ok=True)
|
| 80 |
+
return 0
|
| 81 |
+
bad = suspicious(events(r.stdout), remap_base() + SIGNER_UID_OFFSET)
|
| 82 |
+
if bad:
|
| 83 |
+
lines = [f"{e.get('exe')} (uid {e.get('uid')}, comm {e.get('comm')}, syscall {e.get('syscall')})" for e in bad[:10]]
|
| 84 |
+
alert(f"WALLET KEY READ by something other than the signer ({len(bad)} event(s)):\n" + "\n".join(lines)
|
| 85 |
+
+ "\nIf this was not you: revoke L1 and L2 in the owner wallet now, then investigate.")
|
| 86 |
+
return len(bad)
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
def aide() -> int:
|
| 90 |
+
"""AIDE's exit code: 0 no change, 1-7 added/removed/changed files, >= 14 an error."""
|
| 91 |
+
r = subprocess.run(["aide", "--check", "--config", "/etc/aide/aide.conf"], capture_output=True, text=True)
|
| 92 |
+
if r.returncode == 0:
|
| 93 |
+
return 0
|
| 94 |
+
summary = [l for l in r.stdout.splitlines() if l.strip() and (l.startswith(("Added", "Removed", "Changed", "f", "d", "l"))
|
| 95 |
+
or "entries" in l)][:25]
|
| 96 |
+
what = "an ERROR" if r.returncode >= 14 else "file changes"
|
| 97 |
+
alert(f"AIDE found {what} (exit {r.returncode}). If you did not make them (a release, apt upgrades), investigate:\n"
|
| 98 |
+
+ "\n".join(summary) + "\nFull report: sudo aide --check")
|
| 99 |
+
return r.returncode
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
if __name__ == "__main__":
|
| 103 |
+
cmd = sys.argv[1] if len(sys.argv) > 1 else "run"
|
| 104 |
+
if cmd == "test":
|
| 105 |
+
alert("audit watch test message")
|
| 106 |
+
elif cmd == "aide":
|
| 107 |
+
aide()
|
| 108 |
+
else:
|
| 109 |
+
run()
|
deploy/fly_update.py
CHANGED
|
@@ -3,7 +3,8 @@
|
|
| 3 |
|
| 4 |
Runs on the HOST (root, systemd timer, standard library only), never inside the container, because it rebuilds and
|
| 5 |
restarts the container. Everything it trusts comes from files on this host that no release can change:
|
| 6 |
-
``/etc/fly/allowed_signers`` (Bryce's release
|
|
|
|
| 7 |
|
| 8 |
One round:
|
| 9 |
1. poll the HF model repo's commit sha; nothing to do if it is the installed one
|
|
@@ -12,13 +13,19 @@ One round:
|
|
| 12 |
no forks)
|
| 13 |
3. download exactly the listed paths (strict path check), verify size + sha256 of each, into a fresh directory
|
| 14 |
4. compare file classes with the installed release:
|
| 15 |
-
only model/seed -> ``docker compose exec fly fly-trader apply-release /releases/<seq>`` (hot swap
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 22 |
"""
|
| 23 |
from __future__ import annotations
|
| 24 |
|
|
@@ -37,13 +44,17 @@ from pathlib import Path
|
|
| 37 |
CONF = Path(os.environ.get("FLY_UPDATE_CONF", "/etc/fly/update.json"))
|
| 38 |
PATH_RE = re.compile(r"^(?!.*\.\.)(?!/)[A-Za-z0-9_.\-/]+$")
|
| 39 |
INFRA = ("Dockerfile", "docker-compose.yml", "docker-compose.cuda.yml", ".dockerignore", "docker/", "deploy/")
|
|
|
|
|
|
|
|
|
|
| 40 |
|
| 41 |
|
| 42 |
def conf() -> dict:
|
| 43 |
c = json.loads(CONF.read_text())
|
| 44 |
c.setdefault("repo", "bryceweiner/fly-trader")
|
| 45 |
c.setdefault("allowed_signers", "/etc/fly/allowed_signers")
|
| 46 |
-
c.setdefault("principal", "bryce")
|
|
|
|
| 47 |
c.setdefault("namespace", "fly-trader-release")
|
| 48 |
c.setdefault("root", "/srv/fly")
|
| 49 |
c.setdefault("project", "fly")
|
|
@@ -95,13 +106,19 @@ def classify(path: str) -> str:
|
|
| 95 |
return "code"
|
| 96 |
|
| 97 |
|
| 98 |
-
def verify_signature(c: dict, manifest: bytes, sig: bytes) ->
|
|
|
|
|
|
|
|
|
|
| 99 |
with tempfile.TemporaryDirectory() as d:
|
| 100 |
sp = Path(d) / "release.json.sig"; sp.write_bytes(sig)
|
| 101 |
-
|
| 102 |
-
|
| 103 |
-
|
| 104 |
-
|
|
|
|
|
|
|
|
|
|
| 105 |
|
| 106 |
|
| 107 |
def check_manifest(m: dict) -> None:
|
|
@@ -113,6 +130,8 @@ def check_manifest(m: dict) -> None:
|
|
| 113 |
raise SystemExit(f"refusing path {p!r}")
|
| 114 |
if f.get("class") != classify(p):
|
| 115 |
raise SystemExit(f"{p}: class {f.get('class')} does not match {classify(p)}")
|
|
|
|
|
|
|
| 116 |
if not re.fullmatch(r"[0-9a-f]{64}", f.get("sha256", "")) or not isinstance(f.get("size"), int):
|
| 117 |
raise SystemExit(f"{p}: bad hash or size")
|
| 118 |
|
|
@@ -146,19 +165,27 @@ def save_state(c: dict, st: dict) -> None:
|
|
| 146 |
|
| 147 |
|
| 148 |
def compose(c: dict, *args: str, check: bool = True, env: dict | None = None) -> subprocess.CompletedProcess:
|
| 149 |
-
|
|
|
|
|
|
|
| 150 |
return subprocess.run(["docker", "compose", "-p", c["project"], "-f", str(Path(c["root"]) / "docker-compose.yml"), *args],
|
| 151 |
cwd=c["root"], check=check, capture_output=True, text=True, env=e)
|
| 152 |
|
| 153 |
|
| 154 |
-
def fetch(c: dict, sha: str) -> tuple[dict, Path, bytes]:
|
| 155 |
base = f"https://huggingface.co/{c['repo']}/resolve/{sha}/"
|
| 156 |
raw = http(base + "release.json"); sig = http(base + "release.json.sig")
|
| 157 |
-
verify_signature(c, raw, sig)
|
| 158 |
m = json.loads(raw)
|
| 159 |
check_manifest(m)
|
| 160 |
dest = Path(c["root"]) / "releases" / str(m["seq"])
|
| 161 |
if dest.exists():
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 162 |
shutil.rmtree(dest)
|
| 163 |
part = dest.with_name(dest.name + ".part")
|
| 164 |
if part.exists():
|
|
@@ -170,7 +197,7 @@ def fetch(c: dict, sha: str) -> tuple[dict, Path, bytes]:
|
|
| 170 |
(part / "release.json").write_bytes(raw); (part / "release.json.sig").write_bytes(sig)
|
| 171 |
verify_dir(part, m)
|
| 172 |
part.rename(dest)
|
| 173 |
-
return m, dest, raw
|
| 174 |
|
| 175 |
|
| 176 |
def installed_manifest(c: dict, st: dict) -> dict | None:
|
|
@@ -212,82 +239,155 @@ def healthy(c: dict, limit_s: int = 900) -> bool:
|
|
| 212 |
return False
|
| 213 |
|
| 214 |
|
| 215 |
-
def deploy_code(c: dict, st: dict, m: dict, d: Path, infra: bool) ->
|
|
|
|
| 216 |
tag = f"fly-trader:{m['seq']}"
|
| 217 |
-
|
| 218 |
-
|
| 219 |
-
|
| 220 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 221 |
alert(c, f"release {m['seq']}: the fly never became ready for a restart (orders/claims/settlement in flight); will retry")
|
| 222 |
-
return
|
|
|
|
|
|
|
|
|
|
| 223 |
prev = st.get("image")
|
| 224 |
-
|
|
|
|
| 225 |
if healthy(c):
|
| 226 |
-
return
|
| 227 |
alert(c, f"release {m['seq']} did not become healthy; rolling back to {prev}")
|
|
|
|
|
|
|
| 228 |
if prev:
|
| 229 |
-
compose(c, "up", "-d", env={"FLY_RELEASE": str(prev)
|
| 230 |
-
return
|
|
|
|
| 231 |
|
|
|
|
|
|
|
| 232 |
|
| 233 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 234 |
old = installed_manifest(c, st)
|
| 235 |
-
if old is
|
| 236 |
-
|
| 237 |
-
|
| 238 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 239 |
classes = changed_classes(old, m)
|
| 240 |
-
if
|
| 241 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 242 |
save_state(c, st)
|
| 243 |
-
|
| 244 |
-
|
|
|
|
|
|
|
| 245 |
if classes & {"code", "infra"} or st.get("image") is None:
|
| 246 |
-
|
|
|
|
| 247 |
return
|
|
|
|
|
|
|
| 248 |
st["image"] = m["seq"]
|
| 249 |
elif classes & {"model", "seed"}:
|
| 250 |
r = compose(c, "exec", "-T", "fly", "fly-trader", "apply-release", f"/releases/{m['seq']}", check=False)
|
| 251 |
if r.returncode != 0:
|
| 252 |
-
|
| 253 |
-
|
| 254 |
-
|
|
|
|
|
|
|
|
|
|
| 255 |
save_state(c, st)
|
| 256 |
-
alert(c, f"release {m['seq']} applied ({', '.join(sorted(classes))})")
|
| 257 |
|
| 258 |
|
| 259 |
-
def run(approve_seq: int | None = None) -> None:
|
| 260 |
c = conf()
|
|
|
|
| 261 |
Path(c["root"], "releases").mkdir(parents=True, exist_ok=True)
|
| 262 |
st = load_state(c)
|
| 263 |
-
|
| 264 |
-
sha = info["sha"]
|
| 265 |
if approve_seq is not None:
|
| 266 |
-
pend = st.get("pending") or {}
|
| 267 |
if pend.get("seq") != approve_seq:
|
| 268 |
raise SystemExit(f"no pending release {approve_seq}")
|
| 269 |
sha = pend["hf_sha"]
|
| 270 |
-
|
| 271 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 272 |
try:
|
| 273 |
-
m, d, raw = fetch(c, sha)
|
| 274 |
-
apply(c, st, m, d, raw, sha, approved=approve_seq is not None)
|
| 275 |
except SystemExit as e:
|
|
|
|
|
|
|
|
|
|
| 276 |
st["hf_sha_rejected"] = sha; save_state(c, st)
|
| 277 |
alert(c, f"release at {sha[:10]} rejected: {e}")
|
| 278 |
raise
|
| 279 |
|
| 280 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 281 |
def main(argv: list[str]) -> None:
|
| 282 |
cmd = argv[1] if len(argv) > 1 else "run"
|
| 283 |
if cmd == "run":
|
| 284 |
run()
|
| 285 |
elif cmd == "approve":
|
| 286 |
run(int(argv[2]))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 287 |
elif cmd == "status":
|
| 288 |
print(json.dumps(load_state(conf()), indent=1))
|
| 289 |
elif cmd == "verify": # used by tools/publish_release.py on the Mac before uploading
|
| 290 |
-
d = Path(argv[2]); c = {"allowed_signers": argv[3], "
|
| 291 |
raw = (d / "release.json").read_bytes()
|
| 292 |
verify_signature(c, raw, (d / "release.json.sig").read_bytes())
|
| 293 |
m = json.loads(raw); check_manifest(m); verify_dir(d, m)
|
|
|
|
| 3 |
|
| 4 |
Runs on the HOST (root, systemd timer, standard library only), never inside the container, because it rebuilds and
|
| 5 |
restarts the container. Everything it trusts comes from files on this host that no release can change:
|
| 6 |
+
``/etc/fly/allowed_signers`` (Bryce's release keys, restricted to the release namespace) and ``/etc/fly/update.json``
|
| 7 |
+
(which also pins the very first release by its manifest hash: no trust on first use).
|
| 8 |
|
| 9 |
One round:
|
| 10 |
1. poll the HF model repo's commit sha; nothing to do if it is the installed one
|
|
|
|
| 13 |
no forks)
|
| 14 |
3. download exactly the listed paths (strict path check), verify size + sha256 of each, into a fresh directory
|
| 15 |
4. compare file classes with the installed release:
|
| 16 |
+
only model/seed -> ``docker compose exec fly fly-trader apply-release /releases/<seq>`` (hot swap; models
|
| 17 |
+
cannot run code, fly_trader/train/model_io.py); unhealthy after it -> the previous release's
|
| 18 |
+
models are re-applied
|
| 19 |
+
code -> held for ``code_delay_s`` (24 h) with a Telegram notice, so a stolen signing key cannot ship
|
| 20 |
+
code faster than Bryce can ``veto`` it; ``approve`` applies it at once. Then: wait for
|
| 21 |
+
``ready-for-restart``, build ``fly-trader:<seq>`` from that directory, recreate, wait for
|
| 22 |
+
healthy, else roll back to the previous tag and compose file
|
| 23 |
+
infra -> held until ``approve`` (compose/Dockerfile/entrypoint can grant host access, so a human
|
| 24 |
+
looks first)
|
| 25 |
+
|
| 26 |
+
fly_update.py run | approve <seq> | veto <seq> | status | restart | verify <dir> <allowed_signers> [principal ...]
|
| 27 |
+
|
| 28 |
+
``restart`` recreates the containers on the installed release (after you edit /srv/fly/*.env).
|
| 29 |
"""
|
| 30 |
from __future__ import annotations
|
| 31 |
|
|
|
|
| 44 |
CONF = Path(os.environ.get("FLY_UPDATE_CONF", "/etc/fly/update.json"))
|
| 45 |
PATH_RE = re.compile(r"^(?!.*\.\.)(?!/)[A-Za-z0-9_.\-/]+$")
|
| 46 |
INFRA = ("Dockerfile", "docker-compose.yml", "docker-compose.cuda.yml", ".dockerignore", "docker/", "deploy/")
|
| 47 |
+
# what models/ and seed/ may hold: nothing a loader could execute (fly_trader/train/model_io.SAFE_SUFFIXES)
|
| 48 |
+
MODEL_SUFFIXES = {".pt", ".skops", ".npz", ".json", ".bin", ".parquet", ".txt", ".sha256", ""}
|
| 49 |
+
SEED_SUFFIXES = {".json", ".sql"}
|
| 50 |
|
| 51 |
|
| 52 |
def conf() -> dict:
|
| 53 |
c = json.loads(CONF.read_text())
|
| 54 |
c.setdefault("repo", "bryceweiner/fly-trader")
|
| 55 |
c.setdefault("allowed_signers", "/etc/fly/allowed_signers")
|
| 56 |
+
c.setdefault("principals", [c.get("principal") or "bryce", "bryce-recovery"])
|
| 57 |
+
c.setdefault("code_delay_s", 86400)
|
| 58 |
c.setdefault("namespace", "fly-trader-release")
|
| 59 |
c.setdefault("root", "/srv/fly")
|
| 60 |
c.setdefault("project", "fly")
|
|
|
|
| 106 |
return "code"
|
| 107 |
|
| 108 |
|
| 109 |
+
def verify_signature(c: dict, manifest: bytes, sig: bytes) -> str:
|
| 110 |
+
"""The principal whose key signed ``manifest`` (any of ``principals``), else SystemExit."""
|
| 111 |
+
principals = c.get("principals") or [c["principal"]]
|
| 112 |
+
err = ""
|
| 113 |
with tempfile.TemporaryDirectory() as d:
|
| 114 |
sp = Path(d) / "release.json.sig"; sp.write_bytes(sig)
|
| 115 |
+
for who in principals:
|
| 116 |
+
r = subprocess.run(["ssh-keygen", "-Y", "verify", "-f", c["allowed_signers"], "-I", who, "-n", c["namespace"], "-s", str(sp)],
|
| 117 |
+
input=manifest, capture_output=True)
|
| 118 |
+
if r.returncode == 0:
|
| 119 |
+
return who
|
| 120 |
+
err = r.stderr.decode(errors="replace").strip()
|
| 121 |
+
raise SystemExit(f"release signature does NOT verify: {err[:300]}")
|
| 122 |
|
| 123 |
|
| 124 |
def check_manifest(m: dict) -> None:
|
|
|
|
| 130 |
raise SystemExit(f"refusing path {p!r}")
|
| 131 |
if f.get("class") != classify(p):
|
| 132 |
raise SystemExit(f"{p}: class {f.get('class')} does not match {classify(p)}")
|
| 133 |
+
if f["class"] == "model" and Path(p).suffix not in MODEL_SUFFIXES or f["class"] == "seed" and Path(p).suffix not in SEED_SUFFIXES:
|
| 134 |
+
raise SystemExit(f"{p}: file type not allowed in {f['class']}/ (a model or seed must not be able to run code)")
|
| 135 |
if not re.fullmatch(r"[0-9a-f]{64}", f.get("sha256", "")) or not isinstance(f.get("size"), int):
|
| 136 |
raise SystemExit(f"{p}: bad hash or size")
|
| 137 |
|
|
|
|
| 165 |
|
| 166 |
|
| 167 |
def compose(c: dict, *args: str, check: bool = True, env: dict | None = None) -> subprocess.CompletedProcess:
|
| 168 |
+
"""docker compose on the installed release (FLY_RELEASE = the image tag this host runs, unless ``env`` says otherwise)."""
|
| 169 |
+
installed_tag = load_state(c).get("image")
|
| 170 |
+
e = {**os.environ, "FLY_RELEASE": str(installed_tag) if installed_tag is not None else "none", **(env or {})}
|
| 171 |
return subprocess.run(["docker", "compose", "-p", c["project"], "-f", str(Path(c["root"]) / "docker-compose.yml"), *args],
|
| 172 |
cwd=c["root"], check=check, capture_output=True, text=True, env=e)
|
| 173 |
|
| 174 |
|
| 175 |
+
def fetch(c: dict, sha: str) -> tuple[dict, Path, bytes, str]:
|
| 176 |
base = f"https://huggingface.co/{c['repo']}/resolve/{sha}/"
|
| 177 |
raw = http(base + "release.json"); sig = http(base + "release.json.sig")
|
| 178 |
+
who = verify_signature(c, raw, sig)
|
| 179 |
m = json.loads(raw)
|
| 180 |
check_manifest(m)
|
| 181 |
dest = Path(c["root"]) / "releases" / str(m["seq"])
|
| 182 |
if dest.exists():
|
| 183 |
+
try: # already downloaded (a held or retried release): re-verify, reuse
|
| 184 |
+
if (dest / "release.json").read_bytes() == raw:
|
| 185 |
+
verify_dir(dest, m)
|
| 186 |
+
return m, dest, raw, who
|
| 187 |
+
except (SystemExit, OSError):
|
| 188 |
+
pass
|
| 189 |
shutil.rmtree(dest)
|
| 190 |
part = dest.with_name(dest.name + ".part")
|
| 191 |
if part.exists():
|
|
|
|
| 197 |
(part / "release.json").write_bytes(raw); (part / "release.json.sig").write_bytes(sig)
|
| 198 |
verify_dir(part, m)
|
| 199 |
part.rename(dest)
|
| 200 |
+
return m, dest, raw, who
|
| 201 |
|
| 202 |
|
| 203 |
def installed_manifest(c: dict, st: dict) -> dict | None:
|
|
|
|
| 239 |
return False
|
| 240 |
|
| 241 |
|
| 242 |
+
def deploy_code(c: dict, st: dict, m: dict, d: Path, infra: bool) -> str:
|
| 243 |
+
"""'ok', 'busy' (not ready for a restart: retried next round) or 'failed' (rolled back)."""
|
| 244 |
tag = f"fly-trader:{m['seq']}"
|
| 245 |
+
root = Path(c["root"]); live, backup = root / "docker-compose.yml", root / "docker-compose.yml.prev"
|
| 246 |
+
try: # the brain (default stage) and the key-holding signer, same release
|
| 247 |
+
subprocess.run(["docker", "build", "-t", tag, str(d)], check=True, capture_output=True)
|
| 248 |
+
subprocess.run(["docker", "build", "--target", "signer", "-t", f"fly-signer:{m['seq']}", str(d)], check=True, capture_output=True)
|
| 249 |
+
except subprocess.CalledProcessError as e:
|
| 250 |
+
alert(c, f"release {m['seq']}: image build failed: {(e.stderr or b'').decode(errors='replace')[-300:]}")
|
| 251 |
+
return "failed"
|
| 252 |
+
if st.get("image") is not None and not wait_ready(c): # the first install has nothing running to wait for
|
| 253 |
alert(c, f"release {m['seq']}: the fly never became ready for a restart (orders/claims/settlement in flight); will retry")
|
| 254 |
+
return "busy"
|
| 255 |
+
if infra: # approved: the release's compose file becomes the running one
|
| 256 |
+
shutil.copy2(live, backup)
|
| 257 |
+
shutil.copy2(d / "deploy" / "docker-compose.vault.yml", live)
|
| 258 |
prev = st.get("image")
|
| 259 |
+
env = {"FLY_RELEASES": str(root / "releases")}
|
| 260 |
+
compose(c, "up", "-d", check=False, env={**env, "FLY_RELEASE": str(m["seq"])})
|
| 261 |
if healthy(c):
|
| 262 |
+
return "ok"
|
| 263 |
alert(c, f"release {m['seq']} did not become healthy; rolling back to {prev}")
|
| 264 |
+
if infra and backup.exists():
|
| 265 |
+
shutil.copy2(backup, live)
|
| 266 |
if prev:
|
| 267 |
+
compose(c, "up", "-d", check=False, env={**env, "FLY_RELEASE": str(prev)})
|
| 268 |
+
return "failed"
|
| 269 |
+
|
| 270 |
|
| 271 |
+
def _when(ts: float) -> str:
|
| 272 |
+
return time.strftime("%Y-%m-%d %H:%M UTC", time.gmtime(ts))
|
| 273 |
|
| 274 |
+
|
| 275 |
+
def apply(c: dict, st: dict, m: dict, d: Path, raw: bytes, hf_sha: str, approved: bool = False, who: str = "?",
|
| 276 |
+
now: float | None = None) -> None:
|
| 277 |
+
now = time.time() if now is None else now
|
| 278 |
+
msha = hashlib.sha256(raw).hexdigest()
|
| 279 |
old = installed_manifest(c, st)
|
| 280 |
+
if old is None: # the first release is pinned by hand at bootstrap (no TOFU)
|
| 281 |
+
pin = c.get("expect_manifest_sha256")
|
| 282 |
+
if not pin:
|
| 283 |
+
raise SystemExit("no release installed and update.json pins none (expect_manifest_sha256): refusing to trust the first one seen")
|
| 284 |
+
if msha != pin:
|
| 285 |
+
raise SystemExit(f"first release's manifest {msha[:12]} is not the pinned {str(pin)[:12]}")
|
| 286 |
+
else:
|
| 287 |
+
if m["seq"] <= old["seq"]:
|
| 288 |
+
raise SystemExit(f"release seq {m['seq']} is not newer than installed {old['seq']} (rollback refused)")
|
| 289 |
+
pend = st.get("pending") or {}
|
| 290 |
+
chain = {st.get("manifest_sha256"), pend.get("manifest_sha256"), *(st.get("vetoed") or [])} - {None}
|
| 291 |
+
if m.get("prev_sha256") not in chain:
|
| 292 |
+
raise SystemExit("release does not chain from the installed one (prev_sha256 mismatch)")
|
| 293 |
classes = changed_classes(old, m)
|
| 294 |
+
if old is not None and classes & {"code", "infra"}:
|
| 295 |
+
pend = st.get("pending") or {}
|
| 296 |
+
if pend.get("manifest_sha256") != msha: # newly seen: the delay starts now (a replaced pending one restarts it)
|
| 297 |
+
pend = {"seq": m["seq"], "hf_sha": hf_sha, "manifest_sha256": msha, "classes": sorted(classes), "signer": who,
|
| 298 |
+
"first_seen": now, "due_at": now + float(c.get("code_delay_s", 86400)), "approved": False}
|
| 299 |
+
st["pending"] = pend
|
| 300 |
+
what = ("changes Docker/compose/entrypoint files: review it, then `sudo python3 /usr/local/lib/fly/fly_update.py approve "
|
| 301 |
+
f"{m['seq']}`" if "infra" in classes else
|
| 302 |
+
f"changes code: it applies at {_when(pend['due_at'])} unless you run `sudo python3 /usr/local/lib/fly/fly_update.py veto "
|
| 303 |
+
f"{m['seq']}` (`approve {m['seq']}` applies it now)")
|
| 304 |
+
alert(c, f"release {m['seq']} signed by {who} {what}")
|
| 305 |
+
if approved:
|
| 306 |
+
pend["approved"] = True
|
| 307 |
save_state(c, st)
|
| 308 |
+
if "infra" in classes and not pend["approved"]:
|
| 309 |
+
return
|
| 310 |
+
if not (pend["approved"] or now >= pend["due_at"]):
|
| 311 |
+
return
|
| 312 |
if classes & {"code", "infra"} or st.get("image") is None:
|
| 313 |
+
res = deploy_code(c, st, m, d, "infra" in classes)
|
| 314 |
+
if res == "busy":
|
| 315 |
return
|
| 316 |
+
if res == "failed":
|
| 317 |
+
raise SystemExit("the new image did not come up healthy (rolled back)")
|
| 318 |
st["image"] = m["seq"]
|
| 319 |
elif classes & {"model", "seed"}:
|
| 320 |
r = compose(c, "exec", "-T", "fly", "fly-trader", "apply-release", f"/releases/{m['seq']}", check=False)
|
| 321 |
if r.returncode != 0:
|
| 322 |
+
raise SystemExit(f"apply-release failed: {(r.stderr or r.stdout)[-300:]}")
|
| 323 |
+
if not healthy(c, 300):
|
| 324 |
+
if old is not None:
|
| 325 |
+
compose(c, "exec", "-T", "fly", "fly-trader", "apply-release", f"/releases/{old['seq']}", check=False)
|
| 326 |
+
raise SystemExit("unhealthy after the model release; the previous release's models were re-applied")
|
| 327 |
+
st.update({"seq": m["seq"], "manifest_sha256": msha, "hf_sha": hf_sha, "pending": None})
|
| 328 |
save_state(c, st)
|
| 329 |
+
alert(c, f"release {m['seq']} applied ({', '.join(sorted(classes))}), signed by {who}")
|
| 330 |
|
| 331 |
|
| 332 |
+
def run(approve_seq: int | None = None, now: float | None = None) -> None:
|
| 333 |
c = conf()
|
| 334 |
+
now = time.time() if now is None else now
|
| 335 |
Path(c["root"], "releases").mkdir(parents=True, exist_ok=True)
|
| 336 |
st = load_state(c)
|
| 337 |
+
pend = st.get("pending") or {}
|
|
|
|
| 338 |
if approve_seq is not None:
|
|
|
|
| 339 |
if pend.get("seq") != approve_seq:
|
| 340 |
raise SystemExit(f"no pending release {approve_seq}")
|
| 341 |
sha = pend["hf_sha"]
|
| 342 |
+
else:
|
| 343 |
+
head = json.loads(http(f"https://huggingface.co/api/models/{c['repo']}"))["sha"]
|
| 344 |
+
if head not in (st.get("hf_sha"), st.get("hf_sha_rejected"), pend.get("hf_sha")):
|
| 345 |
+
sha = head # a new release
|
| 346 |
+
elif pend and (pend.get("approved") or now >= pend.get("due_at", float("inf"))):
|
| 347 |
+
sha = pend["hf_sha"] # a held release whose time came (or a retry after 'busy')
|
| 348 |
+
else:
|
| 349 |
+
return # installed, rejected (alerted once), or still held
|
| 350 |
try:
|
| 351 |
+
m, d, raw, who = fetch(c, sha)
|
| 352 |
+
apply(c, st, m, d, raw, sha, approved=approve_seq is not None, who=who, now=now)
|
| 353 |
except SystemExit as e:
|
| 354 |
+
st = load_state(c)
|
| 355 |
+
if (st.get("pending") or {}).get("hf_sha") == sha:
|
| 356 |
+
st["pending"] = None
|
| 357 |
st["hf_sha_rejected"] = sha; save_state(c, st)
|
| 358 |
alert(c, f"release at {sha[:10]} rejected: {e}")
|
| 359 |
raise
|
| 360 |
|
| 361 |
|
| 362 |
+
def veto(seq: int) -> None:
|
| 363 |
+
c = conf(); st = load_state(c); pend = st.get("pending") or {}
|
| 364 |
+
if pend.get("seq") != seq:
|
| 365 |
+
raise SystemExit(f"no pending release {seq}")
|
| 366 |
+
st["vetoed"] = [*(st.get("vetoed") or []), pend["manifest_sha256"]][-20:]
|
| 367 |
+
st["hf_sha_rejected"] = pend["hf_sha"]; st["pending"] = None
|
| 368 |
+
save_state(c, st)
|
| 369 |
+
alert(c, f"release {seq} VETOED: it will never apply. If you did not publish it, your release key is compromised: "
|
| 370 |
+
"remove it from /etc/fly/allowed_signers and rotate.")
|
| 371 |
+
|
| 372 |
+
|
| 373 |
def main(argv: list[str]) -> None:
|
| 374 |
cmd = argv[1] if len(argv) > 1 else "run"
|
| 375 |
if cmd == "run":
|
| 376 |
run()
|
| 377 |
elif cmd == "approve":
|
| 378 |
run(int(argv[2]))
|
| 379 |
+
elif cmd == "veto":
|
| 380 |
+
veto(int(argv[2]))
|
| 381 |
+
elif cmd == "restart":
|
| 382 |
+
c = conf()
|
| 383 |
+
if load_state(c).get("image") is None:
|
| 384 |
+
raise SystemExit("nothing installed yet")
|
| 385 |
+
r = compose(c, "up", "-d", "--force-recreate", check=False, env={"FLY_RELEASES": str(Path(c["root"]) / "releases")})
|
| 386 |
+
print((r.stdout or "") + (r.stderr or ""))
|
| 387 |
elif cmd == "status":
|
| 388 |
print(json.dumps(load_state(conf()), indent=1))
|
| 389 |
elif cmd == "verify": # used by tools/publish_release.py on the Mac before uploading
|
| 390 |
+
d = Path(argv[2]); c = {"allowed_signers": argv[3], "principals": argv[4:] or ["bryce"], "namespace": "fly-trader-release"}
|
| 391 |
raw = (d / "release.json").read_bytes()
|
| 392 |
verify_signature(c, raw, (d / "release.json.sig").read_bytes())
|
| 393 |
m = json.loads(raw); check_manifest(m); verify_dir(d, m)
|
deploy/signer.env.example
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# /srv/fly/signer.env (mode 600): the signer's own settings. Only the signer container reads this file.
|
| 2 |
+
# REQUIRED: the signer does not start without it. Use a different provider from the brain's Helius: it simulates every
|
| 3 |
+
# swap and reads the treasury itself. (Its value check reads Jupiter's public price API, lite-api.jup.ag, no key.)
|
| 4 |
+
SIGNER_RPC_URL=
|
deploy/update.json.example
CHANGED
|
@@ -1,10 +1,15 @@
|
|
| 1 |
{
|
| 2 |
"repo": "bryceweiner/fly-trader",
|
| 3 |
"allowed_signers": "/etc/fly/allowed_signers",
|
| 4 |
-
"
|
|
|
|
|
|
|
|
|
|
| 5 |
"namespace": "fly-trader-release",
|
| 6 |
"root": "/srv/fly",
|
| 7 |
"project": "fly",
|
|
|
|
|
|
|
| 8 |
"telegram_bot_token": "",
|
| 9 |
"telegram_chat_id": ""
|
| 10 |
}
|
|
|
|
| 1 |
{
|
| 2 |
"repo": "bryceweiner/fly-trader",
|
| 3 |
"allowed_signers": "/etc/fly/allowed_signers",
|
| 4 |
+
"principals": [
|
| 5 |
+
"bryce",
|
| 6 |
+
"bryce-recovery"
|
| 7 |
+
],
|
| 8 |
"namespace": "fly-trader-release",
|
| 9 |
"root": "/srv/fly",
|
| 10 |
"project": "fly",
|
| 11 |
+
"code_delay_s": 86400,
|
| 12 |
+
"expect_manifest_sha256": "",
|
| 13 |
"telegram_bot_token": "",
|
| 14 |
"telegram_chat_id": ""
|
| 15 |
}
|
deploy/vault.env.example
CHANGED
|
@@ -1,16 +1,18 @@
|
|
| 1 |
-
# /srv/fly/vault.env for the hosted $FLY vault fly (mode 600). Comments on their own lines only (Compose keeps
|
| 2 |
-
# trailing "# comment" as part of the value).
|
|
|
|
| 3 |
|
| 4 |
-
# --- keys ---
|
| 5 |
HELIUS_API_KEY=
|
| 6 |
JUPITER_API_KEY=
|
|
|
|
|
|
|
| 7 |
|
| 8 |
-
# --- money: the
|
| 9 |
LIVE_ENABLED=1
|
| 10 |
SOLANA_CLUSTER=mainnet-beta
|
|
|
|
| 11 |
CAPITAL_SOL=5
|
| 12 |
-
# only used when the fly has no sizing table yet; the Kelly caps below govern (~0.47 SOL per position on 5 SOL)
|
| 13 |
-
MAX_POSITION_SOL=0.5
|
| 14 |
MAX_POSITION_FRACTION=0.10
|
| 15 |
MAX_POOL_SHARE=0.02
|
| 16 |
MIN_POSITION_SOL=0.02
|
|
@@ -22,15 +24,21 @@ HANDOVER_DAYS=0
|
|
| 22 |
HANDOVER_MIN_TRADES=20
|
| 23 |
HANDOVER_BEAT_SELECTOR=0
|
| 24 |
RESET_ON_START=0
|
|
|
|
| 25 |
DEVICE=cpu
|
| 26 |
-
#
|
| 27 |
-
DEVICE_MEMORY_GB=
|
| 28 |
|
| 29 |
# --- the vault ---
|
| 30 |
-
# SOL from these addresses
|
|
|
|
| 31 |
FUNDING_ADDRESSES=
|
| 32 |
RH_CHAIN_ID=4663
|
| 33 |
RH_RPC_URL=https://rpc.mainnet.chain.robinhood.com
|
|
|
|
|
|
|
|
|
|
|
|
|
| 34 |
# from contracts/deployments/4663.json after you deploy
|
| 35 |
VAULT_ADDRESS=
|
| 36 |
VAULT_TIMELOCK=
|
|
@@ -47,9 +55,11 @@ VAULT_SITE_URI=https://fly-trader.app/vault.html
|
|
| 47 |
# --- alerts ---
|
| 48 |
TELEGRAM_BOT_TOKEN=
|
| 49 |
TELEGRAM_CHAT_ID=
|
|
|
|
|
|
|
|
|
|
|
|
|
| 50 |
|
| 51 |
-
# --- encrypted backups (
|
| 52 |
-
# your age1... or ssh-ed25519 public key
|
| 53 |
-
VAULT_BACKUP_RECIPIENT=
|
| 54 |
VAULT_BACKUP_BUCKET=bryceweiner/fly-vault-backups
|
| 55 |
VAULT_BACKUP_HF_TOKEN=
|
|
|
|
| 1 |
+
# /srv/fly/vault.env for the hosted $FLY vault fly's brain (mode 600). Comments on their own lines only (Compose keeps
|
| 2 |
+
# a trailing "# comment" as part of the value). No key is here or anywhere the brain can read: the two hot keys are
|
| 3 |
+
# files only the signer container mounts. Treasury and cap settings both containers share are in custody.env.
|
| 4 |
|
| 5 |
+
# --- API keys ---
|
| 6 |
HELIUS_API_KEY=
|
| 7 |
JUPITER_API_KEY=
|
| 8 |
+
# a second, independent Solana RPC (Triton, QuickNode, Alchemy...): money-moving reads must agree with it
|
| 9 |
+
SOLANA_CHECK_RPC_URL=
|
| 10 |
|
| 11 |
+
# --- money: the book = the trading wallet (a float, TREASURY_FLOAT_SOL in custody.env) + the Squads treasury ---
|
| 12 |
LIVE_ENABLED=1
|
| 13 |
SOLANA_CLUSTER=mainnet-beta
|
| 14 |
+
# your principal (what you send from FUNDING_ADDRESSES)
|
| 15 |
CAPITAL_SOL=5
|
|
|
|
|
|
|
| 16 |
MAX_POSITION_FRACTION=0.10
|
| 17 |
MAX_POOL_SHARE=0.02
|
| 18 |
MIN_POSITION_SOL=0.02
|
|
|
|
| 24 |
HANDOVER_MIN_TRADES=20
|
| 25 |
HANDOVER_BEAT_SELECTOR=0
|
| 26 |
RESET_ON_START=0
|
| 27 |
+
# the CPU: the fly's scoring is memory-bound, and 16+ cores score as fast as a GPU; no GPU driver stack in the container
|
| 28 |
DEVICE=cpu
|
| 29 |
+
# what the fly's batches may use, leaving room for Postgres and the wallet-skill table (128 GB host; 16 GB host: 6)
|
| 30 |
+
DEVICE_MEMORY_GB=24
|
| 31 |
|
| 32 |
# --- the vault ---
|
| 33 |
+
# comma-separated; SOL from these addresses (to the treasury or the trading wallet) is your deposit (principal); from
|
| 34 |
+
# anyone else it is profit for the lockers. At least your Solflare owner address.
|
| 35 |
FUNDING_ADDRESSES=
|
| 36 |
RH_CHAIN_ID=4663
|
| 37 |
RH_RPC_URL=https://rpc.mainnet.chain.robinhood.com
|
| 38 |
+
# Robinhood Chain memecoin trading and the vault's ETH pot stay off on the hosted server: its signer container holds no
|
| 39 |
+
# RH key, and the brain holds no wallet key at all.
|
| 40 |
+
RH_ENABLED=0
|
| 41 |
+
RH_LIVE_ENABLED=0
|
| 42 |
# from contracts/deployments/4663.json after you deploy
|
| 43 |
VAULT_ADDRESS=
|
| 44 |
VAULT_TIMELOCK=
|
|
|
|
| 55 |
# --- alerts ---
|
| 56 |
TELEGRAM_BOT_TOKEN=
|
| 57 |
TELEGRAM_CHAT_ID=
|
| 58 |
+
# your own Telegram user id: only its /panic and /status are obeyed (message @userinfobot to learn it)
|
| 59 |
+
TELEGRAM_ADMIN_USER_ID=
|
| 60 |
+
# dead-man switch: a healthchecks.io check URL (its Telegram integration tells you when pings stop)
|
| 61 |
+
HEALTHCHECK_URL=
|
| 62 |
|
| 63 |
+
# --- encrypted backups (hot keys once, by the signer; ledger nightly). The recipient key is in custody.env ---
|
|
|
|
|
|
|
| 64 |
VAULT_BACKUP_BUCKET=bryceweiner/fly-vault-backups
|
| 65 |
VAULT_BACKUP_HF_TOKEN=
|
docker-compose.yml
CHANGED
|
@@ -18,12 +18,19 @@ services:
|
|
| 18 |
|
| 19 |
fly:
|
| 20 |
build: .
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 21 |
env_file: .env
|
| 22 |
environment:
|
| 23 |
DATABASE_URL: postgresql://fly:fly@db:5432/fly_trader
|
| 24 |
DATA_DIR: /app/data
|
| 25 |
LOG_DIR: /app/logs
|
| 26 |
DEVICE: auto # the CPU in this image; an NVIDIA GPU with docker-compose.cuda.yml
|
|
|
|
|
|
|
| 27 |
volumes:
|
| 28 |
- ./data:/app/data # models, the fly's plastic state, its hourly snapshots: survives restarts and rebuilds
|
| 29 |
- ./logs:/app/logs
|
|
|
|
| 18 |
|
| 19 |
fly:
|
| 20 |
build: .
|
| 21 |
+
# root inside the container: the self-updater (AUTO_UPDATE=1) replaces the code in /app, and data/, logs/ and .env
|
| 22 |
+
# are folders and a file of yours bind-mounted from here (the vault server's compose runs an unprivileged user and
|
| 23 |
+
# never self-updates)
|
| 24 |
+
user: "0:0"
|
| 25 |
+
# every key (BOT_PRIVATE_KEY, and RH_BOT_PRIVATE_KEY for Robinhood Chain live) comes from .env at run time, never the image
|
| 26 |
env_file: .env
|
| 27 |
environment:
|
| 28 |
DATABASE_URL: postgresql://fly:fly@db:5432/fly_trader
|
| 29 |
DATA_DIR: /app/data
|
| 30 |
LOG_DIR: /app/logs
|
| 31 |
DEVICE: auto # the CPU in this image; an NVIDIA GPU with docker-compose.cuda.yml
|
| 32 |
+
# Robinhood Chain is opt-in: set RH_ENABLED=1 in .env (it needs ENVIO_API_TOKEN and room for its history)
|
| 33 |
+
RH_ENABLED: ${RH_ENABLED:-0}
|
| 34 |
volumes:
|
| 35 |
- ./data:/app/data # models, the fly's plastic state, its hourly snapshots: survives restarts and rebuilds
|
| 36 |
- ./logs:/app/logs
|
docker/bootstrap.sh
CHANGED
|
@@ -1,24 +1,58 @@
|
|
| 1 |
#!/bin/sh
|
| 2 |
# What every boot (and every self-update) does before the console starts: schema, the shipped models into the data
|
| 3 |
-
# volume, the wallet-skill table, the neuron geometry, the
|
| 4 |
-
#
|
| 5 |
set -e
|
| 6 |
cd /app
|
| 7 |
until pg_isready -d "$DATABASE_URL" >/dev/null 2>&1; do echo "fly-trader: waiting for postgres"; sleep 2; done
|
| 8 |
fly-trader init-db
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9 |
for sub in policies selectors connectome; do
|
| 10 |
mkdir -p "data/brain/$sub"
|
| 11 |
for f in models/$sub/*; do [ -e "$f" ] || continue; [ -e "data/brain/$sub/$(basename "$f")" ] || cp "$f" "data/brain/$sub/"; done
|
| 12 |
done
|
|
|
|
|
|
|
|
|
|
| 13 |
mkdir -p data/corpus/wallet_skill
|
| 14 |
-
for f in models/wallet_skill/*; do
|
|
|
|
|
|
|
|
|
|
|
|
|
| 15 |
T=$(cat models/wallet_skill/TABLE 2>/dev/null || true)
|
|
|
|
| 16 |
if [ -n "$T" ] && [ ! -e "data/corpus/wallet_skill/$T" ]; then
|
| 17 |
-
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 21 |
fi
|
|
|
|
|
|
|
| 22 |
mkdir -p fly_trader/ui/static/brain && cp models/geometry/* fly_trader/ui/static/brain/ 2>/dev/null || true
|
| 23 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 24 |
echo "fly-trader: device $(python -c 'from fly_trader.brain import device; print(device.describe(device.resolve()))' 2>/dev/null || echo unknown)"
|
|
|
|
| 1 |
#!/bin/sh
|
| 2 |
# What every boot (and every self-update) does before the console starts: schema, the shipped models into the data
|
| 3 |
+
# volume, the wallet-skill table, the neuron geometry, the rows that admit the fly. Nothing here overwrites a running
|
| 4 |
+
# install: copies skip files that exist, and apply-release reuses rows it already has.
|
| 5 |
set -e
|
| 6 |
cd /app
|
| 7 |
until pg_isready -d "$DATABASE_URL" >/dev/null 2>&1; do echo "fly-trader: waiting for postgres"; sleep 2; done
|
| 8 |
fly-trader init-db
|
| 9 |
+
# models/ holds exactly this release's fly and selector: an earlier release's (the updater before signed releases never
|
| 10 |
+
# deleted a file) would be copied below, and a pickled .joblib makes apply-release refuse the whole release
|
| 11 |
+
if [ -f seed/release_seed.json ]; then
|
| 12 |
+
python - <<'PY'
|
| 13 |
+
import json, pathlib
|
| 14 |
+
keep = {s["file"] for s in json.loads(pathlib.Path("seed/release_seed.json").read_text())["snapshots"]}
|
| 15 |
+
for sub in ("policies", "selectors"):
|
| 16 |
+
for f in pathlib.Path("models", sub).glob("*"):
|
| 17 |
+
if f.is_file() and f.name not in keep:
|
| 18 |
+
f.unlink(); print(f"fly-trader: removed {f} (not in this release)")
|
| 19 |
+
PY
|
| 20 |
+
fi
|
| 21 |
for sub in policies selectors connectome; do
|
| 22 |
mkdir -p "data/brain/$sub"
|
| 23 |
for f in models/$sub/*; do [ -e "$f" ] || continue; [ -e "data/brain/$sub/$(basename "$f")" ] || cp "$f" "data/brain/$sub/"; done
|
| 24 |
done
|
| 25 |
+
# The wallet-skill inputs: the fitted config and the newest table. The table (~300 MB) is too big for git; it comes from
|
| 26 |
+
# a verified release (the vault server's /releases), or is downloaded from Hugging Face and used only if its sha256 is
|
| 27 |
+
# the one pinned in models/wallet_skill/TABLE.sha256. Nothing unverified reaches the engine.
|
| 28 |
mkdir -p data/corpus/wallet_skill
|
| 29 |
+
for f in models/wallet_skill/*; do
|
| 30 |
+
[ -e "$f" ] || continue
|
| 31 |
+
case "$(basename "$f")" in TABLE|TABLE.sha256) continue;; esac
|
| 32 |
+
[ -e "data/corpus/wallet_skill/$(basename "$f")" ] || cp "$f" data/corpus/wallet_skill/
|
| 33 |
+
done
|
| 34 |
T=$(cat models/wallet_skill/TABLE 2>/dev/null || true)
|
| 35 |
+
WANT=$(cat models/wallet_skill/TABLE.sha256 2>/dev/null || true)
|
| 36 |
if [ -n "$T" ] && [ ! -e "data/corpus/wallet_skill/$T" ]; then
|
| 37 |
+
src=$(ls -d /releases/*/models/wallet_skill/"$T" 2>/dev/null | tail -1 || true)
|
| 38 |
+
if [ -n "$src" ]; then
|
| 39 |
+
cp "$src" "data/corpus/wallet_skill/$T"
|
| 40 |
+
elif [ -n "$WANT" ]; then
|
| 41 |
+
echo "fly-trader: fetching the wallet-skill table $T (~300 MB, once; checked against its pinned sha256)"
|
| 42 |
+
if curl -fsSL --retry 5 -o "data/corpus/wallet_skill/$T.part" "https://huggingface.co/${HF_REPO:-bryceweiner/fly-trader}/resolve/main/models/wallet_skill/$T" \
|
| 43 |
+
&& [ "$(sha256sum "data/corpus/wallet_skill/$T.part" | cut -d' ' -f1)" = "$WANT" ]; then
|
| 44 |
+
mv "data/corpus/wallet_skill/$T.part" "data/corpus/wallet_skill/$T"
|
| 45 |
+
else
|
| 46 |
+
rm -f "data/corpus/wallet_skill/$T.part"
|
| 47 |
+
fi
|
| 48 |
+
fi
|
| 49 |
fi
|
| 50 |
+
[ -z "$T" ] || [ -e "data/corpus/wallet_skill/$T" ] || \
|
| 51 |
+
echo "fly-trader: WARNING the wallet-skill table $T is missing or did not match its sha256; the paper selector will not trade and the fly's skill inputs read as zero until it is present (data/corpus/wallet_skill/$T)"
|
| 52 |
mkdir -p fly_trader/ui/static/brain && cp models/geometry/* fly_trader/ui/static/brain/ 2>/dev/null || true
|
| 53 |
+
if [ -f seed/release_seed.json ]; then
|
| 54 |
+
fly-trader apply-release /app --bootstrap # the shipped models, registered by file hash (fresh ids), idempotently
|
| 55 |
+
else
|
| 56 |
+
psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -q -f seed/seed.sql
|
| 57 |
+
fi
|
| 58 |
echo "fly-trader: device $(python -c 'from fly_trader.brain import device; print(device.describe(device.resolve()))' 2>/dev/null || echo unknown)"
|
docker/signer-requirements.txt
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
anyio==4.15.1 \
|
| 2 |
+
--hash=sha256:6152fdbbf9a77fdec97731721bebf7c4c44f7c29b424b0065826173efc7ed101 \
|
| 3 |
+
--hash=sha256:9f28306018cbd6d329e64a36d58256edff76dd996fe423bc957326e578b82a94
|
| 4 |
+
# via httpx
|
| 5 |
+
base58==2.1.1 \
|
| 6 |
+
--hash=sha256:11a36f4d3ce51dfc1043f3218591ac4eb1ceb172919cebe05b52a5bcc8d245c2 \
|
| 7 |
+
--hash=sha256:c5d0cb3f5b6e81e8e35da5754388ddcc6d0d14b6c6a132cb93d69ed580a7278c
|
| 8 |
+
# via -r /private/tmp/claude-501/-Users-bryce-Documents-code-fly-trader/28aad393-6f38-484a-a2b2-a06a8f112cf1/scratchpad/signer.in
|
| 9 |
+
certifi==2026.7.22 \
|
| 10 |
+
--hash=sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775 \
|
| 11 |
+
--hash=sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55
|
| 12 |
+
# via
|
| 13 |
+
# httpcore
|
| 14 |
+
# httpx
|
| 15 |
+
h11==0.16.0 \
|
| 16 |
+
--hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \
|
| 17 |
+
--hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86
|
| 18 |
+
# via httpcore
|
| 19 |
+
httpcore==1.0.9 \
|
| 20 |
+
--hash=sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55 \
|
| 21 |
+
--hash=sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8
|
| 22 |
+
# via httpx
|
| 23 |
+
httpx==0.28.1 \
|
| 24 |
+
--hash=sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc \
|
| 25 |
+
--hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad
|
| 26 |
+
# via -r /private/tmp/claude-501/-Users-bryce-Documents-code-fly-trader/28aad393-6f38-484a-a2b2-a06a8f112cf1/scratchpad/signer.in
|
| 27 |
+
idna==3.20 \
|
| 28 |
+
--hash=sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44 \
|
| 29 |
+
--hash=sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c
|
| 30 |
+
# via
|
| 31 |
+
# anyio
|
| 32 |
+
# httpx
|
| 33 |
+
jsonalias==0.1.1 \
|
| 34 |
+
--hash=sha256:64f04d935397d579fc94509e1fcb6212f2d081235d9d6395bd10baedf760a769 \
|
| 35 |
+
--hash=sha256:a56d2888e6397812c606156504e861e8ec00e188005af149f003c787db3d3f18
|
| 36 |
+
# via solders
|
| 37 |
+
solders==0.29.0 \
|
| 38 |
+
--hash=sha256:00524baf575ae5edb2ba43cdb1d80491c59efdc16b48fec96a6f929c0401d0e4 \
|
| 39 |
+
--hash=sha256:09e80c545c728f434f627d3b4714675a0f654c4d387f3339d54f7d09d59f3f1c \
|
| 40 |
+
--hash=sha256:62e8a5bb4bc8628d5fe49faa56aca149dce284a9f8bf5f3c56c4721e561be84b \
|
| 41 |
+
--hash=sha256:785841623bdd7b8dedc68e7c794b54327d3b03e1ab9697eda90a139ca3c98cfe \
|
| 42 |
+
--hash=sha256:9d2e011ff1d5f82a7de81d6eb89f54d002189f50ee48d4a15d7b51849e04b1c3 \
|
| 43 |
+
--hash=sha256:c17098cab72a1c202ec4cc0ce3ecbff4296c9de97000e05944337ba5ddb681a8 \
|
| 44 |
+
--hash=sha256:f36db7b43088866081b5d37ff62668551e30dd136b9ee58a5fb524fcd828082d
|
| 45 |
+
# via -r /private/tmp/claude-501/-Users-bryce-Documents-code-fly-trader/28aad393-6f38-484a-a2b2-a06a8f112cf1/scratchpad/signer.in
|
| 46 |
+
typing-extensions==4.16.0 \
|
| 47 |
+
--hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \
|
| 48 |
+
--hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5
|
| 49 |
+
# via
|
| 50 |
+
# anyio
|
| 51 |
+
# solders
|
docs/vault/RUNBOOK.md
CHANGED
|
@@ -1,120 +1,435 @@
|
|
| 1 |
# $FLY vault: launch runbook
|
| 2 |
|
| 3 |
-
|
| 4 |
-
handles your keys.
|
| 5 |
-
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
-
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
``
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
``
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 88 |
1. Build the upload bundle:
|
| 89 |
```
|
| 90 |
-
cd web
|
| 91 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 92 |
```
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 96 |
```
|
| 97 |
-
|
| 98 |
```
|
| 99 |
-
|
| 100 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 101 |
|
| 102 |
## 5. Go live
|
| 103 |
-
1.
|
| 104 |
-
|
| 105 |
-
|
| 106 |
-
|
| 107 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 108 |
|
| 109 |
## Day to day
|
| 110 |
-
- **
|
| 111 |
-
|
| 112 |
-
`sudo python3 /usr/local/lib/fly/fly_update.py
|
| 113 |
-
|
| 114 |
-
- **
|
| 115 |
-
|
| 116 |
-
|
| 117 |
-
|
| 118 |
-
|
| 119 |
-
|
| 120 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
# $FLY vault: launch runbook
|
| 2 |
|
| 3 |
+
The steps only you can do, because they need your keys, accounts or devices, in the order to do them. The agent never
|
| 4 |
+
handles your keys. How the system works is in [SPEC.md](SPEC.md); this file is what to type.
|
| 5 |
+
|
| 6 |
+
**Who holds what** (until the Ledger is back; the "Ledger" section moves everything to it without redeploying):
|
| 7 |
+
|
| 8 |
+
| Key | Where | Can do |
|
| 9 |
+
|---|---|---|
|
| 10 |
+
| Solana owner | Solflare on your phone | Everything with the treasury: create/remove its spending limits, take principal out |
|
| 11 |
+
| EVM owner | MetaMask | Pause new $FLY locks; propose upgrades (public for 8 days); hand the roles to the Ledger |
|
| 12 |
+
| SSH | Secretive on the Mac (Secure Enclave, Touch ID) | Log in to the server over Tailscale |
|
| 13 |
+
| Release signing | Secretive (Touch ID) + an offline recovery key | Sign releases; the server still holds code changes 24 h |
|
| 14 |
+
| Backup decryption | an age key, offline | Read the encrypted backups |
|
| 15 |
+
| Trading / payout | on the server, readable only by the signer container | Refill the float (L1, per day), pay claims (L2, per week) |
|
| 16 |
+
|
| 17 |
+
A full takeover of the server can lose at most the trading float plus what is left of this day's L1 and this week's
|
| 18 |
+
L2, and only until you revoke them from your phone ("Emergency").
|
| 19 |
+
|
| 20 |
+
Conventions below:
|
| 21 |
+
- `$` lines run on the **Mac** in the fly-trader checkout (the repo root). Commands for a subfolder run in a subshell,
|
| 22 |
+
`$ (cd web && …)`, so you are always back at the root.
|
| 23 |
+
- `server$` lines run on the **server** as `fly-admin`.
|
| 24 |
+
- `sudo fly <command>` runs `fly-trader <command>` inside the running fly container; `sudo fly signer <command>` runs
|
| 25 |
+
in the signer container.
|
| 26 |
+
|
| 27 |
+
## 0. Once: tools, repos, accounts, keys
|
| 28 |
+
|
| 29 |
+
### 0.1 Mac tools
|
| 30 |
+
```
|
| 31 |
+
$ brew install age node uv git
|
| 32 |
+
$ brew install --cask secretive tailscale
|
| 33 |
+
$ curl -L https://foundry.paradigm.xyz | bash && foundryup # forge, cast
|
| 34 |
+
$ sh -c "$(curl -sSfL https://release.anza.xyz/stable/install)" # solana CLI + solana-test-validator
|
| 35 |
+
$ uv sync --extra dev # the Python environment (.venv)
|
| 36 |
+
$ (cd web && npm ci)
|
| 37 |
+
$ (cd contracts && forge soldeer install)
|
| 38 |
+
$ git worktree add ../fly-trader-dist distribution # the branch the server runs
|
| 39 |
+
```
|
| 40 |
+
|
| 41 |
+
### 0.2 Accounts
|
| 42 |
+
- **2FA everywhere.** Use a hardware key or TOTP on Gandi, Hugging Face, GitHub,
|
| 43 |
+
your Tailscale identity provider and your Apple ID. Set a Telegram two-step password.
|
| 44 |
+
- **Phone wallets.** Write the Solflare and MetaMask seed phrases on paper and keep them offline. The phone needs a
|
| 45 |
+
passcode and biometrics. Keep a little SOL in Solflare (about 0.05 SOL pays the treasury's setup and any later
|
| 46 |
+
change) and a little ETH on Robinhood Chain in MetaMask (for `pause()`).
|
| 47 |
+
- **Reown project id.** Create it at dashboard.reown.com. Allowlist `fly-trader.app`, and `localhost:5173` for local
|
| 48 |
+
testing.
|
| 49 |
+
- **Telegram.**
|
| 50 |
+
1. Create a bot with @BotFather and note its token.
|
| 51 |
+
2. Send `/start` to your new bot; it cannot message you before that.
|
| 52 |
+
3. Message @userinfobot to learn your user id. In a private chat with the bot, the chat id is that same number.
|
| 53 |
+
Only that user's `/panic` and `/status` count.
|
| 54 |
+
- **Hugging Face.**
|
| 55 |
+
1. Create a **public** model repo `bryceweiner/fly-trader`: the server and self-hosted installs read releases
|
| 56 |
+
without logging in.
|
| 57 |
+
2. Create a **private** storage bucket `bryceweiner/fly-vault-backups` for the encrypted backups.
|
| 58 |
+
3. Create two tokens: one with write access to the model repo, for publishing from the Mac; one fine-grained,
|
| 59 |
+
write-only to the bucket, for the server's backups.
|
| 60 |
+
4. `$ uv run hf auth login` with the first token.
|
| 61 |
+
- **GitHub.** Push access to the fly-trader repo: publishing pushes the `distribution` branch.
|
| 62 |
+
- **Second Solana RPC.** Get an API URL from a provider other than Helius (Triton, QuickNode or Alchemy). It is used
|
| 63 |
+
twice:
|
| 64 |
+
- `SOLANA_CHECK_RPC_URL`: settlement balances and halts must agree with it.
|
| 65 |
+
- `SIGNER_RPC_URL`: the signer's own reads and simulations.
|
| 66 |
+
- **healthchecks.io.** Create a check with period 5 minutes and grace 10 minutes, and connect its Telegram
|
| 67 |
+
integration. Its ping URL is `HEALTHCHECK_URL`.
|
| 68 |
+
|
| 69 |
+
### 0.3 Tailscale
|
| 70 |
+
1. Install Tailscale on the Mac, log in, and turn on **MagicDNS** in the admin console, so `fly-vault` resolves.
|
| 71 |
+
2. In the admin console's access controls, add:
|
| 72 |
+
```
|
| 73 |
+
"tagOwners": {"tag:fly": ["autogroup:admin"]},
|
| 74 |
+
"acls": [{"action": "accept", "src": ["<your Tailscale login>"], "dst": ["tag:fly:22"]}]
|
| 75 |
+
```
|
| 76 |
+
Add no rule whose source is `tag:fly`: the server needs to reach nothing on your tailnet.
|
| 77 |
+
3. Turn on **Tailnet Lock** from the Mac:
|
| 78 |
+
```
|
| 79 |
+
$ tailscale lock status # shows this Mac's tlpub:… key
|
| 80 |
+
$ tailscale lock init --gen-disablements 3 --confirm tlpub:<that key>
|
| 81 |
+
```
|
| 82 |
+
Store the printed disablement secrets offline, like a seed phrase.
|
| 83 |
+
4. Create a one-off, pre-authorised auth key tagged `tag:fly` in the admin console. Sign it so the server may join the
|
| 84 |
+
locked tailnet, and keep the output (this is `TS_AUTHKEY` in section 4.2):
|
| 85 |
+
```
|
| 86 |
+
$ tailscale lock sign tskey-auth-…
|
| 87 |
+
```
|
| 88 |
+
|
| 89 |
+
### 0.4 Your keys (Secretive, recovery, backup)
|
| 90 |
+
1. In Secretive create two keys, `fly-ssh` and `fly-release`, each requiring Touch ID. Save their public keys as
|
| 91 |
+
`~/.ssh/fly-ssh.pub` and `~/.ssh/fly-release.pub`.
|
| 92 |
+
2. Put the path of Secretive's agent in your shell profile (`~/.zshrc`), since several steps use it:
|
| 93 |
+
`export SECRETIVE=~/Library/Containers/com.maxgoedjen.Secretive.SecretAgent/Data/socket.ssh`.
|
| 94 |
+
Then tell ssh to use Secretive for the server, in `~/.ssh/config`:
|
| 95 |
+
```
|
| 96 |
+
Host fly-vault
|
| 97 |
+
User fly-admin
|
| 98 |
+
IdentityAgent ~/Library/Containers/com.maxgoedjen.Secretive.SecretAgent/Data/socket.ssh
|
| 99 |
+
```
|
| 100 |
+
3. The offline recovery release signer:
|
| 101 |
+
```
|
| 102 |
+
$ mkdir -p ~/fly-keys
|
| 103 |
+
$ ssh-keygen -t ed25519 -f ~/fly-keys/fly-release-recovery -C fly-release-recovery
|
| 104 |
+
$ age -p ~/fly-keys/fly-release-recovery > ~/fly-keys/fly-release-recovery.age && rm ~/fly-keys/fly-release-recovery
|
| 105 |
+
```
|
| 106 |
+
Put `fly-release-recovery.age` on a USB stick and in your password manager. Keep `fly-release-recovery.pub`.
|
| 107 |
+
4. The backup key: `$ age-keygen -o ~/fly-keys/fly-backup.key`. Store it like the recovery key; its public line
|
| 108 |
+
(`age1…`, printed and at the top of the file) is `VAULT_BACKUP_RECIPIENT`. It is deliberately not the release key.
|
| 109 |
+
5. Trust both release keys in the distribution, so that self-hosted installs accept your releases:
|
| 110 |
+
```
|
| 111 |
+
$ echo "bryce namespaces=\"fly-trader-release\" $(cat ~/.ssh/fly-release.pub)" >> ../fly-trader-dist/release/allowed_signers
|
| 112 |
+
$ echo "bryce-recovery namespaces=\"fly-trader-release\" $(cat ~/fly-keys/fly-release-recovery.pub)" >> ../fly-trader-dist/release/allowed_signers
|
| 113 |
+
$ git -C ../fly-trader-dist status # must be clean apart from this file (no merge in progress)
|
| 114 |
+
$ git -C ../fly-trader-dist commit -m "Release keys" release/allowed_signers
|
| 115 |
+
```
|
| 116 |
+
|
| 117 |
+
## 1. Rehearse locally (no real money)
|
| 118 |
+
- **Custody against the real Squads program:**
|
| 119 |
+
`$ tools/custody_e2e.sh <your Helius mainnet RPC URL>`. It dumps the program from mainnet (read-only) and runs it on
|
| 120 |
+
a throwaway validator on ports 11xxx. It checks:
|
| 121 |
+
- the owner page's code creates the treasury and both limits, changes L2 and revokes them
|
| 122 |
+
- the signer refills through L1 and pays a claim through L2 up to its cap
|
| 123 |
+
- a stolen key cannot send L1 anywhere but the trading wallet (refused on chain)
|
| 124 |
+
- **The whole vault on this Mac:** `$ .venv/bin/python tools/vault_demo.py up`. It runs anvil, the contracts, the relay,
|
| 125 |
+
the vault worker, a treasury on a local validator and the site at http://localhost:5173. The file's docstring lists
|
| 126 |
+
the wallet settings and commands (`fund-evm`, `profit`, `status`, `down`).
|
| 127 |
+
- **Optional: RH testnet + Solana devnet.**
|
| 128 |
+
1. Deploy the contracts:
|
| 129 |
+
```
|
| 130 |
+
$ cast wallet new ~/.foundry/keystores rehearsal # fund it from faucet.testnet.chain.robinhood.com
|
| 131 |
+
$ (cd contracts && forge script script/DeployTestnet.s.sol --rpc-url robinhood_testnet --account rehearsal \
|
| 132 |
+
--sender $(cast wallet address --account rehearsal) --broadcast)
|
| 133 |
+
$ cp contracts/deployments/46630.json ~/fly-keys/testnet-deployment.json
|
| 134 |
+
```
|
| 135 |
+
The addresses land in `contracts/deployments/46630.json`: `vault`, `timelock`, `token` (MockFLY), `blockNumber`.
|
| 136 |
+
Keep the copy: `vault_demo.py` deploys to anvil, which uses the same chain id, and overwrites that file.
|
| 137 |
+
2. Run the site against it:
|
| 138 |
+
Run a relay as in `web/relay/README.md` (e.g. `tools/vault_demo.py` shows how it starts one on port 8612), with
|
| 139 |
+
`"domain": "localhost:5173"`, `"uri": "http://localhost:5173/vault.html"`, `"chain_id": 46630`,
|
| 140 |
+
`"sol_chain": "devnet"`. Then:
|
| 141 |
+
```
|
| 142 |
+
$ (cd web && VITE_NETWORK=testnet VITE_REOWN_PROJECT_ID=… VITE_VAULT_ADDRESS=<vault> VITE_TIMELOCK_ADDRESS=<timelock> \
|
| 143 |
+
VITE_FLY_ADDRESS=<token> VITE_SOLANA_RPC=https://api.devnet.solana.com VITE_RELAY_PROXY=http://127.0.0.1:8612 npm run dev)
|
| 144 |
+
```
|
| 145 |
+
3. Make two throwaway devnet keys (`$ solana-keygen new -o /tmp/devnet-trading.json`, the same for payout; fund
|
| 146 |
+
the trading one at faucet.solana.com). Create the treasury: http://localhost:5173/owner.html with Solflare on
|
| 147 |
+
devnet, as in section 4.3, with those two public keys.
|
| 148 |
+
4. Run the vault worker on the Mac with these settings in the environment:
|
| 149 |
+
- `VAULT_ENABLED=1`, `VAULT_CLUSTER=devnet`, `VAULT_SOLANA_RPC_URL=https://api.devnet.solana.com`,
|
| 150 |
+
`SIGNER_RPC_URL=https://api.devnet.solana.com`, `VAULT_PERIOD_S=900`, `LIVE_ENABLED=0`
|
| 151 |
+
- `BOT_PRIVATE_KEY_FILE=/tmp/devnet-trading.json`, `PAYOUT_PRIVATE_KEY_FILE=/tmp/devnet-payout.json`
|
| 152 |
+
- `VAULT_MULTISIG`, `VAULT_LIMIT_TRADING`, `VAULT_LIMIT_PAYOUT` (from the owner page)
|
| 153 |
+
- `RH_CHAIN_ID=46630`, `RH_RPC_URL=https://rpc.testnet.chain.robinhood.com`, `VAULT_ADDRESS`,
|
| 154 |
+
`VAULT_TIMELOCK`, `VAULT_START_BLOCK`
|
| 155 |
+
- `VAULT_SITE_DOMAIN=localhost:5173`, `VAULT_SITE_URI=http://localhost:5173/vault.html`
|
| 156 |
+
- `RELAY_URL=http://127.0.0.1:8612`, `RELAY_SECRET` (the relay's), and a separate `DATABASE_URL`
|
| 157 |
+
(`createdb fly_vault_rehearsal; DATABASE_URL=postgresql:///fly_vault_rehearsal uv run fly-trader init-db`)
|
| 158 |
+
|
| 159 |
+
```
|
| 160 |
+
$ uv run python -c "from fly_trader.vault import worker; worker.main()"
|
| 161 |
+
```
|
| 162 |
+
|
| 163 |
+
## 2. Mainnet contracts (EVM owner = MetaMask)
|
| 164 |
+
1. Deploy from a throwaway keystore. It pays the gas and holds no role afterwards:
|
| 165 |
+
```
|
| 166 |
+
$ cast wallet new ~/.foundry/keystores deployer # fund with a little ETH on Robinhood Chain
|
| 167 |
+
$ (cd contracts && FLY_TOKEN=0x2fC7f9E2911f20b2C4660d2AEf808aa91bDdb3D3 OWNER=<your MetaMask address> \
|
| 168 |
+
forge script script/Deploy.s.sol --rpc-url robinhood --account deployer \
|
| 169 |
+
--sender $(cast wallet address --account deployer) --broadcast \
|
| 170 |
+
--verify --verifier blockscout --verifier-url https://robinhoodchain.blockscout.com/api/)
|
| 171 |
+
```
|
| 172 |
+
The script refuses to run on chain 4663 unless the token is the real $FLY and the delays are 8 days (timelock) and
|
| 173 |
+
7 days (withdrawal). It writes `contracts/deployments/4663.json`: `vault`, `implementation`, `timelock`,
|
| 174 |
+
`blockNumber`. You need all four below.
|
| 175 |
+
2. Check that the deployer holds nothing. Each of these must print `false`:
|
| 176 |
+
```
|
| 177 |
+
$ D=$(cast wallet address --account deployer)
|
| 178 |
+
$ R=https://rpc.mainnet.chain.robinhood.com
|
| 179 |
+
$ cast call <timelock> "hasRole(bytes32,address)(bool)" $(cast keccak PROPOSER_ROLE) $D --rpc-url $R
|
| 180 |
+
$ cast call <timelock> "hasRole(bytes32,address)(bool)" 0x0000000000000000000000000000000000000000000000000000000000000000 $D --rpc-url $R
|
| 181 |
+
$ cast call <vault> "hasRole(bytes32,address)(bool)" $(cast keccak PAUSER_ROLE) $D --rpc-url $R
|
| 182 |
+
```
|
| 183 |
+
Then delete `~/.foundry/keystores/deployer`.
|
| 184 |
+
3. Check the contracts show as verified on Blockscout; `contracts/README.md` has a manual fallback if Cloudflare
|
| 185 |
+
blocked forge's verification.
|
| 186 |
+
4. Record the implementation code hash; it becomes `VAULT_IMPL_CODEHASHES`:
|
| 187 |
+
`$ cast keccak $(cast code <implementation> --rpc-url https://rpc.mainnet.chain.robinhood.com)`
|
| 188 |
+
5. From your phone (MetaMask), you use Blockscout's page for each contract, under "Write proxy" / "Write contract":
|
| 189 |
+
- **Pause new locks:** FlyVault, `pause()`; later `unpause()`. Withdrawals always stay open.
|
| 190 |
+
- **Cancel a timelock operation you did not schedule:** Timelock, `cancel(<id>)`. The Telegram alert gives the full
|
| 191 |
+
id.
|
| 192 |
+
|
| 193 |
+
## 3. Site (Gandi)
|
| 194 |
+
The owner page is part of the site, and you need it for the treasury, so the site goes up before the server.
|
| 195 |
1. Build the upload bundle:
|
| 196 |
```
|
| 197 |
+
$ (cd web && VITE_REOWN_PROJECT_ID=… VITE_VAULT_ADDRESS=<vault> VITE_TIMELOCK_ADDRESS=<timelock> bash scripts/bundle_gandi.sh)
|
| 198 |
+
```
|
| 199 |
+
The bundle lands in `build/gandi/` at the repo root: `wsgi.py`, `site-headers.json`, `relay/` and `site/`.
|
| 200 |
+
`owner.html` is included; it is not linked anywhere, it is noindex, and it can do nothing without your owner wallet.
|
| 201 |
+
2. Upload the contents of `build/gandi/` over Gandi SFTP (or git) into the instance's deploy directory (where
|
| 202 |
+
`wsgi.py` lives), replacing `site/` entirely.
|
| 203 |
+
3. Make the relay secret: `$ python3 -c "import secrets;print(secrets.token_hex(32))"`. The fly gets the same value as
|
| 204 |
+
`RELAY_SECRET`.
|
| 205 |
+
4. Upload `relay.json` next to `wsgi.py` (never into `site/`, never into git). The database goes on the instance's
|
| 206 |
+
persistent disk, outside the deploy directory:
|
| 207 |
+
```json
|
| 208 |
+
{"keys": {"k1": "<the secret>"}, "db_path": "/srv/data/home/relay/relay.sqlite3", "domain": "fly-trader.app",
|
| 209 |
+
"uri": "https://fly-trader.app/vault.html", "chain_id": 4663, "sol_chain": "mainnet", "claim_ttl_s": 900,
|
| 210 |
+
"min_lamports": 2000000, "client_ip_header": null}
|
| 211 |
```
|
| 212 |
+
Restart the instance. If a later deploy replaces the directory, upload `relay.json` again.
|
| 213 |
+
5. Probe the relay and work through the 7-point checklist in `web/relay/README.md` (writable, WAL, client IP header,
|
| 214 |
+
no caching, clock):
|
| 215 |
+
`$ FLY_RELAY_SECRET=<the secret> python3 web/relay/hmacauth.py https://fly-trader.app/api/fly/probe k1`
|
| 216 |
+
|
| 217 |
+
## 4. Server (a dedicated machine at home, Ubuntu Server 24.04)
|
| 218 |
+
|
| 219 |
+
### 4.1 Publish the first release
|
| 220 |
+
1. The distribution worktree must hold exactly what you want to ship. Code is ported there from master by hand (or with
|
| 221 |
+
a merge); publishing does not port it. Check it is clean: `$ git -C ../fly-trader-dist status`.
|
| 222 |
+
2. `--models` refreshes the models from this Mac; use it for the first release. That needs:
|
| 223 |
+
- a deployable fly (`fly-trader train-fly-selector`) that passed `fly-trader fly-replay`
|
| 224 |
+
- a deployed selector (pinned, or the newest that qualifies)
|
| 225 |
+
- fitted wallet skill (`fly-trader fit-wallet-skill`) with its tables, and a built connectome
|
| 226 |
+
Without `--models` the release carries the models already in the worktree, and publishing refuses if the
|
| 227 |
+
wallet-skill table named in `models/wallet_skill/TABLE` is not there.
|
| 228 |
+
3. Publish. `SSH_AUTH_SOCK` points `ssh-keygen` at Secretive, so Touch ID confirms the signature:
|
| 229 |
```
|
| 230 |
+
$ SSH_AUTH_SOCK=$SECRETIVE .venv/bin/python tools/publish_release.py --worktree ../fly-trader-dist --key ~/.ssh/fly-release.pub --models
|
| 231 |
```
|
| 232 |
+
It runs the tests, commits, signs, verifies with the server's own verifier, pushes the `distribution` branch and
|
| 233 |
+
uploads one Hugging Face commit. It refuses unless the key is in the worktree's `release/allowed_signers` (0.4 step
|
| 234 |
+
5). Note the printed **manifest sha256**; it pins this first release on the server.
|
| 235 |
+
4. **Do not publish again until the server has installed this release** (section 4.4). A new server accepts only the
|
| 236 |
+
pinned manifest, and it permanently refuses any other head it sees first.
|
| 237 |
+
|
| 238 |
+
### 4.2 The machine and its bootstrap
|
| 239 |
+
A dedicated machine at home (the reference is a GMKtec EVO-X3: Ryzen AI Max+ 395, 128 GB, 2 TB NVMe, 2.5 GbE). It
|
| 240 |
+
runs nothing but the vault fly. The fly runs on its CPU; the Radeon GPU is not used (no GPU driver stack in reach of
|
| 241 |
+
the keys).
|
| 242 |
+
|
| 243 |
+
1. **Firmware** (press Del or F7 at power-on for the setup screen):
|
| 244 |
+
- Update the BIOS from the vendor's support site first.
|
| 245 |
+
- Set a supervisor (admin) password.
|
| 246 |
+
- Turn on the firmware TPM (AMD fTPM / "Security Device Support"), Secure Boot (standard keys) and the IOMMU
|
| 247 |
+
(AMD-Vi / "IOMMU: Enabled").
|
| 248 |
+
- Turn off Wi-Fi and Bluetooth if the menu offers it (the bootstrap also blocks their drivers).
|
| 249 |
+
- "Restore on AC power loss": Power On.
|
| 250 |
+
- Boot order: the internal disk first. After installing (step 2), turn off booting from USB.
|
| 251 |
+
2. **A clean Ubuntu Server 24.04 LTS install** from a USB stick made from ubuntu.com's image. Erase the preinstalled
|
| 252 |
+
Windows; some mini-PC vendors have shipped malware in theirs.
|
| 253 |
+
- Storage: "Use an entire disk", "Set up this disk as an LVM group", **"Encrypt the LVM group with LUKS"**. Keep
|
| 254 |
+
the passphrase in your password manager.
|
| 255 |
+
- Profile: server name `fly-vault`, user **`fly-admin`**, any password (the bootstrap locks it).
|
| 256 |
+
- SSH: "Install OpenSSH server", with password authentication (only until the bootstrap).
|
| 257 |
+
- No featured snaps.
|
| 258 |
+
3. **Network:** plug it into the router by cable, on its own VLAN or guest port if the router can. Forward no ports to
|
| 259 |
+
it, and turn off UPnP on the router. Put it on a UPS: during a power or internet outage the fly cannot sell,
|
| 260 |
+
refill or pay, and healthchecks.io tells you it went quiet.
|
| 261 |
+
4. From the Mac, on the same network (the machine's address is on its console, `ip -4 addr`):
|
| 262 |
+
```
|
| 263 |
+
$ ssh-copy-id -o IdentityAgent=$SECRETIVE -f -i ~/.ssh/fly-ssh.pub fly-admin@<LAN address>
|
| 264 |
+
$ scp -o IdentityAgent=$SECRETIVE -r ../fly-trader-dist/deploy ~/.ssh/fly-release.pub ~/fly-keys/fly-release-recovery.pub fly-admin@<LAN address>:
|
| 265 |
+
$ ssh -o IdentityAgent=$SECRETIVE fly-admin@<LAN address>
|
| 266 |
+
```
|
| 267 |
+
5. The first pass. `TPM_UNLOCK=1` binds the disk encryption to the TPM, so security-update reboots come back up on
|
| 268 |
+
their own; it asks once for the disk passphrase. Without it, every boot waits for the passphrase at the console,
|
| 269 |
+
and automatic reboots are switched off.
|
| 270 |
+
```
|
| 271 |
+
server$ sudo TS_AUTHKEY=<signed key from 0.3 step 4> RECOVERY_PUBKEY="$(cat fly-release-recovery.pub)" \
|
| 272 |
+
EXPECT_MANIFEST=<manifest sha256 from 4.1> TPM_UNLOCK=1 ADMIN_USER=fly-admin bash deploy/bootstrap-host.sh "$(cat fly-release.pub)"
|
| 273 |
+
```
|
| 274 |
+
It hardens the box (roughly CIS level 1):
|
| 275 |
+
- **The machine:** Wi-Fi and Bluetooth drivers can never load; it warns if the IOMMU is off or the disk is not
|
| 276 |
+
encrypted.
|
| 277 |
+
- **SSH:** only `fly-admin`, key only (ed25519, P-256, FIDO), root off.
|
| 278 |
+
- **Firewall:** in and out; containers reach only HTTPS, HTTP (apt) and DNS.
|
| 279 |
+
- **Kernel and Docker:** user namespaces, no new privileges.
|
| 280 |
+
- **Monitoring:** fail2ban; an audit log of every read of the wallet keys, with a Telegram alert when anything but
|
| 281 |
+
the signer reads them; a daily AIDE file-integrity check (05:30 UTC) that reports to Telegram when files changed;
|
| 282 |
+
automatic security updates (reboot 04:00 UTC when needed and the TPM unlocks the disk).
|
| 283 |
+
- **Keys:** it makes the trading and payout keys and prints them, **once**, as `bot_key: <address>` (the trading
|
| 284 |
+
key) and `payout_key: <address>`. Copy both lines now. Later, once the containers run:
|
| 285 |
+
`server$ sudo fly signer status` (as `"trading"` and `"payout"`).
|
| 286 |
+
6. **Keep this session open.** From a second terminal on the Mac, log in over Tailscale: `$ ssh fly-vault`. Only when
|
| 287 |
+
that works, run the second pass in the first session. It removes SSH from the local network, leaving Tailscale as
|
| 288 |
+
the only way in:
|
| 289 |
+
```
|
| 290 |
+
server$ sudo CLOSE_PUBLIC_SSH=1 bash deploy/bootstrap-host.sh "$(cat fly-release.pub)"
|
| 291 |
+
```
|
| 292 |
+
The recovery key already installed is kept. After this the machine has no open port except SSH over Tailscale.
|
| 293 |
+
7. Physical access is now the way around every other control: keep the machine somewhere only you reach. The disk is
|
| 294 |
+
useless without the TPM on this board (or the passphrase), and the firmware password and USB-boot-off stop anyone
|
| 295 |
+
booting another system on it. Unbinding the TPM (a new board, a firmware reset) means typing the passphrase at the
|
| 296 |
+
console once and re-running step 5 with `TPM_UNLOCK=1`.
|
| 297 |
+
|
| 298 |
+
### 4.3 The treasury (your phone)
|
| 299 |
+
1. Open `https://fly-trader.app/owner.html` in Solflare's browser and connect Solflare.
|
| 300 |
+
2. Paste the trading key (`bot_key`) and the payout key (`payout_key`), leave the multisig field empty, and tap
|
| 301 |
+
"Read the treasury".
|
| 302 |
+
3. Tap "Create the treasury multisig": you are its only member, threshold 1, no time lock.
|
| 303 |
+
4. Tap "Create L1" (2 SOL a day, to the trading wallet only) and "Create L2" (1 SOL a week, claims).
|
| 304 |
+
5. Every check the page shows must pass. It prints the three lines for `custody.env`. **Save the multisig, L1 and L2
|
| 305 |
+
addresses in your password manager.** The page remembers them only in that browser, and the Revoke button needs
|
| 306 |
+
them.
|
| 307 |
+
|
| 308 |
+
### 4.4 Settings, start, check
|
| 309 |
+
1. Fill in the settings with `sudoedit`: `server$ sudoedit /srv/fly/custody.env`, and so on. The files are root-only
|
| 310 |
+
(mode 600).
|
| 311 |
+
- **`/srv/fly/custody.env`:** the three lines from the owner page (`VAULT_MULTISIG`, `VAULT_LIMIT_TRADING`,
|
| 312 |
+
`VAULT_LIMIT_PAYOUT`) and `VAULT_BACKUP_RECIPIENT` (your `age1…`). The float and caps have working defaults.
|
| 313 |
+
- **`/srv/fly/signer.env`:** `SIGNER_RPC_URL` (the second provider). Required: the signer does not start without it.
|
| 314 |
+
- **`/srv/fly/vault.env`:**
|
| 315 |
+
- `HELIUS_API_KEY`, `JUPITER_API_KEY`, `SOLANA_CHECK_RPC_URL`
|
| 316 |
+
- `CAPITAL_SOL` (your principal, e.g. 5)
|
| 317 |
+
- `FUNDING_ADDRESSES`: the addresses you will send principal from, comma-separated; at least your Solflare address
|
| 318 |
+
- `VAULT_ADDRESS`, `VAULT_TIMELOCK`, `VAULT_START_BLOCK` (`blockNumber`), `VAULT_IMPL_CODEHASHES`, all from step 2
|
| 319 |
+
- `RELAY_SECRET` (the relay's secret from step 3)
|
| 320 |
+
- `TELEGRAM_BOT_TOKEN`, `TELEGRAM_CHAT_ID`, `TELEGRAM_ADMIN_USER_ID`, `HEALTHCHECK_URL`
|
| 321 |
+
- `VAULT_BACKUP_BUCKET`, `VAULT_BACKUP_HF_TOKEN` (the bucket-only token)
|
| 322 |
+
- **`/etc/fly/update.json`:** `telegram_bot_token` and `telegram_chat_id` (the host updater, the key-read watcher and
|
| 323 |
+
AIDE report through these). `expect_manifest_sha256` was set by the bootstrap.
|
| 324 |
+
|
| 325 |
+
Everything must be filled in before the next step: a first release that does not come up healthy is refused for
|
| 326 |
+
good. If that happens: fix the cause, publish a new release, and put its manifest sha256 into
|
| 327 |
+
`expect_manifest_sha256` in `/etc/fly/update.json`.
|
| 328 |
+
2. Start the updater. It installs the pinned release, builds the brain and signer images and starts them (the first
|
| 329 |
+
build takes several minutes):
|
| 330 |
+
```
|
| 331 |
+
server$ sudo systemctl enable --now fly-update.timer
|
| 332 |
+
server$ sudo systemctl start fly-update.service # the first round now, instead of in 2 minutes
|
| 333 |
+
server$ sudo journalctl -u fly-update -f
|
| 334 |
+
```
|
| 335 |
+
3. Check everything:
|
| 336 |
+
- `server$ sudo /usr/local/lib/fly/check-host.sh`: every control must pass (keys only in the signer, Tailnet Lock on,
|
| 337 |
+
no public SSH…)
|
| 338 |
+
- `server$ sudo fly vault custody`: no problems
|
| 339 |
+
- `server$ sudo python3 /usr/local/lib/fly/fly_audit_watch.py test`: the Telegram test message arrives
|
| 340 |
+
4. The console: `$ ssh -L 8501:127.0.0.1:8501 fly-vault`, then http://localhost:8501/vault.
|
| 341 |
|
| 342 |
## 5. Go live
|
| 343 |
+
1. From a funding address, send **0.05 SOL to the trading wallet** (the `bot_key` address). It pays the fees of its own
|
| 344 |
+
refills and of claims.
|
| 345 |
+
2. Send your principal to the **treasury** (the vault 0 address on the owner page) from a funding address. The fly
|
| 346 |
+
refills its 2 SOL float from it through L1 on its own.
|
| 347 |
+
3. `server$ sudo fly vault status`: `"ledger": {"deposits": …}` must equal the SOL you sent from funding addresses. If
|
| 348 |
+
some came from another address, it counts as profit: `server$ sudo fly vault reclassify <signature> deposit`.
|
| 349 |
+
4. The fly trades live after the handover (at least 20 paper trades). Settlements run every Monday 00:00 UTC; claims
|
| 350 |
+
open from the first one.
|
| 351 |
+
5. When claims are larger than L2 allows in a week, Telegram says a claim is waiting. Raise L2 as in "Day to day".
|
| 352 |
+
Waiting claims never fail.
|
| 353 |
+
|
| 354 |
+
## Emergency
|
| 355 |
+
- **Something is wrong (possible compromise):**
|
| 356 |
+
1. On your phone, open the owner page and tap **Revoke L1 and L2 now**. From then on the server cannot move any SOL
|
| 357 |
+
out of the treasury; trading keeps only its float.
|
| 358 |
+
2. Send `/panic` to the bot. The fly halts; the signer refuses buys, top-ups and claims; open positions are sold; and
|
| 359 |
+
the float goes back to the treasury, all but the 0.3 SOL gas reserve.
|
| 360 |
+
3. If $FLY locking should stop too: MetaMask, Blockscout, FlyVault `pause()`.
|
| 361 |
+
- **A key-read, halt, AIDE or setup-change alert:** look before resuming. Resuming is SSH-only:
|
| 362 |
+
```
|
| 363 |
+
server$ sudo fly vault resume
|
| 364 |
+
server$ sudo fly reset-circuit --kill
|
| 365 |
+
server$ sudo fly resume-entries
|
| 366 |
+
server$ sudo fly signer clear-panic
|
| 367 |
+
```
|
| 368 |
+
- **Rebuild after losing the server:**
|
| 369 |
+
1. On the owner page, revoke L1 and L2 of the old keys.
|
| 370 |
+
2. Publish nothing; take the manifest sha256 of the current release:
|
| 371 |
+
`$ curl -sL https://huggingface.co/bryceweiner/fly-trader/resolve/main/release.json | shasum -a 256`.
|
| 372 |
+
3. New server: sections 4.2 (with that pin) to 4.4. On the owner page, clear the old L1/L2 fields and create new
|
| 373 |
+
limits for the new keys; put the new addresses in `custody.env`.
|
| 374 |
+
4. Restore the ledger. Download the newest `ledger/<day>.dump.age` from the backup bucket, then:
|
| 375 |
+
```
|
| 376 |
+
$ age -d -i ~/fly-keys/fly-backup.key <day>.dump.age > ledger.dump
|
| 377 |
+
$ scp ledger.dump fly-vault: && rm ledger.dump
|
| 378 |
+
server$ sudo docker stop fly-fly-1 # the brain must not write while the tables are replaced
|
| 379 |
+
server$ sudo docker exec -i fly-db-1 pg_restore -U fly -d fly_trader --clean < ledger.dump && rm ledger.dump
|
| 380 |
+
server$ sudo docker exec fly-db-1 psql -U fly -d fly_trader -c "DELETE FROM vault_kv WHERE key IN ('backup_key_done','custody','halt','panic')"
|
| 381 |
+
server$ sudo python3 /usr/local/lib/fly/fly_update.py restart
|
| 382 |
+
```
|
| 383 |
+
5. The old trading wallet's float and positions: `$ age -d -i ~/fly-keys/fly-backup.key keys/trading-<address>.age`
|
| 384 |
+
gives its secret (base58, 64 bytes), which you can import into a wallet to move them to the treasury.
|
| 385 |
|
| 386 |
## Day to day
|
| 387 |
+
- **Changing a limit** (e.g. raising L2 after a big settlement): on the owner page, "Set L2 to this amount". A new
|
| 388 |
+
limit has a **new address**, so put it in `/srv/fly/custody.env` and restart:
|
| 389 |
+
`server$ sudo python3 /usr/local/lib/fly/fly_update.py restart`. Until then claims wait. The same applies to any L1
|
| 390 |
+
change.
|
| 391 |
+
- **After editing any `/srv/fly/*.env`:** `server$ sudo python3 /usr/local/lib/fly/fly_update.py restart`.
|
| 392 |
+
- **New models or code:** publish as in 4.1 step 3 (`--models` to refresh the models).
|
| 393 |
+
- Model-only releases apply within 5 minutes; they cannot carry code (weights-only torch files, skops selectors).
|
| 394 |
+
- A release that changes **code** is held 24 h, and Telegram says so. To apply it now:
|
| 395 |
+
`server$ sudo python3 /usr/local/lib/fly/fly_update.py approve <seq>`. To stop it:
|
| 396 |
+
`server$ sudo python3 /usr/local/lib/fly/fly_update.py veto <seq>`. If you did not publish it, the release key is
|
| 397 |
+
compromised: remove it from `/etc/fly/allowed_signers` and rotate.
|
| 398 |
+
- A release that changes `deploy/`, `docker/`, the Dockerfile, `.dockerignore` or a compose file waits for
|
| 399 |
+
`approve`. It replaces only the compose file. For changed host scripts (bootstrap, `fly_update.py`,
|
| 400 |
+
`check-host.sh`, the watchers), copy them over and re-run the bootstrap **with `CLOSE_PUBLIC_SSH=1`** (without it
|
| 401 |
+
the bootstrap opens public SSH again); your recovery key is kept:
|
| 402 |
+
```
|
| 403 |
+
$ scp -r ../fly-trader-dist/deploy ~/.ssh/fly-release.pub fly-vault:
|
| 404 |
+
server$ sudo CLOSE_PUBLIC_SSH=1 bash deploy/bootstrap-host.sh "$(cat fly-release.pub)"
|
| 405 |
+
```
|
| 406 |
+
- To sign with the recovery key: decrypt it (`$ age -d ~/fly-keys/fly-release-recovery.age > /tmp/k && chmod 600
|
| 407 |
+
/tmp/k`), pass `--key /tmp/k`, then `rm /tmp/k`.
|
| 408 |
+
- **Take principal out:** `server$ sudo fly vault withdraw` prints how much you may take and how. You send it from the
|
| 409 |
+
treasury in the Squads app with your owner wallet; the fly books and reports it.
|
| 410 |
+
- **Watch it:** `/status` to the bot at any time. A daily report arrives at 09:00 UTC. If it does not, or
|
| 411 |
+
healthchecks.io alerts, the server or the fly is down. `server$ sudo fly vault status` gives the full ledger.
|
| 412 |
+
- **After an AIDE alert** you expected (a release, apt upgrades): `server$ sudo aide --update` then
|
| 413 |
+
`server$ sudo cp /var/lib/aide/aide.db.new /var/lib/aide/aide.db`.
|
| 414 |
+
- **Backups:** hot keys once, ledger nightly, both age-encrypted in the bucket. Until `VAULT_BACKUP_*` and the recipient
|
| 415 |
+
are set, Telegram reminds you daily. To read them:
|
| 416 |
+
- Hot keys: `$ age -d -i ~/fly-keys/fly-backup.key keys/<name>-<address>.age`
|
| 417 |
+
- Ledger: see "Rebuild", step 4.
|
| 418 |
+
|
| 419 |
+
## Ledger (when you're home): no redeploys
|
| 420 |
+
- **Solana:** in the Squads app, add the Ledger as a member, then remove Solflare (or keep both at threshold 2; the
|
| 421 |
+
owner page then proposes and approves, and the other member executes in the Squads app). L1 and L2 stay as they are.
|
| 422 |
+
- **EVM:** move MetaMask's roles to the Ledger through the timelock (8 days):
|
| 423 |
+
1. Print the calldata:
|
| 424 |
+
```
|
| 425 |
+
$ (cd contracts && VAULT=<vault> TIMELOCK=<timelock> OLD_OWNER=<MetaMask> NEW_OWNER=<Ledger> MODE=calldata \
|
| 426 |
+
forge script script/HandOver.s.sol --rpc-url robinhood)
|
| 427 |
+
```
|
| 428 |
+
2. Send the printed "schedule" data from MetaMask to the timelock (Blockscout "Write contract", `scheduleBatch`, or
|
| 429 |
+
any raw-data send).
|
| 430 |
+
3. After 8 days, send the "execute" data from any wallet.
|
| 431 |
+
- **SSH and releases:** add the Ledger's SSH key (ledger-agent) to `~fly-admin/.ssh/authorized_keys`. For releases,
|
| 432 |
+
make the Ledger's key the release key: re-run the bootstrap (as in "Day to day") with the Ledger's public key as the
|
| 433 |
+
argument (the bootstrap rewrites `/etc/fly/allowed_signers` from its arguments; a line added by hand would be lost),
|
| 434 |
+
and replace the `bryce` line in the distribution's `release/allowed_signers` (publish that change while the old key
|
| 435 |
+
still signs). Then remove the Secretive keys.
|
docs/vault/SPEC.md
CHANGED
|
@@ -1,11 +1,12 @@
|
|
| 1 |
# $FLY vault — interface spec v1
|
| 2 |
|
| 3 |
-
The contract between the
|
| 4 |
|
| 5 |
- **FlyVault** (`contracts/`, Robinhood Chain): holds locked $FLY.
|
| 6 |
- **The vault fly** (`fly_trader/vault/`, the hosted server): trades, keeps the ledger, settles weekly and pays claims.
|
|
|
|
| 7 |
- **The relay** (`web/relay/`, on the site's Gandi app): passes stats and claims between the fly and browsers.
|
| 8 |
-
- **The site** (`web/`): the Trading and
|
| 9 |
|
| 10 |
Units, everywhere:
|
| 11 |
- SOL amounts are integer **lamports**.
|
|
@@ -19,18 +20,18 @@ Solidity `0.8.24+` on OpenZeppelin Contracts Upgradeable v5, using UUPS and ERC-
|
|
| 19 |
(`fly.storage.FlyVault`). The constructor calls `_disableInitializers()`.
|
| 20 |
|
| 21 |
```solidity
|
| 22 |
-
uint256 public immutable WITHDRAW_DELAY; // constructor arg of the implementation: 7 days on mainnet
|
| 23 |
-
//
|
| 24 |
bytes32 public constant PAUSER_ROLE = keccak256("PAUSER_ROLE");
|
| 25 |
bytes32 public constant UPGRADER_ROLE = keccak256("UPGRADER_ROLE");
|
| 26 |
|
| 27 |
-
function initialize(address fly, address pauser, address timelock) external initializer;
|
| 28 |
// DEFAULT_ADMIN_ROLE + UPGRADER_ROLE -> timelock ; PAUSER_ROLE -> pauser
|
| 29 |
|
| 30 |
-
function lock(uint256 amount) external; // whenNotPaused; reverts unless received == amount
|
| 31 |
function requestWithdrawal(uint256 amount) external returns (uint256 id); // never paused
|
| 32 |
function cancelRequest(uint256 id) external; // whenNotPaused (it re-locks)
|
| 33 |
-
function withdraw(uint256 id) external; // never paused; block.timestamp >= readyAt
|
| 34 |
function pause() external; function unpause() external; // PAUSER_ROLE
|
| 35 |
|
| 36 |
function token() external view returns (address);
|
|
@@ -47,16 +48,29 @@ event WithdrawRequested(address indexed user, uint256 indexed id, uint256 amount
|
|
| 47 |
event RequestCancelled(address indexed user, uint256 indexed id, uint256 amount, uint256 lockedAfter);
|
| 48 |
event Withdrawn(address indexed user, uint256 indexed id, uint256 amount);
|
| 49 |
// plus OZ: Paused(address), Unpaused(address), Upgraded(address indexed implementation), RoleGranted/RoleRevoked
|
|
|
|
|
|
|
|
|
|
|
|
|
| 50 |
```
|
| 51 |
|
| 52 |
- **Request ids** start at 1 and are global.
|
| 53 |
- **`lockedAfter`** is the user's earning balance after the event, so the indexer can resync to an absolute value.
|
| 54 |
-
- **Timelock.** Upgrades go through an OZ `TimelockController`:
|
| 55 |
-
- minDelay = 8 days
|
| 56 |
-
- proposer = canceller = the
|
| 57 |
- executor = anyone (`address(0)`)
|
| 58 |
-
- admin = `address(0)`
|
|
|
|
|
|
|
| 59 |
- **No rescue function for FLY.** The token is set only in `initialize`.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 60 |
|
| 61 |
## 2. Claim messages (v1)
|
| 62 |
|
|
@@ -66,13 +80,15 @@ newline. Vectors: `tests/vectors/claim_v1.json`. Python (`fly_trader/vault/claim
|
|
| 66 |
|
| 67 |
Fields:
|
| 68 |
- `domain`: e.g. `fly-trader.app`, or `localhost:5173` on staging
|
| 69 |
-
- `uri`: e.g. `https://fly-trader.app/vault.html`
|
|
|
|
| 70 |
- `chain_id`: `4663`, or `46630` on testnet
|
| 71 |
- `sol_chain`: `mainnet` or `devnet`
|
| 72 |
- `evm`: EIP-55 checksummed
|
| 73 |
- `sol`: base58 pubkey
|
| 74 |
- `nonce`: 32 lowercase hex characters, issued by the relay
|
| 75 |
-
- `issued_at`, `expires_at`: RFC 3339 `YYYY-MM-DDTHH:MM:SSZ`; expiry = issue + 900 s
|
|
|
|
| 76 |
- `request_id`: always `fly-vault-claim-v1`
|
| 77 |
|
| 78 |
EVM text, signed with `personal_sign` (EIP-191); the signature is `0x…` hex, 65 bytes for an EOA and up to 1024 bytes
|
|
@@ -108,60 +124,95 @@ Request ID: fly-vault-claim-v1
|
|
| 108 |
```
|
| 109 |
|
| 110 |
What the fly checks:
|
| 111 |
-
- **The texts.**
|
|
|
|
|
|
|
|
|
|
| 112 |
- **The EVM signature.**
|
| 113 |
-
-
|
| 114 |
-
-
|
| 115 |
-
|
| 116 |
-
- **The
|
| 117 |
-
- **The
|
| 118 |
-
|
| 119 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 120 |
|
| 121 |
## 3. Relay HTTP API (v1)
|
| 122 |
|
| 123 |
The base is `{origin}/api`. Every response is JSON with `Cache-Control: no-store`. Errors look like `{"error": "<text>"}`
|
| 124 |
-
with status 400, 401, 404, 409, 413, 429 or 503.
|
|
|
|
| 125 |
|
| 126 |
### Public
|
| 127 |
|
| 128 |
- **`GET /api/stats`** returns the latest snapshot (§4), or 503 before the first push.
|
| 129 |
- **`GET /api/history?kind=nav|trades|flows|settlements&before=<cursor>&limit=<1..500, default 100>`** returns
|
| 130 |
-
`{"kind", "items": [...], "next": <cursor|null>}`, newest first. The
|
| 131 |
-
|
|
|
|
|
|
|
|
|
|
| 132 |
- **`GET /api/claim/challenge?evm=0x…&sol=<base58>`**:
|
| 133 |
- Returns `{"nonce", "issued_at", "expires_at", "domain", "uri", "chain_id", "sol_chain", "request_id", "min_lamports"}`.
|
| 134 |
-
- The relay stores the challenge
|
| 135 |
- **`POST /api/claim`** with body `{"nonce", "evm", "sol", "evm_sig", "sol_sig"}` (≤ 4 KB):
|
| 136 |
- Returns `202 {"id", "status": "received"}`.
|
| 137 |
- Checks, all cheap and none cryptographic:
|
| 138 |
- the nonce is known, unexpired, unused, and issued for this `evm` + `sol`
|
| 139 |
- the signatures have valid syntax: `evm_sig` matches `^0x([0-9a-fA-F]{2}){65,1024}$`; `sol_sig` is base58 of 64 bytes
|
| 140 |
-
- the published account's owed ≥
|
| 141 |
-
- no claim for this EVM address is in flight
|
| 142 |
-
|
| 143 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 144 |
- Status is one of `received | verified | waiting_liquidity | sending | paid | rejected | failed`.
|
| 145 |
-
-
|
|
|
|
| 146 |
- challenge: 30 per hour per IP
|
| 147 |
-
- claim: 10 per hour per IP and 5 per hour per EVM address
|
| 148 |
-
|
|
|
|
|
|
|
|
|
|
| 149 |
|
| 150 |
### Fly only (HMAC)
|
| 151 |
|
| 152 |
- **`POST /api/fly/push`**: upserts.
|
| 153 |
- Body: `{"stats": {…§4…}?, "history": {"nav": [...], "trades": [...], "flows": [...], "settlements": [...]}?, "accounts": [ …§4.3… ]?}`.
|
| 154 |
-
- History items upsert by `id` (nav by `ts`);
|
|
|
|
|
|
|
|
|
|
| 155 |
- Returns `{"ok": true}`. Body ≤ 2 MB.
|
| 156 |
-
- **`GET /api/fly/claims?after=<id>&limit=<1..50>`**:
|
| 157 |
- Returns `{"claims": [{"id", "nonce", "evm", "sol", "evm_sig", "sol_sig", "domain", "uri", "chain_id", "sol_chain", "issued_at", "expires_at", "created_at"}]}`.
|
| 158 |
-
- Only claims in status `received`, in id order.
|
| 159 |
-
- **`POST /api/fly/claims/result`**: body `{"results": [{"id", "status", "reason"?, "lamports"?, "tx"?}]}` returns
|
| 160 |
-
|
|
|
|
|
|
|
|
|
|
| 161 |
|
| 162 |
HMAC headers:
|
| 163 |
- `X-Fly-Key`: the key id
|
| 164 |
-
- `X-Fly-Ts`: unix seconds
|
| 165 |
- `X-Fly-Nonce`: 32 hex characters
|
| 166 |
- `X-Fly-Sig`: hex HMAC-SHA256(key = the UTF-8 bytes of the configured secret string, canonical)
|
| 167 |
|
|
@@ -169,12 +220,15 @@ The canonical string:
|
|
| 169 |
```
|
| 170 |
FLY-RELAY-1\n{METHOD}\n{PATH}\n{QUERY}\n{TS}\n{NONCE}\n{sha256hex(body)}
|
| 171 |
```
|
| 172 |
-
- `PATH` excludes the query string.
|
| 173 |
- `QUERY` is `&`.join(`quote(k,safe='')=quote(v,safe='')` for (k, v) in sorted(parse_qsl(qs, keep_blank_values=True))).
|
| 174 |
- The body is the raw bytes; an empty body hashes as sha256 of `b""`.
|
| 175 |
-
- The timestamp must be within ±300 s. Nonces are remembered for 15 minutes, and a reused nonce
|
|
|
|
| 176 |
|
| 177 |
-
Relay config: `relay.json`, kept next to `wsgi.py` and never inside `site/`
|
|
|
|
|
|
|
| 178 |
```json
|
| 179 |
{"keys": {"k1": "<64 hex>"}, "db_path": "<writable path>", "domain": "fly-trader.app", "uri": "https://fly-trader.app/vault.html",
|
| 180 |
"chain_id": 4663, "sol_chain": "mainnet", "claim_ttl_s": 900, "min_lamports": 2000000, "client_ip_header": null}
|
|
@@ -182,33 +236,56 @@ Relay config: `relay.json`, kept next to `wsgi.py` and never inside `site/`:
|
|
| 182 |
|
| 183 |
## 4. Payloads
|
| 184 |
|
| 185 |
-
### 4.1 Stats snapshot (`stats`, pushed every
|
| 186 |
```json
|
| 187 |
{
|
| 188 |
-
"v": 1, "ts": 0, "cluster": "mainnet",
|
| 189 |
-
"fly": {"state": "starting|paper|live|halted", "
|
| 190 |
"entries_paused": false, "model": {"fly": 62, "selector": 59, "release": 3}},
|
| 191 |
-
"wallet": {"native": 0, "
|
| 192 |
"ledger": {"deposits": 0, "withdrawals": 0, "claims_paid": 0, "realized": 0, "booked_realized": 0,
|
| 193 |
"allocated": 0, "reserved": 0, "pot": 0},
|
| 194 |
"prices": {"sol_usd": 0.0, "fly_usd": 0.0, "ts": 0},
|
| 195 |
"vault": {"address": "0x…", "chain_id": 4663, "total_locked": "0", "total_pending": "0", "earners": 0,
|
| 196 |
"finalized_block": 0, "paused": false, "impl": "0x…", "upgrade_scheduled": null},
|
| 197 |
-
"settlement": {"
|
| 198 |
-
"positions": [{"mint": "", "symbol": "", "opened_at": 0, "cost": 0, "value": 0, "entry_price": 0.0, "mark_price": 0.0, "hold_min": 0}],
|
| 199 |
-
"index": {"value": 1.0, "peak": 1.0, "drawdown": 0.0}
|
| 200 |
}
|
| 201 |
```
|
| 202 |
-
-
|
| 203 |
-
-
|
| 204 |
-
- `
|
| 205 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 206 |
|
| 207 |
### 4.2 History items
|
| 208 |
-
- `nav`: `{"ts", "nav", "
|
| 209 |
-
|
| 210 |
-
- `
|
| 211 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 212 |
|
| 213 |
### 4.3 Account
|
| 214 |
```json
|
|
@@ -216,15 +293,114 @@ Relay config: `relay.json`, kept next to `wsgi.py` and never inside `site/`:
|
|
| 216 |
"allocations": [{"period_end": 0, "lamports": 0, "weight": "<str>", "share": 0.0}],
|
| 217 |
"claims": [{"id": 0, "status": "", "lamports": 0, "sol": "", "tx": "", "created_at": 0, "updated_at": 0}]}
|
| 218 |
```
|
| 219 |
-
`owed` = allocated − claimed − in_flight
|
|
|
|
|
|
|
|
|
|
| 220 |
|
| 221 |
## 5. Networks
|
| 222 |
|
| 223 |
| | mainnet | rehearsal |
|
| 224 |
|---|---|---|
|
|
|
|
| 225 |
| EVM chain | Robinhood Chain 4663, `https://rpc.mainnet.chain.robinhood.com` | RH testnet 46630, `https://rpc.testnet.chain.robinhood.com` |
|
| 226 |
| Explorer | `https://robinhoodchain.blockscout.com` | `https://explorer.testnet.chain.robinhood.com` |
|
| 227 |
| $FLY | `0x2fC7f9E2911f20b2C4660d2AEf808aa91bDdb3D3` | `MockFLY` (deployed for the rehearsal) |
|
|
|
|
|
|
|
| 228 |
| Solana | mainnet-beta | devnet |
|
|
|
|
|
|
|
| 229 |
| Kyber | `https://aggregator-api.kyberswap.com/robinhood/api/v1/`, router `0x6131B5fae19EA4f9D964eAc0408E4408b66337b5` | — |
|
| 230 |
| Chart | GeckoTerminal network `robinhood`, pool `0xe6925f7bdedf22c2714c749d0ff208ce9e4bc1540938fa2b9ab2379486322edd` | fixtures |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
# $FLY vault — interface spec v1
|
| 2 |
|
| 3 |
+
The contract between the parts of the vault system. Change it here first, then in code.
|
| 4 |
|
| 5 |
- **FlyVault** (`contracts/`, Robinhood Chain): holds locked $FLY.
|
| 6 |
- **The vault fly** (`fly_trader/vault/`, the hosted server): trades, keeps the ledger, settles weekly and pays claims.
|
| 7 |
+
- **The signer** (`fly_trader/signer/`, its own container on the server): the only holder of the two hot keys (§6).
|
| 8 |
- **The relay** (`web/relay/`, on the site's Gandi app): passes stats and claims between the fly and browsers.
|
| 9 |
+
- **The site** (`web/`): the Trading, Vault and (unlisted) Owner pages.
|
| 10 |
|
| 11 |
Units, everywhere:
|
| 12 |
- SOL amounts are integer **lamports**.
|
|
|
|
| 20 |
(`fly.storage.FlyVault`). The constructor calls `_disableInitializers()`.
|
| 21 |
|
| 22 |
```solidity
|
| 23 |
+
uint256 public immutable WITHDRAW_DELAY; // constructor arg of the implementation: 7 days on mainnet, 300 s by default
|
| 24 |
+
// on the rehearsal (DeployTestnet.s.sol). Changing it = an upgrade.
|
| 25 |
bytes32 public constant PAUSER_ROLE = keccak256("PAUSER_ROLE");
|
| 26 |
bytes32 public constant UPGRADER_ROLE = keccak256("UPGRADER_ROLE");
|
| 27 |
|
| 28 |
+
function initialize(address fly, address pauser, address timelock) external initializer; // reverts on a zero address
|
| 29 |
// DEFAULT_ADMIN_ROLE + UPGRADER_ROLE -> timelock ; PAUSER_ROLE -> pauser
|
| 30 |
|
| 31 |
+
function lock(uint256 amount) external; // whenNotPaused, nonReentrant; reverts unless received == amount
|
| 32 |
function requestWithdrawal(uint256 amount) external returns (uint256 id); // never paused
|
| 33 |
function cancelRequest(uint256 id) external; // whenNotPaused (it re-locks)
|
| 34 |
+
function withdraw(uint256 id) external; // never paused, nonReentrant; block.timestamp >= readyAt
|
| 35 |
function pause() external; function unpause() external; // PAUSER_ROLE
|
| 36 |
|
| 37 |
function token() external view returns (address);
|
|
|
|
| 48 |
event RequestCancelled(address indexed user, uint256 indexed id, uint256 amount, uint256 lockedAfter);
|
| 49 |
event Withdrawn(address indexed user, uint256 indexed id, uint256 amount);
|
| 50 |
// plus OZ: Paused(address), Unpaused(address), Upgraded(address indexed implementation), RoleGranted/RoleRevoked
|
| 51 |
+
|
| 52 |
+
error ZeroAddress(); error ZeroAmount(); error TransferAmountMismatch(uint256 expected, uint256 received);
|
| 53 |
+
error InsufficientLocked(uint256 requested, uint256 available); error UnknownRequest(uint256 id);
|
| 54 |
+
error NotRequestOwner(uint256 id); error RequestNotPending(uint256 id); error WithdrawalNotReady(uint256 id, uint64 readyAt);
|
| 55 |
```
|
| 56 |
|
| 57 |
- **Request ids** start at 1 and are global.
|
| 58 |
- **`lockedAfter`** is the user's earning balance after the event, so the indexer can resync to an absolute value.
|
| 59 |
+
- **Timelock.** Upgrades and role changes go through an OZ `TimelockController`:
|
| 60 |
+
- minDelay = 8 days on mainnet (600 s by default on the rehearsal)
|
| 61 |
+
- proposer = canceller = the owner (MetaMask until the Ledger takes over; `OWNER` in the deploy)
|
| 62 |
- executor = anyone (`address(0)`)
|
| 63 |
+
- admin = `address(0)`: the timelock administers itself, so role changes also wait the full delay
|
| 64 |
+
- **Mainnet deploy rules.** On chain 4663 the deploy (`FlyVaultStack.enforceChainRules`) refuses anything but the real
|
| 65 |
+
$FLY, an 8-day timelock and a 7-day withdraw delay. `PAUSER` defaults to `OWNER`.
|
| 66 |
- **No rescue function for FLY.** The token is set only in `initialize`.
|
| 67 |
+
- **The fly indexes** the vault's events and the timelock's `CallScheduled`, `CallExecuted` and `Cancelled` for
|
| 68 |
+
operations aimed at the vault or the timelock (an alert and `vault.upgrade_scheduled`, §4.1).
|
| 69 |
+
- **Scripts.**
|
| 70 |
+
- `HandOver.s.sol`: one timelock batch that grants PROPOSER, CANCELLER and PAUSER to a new owner and revokes them
|
| 71 |
+
from the old one. `MODE=schedule|execute`, or `MODE=calldata` to print the calldata for a browser wallet.
|
| 72 |
+
- `ScheduleUpgrade.s.sol` rehearses an upgrade locally and schedules `upgradeToAndCall`; `ExecuteUpgrade.s.sol` runs
|
| 73 |
+
it after the delay.
|
| 74 |
|
| 75 |
## 2. Claim messages (v1)
|
| 76 |
|
|
|
|
| 80 |
|
| 81 |
Fields:
|
| 82 |
- `domain`: e.g. `fly-trader.app`, or `localhost:5173` on staging
|
| 83 |
+
- `uri`: e.g. `https://fly-trader.app/vault.html`, or `http://localhost:5173/vault.html` on staging; https (http only
|
| 84 |
+
for localhost/127.0.0.1), no spaces or line breaks
|
| 85 |
- `chain_id`: `4663`, or `46630` on testnet
|
| 86 |
- `sol_chain`: `mainnet` or `devnet`
|
| 87 |
- `evm`: EIP-55 checksummed
|
| 88 |
- `sol`: base58 pubkey
|
| 89 |
- `nonce`: 32 lowercase hex characters, issued by the relay
|
| 90 |
+
- `issued_at`, `expires_at`: RFC 3339 `YYYY-MM-DDTHH:MM:SSZ`; expiry = issue + 900 s. The fly accepts no other TTL, so
|
| 91 |
+
the relay's `claim_ttl_s` must stay 900.
|
| 92 |
- `request_id`: always `fly-vault-claim-v1`
|
| 93 |
|
| 94 |
EVM text, signed with `personal_sign` (EIP-191); the signature is `0x…` hex, 65 bytes for an EOA and up to 1024 bytes
|
|
|
|
| 124 |
```
|
| 125 |
|
| 126 |
What the fly checks:
|
| 127 |
+
- **The texts.** Both are re-rendered from the fields; relay-supplied text is never used. `domain`/`uri` must equal
|
| 128 |
+
`VAULT_SITE_DOMAIN`/`VAULT_SITE_URI`, `chain_id` must equal `RH_CHAIN_ID`, and `sol_chain` must match `VAULT_CLUSTER`.
|
| 129 |
+
- **Freshness.** A claim is rejected if the fly first stored it more than 3600 s after `expires_at`, or if `issued_at`
|
| 130 |
+
is more than 300 s in the future. The relay already refuses expired nonces at submission.
|
| 131 |
- **The EVM signature.**
|
| 132 |
+
- ecrecover first: exactly 65 bytes, 0 < r < n, low-s, v ∈ {27, 28} (0/1 are normalized).
|
| 133 |
+
- Only if that does not recover the address and the address has code: EIP-1271 `isValidSignature(hashMessage(text), sig)`
|
| 134 |
+
(65..1024 bytes, 200,000 gas cap) must return exactly `0x1626ba7e`.
|
| 135 |
+
- **The Solana signature.** 64 bytes, ed25519 over the UTF-8 text; `sol` must be an on-curve pubkey.
|
| 136 |
+
- **The nonce.** Unique per EVM address.
|
| 137 |
+
- **One in flight.** A claim is rejected while another claim for the address is `verified | waiting_liquidity |
|
| 138 |
+
sending`.
|
| 139 |
+
- **The amount.** Owed ≥ `CLAIM_MIN_LAMPORTS` (2,000,000), read when the claim moves to `sending`; the amount is fixed
|
| 140 |
+
then. The relay's `min_lamports` and the site's `claimMinLamports` must hold the same value.
|
| 141 |
+
- **Halts.** Nothing is verified or paid while the vault is halted (claims are still pulled from the relay).
|
| 142 |
+
|
| 143 |
+
A valid claim pays **all** SOL owed to the EVM address, sent to `sol` from the **treasury** (Squads vault 0) under
|
| 144 |
+
spending limit L2 (§6): the payout key signs `spendingLimitUse` and the trading key pays the fee. Nothing is ever paid
|
| 145 |
+
from the trading wallet's own SOL. Neither the treasury, the trading wallet nor any payout signature is published.
|
| 146 |
+
|
| 147 |
+
**Statuses:** `received → verified → (waiting_liquidity) → sending → paid | rejected | failed`.
|
| 148 |
+
- `waiting_liquidity`: L2 has too little left this week, the treasury is short, or the signer is in panic. The claim
|
| 149 |
+
retries every loop and alerts the operator; it never fails for these reasons.
|
| 150 |
+
- `failed`: four unlanded attempts, a failure on chain, or a signer policy refusal.
|
| 151 |
+
- The payment's signature and last valid block height are stored before broadcast; a claim is re-signed only once its
|
| 152 |
+
blockhash has expired unlanded, and the signer keeps its own once-per-claim ledger (§6).
|
| 153 |
|
| 154 |
## 3. Relay HTTP API (v1)
|
| 155 |
|
| 156 |
The base is `{origin}/api`. Every response is JSON with `Cache-Control: no-store`. Errors look like `{"error": "<text>"}`
|
| 157 |
+
with status 400, 401, 404, 405 (with `Allow`), 409, 413, 429 (with `Retry-After`), 500 or 503. 503 means either
|
| 158 |
+
`relay not configured` (relay.json missing or invalid) or `storage unavailable` (SQLite error, with `Retry-After: 5`).
|
| 159 |
|
| 160 |
### Public
|
| 161 |
|
| 162 |
- **`GET /api/stats`** returns the latest snapshot (§4), or 503 before the first push.
|
| 163 |
- **`GET /api/history?kind=nav|trades|flows|settlements&before=<cursor>&limit=<1..500, default 100>`** returns
|
| 164 |
+
`{"kind", "items": [...], "next": <cursor|null>}`, newest first by (time, id). The time is nav `ts`, trades
|
| 165 |
+
`closed_at`, flows `ts`, settlements `period_end`. The cursor is opaque; send it back as `before`. An unknown `kind`,
|
| 166 |
+
a bad `limit` or a bad cursor gets 400.
|
| 167 |
+
- **`GET /api/account?evm=0x…`** returns the account (§4.3). `evm` is accepted in any case. An unknown address gets all
|
| 168 |
+
zeros; the fly pushes only addresses with at least one allocation.
|
| 169 |
- **`GET /api/claim/challenge?evm=0x…&sol=<base58>`**:
|
| 170 |
- Returns `{"nonce", "issued_at", "expires_at", "domain", "uri", "chain_id", "sol_chain", "request_id", "min_lamports"}`.
|
| 171 |
+
- The relay stores the challenge; a bad `evm` or `sol` gets 400 (checked before the rate limit).
|
| 172 |
- **`POST /api/claim`** with body `{"nonce", "evm", "sol", "evm_sig", "sol_sig"}` (≤ 4 KB):
|
| 173 |
- Returns `202 {"id", "status": "received"}`.
|
| 174 |
- Checks, all cheap and none cryptographic:
|
| 175 |
- the nonce is known, unexpired, unused, and issued for this `evm` + `sol`
|
| 176 |
- the signatures have valid syntax: `evm_sig` matches `^0x([0-9a-fA-F]{2}){65,1024}$`; `sol_sig` is base58 of 64 bytes
|
| 177 |
+
- the published account's owed ≥ `min_lamports`
|
| 178 |
+
- no claim for this EVM address that the fly has verified is in flight (`verified | waiting_liquidity |
|
| 179 |
+
sending`). An unverified `received` claim does not block, since it may be a stranger's junk; the fly pays
|
| 180 |
+
each address at most once whatever the relay accepts.
|
| 181 |
+
- 400 for bad syntax, an unknown, expired or mismatched nonce, or nothing to claim; 409 if the nonce was already used
|
| 182 |
+
or a verified claim is in flight; 429 per the limits below.
|
| 183 |
+
- **`GET /api/claim/<id>`** (the relay's id, 1-18 digits) returns `{"id", "status", "reason", "lamports", "tx",
|
| 184 |
+
"created_at", "updated_at"}`, or 404.
|
| 185 |
- Status is one of `received | verified | waiting_liquidity | sending | paid | rejected | failed`.
|
| 186 |
+
- `tx` is always null: the fly never reports the payout signature.
|
| 187 |
+
- **Rate limits (token buckets; a 429 carries `Retry-After`):**
|
| 188 |
- challenge: 30 per hour per IP
|
| 189 |
+
- claim: 10 per hour per IP, charged after the syntax checks; and 5 per hour per EVM address and IP, charged only
|
| 190 |
+
after every other check passes (so junk from one client cannot lock a holder out)
|
| 191 |
+
- reads (stats, history, account, claim/<id>): 600 per hour per IP
|
| 192 |
+
- IPv6 clients are bucketed per /64. Behind a proxy, `client_ip_header` names the header to read, and the relay takes
|
| 193 |
+
its rightmost entry (the one the proxy appended). The fly's HMAC routes are not rate-limited.
|
| 194 |
|
| 195 |
### Fly only (HMAC)
|
| 196 |
|
| 197 |
- **`POST /api/fly/push`**: upserts.
|
| 198 |
- Body: `{"stats": {…§4…}?, "history": {"nav": [...], "trades": [...], "flows": [...], "settlements": [...]}?, "accounts": [ …§4.3… ]?}`.
|
| 199 |
+
- Unknown top-level keys get 400. History items upsert by `id` (nav by `ts`); each needs an integer time field (nav
|
| 200 |
+
`ts`; trades `closed_at` or `opened_at`; settlements `period_end` or `period_start`; flows `ts`) and, except nav, an
|
| 201 |
+
`id` (an integer ≥ 0 or a string ≤ 200 characters). Accounts upsert by `evm` and need an integer `owed`. `stats` is
|
| 202 |
+
stored as is; NaN and Infinity become null.
|
| 203 |
- Returns `{"ok": true}`. Body ≤ 2 MB.
|
| 204 |
+
- **`GET /api/fly/claims?after=<id, default 0>&limit=<1..50, default 50>`**:
|
| 205 |
- Returns `{"claims": [{"id", "nonce", "evm", "sol", "evm_sig", "sol_sig", "domain", "uri", "chain_id", "sol_chain", "issued_at", "expires_at", "created_at"}]}`.
|
| 206 |
+
- Only claims in status `received`, in id order. `evm` is lowercase; `created_at` is unix seconds.
|
| 207 |
+
- **`POST /api/fly/claims/result`**: body `{"results": [{"id", "status", "reason"?, "lamports"?, "tx"?}]}` returns
|
| 208 |
+
`{"ok": true}`. `status` must be one of the seven statuses; `reason` is cut to 500 characters; omitted fields keep
|
| 209 |
+
their value; unknown ids are ignored.
|
| 210 |
+
- **`GET /api/fly/probe`** returns `{"python", "db_path", "writable", "wal", "pid", "remote_addr", "headers": {...},
|
| 211 |
+
"sqlite", "time", "client_ip", "multiprocess", "multithread"}` (headers: every `HTTP_*` except Cookie).
|
| 212 |
|
| 213 |
HMAC headers:
|
| 214 |
- `X-Fly-Key`: the key id
|
| 215 |
+
- `X-Fly-Ts`: unix seconds (1-15 digits)
|
| 216 |
- `X-Fly-Nonce`: 32 hex characters
|
| 217 |
- `X-Fly-Sig`: hex HMAC-SHA256(key = the UTF-8 bytes of the configured secret string, canonical)
|
| 218 |
|
|
|
|
| 220 |
```
|
| 221 |
FLY-RELAY-1\n{METHOD}\n{PATH}\n{QUERY}\n{TS}\n{NONCE}\n{sha256hex(body)}
|
| 222 |
```
|
| 223 |
+
- `PATH` starts at `/api/…` and excludes the query string.
|
| 224 |
- `QUERY` is `&`.join(`quote(k,safe='')=quote(v,safe='')` for (k, v) in sorted(parse_qsl(qs, keep_blank_values=True))).
|
| 225 |
- The body is the raw bytes; an empty body hashes as sha256 of `b""`.
|
| 226 |
+
- The timestamp must be within ±300 s. Nonces are remembered for 15 minutes (case-insensitively), and a reused nonce
|
| 227 |
+
gets 401. The body is verified before the nonce is recorded.
|
| 228 |
|
| 229 |
+
Relay config: `relay.json`, kept next to `wsgi.py` and never inside `site/`, or wherever `FLY_RELAY_CONFIG` points. It
|
| 230 |
+
is re-read when it changes; a relative `db_path` is resolved against its folder. Secrets must be at least 32 characters
|
| 231 |
+
(64 hex recommended).
|
| 232 |
```json
|
| 233 |
{"keys": {"k1": "<64 hex>"}, "db_path": "<writable path>", "domain": "fly-trader.app", "uri": "https://fly-trader.app/vault.html",
|
| 234 |
"chain_id": 4663, "sol_chain": "mainnet", "claim_ttl_s": 900, "min_lamports": 2000000, "client_ip_header": null}
|
|
|
|
| 236 |
|
| 237 |
## 4. Payloads
|
| 238 |
|
| 239 |
+
### 4.1 Stats snapshot (`stats`, pushed every vault-worker loop, about every 10 s)
|
| 240 |
```json
|
| 241 |
{
|
| 242 |
+
"v": 1, "ts": 0, "book": "live", "cluster": "mainnet",
|
| 243 |
+
"fly": {"state": "starting|paper|live|halted", "handover": false, "kill_switch": false,
|
| 244 |
"entries_paused": false, "model": {"fly": 62, "selector": 59, "release": 3}},
|
| 245 |
+
"wallet": {"native": 0, "nav": 0},
|
| 246 |
"ledger": {"deposits": 0, "withdrawals": 0, "claims_paid": 0, "realized": 0, "booked_realized": 0,
|
| 247 |
"allocated": 0, "reserved": 0, "pot": 0},
|
| 248 |
"prices": {"sol_usd": 0.0, "fly_usd": 0.0, "ts": 0},
|
| 249 |
"vault": {"address": "0x…", "chain_id": 4663, "total_locked": "0", "total_pending": "0", "earners": 0,
|
| 250 |
"finalized_block": 0, "paused": false, "impl": "0x…", "upgrade_scheduled": null},
|
| 251 |
+
"settlement": {"last": null}
|
|
|
|
|
|
|
| 252 |
}
|
| 253 |
```
|
| 254 |
+
- **Nothing that helps trade ahead of the fly.** `ts` is rounded down to the hour; `wallet.native` and `wallet.nav`
|
| 255 |
+
come from the last 5-minute mark and are rounded to 0.1 SOL (NAV history points likewise), so one buy cannot be
|
| 256 |
+
matched to an on-chain trade; `ledger.realized` is R at that same mark. Open positions, the performance index and
|
| 257 |
+
the next settlement time are not published; a trade appears in `trades` history once it closes. No address of the
|
| 258 |
+
fly's (trading wallet, treasury, keys) is published, nor anything that leads to one: no transaction signatures and no
|
| 259 |
+
deposit senders.
|
| 260 |
+
- `book` is `VAULT_BOOK` (`live`, or a paper book such as `paper_fly` on a dry run). `cluster` is `mainnet` or
|
| 261 |
+
`VAULT_CLUSTER` as is.
|
| 262 |
+
- `fly.state`: `halted` if the vault is halted or the kill switch is on; `live` after the handover once the live wallet
|
| 263 |
+
reports; `paper` once the fly reports status; else `starting`. `fly.model` = `{fly, selector, release}`, each may be
|
| 264 |
+
null.
|
| 265 |
+
- `wallet.native` = trading wallet + treasury lamports; `wallet.nav` = native + open positions' value − exit cost.
|
| 266 |
+
- `ledger`: `realized` is R (§7); `pot` = max(0, R − allocated); `reserved` = max(0, allocated − claims_paid);
|
| 267 |
+
`booked_realized` is the book's summed closed-trade profit; `deposits`, `withdrawals` (fees included) and
|
| 268 |
+
`claims_paid` are running totals. On a paper book `realized` is closed-trade profit since the vault started and
|
| 269 |
+
`deposits` is `CAPITAL_SOL`.
|
| 270 |
+
- `prices`: `sol_usd` from Jupiter Price v3, `fly_usd` from the GeckoTerminal pool, cached 60 s (a failed fetch keeps
|
| 271 |
+
the last value); `ts` is the older of the two fetch times, or 0.
|
| 272 |
+
- `vault`: `total_locked`, `total_pending`, `earners` and `paused` come from events in finalized blocks (about 18 minutes
|
| 273 |
+
behind the head), up to `finalized_block`. `address` is null until `VAULT_ADDRESS` is set.
|
| 274 |
+
- `settlement.last` is the newest `allocated` settlement (§4.2), or null.
|
| 275 |
+
- `vault.upgrade_scheduled` is `{"eta": ts, "id": "0x…"}` or null: the soonest pending timelock operation whose target is the
|
| 276 |
+
vault (upgrade, role change) or the timelock itself (`updateDelay`, proposer changes). `eta` is the block time of
|
| 277 |
+
`CallScheduled` plus its `delay`.
|
| 278 |
|
| 279 |
### 4.2 History items
|
| 280 |
+
- `nav`: `{"ts", "nav", "sol_usd"}`: one point per 5-minute boundary (`ts`), from the newest consistent mark at or before
|
| 281 |
+
it, nav rounded to 0.1 SOL; `sol_usd` is the price at push time.
|
| 282 |
+
- `trades`: `{"id", "mint", "symbol", "opened_at", "closed_at", "cost", "proceeds", "realized", "exit_kind"}`: closed
|
| 283 |
+
positions of `VAULT_BOOK` only; `proceeds` = cost + realized; `exit_kind` is the forced-exit kind or `hold`.
|
| 284 |
+
- `flows`: `{"id", "ts", "direction": "in|out", "kind": "deposit|profit|withdrawal|claim", "lamports"}`. Internal moves
|
| 285 |
+
(`sweep`, `topup`) and `unknown_outbound`/`anomaly` rows are never published.
|
| 286 |
+
- `settlements`: `{"id", "period_start", "period_end", "realized", "pot", "allocated", "carried", "earners",
|
| 287 |
+
"total_weight": "<str>", "status"}`: `status` is always `allocated`; `carried` = pot − allocated; `total_weight` is
|
| 288 |
+
the sum of wei·seconds; `realized` is R at the snapshot.
|
| 289 |
|
| 290 |
### 4.3 Account
|
| 291 |
```json
|
|
|
|
| 293 |
"allocations": [{"period_end": 0, "lamports": 0, "weight": "<str>", "share": 0.0}],
|
| 294 |
"claims": [{"id": 0, "status": "", "lamports": 0, "sol": "", "tx": "", "created_at": 0, "updated_at": 0}]}
|
| 295 |
```
|
| 296 |
+
- `owed` = allocated − claimed − in_flight. `in_flight` counts claims from `sending` on (a `verified` claim has no amount
|
| 297 |
+
yet).
|
| 298 |
+
- `allocations`: the 52 most recent; `share` = weight / that settlement's total weight.
|
| 299 |
+
- `claims`: the 20 most recent; `id` is the relay's claim id (as in `GET /api/claim/<id>`); `tx` is always `""`.
|
| 300 |
|
| 301 |
## 5. Networks
|
| 302 |
|
| 303 |
| | mainnet | rehearsal |
|
| 304 |
|---|---|---|
|
| 305 |
+
| Site build | `VITE_NETWORK=mainnet` (default) | `VITE_NETWORK=testnet` |
|
| 306 |
| EVM chain | Robinhood Chain 4663, `https://rpc.mainnet.chain.robinhood.com` | RH testnet 46630, `https://rpc.testnet.chain.robinhood.com` |
|
| 307 |
| Explorer | `https://robinhoodchain.blockscout.com` | `https://explorer.testnet.chain.robinhood.com` |
|
| 308 |
| $FLY | `0x2fC7f9E2911f20b2C4660d2AEf808aa91bDdb3D3` | `MockFLY` (deployed for the rehearsal) |
|
| 309 |
+
| USDG | `0x5fc5360D0400a0Fd4f2af552ADD042D716F1d168` (6 decimals) | — |
|
| 310 |
+
| Multicall3 | `0xcA11bde05977b3631167028862bE2a173976CA11` | same |
|
| 311 |
| Solana | mainnet-beta | devnet |
|
| 312 |
+
| Squads v4 | `SQDS4ep65T869zMMBKyuUq6aD6EgTu8psMjkvj52pCf` | same |
|
| 313 |
+
| Settlement period | 7 days (`VAULT_PERIOD_S=604800`) | 900 s |
|
| 314 |
| Kyber | `https://aggregator-api.kyberswap.com/robinhood/api/v1/`, router `0x6131B5fae19EA4f9D964eAc0408E4408b66337b5` | — |
|
| 315 |
| Chart | GeckoTerminal network `robinhood`, pool `0xe6925f7bdedf22c2714c749d0ff208ce9e4bc1540938fa2b9ab2379486322edd` | fixtures |
|
| 316 |
+
|
| 317 |
+
- The fly's defaults are mainnet. A rehearsal sets `RH_CHAIN_ID=46630`, `RH_RPC_URL`, `VAULT_CLUSTER=devnet`,
|
| 318 |
+
`VAULT_SOLANA_RPC_URL`, `VAULT_PERIOD_S=900` and `VAULT_SITE_DOMAIN`/`VAULT_SITE_URI`.
|
| 319 |
+
- The site's Solana RPC (the Owner page) is WalletConnect's unless `VITE_SOLANA_RPC` is set.
|
| 320 |
+
|
| 321 |
+
## 6. Custody (Solana)
|
| 322 |
+
|
| 323 |
+
The fly's SOL is one book in two accounts:
|
| 324 |
+
- **The treasury** is vault 0 of a Squads v4 multisig. Its members are the owner's wallets only (Solflare until the
|
| 325 |
+
Ledger takes over); threshold 1 and no time lock while there is one member.
|
| 326 |
+
- **The trading wallet** is a hot key that keeps a float of about `TREASURY_FLOAT_SOL` (default 2 SOL).
|
| 327 |
+
|
| 328 |
+
| Spending limit | Member | Destinations | Period | Used for |
|
| 329 |
+
|---|---|---|---|---|
|
| 330 |
+
| L1 | trading key | the trading wallet only | Day | refilling the float (`topup` flows, memo `fly-topup`) |
|
| 331 |
+
| L2 | payout key | any | Week | claims (`spendingLimitUse` from the treasury, memo `fly-vault-claim:<fly claim id>:<nonce>`) |
|
| 332 |
+
|
| 333 |
+
- **Rebalancing** (every minute, `vault/payout.py`): first the treasury must hold everything owed plus its 890,880-lamport
|
| 334 |
+
rent minimum, sweeping from the trading wallet if it is short; then the trading wallet is brought to the float, by a
|
| 335 |
+
top-up under L1 when low or a sweep when high. Moves under `TOPUP_MIN_SOL` (0.1) are skipped, except a sweep the
|
| 336 |
+
treasury needs. A sweep never takes the trading wallet below `GAS_RESERVE_SOL` + fee. Under `/panic` the float target
|
| 337 |
+
is 0.
|
| 338 |
+
- **Flows.** A trading → treasury transfer is a `sweep`, a treasury → trading one a `topup`: both internal, excluded
|
| 339 |
+
from R, the index and published history. SOL or wSOL arriving at either account from `FUNDING_ADDRESSES` is a
|
| 340 |
+
`deposit`; from anyone else it is `profit` (a gift). A treasury outflow signed by a multisig member is a `withdrawal`
|
| 341 |
+
(the owner's principal; alerted, louder when the destination is not a funding address). A transaction one of our
|
| 342 |
+
keys signed that no table knows is `unknown_outbound`; lamports leaving without our or an owner's signature are an
|
| 343 |
+
`anomaly`. Both halt the vault (§7), unless the second RPC provider sees the transaction differently.
|
| 344 |
+
- **Accounting.** R, NAV and the performance index count both accounts together. Positions are sized on the bankroll
|
| 345 |
+
= trading free SOL + open positions at cost + (treasury − owed); buys spend only the trading wallet's own SOL, less
|
| 346 |
+
any owed SOL the treasury does not hold.
|
| 347 |
+
- **The signer** (`fly_trader/signer`) is the only holder of the two hot keys. It builds and signs:
|
| 348 |
+
- Jupiter swaps: SOL ↔ one token only; Metis routes only (`excludeRouters=jupiterz,dflow,okx`); static rules on every
|
| 349 |
+
instruction (programs, the trading wallet's roles, a priority-fee cap of `SIGNER_MAX_PRIORITY_SOL`, 0.005); a
|
| 350 |
+
simulation, through its own RPC, of the trading wallet's SOL and every token account it owns (a buy may spend at
|
| 351 |
+
most the ordered SOL plus fees and must deliver the token to the trading wallet; a sell may spend at most the
|
| 352 |
+
ordered tokens and must return SOL; no other holding may shrink); a value floor of `SIGNER_MIN_VALUE_RATIO` (0.6)
|
| 353 |
+
at the signer's own Jupiter prices. Buys are capped per trade (`SIGNER_MAX_BUY_SOL`, default `MAX_POSITION_SOL`) and
|
| 354 |
+
per day (`SIGNER_DAILY_BUY_SOL`, 10); all swaps per minute (`SIGNER_SWAPS_PER_MIN`, 6).
|
| 355 |
+
- top-ups under L1 (at most `TREASURY_FLOAT_SOL` each; L1's destinations must be exactly the trading wallet) and
|
| 356 |
+
sweeps (a plain transfer to the treasury)
|
| 357 |
+
- claims under L2, once per claim id, with the same destination and amount; re-signed only after the earlier
|
| 358 |
+
signature failed or its blockhash expired
|
| 359 |
+
- closes of empty token accounts of the trading wallet (batches of 10)
|
| 360 |
+
- the hot keys, age-encrypted to `VAULT_BACKUP_RECIPIENT`, for the off-box backup
|
| 361 |
+
- after `panic`: no buys, top-ups or claims; `clear-panic` is a CLI command inside its container only
|
| 362 |
+
|
| 363 |
+
On the server the brain reaches it over a Unix socket (one JSON request per line, 256 KB max, one at a time) and never
|
| 364 |
+
sees a key. The signer keeps its own SQLite ledger. Without `SIGNER_SOCKET` (self-hosting, tests) the same signer runs
|
| 365 |
+
inside the brain's process.
|
| 366 |
+
- **Custody checks** (every minute, `vault/custody.py`), alerted: neither server key is a multisig member; no config
|
| 367 |
+
authority; L1 is a SOL limit with member = the trading key and sole destination = the trading wallet; L2 is a SOL
|
| 368 |
+
limit with member = the payout key. Any change to members, threshold, time lock or the limits is reported.
|
| 369 |
+
- **The owner** changes the limits from `owner.html` (`web/src/lib/squads.ts`). Every new limit has a fresh address:
|
| 370 |
+
the server's `VAULT_LIMIT_TRADING` / `VAULT_LIMIT_PAYOUT` must be updated to it (the page prints the lines). A
|
| 371 |
+
limit's amount is changed by removing it and adding a new one, in two config transactions: the program rejects both
|
| 372 |
+
in one.
|
| 373 |
+
|
| 374 |
+
## 7. Settlement, halts, kill switch
|
| 375 |
+
|
| 376 |
+
- **Schedule.** Period ends fall at `VAULT_EPOCH + k·VAULT_PERIOD_S` (345600 = Monday 1970-01-05 00:00 UTC; 7 days).
|
| 377 |
+
The first settlement covers the period before the first end after the vault started; missed periods are settled as
|
| 378 |
+
one window. Settlements advance one step per minute, in period order.
|
| 379 |
+
- **Two phases.**
|
| 380 |
+
1. Snapshot at the period end T1: both accounts at a finalized slot that contains every transaction the fly sent,
|
| 381 |
+
read under the wallet lock. The second RPC provider (`SOLANA_CHECK_RPC_URL`) must report the same balance, or the
|
| 382 |
+
settlement pauses, alerts and retries. Flows are scanned up to that slot.
|
| 383 |
+
2. Allocate once Robinhood Chain is finalized past T1, and only if the vault's implementation code hash is in
|
| 384 |
+
`VAULT_IMPL_CODEHASHES` (else the vault halts).
|
| 385 |
+
|
| 386 |
+
Statuses: `snapshotted | allocated | failed`.
|
| 387 |
+
- **Formulas.**
|
| 388 |
+
```
|
| 389 |
+
R = N + K + C − D + Wd + P N: both accounts' SOL; K: wSOL and empty token-account rent; C: open positions at
|
| 390 |
+
cost; D: deposits; Wd: withdrawals incl. fees; P: claims paid
|
| 391 |
+
pot = max(0, R − A) A: everything ever allocated
|
| 392 |
+
out = min(pot, N − (A − P) − GAS_RESERVE_SOL)
|
| 393 |
+
weight_i = ∫ lockedAfter_i dt over [T0, T1) (wei·s, from FlyVault events)
|
| 394 |
+
alloc_i = floor(out · weight_i / Σ weight); the remainder, dust and empty weeks carry forward (A grows only by alloc)
|
| 395 |
+
```
|
| 396 |
+
- **Halts.** The vault halts on `unknown_outbound`, `anomaly`, an unknown implementation code hash, or Telegram
|
| 397 |
+
`/panic` (the operator's user only). A halt stops claim processing, settlement and new entries. `/panic` also trips
|
| 398 |
+
the kill switch with liquidation, sets the float target to 0 and puts the signer in panic. Resume is SSH-only:
|
| 399 |
+
`fly-trader vault resume` (clears halt and panic), `fly-trader reset-circuit --kill`, `fly-trader resume-entries`,
|
| 400 |
+
and the signer's `clear-panic`.
|
| 401 |
+
- **Kill switch.** A flow-adjusted index `I_t = I_{t−1}·(W_t − F_t)/W_{t−1}` over consistent NAV marks of both
|
| 402 |
+
accounts, where F is every deposit/profit in and withdrawal/claim out since the previous mark. Its peak uses marks at
|
| 403 |
+
least 600 s old, over 30 days. It trips at `KILL_SWITCH_DRAWDOWN` (default 0.30; the vault server sets 0.20) and, with
|
| 404 |
+
`KILL_SWITCH_LIQUIDATE`, sells every open position.
|
| 405 |
+
- **Principal withdrawal.** Only the owner, from the Squads app. The cap `fly-trader vault withdraw` prints is
|
| 406 |
+
min(D − Wd + min(0, R − A), treasury − owed − 890,880).
|
fly_trader/__main__.py
CHANGED
|
@@ -144,7 +144,8 @@ def _cmd_fly_replay(args):
|
|
| 144 |
from .logging_setup import setup
|
| 145 |
from .train import fly_replay
|
| 146 |
setup("train"); _use_device(args)
|
| 147 |
-
out = fly_replay.main(days=args.days, start_day=args.start_day if args.start_day is not None else fly_replay.DEFAULT_START
|
|
|
|
| 148 |
print(json.dumps({k: out.get(k) for k in ("S", "passed", "reason", "alpha", "half_life_days", "evaluation", "random", "frozen", "plastic_beats_frozen")}, indent=1, default=str))
|
| 149 |
|
| 150 |
|
|
@@ -183,6 +184,123 @@ def _cmd_worker(args):
|
|
| 183 |
procs.worker_cli(args.action, args.name)
|
| 184 |
|
| 185 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 186 |
def build_parser() -> argparse.ArgumentParser:
|
| 187 |
p = argparse.ArgumentParser(prog="fly-trader")
|
| 188 |
sub = p.add_subparsers(dest="command", required=True)
|
|
@@ -211,6 +329,7 @@ def build_parser() -> argparse.ArgumentParser:
|
|
| 211 |
fr.add_argument("--start-day", type=int, default=None,
|
| 212 |
help="day index of the bootstrap (the selector teaches on the days before it); default: train/fly_replay.DEFAULT_START, "
|
| 213 |
"the earliest day that leaves the teacher's walk-forward enough days in each half")
|
|
|
|
| 214 |
fr.set_defaults(fn=_cmd_fly_replay)
|
| 215 |
bc = sub.add_parser("backtest-corpus"); bc.add_argument("--max-tokens", type=int, default=None); bc.add_argument("--days", type=int, default=None)
|
| 216 |
bc.add_argument("--fees", default="0.003,0.0055"); bc.add_argument("--only", default=None); bc.add_argument("--universe", default="graduation", choices=["graduation", "mature"]); bc.set_defaults(fn=_cmd_backtest_corpus)
|
|
@@ -229,9 +348,42 @@ def build_parser() -> argparse.ArgumentParser:
|
|
| 229 |
ar.add_argument("dir"); ar.add_argument("--bootstrap", action="store_true"); ar.set_defaults(fn=_cmd_apply_release)
|
| 230 |
sub.add_parser("health", help="exit 0 when the database answers and workers run (container HEALTHCHECK)").set_defaults(fn=_cmd_health)
|
| 231 |
sub.add_parser("ready-for-restart", help="exit 0 when no order, claim payment or settlement is in flight").set_defaults(fn=_cmd_ready)
|
|
|
|
|
|
|
| 232 |
from .vault import cli as vault_cli
|
| 233 |
vault_cli.add(sub)
|
| 234 |
wk = sub.add_parser("worker"); wk.add_argument("action", choices=["start", "stop", "status"]); wk.add_argument("name", nargs="?"); wk.set_defaults(fn=_cmd_worker)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 235 |
return p
|
| 236 |
|
| 237 |
|
|
@@ -248,6 +400,12 @@ def _cmd_health(args) -> None:
|
|
| 248 |
raise SystemExit(0 if ok else 1)
|
| 249 |
|
| 250 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 251 |
def _cmd_ready(args) -> None:
|
| 252 |
from .ops import release
|
| 253 |
ok, why = release.ready_for_restart(); print(why)
|
|
|
|
| 144 |
from .logging_setup import setup
|
| 145 |
from .train import fly_replay
|
| 146 |
setup("train"); _use_device(args)
|
| 147 |
+
out = fly_replay.main(days=args.days, start_day=args.start_day if args.start_day is not None else fly_replay.DEFAULT_START,
|
| 148 |
+
only_chains=tuple(args.chains.split(",")) if args.chains else None)
|
| 149 |
print(json.dumps({k: out.get(k) for k in ("S", "passed", "reason", "alpha", "half_life_days", "evaluation", "random", "frozen", "plastic_beats_frozen")}, indent=1, default=str))
|
| 150 |
|
| 151 |
|
|
|
|
| 184 |
procs.worker_cli(args.action, args.name)
|
| 185 |
|
| 186 |
|
| 187 |
+
def _cmd_kalshi_history(args):
|
| 188 |
+
from .kalshi import history
|
| 189 |
+
history.main()
|
| 190 |
+
|
| 191 |
+
|
| 192 |
+
def _cmd_kalshi_build(args):
|
| 193 |
+
from .kalshi import mature
|
| 194 |
+
mature.main()
|
| 195 |
+
|
| 196 |
+
|
| 197 |
+
def _cmd_kalshi_train_selector(args):
|
| 198 |
+
from .kalshi import selector
|
| 199 |
+
from .logging_setup import setup
|
| 200 |
+
setup("kalshi_train")
|
| 201 |
+
out = selector.main(days=args.days, holdout_days=args.holdout_days)
|
| 202 |
+
print(json.dumps({k: out.get(k) for k in ("snapshot_id", "deployable", "deploy_reason")}, indent=1, default=str))
|
| 203 |
+
|
| 204 |
+
|
| 205 |
+
def _cmd_kalshi_train_fly(args):
|
| 206 |
+
from .kalshi import fly
|
| 207 |
+
from .logging_setup import setup
|
| 208 |
+
setup("kalshi_train"); _use_device(args)
|
| 209 |
+
out = fly.main(days=args.days, epochs=args.epochs)
|
| 210 |
+
print(json.dumps({k: out.get(k) for k in ("snapshot_id", "S", "lines", "calibration", "diagnostics", "gates_ok", "gate_failures", "teacher")}, indent=1, default=str))
|
| 211 |
+
|
| 212 |
+
|
| 213 |
+
def _cmd_kalshi_fly_replay(args):
|
| 214 |
+
from .kalshi import fly_replay
|
| 215 |
+
from .logging_setup import setup
|
| 216 |
+
setup("kalshi_train"); _use_device(args)
|
| 217 |
+
out = fly_replay.main(days=args.days, start_day=args.start_day if args.start_day is not None else fly_replay.DEFAULT_START,
|
| 218 |
+
only_chains=tuple(args.chains.split(",")) if args.chains else None)
|
| 219 |
+
print(json.dumps({k: out.get(k) for k in ("S", "passed", "reason", "alpha", "half_life_days", "evaluation", "random", "frozen", "plastic_beats_frozen")}, indent=1, default=str))
|
| 220 |
+
|
| 221 |
+
|
| 222 |
+
def _cmd_kalshi_train(args):
|
| 223 |
+
from .kalshi import pipeline
|
| 224 |
+
pipeline.main()
|
| 225 |
+
|
| 226 |
+
|
| 227 |
+
def _cmd_kalshi_stream(args):
|
| 228 |
+
from .kalshi import stream
|
| 229 |
+
stream.main()
|
| 230 |
+
|
| 231 |
+
|
| 232 |
+
def _cmd_kalshi_run(args):
|
| 233 |
+
from .kalshi import engine
|
| 234 |
+
engine.main()
|
| 235 |
+
|
| 236 |
+
|
| 237 |
+
def _cmd_kalshi_subaccount(args):
|
| 238 |
+
from .kalshi import client
|
| 239 |
+
if args.action == "create":
|
| 240 |
+
print(f"subaccount {client.subaccount_create()} created and written to .env")
|
| 241 |
+
else:
|
| 242 |
+
print(json.dumps(client.rest().subaccount_balances(), indent=1, default=str))
|
| 243 |
+
|
| 244 |
+
|
| 245 |
+
def _cmd_kalshi_fund(args):
|
| 246 |
+
from .kalshi import client
|
| 247 |
+
print(json.dumps(client.fund(args.usd), indent=1, default=str))
|
| 248 |
+
|
| 249 |
+
|
| 250 |
+
def _cmd_rh_backfill(args):
|
| 251 |
+
from .rh import stream
|
| 252 |
+
stream.main()
|
| 253 |
+
|
| 254 |
+
|
| 255 |
+
def _cmd_rh_status(args):
|
| 256 |
+
from .rh import stream
|
| 257 |
+
print(json.dumps(stream.status(), indent=1, default=str))
|
| 258 |
+
|
| 259 |
+
|
| 260 |
+
def _cmd_rh_probe(args):
|
| 261 |
+
from .rh import probe
|
| 262 |
+
print(json.dumps(probe.run(sample=args.sample), indent=1, default=str))
|
| 263 |
+
|
| 264 |
+
|
| 265 |
+
def _cmd_rh_wallet(args):
|
| 266 |
+
from .rh import smoke
|
| 267 |
+
if args.action == "new":
|
| 268 |
+
print(f"RH bot wallet {smoke.new_key()} written to .env (RH_BOT_PRIVATE_KEY, RH_BOT_ADDRESS). Fund it with ETH on Robinhood Chain.")
|
| 269 |
+
else:
|
| 270 |
+
from . import config
|
| 271 |
+
from .rh.rpc import RhRpc
|
| 272 |
+
addr = (config.RH_BOT_ADDRESS or "").lower()
|
| 273 |
+
print(json.dumps({"address": addr or None, "eth": RhRpc().get_balance(addr) / 1e18 if addr else None,
|
| 274 |
+
"live_prerequisites_missing": config.rh_live_prerequisites_missing()}, indent=1))
|
| 275 |
+
|
| 276 |
+
|
| 277 |
+
def _cmd_rh_smoke(args):
|
| 278 |
+
if not args.yes:
|
| 279 |
+
raise SystemExit("rh-smoke sends real Robinhood Chain transactions: pass --yes")
|
| 280 |
+
from .rh import smoke
|
| 281 |
+
print(json.dumps(smoke.run(args.size, args.cap), indent=1, default=str))
|
| 282 |
+
|
| 283 |
+
|
| 284 |
+
def _cmd_kalshi_status(args):
|
| 285 |
+
from .kalshi import client
|
| 286 |
+
print(json.dumps(client.status(), indent=1, default=str))
|
| 287 |
+
|
| 288 |
+
|
| 289 |
+
def _cmd_kalshi_pause(args):
|
| 290 |
+
from .agent import rails
|
| 291 |
+
rails.set_entries_paused(True, rails.KALSHI_CIRCUIT)
|
| 292 |
+
|
| 293 |
+
|
| 294 |
+
def _cmd_kalshi_resume(args):
|
| 295 |
+
from .agent import rails
|
| 296 |
+
rails.set_entries_paused(False, rails.KALSHI_CIRCUIT)
|
| 297 |
+
|
| 298 |
+
|
| 299 |
+
def _cmd_kalshi_reset_circuit(args):
|
| 300 |
+
from .agent import rails
|
| 301 |
+
rails.reset_circuit(kill=args.kill, circuit_id=rails.KALSHI_CIRCUIT)
|
| 302 |
+
|
| 303 |
+
|
| 304 |
def build_parser() -> argparse.ArgumentParser:
|
| 305 |
p = argparse.ArgumentParser(prog="fly-trader")
|
| 306 |
sub = p.add_subparsers(dest="command", required=True)
|
|
|
|
| 329 |
fr.add_argument("--start-day", type=int, default=None,
|
| 330 |
help="day index of the bootstrap (the selector teaches on the days before it); default: train/fly_replay.DEFAULT_START, "
|
| 331 |
"the earliest day that leaves the teacher's walk-forward enough days in each half")
|
| 332 |
+
fr.add_argument("--chains", default=None, help="store the verdict of these chains only (comma list, e.g. rh); the others keep theirs")
|
| 333 |
fr.set_defaults(fn=_cmd_fly_replay)
|
| 334 |
bc = sub.add_parser("backtest-corpus"); bc.add_argument("--max-tokens", type=int, default=None); bc.add_argument("--days", type=int, default=None)
|
| 335 |
bc.add_argument("--fees", default="0.003,0.0055"); bc.add_argument("--only", default=None); bc.add_argument("--universe", default="graduation", choices=["graduation", "mature"]); bc.set_defaults(fn=_cmd_backtest_corpus)
|
|
|
|
| 348 |
ar.add_argument("dir"); ar.add_argument("--bootstrap", action="store_true"); ar.set_defaults(fn=_cmd_apply_release)
|
| 349 |
sub.add_parser("health", help="exit 0 when the database answers and workers run (container HEALTHCHECK)").set_defaults(fn=_cmd_health)
|
| 350 |
sub.add_parser("ready-for-restart", help="exit 0 when no order, claim payment or settlement is in flight").set_defaults(fn=_cmd_ready)
|
| 351 |
+
mc = sub.add_parser("models", help="convert: rewrite this machine's own .joblib selectors as .skops (pickles can run code)")
|
| 352 |
+
mc.add_argument("action", choices=["convert"]); mc.set_defaults(fn=_cmd_models)
|
| 353 |
from .vault import cli as vault_cli
|
| 354 |
vault_cli.add(sub)
|
| 355 |
wk = sub.add_parser("worker"); wk.add_argument("action", choices=["start", "stop", "status"]); wk.add_argument("name", nargs="?"); wk.set_defaults(fn=_cmd_worker)
|
| 356 |
+
# ---- Kalshi prediction markets (fly_trader/kalshi) ----
|
| 357 |
+
ks = sub.add_parser("kalshi-subaccount", help="create the fly's dedicated Kalshi subaccount (writes KALSHI_SUBACCOUNT to .env) or list balances")
|
| 358 |
+
ks.add_argument("action", choices=["create", "list"]); ks.set_defaults(fn=_cmd_kalshi_subaccount)
|
| 359 |
+
kf = sub.add_parser("kalshi-fund", help="move dollars from the primary Kalshi account into the fly's subaccount"); kf.add_argument("--usd", type=float, required=True); kf.set_defaults(fn=_cmd_kalshi_fund)
|
| 360 |
+
sub.add_parser("rh-backfill", help="the Robinhood Chain index from Pons V2's deployment to the head, then live (the rh_stream worker)").set_defaults(fn=_cmd_rh_backfill)
|
| 361 |
+
sub.add_parser("rh-status", help="Robinhood Chain index cursors, row counts and stream status").set_defaults(fn=_cmd_rh_status)
|
| 362 |
+
rp = sub.add_parser("rh-probe", help="read-only check of the Robinhood Chain path: contracts, Kyber / v4 routability and gas on recent Pons graduates")
|
| 363 |
+
rp.add_argument("--sample", type=int, default=12); rp.set_defaults(fn=_cmd_rh_probe)
|
| 364 |
+
rw = sub.add_parser("rh-wallet", help="new: create the RH bot key in .env (refuses to replace one); show: its address, ETH and what live trading still needs")
|
| 365 |
+
rw.add_argument("action", choices=["new", "show"]); rw.set_defaults(fn=_cmd_rh_wallet)
|
| 366 |
+
rs = sub.add_parser("rh-smoke", help="three tiny real RH round trips (Kyber, v4 fallback, non-ETH quote) booked and reconciled to the wei")
|
| 367 |
+
rs.add_argument("--size", type=float, default=0.002); rs.add_argument("--cap", type=float, default=0.01); rs.add_argument("--yes", action="store_true")
|
| 368 |
+
rs.set_defaults(fn=_cmd_rh_smoke)
|
| 369 |
+
sub.add_parser("kalshi-status", help="exchange status, subaccount balance, resting orders, positions, settlements").set_defaults(fn=_cmd_kalshi_status)
|
| 370 |
+
sub.add_parser("kalshi-history", help="the Kalshi corpus worker: dataset seed, settled markets, candles, trades").set_defaults(fn=_cmd_kalshi_history)
|
| 371 |
+
sub.add_parser("kalshi-build", help="build the Kalshi feature parts from the corpus").set_defaults(fn=_cmd_kalshi_build)
|
| 372 |
+
kts = sub.add_parser("kalshi-train-selector", help="fit the Kalshi strategy stack walk-forward and save it"); kts.add_argument("--days", type=int, default=None)
|
| 373 |
+
kts.add_argument("--holdout-days", type=int, default=None, help="last days withheld from every fit and scored once (default 21)")
|
| 374 |
+
kts.set_defaults(fn=_cmd_kalshi_train_selector)
|
| 375 |
+
ktf = sub.add_parser("kalshi-train-fly", help="bootstrap the Kalshi fly (optic lobes) from the deployable Kalshi selector")
|
| 376 |
+
ktf.add_argument("--days", type=int, default=None); ktf.add_argument("--epochs", type=int, default=None); ktf.add_argument("--device", default=None)
|
| 377 |
+
ktf.set_defaults(fn=_cmd_kalshi_train_fly)
|
| 378 |
+
kfr = sub.add_parser("kalshi-fly-replay", help="bootstrap the Kalshi fly once, let it learn from settlements over the corpus, judge it")
|
| 379 |
+
kfr.add_argument("--days", type=int, default=None); kfr.add_argument("--device", default=None); kfr.add_argument("--start-day", type=int, default=None)
|
| 380 |
+
kfr.set_defaults(fn=_cmd_kalshi_fly_replay)
|
| 381 |
+
sub.add_parser("kalshi-train", help="the Kalshi training pipeline worker (selector weekly, fly bootstrap when needed)").set_defaults(fn=_cmd_kalshi_train)
|
| 382 |
+
sub.add_parser("kalshi-stream", help="the Kalshi live feed worker (websocket → kalshi_minutes / kalshi_quotes)").set_defaults(fn=_cmd_kalshi_stream)
|
| 383 |
+
sub.add_parser("kalshi-run", help="the Kalshi trading engine worker (the visual fly's paper arms, live mirror behind KALSHI_LIVE_ENABLED)").set_defaults(fn=_cmd_kalshi_run)
|
| 384 |
+
sub.add_parser("kalshi-pause-entries").set_defaults(fn=_cmd_kalshi_pause)
|
| 385 |
+
sub.add_parser("kalshi-resume-entries").set_defaults(fn=_cmd_kalshi_resume)
|
| 386 |
+
kr = sub.add_parser("kalshi-reset-circuit"); kr.add_argument("--kill", action="store_true"); kr.set_defaults(fn=_cmd_kalshi_reset_circuit)
|
| 387 |
return p
|
| 388 |
|
| 389 |
|
|
|
|
| 400 |
raise SystemExit(0 if ok else 1)
|
| 401 |
|
| 402 |
|
| 403 |
+
def _cmd_models(args) -> None:
|
| 404 |
+
from .train import model_io
|
| 405 |
+
for r in model_io.convert_legacy():
|
| 406 |
+
print(f"selector #{r['id']} -> {r['path']}")
|
| 407 |
+
|
| 408 |
+
|
| 409 |
def _cmd_ready(args) -> None:
|
| 410 |
from .ops import release
|
| 411 |
ok, why = release.ready_for_restart(); print(why)
|
fly_trader/agent/fly_live.py
CHANGED
|
@@ -79,15 +79,18 @@ class LiveMirror:
|
|
| 79 |
opens = ledger.open_positions(conn, BOOK); held_mints = {p["mint"] for p in opens} | set(inflight)
|
| 80 |
c = rails.load_circuit(conn)
|
| 81 |
blocked = "kill switch" if c.kill_switch else "circuit tripped" if c.tripped else "paused" if c.entries_paused else None
|
| 82 |
-
reserved = 0.0
|
| 83 |
-
if config.VAULT_ENABLED: # SOL owed to vault lockers
|
| 84 |
from ..vault import nav as vault_nav
|
| 85 |
-
reserved = vault_nav.
|
| 86 |
-
|
|
|
|
|
|
|
| 87 |
for e in entries:
|
| 88 |
if e["mint"] in held_mints:
|
| 89 |
skipped.append((e["mint"], "held")); continue
|
| 90 |
-
size, why = sizing.size_position(e["score"], e.get("threshold", line), e.get("table") or table or [], bankroll, cash, e["info"]["resq"]
|
|
|
|
| 91 |
if blocked or size <= 0:
|
| 92 |
skipped.append((e["mint"], blocked or why)); continue
|
| 93 |
if self.worker.submit(ExecRequest(decision_id=e["decision_id"], mint=e["mint"], pool=e["info"]["pool"], side="buy", amount_in=int(size * config.LAMPORTS_PER_SOL),
|
|
@@ -109,13 +112,18 @@ class LiveMirror:
|
|
| 109 |
"ON CONFLICT (beat_id, book) DO NOTHING",
|
| 110 |
(beat_id, BOOK, m1, sol_free, gross_v - exit_cost, exit_cost, wealth, peak, (1.0 - wealth / peak) if peak > 0 else 0.0, exposure, len(opens)))
|
| 111 |
if config.VAULT_ENABLED: # deposits, withdrawals and payouts are not performance
|
| 112 |
-
|
|
|
|
| 113 |
int(round(exit_cost * config.LAMPORTS_PER_SOL)))
|
| 114 |
ix = vault_nav.status(conn, since=rebase)
|
| 115 |
-
killed = rails.check_drawdown(conn, ix["value"], ix["peak"])
|
| 116 |
else:
|
| 117 |
killed = rails.check_drawdown(conn, wealth, peak)
|
| 118 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 119 |
gap = self.gap(conn)
|
| 120 |
if ctx.m1_epoch - self.last_sweep >= SWEEP_S:
|
| 121 |
self.last_sweep = ctx.m1_epoch; self.sweep(conn, snap, inflight, m1)
|
|
|
|
| 79 |
opens = ledger.open_positions(conn, BOOK); held_mints = {p["mint"] for p in opens} | set(inflight)
|
| 80 |
c = rails.load_circuit(conn)
|
| 81 |
blocked = "kill switch" if c.kill_switch else "circuit tripped" if c.tripped else "paused" if c.entries_paused else None
|
| 82 |
+
reserved = treasury = 0.0
|
| 83 |
+
if config.VAULT_ENABLED: # SOL owed to vault lockers is never traded; the treasury's idle SOL is bankroll
|
| 84 |
from ..vault import nav as vault_nav
|
| 85 |
+
reserved = vault_nav.reserved_in_trading(conn) / config.LAMPORTS_PER_SOL
|
| 86 |
+
treasury = vault_nav.treasury_free(conn) / config.LAMPORTS_PER_SOL
|
| 87 |
+
# sized on the whole book, spent only from the trading wallet's own float (cash)
|
| 88 |
+
bankroll = sol_free + sum(float(p["cost_sol"]) for p in opens) - reserved + treasury; cash = sol_free - reserved; n_enter = 0; skipped = []
|
| 89 |
for e in entries:
|
| 90 |
if e["mint"] in held_mints:
|
| 91 |
skipped.append((e["mint"], "held")); continue
|
| 92 |
+
size, why = sizing.size_position(e["score"], e.get("threshold", line), e.get("table") or table or [], bankroll, cash, e["info"]["resq"],
|
| 93 |
+
flat=config.FLY_SIZING == "flat")
|
| 94 |
if blocked or size <= 0:
|
| 95 |
skipped.append((e["mint"], blocked or why)); continue
|
| 96 |
if self.worker.submit(ExecRequest(decision_id=e["decision_id"], mint=e["mint"], pool=e["info"]["pool"], side="buy", amount_in=int(size * config.LAMPORTS_PER_SOL),
|
|
|
|
| 112 |
"ON CONFLICT (beat_id, book) DO NOTHING",
|
| 113 |
(beat_id, BOOK, m1, sol_free, gross_v - exit_cost, exit_cost, wealth, peak, (1.0 - wealth / peak) if peak > 0 else 0.0, exposure, len(opens)))
|
| 114 |
if config.VAULT_ENABLED: # deposits, withdrawals and payouts are not performance
|
| 115 |
+
from ..vault import payout as vault_payout # trading wallet + treasury are one book: a top-up or sweep is no loss
|
| 116 |
+
vault_nav.mark(conn, m1, snap.lamports + vault_payout.cached_balance(), int(round(gross_v * config.LAMPORTS_PER_SOL)),
|
| 117 |
int(round(exit_cost * config.LAMPORTS_PER_SOL)))
|
| 118 |
ix = vault_nav.status(conn, since=rebase)
|
| 119 |
+
killed = rails.check_drawdown(conn, ix["value"], ix["peak"], unit="index")
|
| 120 |
else:
|
| 121 |
killed = rails.check_drawdown(conn, wealth, peak)
|
| 122 |
+
panic = False
|
| 123 |
+
if config.VAULT_ENABLED: # Telegram /panic: sell everything whatever KILL_SWITCH_LIQUIDATE says
|
| 124 |
+
from ..vault import state as vault_state
|
| 125 |
+
panic = bool(vault_state.get("panic"))
|
| 126 |
+
n_liq = self.liquidate(conn, ctx, run_id, beat_id) if killed and (config.KILL_SWITCH_LIQUIDATE or panic) else 0
|
| 127 |
gap = self.gap(conn)
|
| 128 |
if ctx.m1_epoch - self.last_sweep >= SWEEP_S:
|
| 129 |
self.last_sweep = ctx.m1_epoch; self.sweep(conn, snap, inflight, m1)
|
fly_trader/agent/fly_session.py
CHANGED
|
@@ -35,7 +35,7 @@ from pathlib import Path
|
|
| 35 |
import numpy as np
|
| 36 |
import torch
|
| 37 |
|
| 38 |
-
from .. import config
|
| 39 |
from ..brain import activity, device as brain_device, plastic
|
| 40 |
from ..brain.connectome import current_connectome_path
|
| 41 |
from ..db.apilog import record_event
|
|
@@ -43,12 +43,12 @@ from ..db.connection import transaction
|
|
| 43 |
from ..execution import ledger
|
| 44 |
from ..execution.broker_paper import PaperBroker
|
| 45 |
from ..ops.reset import activity_dir, fly_state_dir
|
| 46 |
-
from ..train import fly_calibrate, fly_governance as gov, fly_selector
|
| 47 |
from ..train.decisions import X_COLS
|
| 48 |
from . import handover, paper_trading
|
| 49 |
|
| 50 |
log = logging.getLogger(__name__)
|
| 51 |
-
BOOK, KIND = "paper_fly", "fly"
|
| 52 |
BAR_KEEP = 270 # minutes of bars kept per watched mint (the longest strategy hold, 240, + next-open window + margin)
|
| 53 |
NEXT_OPEN_S = 120.0 # entry at the next traded minute's open when it comes within two minutes (decisions.build)
|
| 54 |
JUMP = 50.0
|
|
@@ -90,37 +90,57 @@ def resolve_label(bars: list, t: float, horizon_s: float) -> float | None:
|
|
| 90 |
return float(exit_px / entry * (1 - rows[0][3]) * (1 - rows[j][3]) - 1)
|
| 91 |
|
| 92 |
|
| 93 |
-
def
|
| 94 |
-
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 95 |
from ..train import fly_replay
|
| 96 |
v = fly_replay.verdict()
|
| 97 |
if not v:
|
| 98 |
return None, "the fly's replay has not run on the current definitions (fly-trader fly-replay)"
|
| 99 |
-
|
| 100 |
-
|
|
|
|
| 101 |
with transaction() as conn:
|
| 102 |
boot = fly_selector.latest_deployable(conn)
|
| 103 |
if boot is None:
|
| 104 |
return None, "no deployable fly bootstrap (the training pipeline bootstraps one)"
|
| 105 |
try:
|
| 106 |
-
return FlyBook(boot, v, live=live), "ok"
|
| 107 |
except Exception as e:
|
| 108 |
log.exception("fly book failed to start")
|
| 109 |
return None, f"fly book failed to start: {type(e).__name__}: {e}"
|
| 110 |
|
| 111 |
|
| 112 |
-
|
| 113 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 114 |
|
| 115 |
-
|
|
|
|
| 116 |
self.live = live; self.done = False; self.mirror = None
|
| 117 |
self.verdict = verdict
|
| 118 |
-
self.
|
|
|
|
|
|
|
|
|
|
| 119 |
self._load_bootstrap(boot)
|
| 120 |
-
self.run_id = str(uuid.uuid4()); self.beat_no = 0; self.broker = PaperBroker(BOOK)
|
| 121 |
with transaction() as conn:
|
| 122 |
conn.execute("INSERT INTO runs (run_id, kind, config, brain_snapshot_id, status) VALUES (%s,%s,%s,%s,'running')",
|
| 123 |
-
(self.run_id, KIND, json.dumps({"channels": self.cfg, "book": BOOK}, default=str), self.boot_id))
|
| 124 |
self._restore_tags(conn)
|
| 125 |
record_event("info", "fly", "fly session started", {"run_id": self.run_id, "bootstrap": self.boot_id, "channels": self.cfg, "lines": self.lines["plastic"],
|
| 126 |
"pending": len(self.pending), "learning_frozen": self.learning_frozen})
|
|
@@ -128,54 +148,126 @@ class FlyBook:
|
|
| 128 |
self.lines["plastic"], len(self.pending))
|
| 129 |
|
| 130 |
# ---- model and state ----
|
| 131 |
-
def _load_bootstrap(self, boot: dict) -> None:
|
| 132 |
-
|
| 133 |
-
|
| 134 |
-
|
| 135 |
-
self.
|
| 136 |
-
|
| 137 |
-
self.
|
| 138 |
-
|
| 139 |
-
|
| 140 |
-
|
| 141 |
-
|
| 142 |
-
|
| 143 |
-
|
| 144 |
-
|
| 145 |
-
|
| 146 |
-
|
| 147 |
-
|
| 148 |
-
|
| 149 |
-
self.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 150 |
self.pending = plastic.PendingTags(net.n_kc, net.k_active)
|
| 151 |
self.bars: dict[str, deque] = {}; self.watch: dict[str, int] = {}
|
| 152 |
-
|
| 153 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 154 |
self.applied_through = -math.inf; self.learning_frozen = False; self.checks_paused_until = {k: 0.0 for k in self.names}
|
| 155 |
self.race_started_at: float | None = None; self.last_day: int | None = None; self.last_hour: int | None = None
|
| 156 |
self.rollbacks: dict = {k: [] for k in self.names}; self.nu_set = False; self.hour_stats = self._empty_stats(); self.last_checks: dict = {}
|
| 157 |
s = self._read_state()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 158 |
if s and s.get("bootstrap_id") == self.boot_id and s.get("cfg") == self.cfg:
|
| 159 |
-
|
|
|
|
|
|
|
| 160 |
self.lines, self.sizing = s["lines"], s["sizing"]
|
| 161 |
self.applied_through, self.learning_frozen, self.checks_paused_until = s["applied_through"], s["learning_frozen"], s["checks_paused_until"]
|
| 162 |
self.race_started_at, self.last_day, self.last_hour, self.rollbacks = s["race_started_at"], s["last_day"], s["last_hour"], s["rollbacks"]
|
| 163 |
elif s:
|
| 164 |
self.race_started_at = s.get("race_started_at") # a new bootstrap keeps racing the same book
|
| 165 |
with transaction() as conn:
|
| 166 |
-
conn.execute("UPDATE fly_scored SET state = 'dropped', x = NULL WHERE state = 'pending' AND bootstrap_id IS DISTINCT FROM %s",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 167 |
|
| 168 |
def _read_state(self) -> dict | None:
|
| 169 |
if not self.state_path.exists():
|
| 170 |
return None
|
| 171 |
try:
|
| 172 |
-
return torch.load(self.state_path, map_location="cpu", weights_only=
|
| 173 |
except Exception:
|
| 174 |
log.exception("fly state unreadable; starting from the bootstrap")
|
| 175 |
return None
|
| 176 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 177 |
def _state(self) -> dict:
|
| 178 |
-
return {"bootstrap_id": self.boot_id, "cfg": self.cfg, "bank": self.bank.state(), "lines": self.lines, "sizing": self.sizing,
|
| 179 |
"applied_through": self.applied_through, "learning_frozen": self.learning_frozen, "checks_paused_until": self.checks_paused_until,
|
| 180 |
"race_started_at": self.race_started_at, "last_day": self.last_day, "last_hour": self.last_hour, "rollbacks": self.rollbacks,
|
| 181 |
"saved_at": time.time()}
|
|
@@ -186,7 +278,8 @@ class FlyBook:
|
|
| 186 |
torch.save(self._state(), tmp); os.replace(tmp, self.state_path)
|
| 187 |
|
| 188 |
def _restore_tags(self, conn) -> None:
|
| 189 |
-
rows = conn.execute("SELECT ts, mint, strategy, x, score, line FROM fly_scored WHERE state = 'pending' AND bootstrap_id = %s ORDER BY ts",
|
|
|
|
| 190 |
rows = [r for r in rows if r["strategy"] in self.holds]
|
| 191 |
applied = [(r["ts"], r["mint"], r["strategy"]) for r in rows if r["ts"].timestamp() + self.holds[r["strategy"]] + LABEL_LAG_S <= self.applied_through]
|
| 192 |
if applied:
|
|
@@ -198,14 +291,34 @@ class FlyBook:
|
|
| 198 |
groups.setdefault((r["ts"].timestamp(), r["strategy"]), []).append(r)
|
| 199 |
for (t, name), rs in sorted(groups.items()):
|
| 200 |
j = self.names.index(name)
|
| 201 |
-
|
| 202 |
-
|
| 203 |
-
self.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 204 |
for r in rs:
|
| 205 |
self.watch[r["mint"]] = self.watch.get(r["mint"], 0) + 1
|
| 206 |
|
| 207 |
def maybe_reload(self) -> bool:
|
| 208 |
-
"""A newer deployable bootstrap (a re-bootstrap) replaces this one at once: D = 0, pending tags dropped, the book carries on.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 209 |
from ..train import fly_replay
|
| 210 |
with transaction() as conn:
|
| 211 |
boot = fly_selector.latest_deployable(conn)
|
|
@@ -231,7 +344,7 @@ class FlyBook:
|
|
| 231 |
q.append(b)
|
| 232 |
|
| 233 |
def open_mints(self, conn) -> set[str]:
|
| 234 |
-
return {p["mint"] for p in ledger.open_positions(conn, BOOK)}
|
| 235 |
|
| 236 |
def on_minute(self, ctx) -> dict:
|
| 237 |
conn, t_now = ctx.conn, ctx.t_start
|
|
@@ -254,11 +367,12 @@ class FlyBook:
|
|
| 254 |
self.race_started_at = ctx.m1_epoch
|
| 255 |
self.beat_no += 1
|
| 256 |
d = scored["decision"]
|
| 257 |
-
st = paper_trading.trade_minute(ctx, book=BOOK, run_id=self.run_id, beat_no=self.beat_no, broker=self.broker, kind=KIND, mints=ctx.mints, infos=ctx.infos,
|
| 258 |
scores=d["score"], threshold=d["threshold"], table=None, horizon_s=self.holds[self.names[0]], holds=d["hold_s"],
|
| 259 |
-
strategies=d["strategy"], tables=d["tables"]
|
|
|
|
| 260 |
out.update({k: v for k, v in st.items() if k != "entries"}); out["stage"] = "trading"
|
| 261 |
-
if self.live and handover.state(conn) is not None:
|
| 262 |
out["live"] = self._live(ctx, st)
|
| 263 |
else:
|
| 264 |
out["stage"] = "learning (not trading this minute)" if ctx.trade else "catching up"
|
|
@@ -273,12 +387,15 @@ class FlyBook:
|
|
| 273 |
cur.executemany("UPDATE fly_scored SET label = %s, state = %s, resolved_at = now(), x = NULL WHERE ts = %s AND mint = %s AND strategy = %s", learned.pop("keys"))
|
| 274 |
learned.pop("keys", None)
|
| 275 |
out["checks"] = self.last_checks; out["updated_at"] = datetime.now(timezone.utc).isoformat()
|
| 276 |
-
conn.execute("INSERT INTO ui_settings (key, value) VALUES (
|
| 277 |
-
(json.dumps(out, default=str)
|
| 278 |
return out
|
| 279 |
|
| 280 |
def _live(self, ctx, st: dict) -> dict:
|
| 281 |
"""The fly holds the seat: mirror this minute on the bot wallet (agent/fly_live.py) while signing is allowed."""
|
|
|
|
|
|
|
|
|
|
| 282 |
from ..chain.cluster_guard import signing_allowed
|
| 283 |
if not signing_allowed():
|
| 284 |
return {"stage": "live gated: signing is not allowed (LIVE_ENABLED=1 and the live prerequisites)"}
|
|
@@ -343,21 +460,37 @@ class FlyBook:
|
|
| 343 |
self._capture(H, trig, ctx.t_start); del H
|
| 344 |
if not self.nu_set:
|
| 345 |
self.bank.estimate_nu(Y[:, 0], u0, k); self.nu_set = True
|
| 346 |
-
t = ctx.t_start; ts = datetime.fromtimestamp(t, timezone.utc); rows = []
|
| 347 |
-
V = np.full((n, len(self.names)), -np.inf)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 348 |
for j, name in enumerate(self.names):
|
| 349 |
loc = np.flatnonzero(trig[:, j])
|
|
|
|
|
|
|
| 350 |
if not len(loc):
|
| 351 |
continue
|
| 352 |
li = torch.as_tensor(loc, device=Y.device)
|
|
|
|
| 353 |
with torch.no_grad():
|
| 354 |
-
sc, _ = self.bank.predict(
|
| 355 |
-
fr = self.bank.frozen(
|
| 356 |
-
v, f = sc.float().cpu().numpy(), fr.float().cpu().numpy(); V[loc, j] = v
|
| 357 |
line, fline = self.lines["plastic"][name], self.lines["frozen"][name]
|
| 358 |
-
|
| 359 |
-
|
| 360 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 361 |
for i in loc:
|
| 362 |
m = ctx.mints[i]; self.watch[m] = self.watch.get(m, 0) + 1
|
| 363 |
q_ = self.bars.setdefault(m, deque(maxlen=BAR_KEEP)); b = ctx.bars.get(m)
|
|
@@ -365,12 +498,64 @@ class FlyBook:
|
|
| 365 |
q_.append(b)
|
| 366 |
if rows:
|
| 367 |
with conn.cursor() as cur:
|
| 368 |
-
cur.executemany("INSERT INTO fly_scored (ts, mint, strategy, hold_min, x, score, frozen_score, line, frozen_line, bootstrap_id) "
|
| 369 |
-
"VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT (ts, mint, strategy) DO NOTHING", rows)
|
| 370 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 371 |
d = fly_selector.fly_decide(self.fly, ctx.X, X_COLS, np.where(np.isfinite(V), V, -np.inf))
|
| 372 |
return {"decision": d, "picks": int(sum(1 for x in d["strategy"] if x is not None))}
|
| 373 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 374 |
def _capture(self, H: torch.Tensor, trig: np.ndarray, t: float) -> None:
|
| 375 |
"""The console's brain view: this minute's mean activity over the rows some strategy's trigger fired on (every
|
| 376 |
row when none did), one file per minute (brain/activity.py). Never allowed to interrupt scoring."""
|
|
@@ -387,25 +572,39 @@ class FlyBook:
|
|
| 387 |
# ---- daily and hourly ----
|
| 388 |
def _resolved(self, conn, since_resolved: float, cols: str, name: str) -> list[dict]:
|
| 389 |
"""A strategy's resolved rows whose labels became known after ``since_resolved`` (epoch s)."""
|
| 390 |
-
return conn.execute(f"SELECT {cols} FROM fly_scored WHERE state = 'resolved' AND label IS NOT NULL AND bootstrap_id = %s AND strategy = %s AND
|
| 391 |
-
(self.boot_id, name,
|
|
|
|
| 392 |
|
| 393 |
def _calibrate(self, conn, now: float) -> None:
|
| 394 |
day = datetime.fromtimestamp(now, timezone.utc).date()
|
|
|
|
| 395 |
for name in self.names:
|
| 396 |
-
rows = self._resolved(conn, now - fly_calibrate.WINDOW_DAYS * 86400.0, "ts, mint, score, frozen_score, label", name)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 397 |
if not rows:
|
| 398 |
continue
|
| 399 |
ts = np.array([r["ts"].timestamp() for r in rows]); mint = np.array([r["mint"] for r in rows]); lab = np.array([r["label"] for r in rows], dtype=np.float64)
|
|
|
|
|
|
|
|
|
|
| 400 |
for arm, col in (("plastic", "score"), ("frozen", "frozen_score")):
|
| 401 |
cal = fly_calibrate.calibrate(ts, mint, self.holds[name], np.array([r[col] for r in rows], dtype=np.float64), lab,
|
| 402 |
-
prev_line=self.lines[arm][name], prev_sizing=self.sizing[arm][name])
|
| 403 |
if cal.changed:
|
| 404 |
-
record_event("info", "fly", f"{name} {arm} line {self.lines[arm][name] * 100:+.2f}% → {cal.line * 100:+.2f}%",
|
|
|
|
| 405 |
self.lines[arm][name], self.sizing[arm][name] = cal.line, cal.sizing
|
| 406 |
conn.execute("INSERT INTO fly_calibrations (day, arm, line, sizing, trades, total, mean, window_days) VALUES (%s,%s,%s,%s,%s,%s,%s,%s) "
|
| 407 |
"ON CONFLICT (day, arm) DO UPDATE SET line = EXCLUDED.line, sizing = EXCLUDED.sizing, trades = EXCLUDED.trades, total = EXCLUDED.total, mean = EXCLUDED.mean",
|
| 408 |
-
(day, f"{arm}:{name}", cal.line, json.dumps(cal.sizing), cal.trades,
|
|
|
|
| 409 |
|
| 410 |
def _drift(self, j: int) -> float:
|
| 411 |
return float(self.bank.drift(j)[0]) if self.bank.learn.shape[0] > 1 else float(self.bank.drift()[0])
|
|
@@ -427,6 +626,10 @@ class FlyBook:
|
|
| 427 |
return out
|
| 428 |
|
| 429 |
def _hourly(self, conn, now: float) -> None:
|
|
|
|
|
|
|
|
|
|
|
|
|
| 430 |
hour = datetime.fromtimestamp(math.floor(now / 3600) * 3600 - 3600, timezone.utc)
|
| 431 |
checks = self._checks(conn, now); self.last_checks = checks
|
| 432 |
h = self.hour_stats; first = checks.get(self.names[0]) or {}
|
|
@@ -447,7 +650,7 @@ class FlyBook:
|
|
| 447 |
d = fly_state_dir(); d.mkdir(parents=True, exist_ok=True)
|
| 448 |
path = d / f"snap_{datetime.fromtimestamp(now, timezone.utc).strftime('%Y%m%dT%H%M')}.pt"
|
| 449 |
torch.save({"bank": self.bank.state(), "bootstrap_id": self.boot_id, "lines": self.lines, "sizing": self.sizing}, path)
|
| 450 |
-
note = {"bootstrap_id": self.boot_id, "data": fly_selector.FLY_VERSION, "good": good, "checks": checks,
|
| 451 |
"drift": max((c["drift"] for c in checks.values()), default=0.0), "ic24": {k: c.get("ic") for k, c in checks.items()}}
|
| 452 |
return int(conn.execute("INSERT INTO brain_snapshots (run_id, path, kind, note) VALUES (%s,%s,'fly_plastic',%s) RETURNING id",
|
| 453 |
(self.run_id, str(path), json.dumps(note, default=str))).fetchone()["id"])
|
|
@@ -476,11 +679,11 @@ class FlyBook:
|
|
| 476 |
(datetime.fromtimestamp(now - ROLLBACK_MIN_AGE_S, timezone.utc),)).fetchall():
|
| 477 |
n = json.loads(r["note"] or "{}")
|
| 478 |
ok = n.get("good") or not ((n.get("checks") or {}).get(name) or {}).get("triggers")
|
| 479 |
-
if ok and n.get("bootstrap_id") == self.boot_id and Path(r["path"]).exists():
|
| 480 |
target = r; break
|
| 481 |
if target is not None:
|
| 482 |
snap = plastic.PlasticBank(self.fly.net, [(0.0, math.inf)], fly_selector.SCALE, learn=self.bank.learn.cpu().numpy(), read=self.bank.read.cpu().numpy())
|
| 483 |
-
st = torch.load(target["path"], map_location="cpu", weights_only=
|
| 484 |
snap.D = torch.zeros_like(snap.D); snap.D[:, snap.mask.bool()] = st["D"].to(snap.dev).float()
|
| 485 |
self.bank.D[0][:, cols] = snap.D[0][:, cols]
|
| 486 |
else:
|
|
@@ -519,15 +722,17 @@ class FlyBook:
|
|
| 519 |
whole race), closed ``config.HANDOVER_MIN_TRADES`` paper trades and, when ``config.HANDOVER_BEAT_SELECTOR``,
|
| 520 |
realized at least the paper selector's P&L over that window."""
|
| 521 |
days = config.HANDOVER_DAYS
|
| 522 |
-
if handover.state(conn) is not None or self.race_started_at is None or now - self.race_started_at < days * 86400.0:
|
| 523 |
return
|
| 524 |
since = datetime.fromtimestamp(self.race_started_at if days <= 0 else now - days * 86400.0, timezone.utc)
|
|
|
|
| 525 |
pnl = {b: conn.execute("SELECT COALESCE(sum(realized_sol), 0) AS s, count(*) AS n FROM positions WHERE book = %s AND status = 'closed' AND closed_at >= %s",
|
| 526 |
-
(b, since)).fetchone() for b in (
|
| 527 |
-
fly_pnl, sel_pnl, n = float(pnl[
|
| 528 |
if n >= config.HANDOVER_MIN_TRADES and (fly_pnl >= sel_pnl or not config.HANDOVER_BEAT_SELECTOR):
|
| 529 |
detail = {"at": datetime.fromtimestamp(now, timezone.utc).isoformat(), "fly_pnl_sol": fly_pnl, "selector_pnl_sol": sel_pnl, "fly_trades": n,
|
| 530 |
-
"selector_trades": int(pnl[
|
| 531 |
-
|
|
|
|
| 532 |
record_event("info", "handover", f"the fly takes the selector's seat: {fly_pnl:+.3f} SOL vs {sel_pnl:+.3f} SOL over {n} trades"
|
| 533 |
+ (f" in {days} days" if days > 0 else ""), detail)
|
|
|
|
| 35 |
import numpy as np
|
| 36 |
import torch
|
| 37 |
|
| 38 |
+
from .. import config, markets
|
| 39 |
from ..brain import activity, device as brain_device, plastic
|
| 40 |
from ..brain.connectome import current_connectome_path
|
| 41 |
from ..db.apilog import record_event
|
|
|
|
| 43 |
from ..execution import ledger
|
| 44 |
from ..execution.broker_paper import PaperBroker
|
| 45 |
from ..ops.reset import activity_dir, fly_state_dir
|
| 46 |
+
from ..train import fly_calibrate, fly_governance as gov, fly_selector, selector, strategies
|
| 47 |
from ..train.decisions import X_COLS
|
| 48 |
from . import handover, paper_trading
|
| 49 |
|
| 50 |
log = logging.getLogger(__name__)
|
| 51 |
+
BOOK, KIND = "paper_fly", "fly" # the Solana book's names (another chain's book takes its own from fly_trader/markets)
|
| 52 |
BAR_KEEP = 270 # minutes of bars kept per watched mint (the longest strategy hold, 240, + next-open window + margin)
|
| 53 |
NEXT_OPEN_S = 120.0 # entry at the next traded minute's open when it comes within two minutes (decisions.build)
|
| 54 |
JUMP = 50.0
|
|
|
|
| 90 |
return float(exit_px / entry * (1 - rows[0][3]) * (1 - rows[j][3]) - 1)
|
| 91 |
|
| 92 |
|
| 93 |
+
def chain_passed(v: dict, chain: str) -> tuple[bool, str]:
|
| 94 |
+
"""Whether the replay lets this chain's book trade: its own per-chain verdict (a combined replay), else the one verdict
|
| 95 |
+
for Solana (a Solana-only replay says nothing about another chain)."""
|
| 96 |
+
if "chains" in v:
|
| 97 |
+
c = v["chains"].get(chain)
|
| 98 |
+
return (bool(c and c.get("passed")), (c or {}).get("reason") or f"the replay has no {chain} verdict")
|
| 99 |
+
return (bool(v.get("passed")) and chain == "sol", v.get("reason") if chain == "sol" else f"the replay judged no {chain} trades")
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
def try_start(live: bool = False, chain: str = "sol", brain: "FlyBook | None" = None) -> tuple["FlyBook | None", str]:
|
| 103 |
+
"""A FlyBook for ``chain`` when the fly may trade there, else (None, why). ``brain``: the Solana book whose fly and
|
| 104 |
+
plastic memory another chain's book shares (one model learns from both chains)."""
|
| 105 |
from ..train import fly_replay
|
| 106 |
v = fly_replay.verdict()
|
| 107 |
if not v:
|
| 108 |
return None, "the fly's replay has not run on the current definitions (fly-trader fly-replay)"
|
| 109 |
+
ok, why = chain_passed(v, chain)
|
| 110 |
+
if not ok:
|
| 111 |
+
return None, f"the fly's replay did not pass on {markets.for_chain(chain).name}: {why}"
|
| 112 |
with transaction() as conn:
|
| 113 |
boot = fly_selector.latest_deployable(conn)
|
| 114 |
if boot is None:
|
| 115 |
return None, "no deployable fly bootstrap (the training pipeline bootstraps one)"
|
| 116 |
try:
|
| 117 |
+
return FlyBook(boot, v, live=live, chain=chain, brain=brain), "ok"
|
| 118 |
except Exception as e:
|
| 119 |
log.exception("fly book failed to start")
|
| 120 |
return None, f"fly book failed to start: {type(e).__name__}: {e}"
|
| 121 |
|
| 122 |
|
| 123 |
+
def no_teacher(n: int) -> dict:
|
| 124 |
+
"""The decision when the teacher gate is on (FLY_TEACHER_GATE) but the fly's teacher could not be loaded: no row is
|
| 125 |
+
allowed, each recorded with why. A fresh install once lost the teacher's snapshot id (ops/release.remap_teacher)."""
|
| 126 |
+
return {"strategy": np.full(n, None, dtype=object), "score": np.full(n, -1.0), "hold_s": np.zeros(n), "threshold": np.full(n, np.inf),
|
| 127 |
+
"tables": [[] for _ in range(n)], "allow": np.zeros(n, bool),
|
| 128 |
+
"reason": np.full(n, "fail closed: the fly's teacher (the selector that taught it) is not loaded", dtype=object)}
|
| 129 |
+
|
| 130 |
|
| 131 |
+
class FlyBook:
|
| 132 |
+
def __init__(self, boot: dict, verdict: dict, live: bool = False, chain: str = "sol", brain: "FlyBook | None" = None):
|
| 133 |
self.live = live; self.done = False; self.mirror = None
|
| 134 |
self.verdict = verdict
|
| 135 |
+
self.chain = chain; self.market = markets.for_chain(chain); self.brain = brain
|
| 136 |
+
self.name = "fly" if chain == "sol" else f"fly_{chain}"
|
| 137 |
+
self.BOOK = self.market.fly_book; self.KIND = self.name; self.status_key = self.market.fly_status_key
|
| 138 |
+
self.state_path = fly_state_dir() / ("state.pt" if chain == "sol" else f"state_{chain}.pt")
|
| 139 |
self._load_bootstrap(boot)
|
| 140 |
+
self.run_id = str(uuid.uuid4()); self.beat_no = 0; self.broker = PaperBroker(self.BOOK)
|
| 141 |
with transaction() as conn:
|
| 142 |
conn.execute("INSERT INTO runs (run_id, kind, config, brain_snapshot_id, status) VALUES (%s,%s,%s,%s,'running')",
|
| 143 |
+
(self.run_id, self.KIND, json.dumps({"channels": self.cfg, "book": self.BOOK, "chain": chain}, default=str), self.boot_id))
|
| 144 |
self._restore_tags(conn)
|
| 145 |
record_event("info", "fly", "fly session started", {"run_id": self.run_id, "bootstrap": self.boot_id, "channels": self.cfg, "lines": self.lines["plastic"],
|
| 146 |
"pending": len(self.pending), "learning_frozen": self.learning_frozen})
|
|
|
|
| 148 |
self.lines["plastic"], len(self.pending))
|
| 149 |
|
| 150 |
# ---- model and state ----
|
| 151 |
+
def _load_bootstrap(self, boot: dict | None) -> None:
|
| 152 |
+
b = self.brain
|
| 153 |
+
if b is not None: # another chain's book: the Solana book's fly and plastic memory, this chain's own lines and tags
|
| 154 |
+
self.boot_id = b.boot_id; self.fly = b.fly; self.connectome_name = b.connectome_name; self.capture_ok = False
|
| 155 |
+
self.teacher_models = b.teacher_models; self.idx = b.idx; self.names = b.names; self.holds = b.holds; self.H = b.H
|
| 156 |
+
self.teacher_live, self.teacher_groups = b.teacher_live, b.teacher_groups
|
| 157 |
+
self.cfg = b.cfg; self.alpha = b.alpha; self.half_life = b.half_life; self.bank = b.bank
|
| 158 |
+
net = self.fly.net
|
| 159 |
+
else:
|
| 160 |
+
self.boot_id = int(boot["id"])
|
| 161 |
+
self.fly = fly_selector.load(boot["path"])
|
| 162 |
+
try:
|
| 163 |
+
self.connectome_name = current_connectome_path().name # stamped on the activity files the console draws
|
| 164 |
+
except Exception:
|
| 165 |
+
self.connectome_name = ""
|
| 166 |
+
self.capture_ok = True
|
| 167 |
+
self.teacher_models = self._teacher_models(boot)
|
| 168 |
+
self.idx = np.asarray([X_COLS.index(c) for c in self.fly.cols], dtype=int)
|
| 169 |
+
self.names = list(self.fly.strategies); self.holds = {k: float(self.fly.rules[k]["hold_min"]) * 60.0 for k in self.names}
|
| 170 |
+
self.H = max(self.holds.values())
|
| 171 |
+
per = self.verdict.get("per_strategy") or {self.names[0]: {"alpha": self.verdict.get("alpha", 0.0), "half_life_days": self.verdict.get("half_life_days")}}
|
| 172 |
+
self.cfg = {k: (float((per.get(k) or {}).get("alpha") or 0.0), (per.get(k) or {}).get("half_life_days")) for k in self.names}
|
| 173 |
+
self.alpha = self.cfg[self.names[0]][0]; self.half_life = float(self.cfg[self.names[0]][1]) if self.cfg[self.names[0]][1] is not None else math.inf
|
| 174 |
+
net = self.fly.net
|
| 175 |
+
self.bank = plastic.PlasticBank(net, [(self.alpha, self.half_life)], fly_selector.SCALE, learn=net.learn.cpu().numpy(), read=net.read.cpu().numpy())
|
| 176 |
+
for j, k in enumerate(self.names): # each strategy's channel learns and forgets at its own replay-chosen rate
|
| 177 |
+
a, h = self.cfg[k]; cols = self.bank.learn[j]
|
| 178 |
+
self.bank.alpha[0, cols] = a; self.bank.half_life_s[0, cols] = float(h) * 86400.0 if h is not None else float("inf")
|
| 179 |
self.pending = plastic.PendingTags(net.n_kc, net.k_active)
|
| 180 |
self.bars: dict[str, deque] = {}; self.watch: dict[str, int] = {}
|
| 181 |
+
own = {k: self.fly.line_for(k, self.chain) for k in self.names}; own_s = {k: list(self.fly.sizing_for(k, self.chain)) for k in self.names}
|
| 182 |
+
if config.FLY_SCORE == "teacher" and self.teacher_live: # it scores on its teacher's scale: it starts on its teacher's lines and bands
|
| 183 |
+
ts_ = self.teacher_live.get("strategies") or {}
|
| 184 |
+
own = {k: float(ts_[k]["line"]) if k in ts_ else v for k, v in own.items()}
|
| 185 |
+
own_s = {k: list(ts_[k].get("sizing") or v) if k in ts_ else v for k, v in own_s.items()}
|
| 186 |
+
self.lines = {"plastic": dict(own), "frozen": dict(own)}
|
| 187 |
+
self.sizing = {"plastic": {k: list(v) for k, v in own_s.items()}, "frozen": {k: list(v) for k, v in own_s.items()}}
|
| 188 |
self.applied_through = -math.inf; self.learning_frozen = False; self.checks_paused_until = {k: 0.0 for k in self.names}
|
| 189 |
self.race_started_at: float | None = None; self.last_day: int | None = None; self.last_hour: int | None = None
|
| 190 |
self.rollbacks: dict = {k: [] for k in self.names}; self.nu_set = False; self.hour_stats = self._empty_stats(); self.last_checks: dict = {}
|
| 191 |
s = self._read_state()
|
| 192 |
+
if b is not None:
|
| 193 |
+
self.nu_set = True # the shared memory is the Solana book's to load and save
|
| 194 |
+
if s and s.get("bootstrap_id") == self.boot_id and s.get("cfg") == self.cfg and s.get("score", "own") != config.FLY_SCORE:
|
| 195 |
+
self._archive_state(s) # learned under another scoring: it starts from its bootstrap
|
| 196 |
+
s = None
|
| 197 |
if s and s.get("bootstrap_id") == self.boot_id and s.get("cfg") == self.cfg:
|
| 198 |
+
if b is None:
|
| 199 |
+
self.bank.load_state(s["bank"])
|
| 200 |
+
self.nu_set = True
|
| 201 |
self.lines, self.sizing = s["lines"], s["sizing"]
|
| 202 |
self.applied_through, self.learning_frozen, self.checks_paused_until = s["applied_through"], s["learning_frozen"], s["checks_paused_until"]
|
| 203 |
self.race_started_at, self.last_day, self.last_hour, self.rollbacks = s["race_started_at"], s["last_day"], s["last_hour"], s["rollbacks"]
|
| 204 |
elif s:
|
| 205 |
self.race_started_at = s.get("race_started_at") # a new bootstrap keeps racing the same book
|
| 206 |
with transaction() as conn:
|
| 207 |
+
conn.execute("UPDATE fly_scored SET state = 'dropped', x = NULL WHERE state = 'pending' AND chain = %s AND bootstrap_id IS DISTINCT FROM %s",
|
| 208 |
+
(self.chain, self.boot_id))
|
| 209 |
+
|
| 210 |
+
teacher_live: dict | None = None
|
| 211 |
+
teacher_groups: set = set()
|
| 212 |
+
|
| 213 |
+
def _teacher_models(self, boot: dict) -> dict | None:
|
| 214 |
+
"""The stack that taught this fly (its snapshot's ``teacher_snapshot``: the holdout-blind copy when that is what
|
| 215 |
+
taught it), whose final decision on each scored row is stored as ``teacher_allow`` -- the match calibration
|
| 216 |
+
rule's budget. None (with a warning) when the fly was taught by a refit stack or the snapshot is gone."""
|
| 217 |
+
meta = boot.get("meta") or {}
|
| 218 |
+
sid = meta.get("teacher_snapshot")
|
| 219 |
+
if sid is None:
|
| 220 |
+
if fly_calibrate.RULE == "match":
|
| 221 |
+
log.warning("fly #%s names no teacher snapshot: the match rule keeps its bootstrap line", self.boot_id)
|
| 222 |
+
return None
|
| 223 |
+
try:
|
| 224 |
+
m = selector.load_snapshot(int(sid))
|
| 225 |
+
except Exception:
|
| 226 |
+
log.exception("fly #%s: teacher snapshot #%s could not be loaded", self.boot_id, sid)
|
| 227 |
+
return None
|
| 228 |
+
blind = getattr(m, "blind", None) or {}
|
| 229 |
+
models = blind["models"] if "blind to the calibration week" in str(meta.get("teacher") or "") and blind.get("models") else getattr(m, "stack", None)
|
| 230 |
+
live = getattr(m, "stack", None) # the gate: the teacher's full stack, as its selector book trades it
|
| 231 |
+
if live and live.get("strategies") and all(c in X_COLS for v in live["strategies"].values() for c in v["cols"]):
|
| 232 |
+
self.teacher_live = live
|
| 233 |
+
self.teacher_groups = set((getattr(m, "metrics", None) or {}).get("groups") or [])
|
| 234 |
+
if not models or not models.get("strategies") or any(c not in X_COLS for v in models["strategies"].values() for c in v["cols"]):
|
| 235 |
+
log.warning("fly #%s: teacher #%s has no usable strategy stack", self.boot_id, sid)
|
| 236 |
+
return None
|
| 237 |
+
return models
|
| 238 |
+
|
| 239 |
+
def _teacher_allow(self, X: np.ndarray, t: float) -> dict[str, np.ndarray]:
|
| 240 |
+
"""Per strategy: would the teacher trade each row (trigger, line and every filter)? Empty when unknown."""
|
| 241 |
+
if self.teacher_models is None:
|
| 242 |
+
return {}
|
| 243 |
+
try:
|
| 244 |
+
per = strategies.decide_all(self.teacher_models, X, X_COLS, t)
|
| 245 |
+
return {k: np.asarray(v["allow"], bool) for k, v in per.items()}
|
| 246 |
+
except Exception:
|
| 247 |
+
log.exception("teacher decisions failed for this minute (the fly still trades)")
|
| 248 |
+
return {}
|
| 249 |
|
| 250 |
def _read_state(self) -> dict | None:
|
| 251 |
if not self.state_path.exists():
|
| 252 |
return None
|
| 253 |
try:
|
| 254 |
+
return torch.load(self.state_path, map_location="cpu", weights_only=True)
|
| 255 |
except Exception:
|
| 256 |
log.exception("fly state unreadable; starting from the bootstrap")
|
| 257 |
return None
|
| 258 |
|
| 259 |
+
def _archive_state(self, s: dict) -> None:
|
| 260 |
+
"""Keep a state learned under another scoring mode beside the new one (never loaded again)."""
|
| 261 |
+
try:
|
| 262 |
+
dst = self.state_path.with_name(f"{self.state_path.stem}_{s.get('score', 'own')}_{time.strftime('%Y%m%dT%H%M%S')}.pt")
|
| 263 |
+
os.replace(self.state_path, dst)
|
| 264 |
+
record_event("info", "fly", f"fly state learned with FLY_SCORE={s.get('score', 'own')} set aside: the fly starts from its bootstrap",
|
| 265 |
+
{"path": str(dst), "drift": None})
|
| 266 |
+
except OSError:
|
| 267 |
+
log.exception("could not set the old fly state aside")
|
| 268 |
+
|
| 269 |
def _state(self) -> dict:
|
| 270 |
+
return {"bootstrap_id": self.boot_id, "cfg": self.cfg, "score": config.FLY_SCORE, "bank": self.bank.state() if self.brain is None else None, "lines": self.lines, "sizing": self.sizing,
|
| 271 |
"applied_through": self.applied_through, "learning_frozen": self.learning_frozen, "checks_paused_until": self.checks_paused_until,
|
| 272 |
"race_started_at": self.race_started_at, "last_day": self.last_day, "last_hour": self.last_hour, "rollbacks": self.rollbacks,
|
| 273 |
"saved_at": time.time()}
|
|
|
|
| 278 |
torch.save(self._state(), tmp); os.replace(tmp, self.state_path)
|
| 279 |
|
| 280 |
def _restore_tags(self, conn) -> None:
|
| 281 |
+
rows = conn.execute("SELECT ts, mint, strategy, x, score, line FROM fly_scored WHERE state = 'pending' AND bootstrap_id = %s AND chain = %s ORDER BY ts",
|
| 282 |
+
(self.boot_id, self.chain)).fetchall()
|
| 283 |
rows = [r for r in rows if r["strategy"] in self.holds]
|
| 284 |
applied = [(r["ts"], r["mint"], r["strategy"]) for r in rows if r["ts"].timestamp() + self.holds[r["strategy"]] + LABEL_LAG_S <= self.applied_through]
|
| 285 |
if applied:
|
|
|
|
| 291 |
groups.setdefault((r["ts"].timestamp(), r["strategy"]), []).append(r)
|
| 292 |
for (t, name), rs in sorted(groups.items()):
|
| 293 |
j = self.names.index(name)
|
| 294 |
+
Xr = np.asarray([r["x"] for r in rs], dtype=np.float32)
|
| 295 |
+
Y, u0, k = self.fly.parts_all(Xr[:, self.idx])
|
| 296 |
+
tv = self._teacher_scores(Xr, t); loc = np.arange(len(rs))
|
| 297 |
+
if tv is not None:
|
| 298 |
+
loc = loc[np.isfinite(tv[:, j])]
|
| 299 |
+
if not len(loc):
|
| 300 |
+
continue
|
| 301 |
+
rs = [rs[i] for i in loc]; Xr = Xr[loc]; li = torch.as_tensor(loc, device=Y.device); Y, u0, k = Y[li], u0[li], k[li]
|
| 302 |
+
y = self._head(Y[:, j], tv, loc, j, u0)
|
| 303 |
+
if config.FLY_TEACHER_GATE and self.teacher_live is not None: # gated: it learns from its teacher's trades, as live
|
| 304 |
+
d = strategies.decide(self.teacher_live, Xr, X_COLS, t)
|
| 305 |
+
w = torch.tensor([[float(d["strategy"][i] == name and bool(d["allow"][i])) for i in range(len(rs))]])
|
| 306 |
+
elif config.FLY_TEACHER_GATE:
|
| 307 |
+
w = torch.zeros((1, len(rs)))
|
| 308 |
+
else:
|
| 309 |
+
w = plastic.row_weights(torch.tensor([[float(r["score"]) for r in rs]]), torch.tensor([float(rs[0]["line"])]))
|
| 310 |
+
self.pending.push([(t, r["mint"], name) for r in rs], np.full(len(rs), t), t + self.holds[name] + LABEL_LAG_S, y, u0, k, w, s=j)
|
| 311 |
for r in rs:
|
| 312 |
self.watch[r["mint"]] = self.watch.get(r["mint"], 0) + 1
|
| 313 |
|
| 314 |
def maybe_reload(self) -> bool:
|
| 315 |
+
"""A newer deployable bootstrap (a re-bootstrap) replaces this one at once: D = 0, pending tags dropped, the book carries on.
|
| 316 |
+
Another chain's book follows its brain (the Solana book reloads first)."""
|
| 317 |
+
if self.brain is not None:
|
| 318 |
+
if self.brain.boot_id == self.boot_id:
|
| 319 |
+
return False
|
| 320 |
+
self._write_state(); self._load_bootstrap(None)
|
| 321 |
+
return True
|
| 322 |
from ..train import fly_replay
|
| 323 |
with transaction() as conn:
|
| 324 |
boot = fly_selector.latest_deployable(conn)
|
|
|
|
| 344 |
q.append(b)
|
| 345 |
|
| 346 |
def open_mints(self, conn) -> set[str]:
|
| 347 |
+
return {p["mint"] for p in ledger.open_positions(conn, self.BOOK)}
|
| 348 |
|
| 349 |
def on_minute(self, ctx) -> dict:
|
| 350 |
conn, t_now = ctx.conn, ctx.t_start
|
|
|
|
| 367 |
self.race_started_at = ctx.m1_epoch
|
| 368 |
self.beat_no += 1
|
| 369 |
d = scored["decision"]
|
| 370 |
+
st = paper_trading.trade_minute(ctx, book=self.BOOK, run_id=self.run_id, beat_no=self.beat_no, broker=self.broker, kind=self.KIND, mints=ctx.mints, infos=ctx.infos,
|
| 371 |
scores=d["score"], threshold=d["threshold"], table=None, horizon_s=self.holds[self.names[0]], holds=d["hold_s"],
|
| 372 |
+
strategies=d["strategy"], tables=d["tables"], flat=config.FLY_SIZING == "flat",
|
| 373 |
+
allow=d.get("allow"), reasons=d.get("reason"))
|
| 374 |
out.update({k: v for k, v in st.items() if k != "entries"}); out["stage"] = "trading"
|
| 375 |
+
if self.live and handover.state(conn, self.chain) is not None:
|
| 376 |
out["live"] = self._live(ctx, st)
|
| 377 |
else:
|
| 378 |
out["stage"] = "learning (not trading this minute)" if ctx.trade else "catching up"
|
|
|
|
| 387 |
cur.executemany("UPDATE fly_scored SET label = %s, state = %s, resolved_at = now(), x = NULL WHERE ts = %s AND mint = %s AND strategy = %s", learned.pop("keys"))
|
| 388 |
learned.pop("keys", None)
|
| 389 |
out["checks"] = self.last_checks; out["updated_at"] = datetime.now(timezone.utc).isoformat()
|
| 390 |
+
conn.execute("INSERT INTO ui_settings (key, value) VALUES (%s, %s) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()",
|
| 391 |
+
(self.status_key, json.dumps({**out, "chain": self.chain}, default=str)))
|
| 392 |
return out
|
| 393 |
|
| 394 |
def _live(self, ctx, st: dict) -> dict:
|
| 395 |
"""The fly holds the seat: mirror this minute on the bot wallet (agent/fly_live.py) while signing is allowed."""
|
| 396 |
+
if self.chain != "sol":
|
| 397 |
+
from ..rh import live as rh_live # Robinhood Chain: the ETH wallet (rh/live.py)
|
| 398 |
+
return rh_live.minute(self, ctx, st)
|
| 399 |
from ..chain.cluster_guard import signing_allowed
|
| 400 |
if not signing_allowed():
|
| 401 |
return {"stage": "live gated: signing is not allowed (LIVE_ENABLED=1 and the live prerequisites)"}
|
|
|
|
| 460 |
self._capture(H, trig, ctx.t_start); del H
|
| 461 |
if not self.nu_set:
|
| 462 |
self.bank.estimate_nu(Y[:, 0], u0, k); self.nu_set = True
|
| 463 |
+
t = ctx.t_start; ts = datetime.fromtimestamp(t, timezone.utc); rows = []; ta = self._teacher_allow(ctx.X, t)
|
| 464 |
+
V = np.full((n, len(self.names)), -np.inf); F = np.full((n, len(self.names)), np.nan)
|
| 465 |
+
gated = config.FLY_TEACHER_GATE
|
| 466 |
+
if config.FLY_SCORE == "teacher" and self.teacher_live is None: # it starts from its teacher's score: none, no score
|
| 467 |
+
return {"decision": no_teacher(n), "picks": 0}
|
| 468 |
+
td = self._teacher_decision(ctx) if gated and self.teacher_live is not None else None
|
| 469 |
+
tv = self._teacher_scores(ctx.X, t)
|
| 470 |
for j, name in enumerate(self.names):
|
| 471 |
loc = np.flatnonzero(trig[:, j])
|
| 472 |
+
if tv is not None:
|
| 473 |
+
loc = loc[np.isfinite(tv[loc, j])] # a candidate its teacher scores
|
| 474 |
if not len(loc):
|
| 475 |
continue
|
| 476 |
li = torch.as_tensor(loc, device=Y.device)
|
| 477 |
+
y = self._head(Y[li, j], tv, loc, j, u0[li])
|
| 478 |
with torch.no_grad():
|
| 479 |
+
sc, _ = self.bank.predict(y, u0[li], k[li], s=np.full(len(loc), j)); sc = sc[0]
|
| 480 |
+
fr = self.bank.frozen(y, u0[li], s=np.full(len(loc), j))
|
| 481 |
+
v, f = sc.float().cpu().numpy(), fr.float().cpu().numpy(); V[loc, j] = v; F[loc, j] = f
|
| 482 |
line, fline = self.lines["plastic"][name], self.lines["frozen"][name]
|
| 483 |
+
al = ta.get(name)
|
| 484 |
+
rows += [(ts, ctx.mints[i], name, int(self.holds[name] // 60), ctx.X[i].tolist(), float(v[q]), float(f[q]), line, fline, self.boot_id,
|
| 485 |
+
None if al is None else bool(al[i])) for q, i in enumerate(loc)]
|
| 486 |
+
if td is not None: # gated: its decisions are the teacher's trades of this strategy (before its own vetoes), so it learns from those
|
| 487 |
+
mine = np.array([td["strategy"][i] == name and bool(td["allow"][i]) for i in loc], dtype=np.float32)
|
| 488 |
+
w = torch.as_tensor(mine[None], dtype=sc.dtype, device=sc.device)
|
| 489 |
+
elif gated: # the gate is on but its teacher is not loaded: nothing to learn from until it is
|
| 490 |
+
w = torch.zeros((1, len(loc)), dtype=sc.dtype, device=sc.device)
|
| 491 |
+
else:
|
| 492 |
+
w = plastic.row_weights(sc[None], torch.tensor([line], dtype=sc.dtype, device=sc.device))
|
| 493 |
+
self.pending.push([(t, ctx.mints[i], name) for i in loc], np.full(len(loc), t), t + self.holds[name] + LABEL_LAG_S, y, u0[li], k[li], w, s=j)
|
| 494 |
for i in loc:
|
| 495 |
m = ctx.mints[i]; self.watch[m] = self.watch.get(m, 0) + 1
|
| 496 |
q_ = self.bars.setdefault(m, deque(maxlen=BAR_KEEP)); b = ctx.bars.get(m)
|
|
|
|
| 498 |
q_.append(b)
|
| 499 |
if rows:
|
| 500 |
with conn.cursor() as cur:
|
| 501 |
+
cur.executemany("INSERT INTO fly_scored (ts, mint, strategy, hold_min, x, score, frozen_score, line, frozen_line, bootstrap_id, teacher_allow, chain) "
|
| 502 |
+
"VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT (ts, mint, strategy) DO NOTHING", [r + (self.chain,) for r in rows])
|
| 503 |
+
if self.chain == "sol":
|
| 504 |
+
self.fly.lines = dict(self.lines["plastic"]); self.fly.sizings = {k: list(v) for k, v in self.sizing["plastic"].items()}
|
| 505 |
+
else: # the shared model decides this chain's rows on this chain's lines
|
| 506 |
+
self.fly.chain_lines[self.chain] = dict(self.lines["plastic"]); self.fly.chain_sizings[self.chain] = {k: list(v) for k, v in self.sizing["plastic"].items()}
|
| 507 |
+
if td is not None:
|
| 508 |
+
d = self._gated(ctx, V, F, td)
|
| 509 |
+
return {"decision": d, "picks": int(d["allow"].sum())}
|
| 510 |
+
if gated: # never the ungated fly: it bought what the selector rejected and lost (2026-10-08)
|
| 511 |
+
return {"decision": no_teacher(n), "picks": 0}
|
| 512 |
d = fly_selector.fly_decide(self.fly, ctx.X, X_COLS, np.where(np.isfinite(V), V, -np.inf))
|
| 513 |
return {"decision": d, "picks": int(sum(1 for x in d["strategy"] if x is not None))}
|
| 514 |
|
| 515 |
+
def _teacher_scores(self, X: np.ndarray, t) -> np.ndarray | None:
|
| 516 |
+
"""[B, S] the teacher's own score of each row per strategy (NaN off its trigger) when the fly scores as its teacher
|
| 517 |
+
plus its learned correction (config.FLY_SCORE = teacher), else None."""
|
| 518 |
+
if config.FLY_SCORE != "teacher" or self.teacher_live is None:
|
| 519 |
+
return None
|
| 520 |
+
return fly_selector.teacher_scores(self.teacher_live, X, t, self.names, X_COLS)
|
| 521 |
+
|
| 522 |
+
def _head(self, y_own: torch.Tensor, tv: np.ndarray | None, loc: np.ndarray, j: int, u0: torch.Tensor) -> torch.Tensor:
|
| 523 |
+
"""The head output the bank scores with: the fly's own decoder output, or (FLY_SCORE = teacher) the one under which
|
| 524 |
+
its frozen score is the teacher's score exactly (PlasticBank.head_for), so only its learning moves it."""
|
| 525 |
+
if tv is None:
|
| 526 |
+
return y_own
|
| 527 |
+
return self.bank.head_for(torch.as_tensor(tv[loc, j], dtype=torch.float32, device=self.bank.dev), u0, s=np.full(len(loc), j))
|
| 528 |
+
|
| 529 |
+
def _teacher_decision(self, ctx) -> dict:
|
| 530 |
+
"""The deployed selector's decision on this minute's rows, with its live fail-closed rules."""
|
| 531 |
+
from .selector_session import fail_closed
|
| 532 |
+
return fail_closed(self.teacher_groups, ctx.infos, strategies.decide(self.teacher_live, ctx.X, X_COLS, ctx.t_start))
|
| 533 |
+
|
| 534 |
+
def _gated(self, ctx, V: np.ndarray, F: np.ndarray, td: dict | None = None) -> dict:
|
| 535 |
+
"""The fly's decision under its teacher: each row's strategy, trigger, line, win filter, gates, dump veto and
|
| 536 |
+
fail-closed rules are the deployed selector's (agent/selector_session.fail_closed), so a fresh bootstrap trades the
|
| 537 |
+
selector's set; the fly drops a teacher trade only where its learning has moved its score more than
|
| 538 |
+
``FLY_LEARNED_VETO`` below its bootstrap score. A row the teacher blocks is recorded with the teacher's reason."""
|
| 539 |
+
n = len(ctx.mints)
|
| 540 |
+
td = td if td is not None else self._teacher_decision(ctx)
|
| 541 |
+
strat = np.full(n, None, dtype=object); score = np.full(n, -1.0); hold = np.zeros(n); thr = np.full(n, np.inf)
|
| 542 |
+
tables = [[] for _ in range(n)]; allow = np.zeros(n, bool); reason = np.full(n, "", dtype=object)
|
| 543 |
+
for i in range(n):
|
| 544 |
+
s_ = td["strategy"][i]
|
| 545 |
+
if s_ is None or s_ not in self.names:
|
| 546 |
+
continue
|
| 547 |
+
j = self.names.index(s_); v, f = V[i, j], F[i, j]
|
| 548 |
+
# the selector's own score, line and sizing table for the row: a gated fly buy is sized exactly as the selector
|
| 549 |
+
# sizes it (agent/sizing.size_position; FLY_SIZING=kelly: the same Kelly band, fraction and caps; flat: flat)
|
| 550 |
+
strat[i] = s_; score[i] = float(td["score"][i]); thr[i] = float(td["threshold"][i]); hold[i] = self.holds[s_]; tables[i] = td["tables"][i]
|
| 551 |
+
if not td["allow"][i]:
|
| 552 |
+
reason[i] = f"selector: {td['reason'][i]}"
|
| 553 |
+
elif np.isfinite(v) and np.isfinite(f) and v < f - config.FLY_LEARNED_VETO:
|
| 554 |
+
reason[i] = f"learned veto: score {v:.3f} < bootstrap {f:.3f} - {config.FLY_LEARNED_VETO:g}"
|
| 555 |
+
else:
|
| 556 |
+
allow[i] = True; reason[i] = "trade"
|
| 557 |
+
return {"strategy": strat, "score": score, "hold_s": hold, "threshold": thr, "tables": tables, "allow": allow, "reason": reason}
|
| 558 |
+
|
| 559 |
def _capture(self, H: torch.Tensor, trig: np.ndarray, t: float) -> None:
|
| 560 |
"""The console's brain view: this minute's mean activity over the rows some strategy's trigger fired on (every
|
| 561 |
row when none did), one file per minute (brain/activity.py). Never allowed to interrupt scoring."""
|
|
|
|
| 572 |
# ---- daily and hourly ----
|
| 573 |
def _resolved(self, conn, since_resolved: float, cols: str, name: str) -> list[dict]:
|
| 574 |
"""A strategy's resolved rows whose labels became known after ``since_resolved`` (epoch s)."""
|
| 575 |
+
return conn.execute(f"SELECT {cols} FROM fly_scored WHERE state = 'resolved' AND label IS NOT NULL AND bootstrap_id = %s AND strategy = %s AND chain = %s "
|
| 576 |
+
"AND ts >= %s ORDER BY ts", (self.boot_id, name, self.chain,
|
| 577 |
+
datetime.fromtimestamp(since_resolved - self.holds[name] - LABEL_LAG_S, timezone.utc))).fetchall()
|
| 578 |
|
| 579 |
def _calibrate(self, conn, now: float) -> None:
|
| 580 |
day = datetime.fromtimestamp(now, timezone.utc).date()
|
| 581 |
+
rule = fly_calibrate.RULE
|
| 582 |
for name in self.names:
|
| 583 |
+
rows = self._resolved(conn, now - fly_calibrate.WINDOW_DAYS * 86400.0, "ts, mint, score, frozen_score, label, teacher_allow", name)
|
| 584 |
+
target = None
|
| 585 |
+
if rule == "match":
|
| 586 |
+
# the teacher's trades over the same rows are the budget; rows scored before it was recorded cannot count
|
| 587 |
+
rows = [r for r in rows if r["teacher_allow"] is not None]
|
| 588 |
+
if not rows:
|
| 589 |
+
log.info("fly %s: no scored rows with the teacher's decision yet; the line stays %.4f", name, self.lines["plastic"][name])
|
| 590 |
+
continue
|
| 591 |
if not rows:
|
| 592 |
continue
|
| 593 |
ts = np.array([r["ts"].timestamp() for r in rows]); mint = np.array([r["mint"] for r in rows]); lab = np.array([r["label"] for r in rows], dtype=np.float64)
|
| 594 |
+
if rule == "match":
|
| 595 |
+
al = np.array([bool(r["teacher_allow"]) for r in rows]) & np.isfinite(lab)
|
| 596 |
+
target = len(fly_calibrate.taken_idx(ts, mint, self.holds[name], np.flatnonzero(al)))
|
| 597 |
for arm, col in (("plastic", "score"), ("frozen", "frozen_score")):
|
| 598 |
cal = fly_calibrate.calibrate(ts, mint, self.holds[name], np.array([r[col] for r in rows], dtype=np.float64), lab,
|
| 599 |
+
prev_line=self.lines[arm][name], prev_sizing=self.sizing[arm][name], rule=rule, target_trades=target)
|
| 600 |
if cal.changed:
|
| 601 |
+
record_event("info", "fly", f"{name} {arm} line {self.lines[arm][name] * 100:+.2f}% → {cal.line * 100:+.2f}%",
|
| 602 |
+
{"trades": cal.trades, "mean": cal.mean, "total": cal.total, "rule": rule, "teacher_trades": target})
|
| 603 |
self.lines[arm][name], self.sizing[arm][name] = cal.line, cal.sizing
|
| 604 |
conn.execute("INSERT INTO fly_calibrations (day, arm, line, sizing, trades, total, mean, window_days) VALUES (%s,%s,%s,%s,%s,%s,%s,%s) "
|
| 605 |
"ON CONFLICT (day, arm) DO UPDATE SET line = EXCLUDED.line, sizing = EXCLUDED.sizing, trades = EXCLUDED.trades, total = EXCLUDED.total, mean = EXCLUDED.mean",
|
| 606 |
+
(day, f"{arm}:{name}" if self.chain == "sol" else f"{arm}:{self.chain}:{name}", cal.line, json.dumps(cal.sizing), cal.trades,
|
| 607 |
+
cal.total, cal.mean, fly_calibrate.WINDOW_DAYS))
|
| 608 |
|
| 609 |
def _drift(self, j: int) -> float:
|
| 610 |
return float(self.bank.drift(j)[0]) if self.bank.learn.shape[0] > 1 else float(self.bank.drift()[0])
|
|
|
|
| 626 |
return out
|
| 627 |
|
| 628 |
def _hourly(self, conn, now: float) -> None:
|
| 629 |
+
if self.brain is not None:
|
| 630 |
+
# another chain's book: its own handover; the shared memory's checks, snapshots and rollbacks are the Solana book's
|
| 631 |
+
self._handover_check(conn, now)
|
| 632 |
+
return
|
| 633 |
hour = datetime.fromtimestamp(math.floor(now / 3600) * 3600 - 3600, timezone.utc)
|
| 634 |
checks = self._checks(conn, now); self.last_checks = checks
|
| 635 |
h = self.hour_stats; first = checks.get(self.names[0]) or {}
|
|
|
|
| 650 |
d = fly_state_dir(); d.mkdir(parents=True, exist_ok=True)
|
| 651 |
path = d / f"snap_{datetime.fromtimestamp(now, timezone.utc).strftime('%Y%m%dT%H%M')}.pt"
|
| 652 |
torch.save({"bank": self.bank.state(), "bootstrap_id": self.boot_id, "lines": self.lines, "sizing": self.sizing}, path)
|
| 653 |
+
note = {"bootstrap_id": self.boot_id, "data": fly_selector.FLY_VERSION, "score": config.FLY_SCORE, "good": good, "checks": checks,
|
| 654 |
"drift": max((c["drift"] for c in checks.values()), default=0.0), "ic24": {k: c.get("ic") for k, c in checks.items()}}
|
| 655 |
return int(conn.execute("INSERT INTO brain_snapshots (run_id, path, kind, note) VALUES (%s,%s,'fly_plastic',%s) RETURNING id",
|
| 656 |
(self.run_id, str(path), json.dumps(note, default=str))).fetchone()["id"])
|
|
|
|
| 679 |
(datetime.fromtimestamp(now - ROLLBACK_MIN_AGE_S, timezone.utc),)).fetchall():
|
| 680 |
n = json.loads(r["note"] or "{}")
|
| 681 |
ok = n.get("good") or not ((n.get("checks") or {}).get(name) or {}).get("triggers")
|
| 682 |
+
if ok and n.get("bootstrap_id") == self.boot_id and n.get("score", "own") == config.FLY_SCORE and Path(r["path"]).exists():
|
| 683 |
target = r; break
|
| 684 |
if target is not None:
|
| 685 |
snap = plastic.PlasticBank(self.fly.net, [(0.0, math.inf)], fly_selector.SCALE, learn=self.bank.learn.cpu().numpy(), read=self.bank.read.cpu().numpy())
|
| 686 |
+
st = torch.load(target["path"], map_location="cpu", weights_only=True)["bank"]
|
| 687 |
snap.D = torch.zeros_like(snap.D); snap.D[:, snap.mask.bool()] = st["D"].to(snap.dev).float()
|
| 688 |
self.bank.D[0][:, cols] = snap.D[0][:, cols]
|
| 689 |
else:
|
|
|
|
| 722 |
whole race), closed ``config.HANDOVER_MIN_TRADES`` paper trades and, when ``config.HANDOVER_BEAT_SELECTOR``,
|
| 723 |
realized at least the paper selector's P&L over that window."""
|
| 724 |
days = config.HANDOVER_DAYS
|
| 725 |
+
if handover.state(conn, self.chain) is not None or self.race_started_at is None or now - self.race_started_at < days * 86400.0:
|
| 726 |
return
|
| 727 |
since = datetime.fromtimestamp(self.race_started_at if days <= 0 else now - days * 86400.0, timezone.utc)
|
| 728 |
+
fb, sb = self.BOOK, self.market.selector_book # this chain's race: its fly book against its selector book
|
| 729 |
pnl = {b: conn.execute("SELECT COALESCE(sum(realized_sol), 0) AS s, count(*) AS n FROM positions WHERE book = %s AND status = 'closed' AND closed_at >= %s",
|
| 730 |
+
(b, since)).fetchone() for b in (fb, sb)}
|
| 731 |
+
fly_pnl, sel_pnl, n = float(pnl[fb]["s"]), float(pnl[sb]["s"]), int(pnl[fb]["n"])
|
| 732 |
if n >= config.HANDOVER_MIN_TRADES and (fly_pnl >= sel_pnl or not config.HANDOVER_BEAT_SELECTOR):
|
| 733 |
detail = {"at": datetime.fromtimestamp(now, timezone.utc).isoformat(), "fly_pnl_sol": fly_pnl, "selector_pnl_sol": sel_pnl, "fly_trades": n,
|
| 734 |
+
"selector_trades": int(pnl[sb]["n"]), "days": days, "beat_selector": config.HANDOVER_BEAT_SELECTOR, "bootstrap": self.boot_id,
|
| 735 |
+
"chain": self.chain, "unit": self.market.unit}
|
| 736 |
+
handover.record(conn, detail, self.chain)
|
| 737 |
record_event("info", "handover", f"the fly takes the selector's seat: {fly_pnl:+.3f} SOL vs {sel_pnl:+.3f} SOL over {n} trades"
|
| 738 |
+ (f" in {days} days" if days > 0 else ""), detail)
|
fly_trader/agent/handover.py
CHANGED
|
@@ -11,14 +11,22 @@ import json
|
|
| 11 |
KEY = "handover"
|
| 12 |
|
| 13 |
|
| 14 |
-
def
|
| 15 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 16 |
if not r or r["value"] is None:
|
| 17 |
return None
|
| 18 |
v = r["value"] if isinstance(r["value"], dict) else json.loads(r["value"])
|
| 19 |
return v or None
|
| 20 |
|
| 21 |
|
| 22 |
-
def record(conn, detail: dict) -> None:
|
| 23 |
conn.execute("INSERT INTO ui_settings (key, value) VALUES (%s, %s) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()",
|
| 24 |
-
(
|
|
|
|
| 11 |
KEY = "handover"
|
| 12 |
|
| 13 |
|
| 14 |
+
def _key(chain: str | None) -> str:
|
| 15 |
+
"""Each chain's fly takes its own chain's seat (fly_trader/markets: 'handover', 'handover_rh')."""
|
| 16 |
+
if chain in (None, "sol"):
|
| 17 |
+
return KEY
|
| 18 |
+
from ..markets import for_chain
|
| 19 |
+
return for_chain(chain).handover_key
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def state(conn, chain: str | None = "sol") -> dict | None:
|
| 23 |
+
r = conn.execute("SELECT value FROM ui_settings WHERE key = %s", (_key(chain),)).fetchone()
|
| 24 |
if not r or r["value"] is None:
|
| 25 |
return None
|
| 26 |
v = r["value"] if isinstance(r["value"], dict) else json.loads(r["value"])
|
| 27 |
return v or None
|
| 28 |
|
| 29 |
|
| 30 |
+
def record(conn, detail: dict, chain: str | None = "sol") -> None:
|
| 31 |
conn.execute("INSERT INTO ui_settings (key, value) VALUES (%s, %s) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()",
|
| 32 |
+
(_key(chain), json.dumps(detail, default=str)))
|
fly_trader/agent/minute_engine.py
CHANGED
|
@@ -30,6 +30,7 @@ from datetime import datetime, timezone
|
|
| 30 |
|
| 31 |
import numpy as np
|
| 32 |
|
|
|
|
| 33 |
from ..db.apilog import record_event
|
| 34 |
from ..db.connection import transaction
|
| 35 |
from ..market.exit_cost import PUMP_SUPPLY
|
|
@@ -78,6 +79,7 @@ class Minute:
|
|
| 78 |
fresh: bool
|
| 79 |
trade: bool
|
| 80 |
engine: "MinuteEngine"
|
|
|
|
| 81 |
|
| 82 |
@property
|
| 83 |
def t_start(self) -> float:
|
|
@@ -91,7 +93,14 @@ class Minute:
|
|
| 91 |
|
| 92 |
|
| 93 |
class MinuteEngine:
|
| 94 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 95 |
self.books = list(books or []); self.live = live
|
| 96 |
self.states: dict[str, MintState] = {}
|
| 97 |
self.meta_cache: dict[str, dict] = {}; self.last_sweep = time.time(); self.last_minute: float | None = None
|
|
@@ -100,21 +109,34 @@ class MinuteEngine:
|
|
| 100 |
def has_book(self, name: str) -> bool:
|
| 101 |
return any(b.name == name for b in self.books)
|
| 102 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 103 |
# ---- lookups ----
|
| 104 |
def _meta(self, conn, mint: str) -> dict | None:
|
| 105 |
"""``corpus_meta`` row (graduation time + creation/creator features), cached for 10 minutes: rows appear at
|
| 106 |
graduation and the archive rebuild later replaces stream values, so the cache is refreshed."""
|
| 107 |
hit = self.meta_cache.get(mint)
|
| 108 |
if hit is None or time.time() - hit["at"] > META_TTL_S:
|
| 109 |
-
|
|
|
|
|
|
|
|
|
|
| 110 |
hit = self.meta_cache[mint] = {"row": dict(r) if r else None, "at": time.time()}
|
| 111 |
return hit["row"]
|
| 112 |
|
| 113 |
def _state(self, conn, mint: str, pool: str | None, program_label: str | None) -> MintState:
|
| 114 |
s = self.states.get(mint)
|
| 115 |
if s is None:
|
| 116 |
-
|
| 117 |
-
|
|
|
|
|
|
|
|
|
|
| 118 |
self.states[mint] = s
|
| 119 |
row = self._meta(conn, mint)
|
| 120 |
s.meta_row = row
|
|
@@ -128,8 +150,11 @@ class MinuteEngine:
|
|
| 128 |
s = self.states.get(mint)
|
| 129 |
if s is not None and s.st.last_res_quote_sol:
|
| 130 |
return s.st.last_res_quote_sol
|
| 131 |
-
|
| 132 |
-
|
|
|
|
|
|
|
|
|
|
| 133 |
|
| 134 |
def _mcap(self, mint: str, price: float | None) -> float | None:
|
| 135 |
"""Market cap in SOL (price × supply), which sets the pool fee tier; None without a price."""
|
|
@@ -138,7 +163,16 @@ class MinuteEngine:
|
|
| 138 |
|
| 139 |
# ---- one minute ----
|
| 140 |
def _aggregate(self, conn, m0: datetime, m1: datetime) -> dict[str, dict]:
|
| 141 |
-
"""PumpAPI minutes: every SOL-quoted PumpSwap pump.fun token, the same fields as the archive.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 142 |
rows = conn.execute("SELECT mint, pool_id, open, high, low, close, buy_sol, sell_sol, n_buys, n_sells, n_traders, resq_sol, fee_rate, "
|
| 143 |
"n_buyers, wash_sol, wash_buy_sol, top_sell_sol, insider_sell_sol, skill_buy FROM pump_minutes WHERE ts = %s", (m0,)).fetchall()
|
| 144 |
return {r["mint"]: {"open": r["open"], "high": r["high"], "low": r["low"], "close": r["close"], "buy": r["buy_sol"] or 0.0, "sell": r["sell_sol"] or 0.0,
|
|
@@ -151,7 +185,9 @@ class MinuteEngine:
|
|
| 151 |
"""The full feature vector (``X_COLS``) of a mint for the minute ending ``t_end``, and what trading needs."""
|
| 152 |
s = self._state(conn, mint, a["pool"], a["program_label"])
|
| 153 |
st, price, resq = s.st, float(a["close"]), a["resq"]
|
| 154 |
-
if
|
|
|
|
|
|
|
| 155 |
s.broken = True # train/decisions.py: a scale break makes the mint ineligible from that minute on
|
| 156 |
s.prev_close = price
|
| 157 |
if a["buy"] > 0:
|
|
@@ -175,21 +211,27 @@ class MinuteEngine:
|
|
| 175 |
"meta_known": 1.0 if meta.get("ttg_min") is not None else 0.0, **s.flow.features(t_end), "mkt_vol_1h": self.market.value(t_end)}
|
| 176 |
by_name = {**{n: float(f[i]) for i, n in enumerate(FEATURES)}, **extra,
|
| 177 |
**{c: float(meta.get(c)) if meta.get(c) is not None else 0.0 for c in META_COLS}}
|
|
|
|
|
|
|
|
|
|
| 178 |
x = np.nan_to_num(np.asarray([by_name.get(c, 0.0) for c in X_COLS], dtype=np.float32), nan=0.0, posinf=0.0, neginf=0.0)
|
| 179 |
info = {"price": price, "open": float(a["open"]) if a.get("open") else price, "resq": resq, "logvol_15m": f[FIDX["logvol_15m"]],
|
| 180 |
# "ec" is the training label's own cost model (market/exit_cost.cost_at_size), at the size the book trades
|
| 181 |
-
"ec": float(cost_at_size(f[FIDX["exit_cost_0p1"]], resq
|
|
|
|
| 182 |
"mcap": price * s.meta.supply, "fee_rate": a.get("fee_rate"), "decimals": s.decimals, "pool": s.pool, "program_label": s.program_label, "broken": s.broken,
|
| 183 |
-
"skill_missing": a.get("skill_buy") is None, "curve_known": bool(meta.get("curve_known"))
|
|
|
|
| 184 |
return x, info
|
| 185 |
|
| 186 |
-
|
| 187 |
-
|
|
|
|
| 188 |
return not info["broken"] and info["resq"] is not None and info["resq"] >= MIN_RESQ_SOL and info["logvol_15m"] >= math.log1p(MIN_VOL_15M_SOL)
|
| 189 |
|
| 190 |
def _stream_through(self) -> float | None:
|
| 191 |
with transaction() as conn:
|
| 192 |
-
r = conn.execute("SELECT value->>'flushed_through' AS ft FROM ui_settings WHERE key =
|
| 193 |
return datetime.fromisoformat(r["ft"]).timestamp() if r and r["ft"] else None
|
| 194 |
|
| 195 |
def ready_through(self, m1_epoch: float) -> float:
|
|
@@ -224,7 +266,7 @@ class MinuteEngine:
|
|
| 224 |
for mint, a in agg.items():
|
| 225 |
_x, info = self._features(conn, mint, a, t1); n += 1
|
| 226 |
bars[mint] = (t1 - 60.0, info["open"], info["price"], info["ec"])
|
| 227 |
-
for b in self.
|
| 228 |
b.on_bars(t1 - 60.0, bars)
|
| 229 |
log.info("warm-up: %d token-minutes over the last %d minutes (%d tokens)", n, minutes, len(self.states))
|
| 230 |
return n
|
|
@@ -247,9 +289,10 @@ class MinuteEngine:
|
|
| 247 |
X, infos, mints, bars = self._minute_rows(conn, agg, m1_epoch)
|
| 248 |
ctx = Minute(conn=conn, m0=m0, m1=m1, m1_epoch=m1_epoch, agg=agg, X=X, infos=infos, mints=mints, bars=bars,
|
| 249 |
prices={m: float(a["close"]) for m, a in agg.items()}, resqs={m: a["resq"] for m, a in agg.items()},
|
| 250 |
-
fees={m: a.get("fee_rate") for m, a in agg.items()}, fresh=trade and self.stream_fresh(m1_epoch - 60), trade=trade, engine=self
|
|
|
|
| 251 |
out.update(mints_traded=len(agg), eligible=len(mints)); held: set[str] = set()
|
| 252 |
-
for b in list(self.
|
| 253 |
try:
|
| 254 |
with conn.transaction(): # a savepoint per book
|
| 255 |
b.on_bars(m1_epoch - 60.0, bars)
|
|
@@ -259,7 +302,7 @@ class MinuteEngine:
|
|
| 259 |
log.exception("%s: minute %s failed", b.name, m1.isoformat())
|
| 260 |
record_event("error", b.name, f"minute failed: {m1.isoformat()}")
|
| 261 |
self._sweep(m1_epoch, held)
|
| 262 |
-
for b in [b for b in self.
|
| 263 |
b.finish(); self.books.remove(b)
|
| 264 |
return out
|
| 265 |
|
|
@@ -288,35 +331,45 @@ def admit_books(engine: MinuteEngine, live: bool) -> list:
|
|
| 288 |
from . import fly_session, handover
|
| 289 |
from .selector_session import NoModel, SelectorBook
|
| 290 |
new = []
|
| 291 |
-
|
| 292 |
-
|
| 293 |
-
if
|
| 294 |
-
new.
|
| 295 |
-
|
| 296 |
-
|
| 297 |
-
|
| 298 |
-
|
| 299 |
-
|
| 300 |
-
|
| 301 |
-
|
| 302 |
-
|
| 303 |
-
|
| 304 |
-
|
|
|
|
|
|
|
|
|
|
| 305 |
return new
|
| 306 |
|
| 307 |
|
| 308 |
def main(stop_event: threading.Event | None = None, live: bool = False) -> None:
|
| 309 |
-
engine
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 310 |
while not (stop_event is not None and stop_event.is_set()):
|
| 311 |
-
|
| 312 |
-
if
|
| 313 |
break
|
| 314 |
log.info("trading engine waiting: no book qualifies yet")
|
| 315 |
(stop_event or threading.Event()).wait(MODEL_CHECK_S)
|
| 316 |
else:
|
| 317 |
return
|
| 318 |
m_start = math.floor(time.time() / 60) * 60
|
| 319 |
-
|
|
|
|
|
|
|
| 320 |
last_release, last_stamp_check = release_stamp(), 0.0
|
| 321 |
try:
|
| 322 |
while not (stop_event is not None and stop_event.is_set()):
|
|
@@ -327,31 +380,35 @@ def main(stop_event: threading.Event | None = None, live: bool = False) -> None:
|
|
| 327 |
last_release, last_check = stamp, 0.0
|
| 328 |
if now - last_check >= MODEL_CHECK_S:
|
| 329 |
last_check = now
|
| 330 |
-
for b in
|
| 331 |
try:
|
| 332 |
b.maybe_reload()
|
| 333 |
except Exception:
|
| 334 |
log.exception("%s: model reload check failed", b.name)
|
| 335 |
try:
|
| 336 |
-
|
| 337 |
except Exception:
|
| 338 |
log.exception("book admission failed")
|
| 339 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 340 |
for minute in range(int(engine.last_minute) + 60, int(upto) + 1, 60):
|
| 341 |
try:
|
| 342 |
-
st = engine.run_minute(float(minute), trade=(minute
|
| 343 |
n_min += 1
|
| 344 |
-
if minute
|
| 345 |
-
log.info("minute %s: traded %d eligible %d | %s", st["minute"], st.get("mints_traded", 0), st.get("eligible", 0),
|
| 346 |
" | ".join(f"{b.name}: picks {(st.get(b.name) or {}).get('picks')} open {(st.get(b.name) or {}).get('open')} wealth {(st.get(b.name) or {}).get('wealth')}"
|
| 347 |
-
for b in engine.
|
| 348 |
except Exception:
|
| 349 |
-
log.exception("minute %s failed", minute); record_event("error", "engine", f"minute failed: {minute}")
|
| 350 |
engine.last_minute = upto
|
| 351 |
time.sleep(0.5)
|
| 352 |
finally:
|
| 353 |
-
for b in
|
| 354 |
try:
|
| 355 |
b.finish()
|
| 356 |
except Exception:
|
| 357 |
-
log.exception("%s: finish failed", b.name)
|
|
|
|
| 30 |
|
| 31 |
import numpy as np
|
| 32 |
|
| 33 |
+
from .. import config, markets
|
| 34 |
from ..db.apilog import record_event
|
| 35 |
from ..db.connection import transaction
|
| 36 |
from ..market.exit_cost import PUMP_SUPPLY
|
|
|
|
| 79 |
fresh: bool
|
| 80 |
trade: bool
|
| 81 |
engine: "MinuteEngine"
|
| 82 |
+
market: object = None # fly_trader/markets.MarketSpec of these rows (None: Solana)
|
| 83 |
|
| 84 |
@property
|
| 85 |
def t_start(self) -> float:
|
|
|
|
| 93 |
|
| 94 |
|
| 95 |
class MinuteEngine:
|
| 96 |
+
"""One market's minutes (``market``: Solana — pump_minutes, corpus_meta — or Robinhood Chain — rh_minutes, rh_meta,
|
| 97 |
+
amounts in ETH). The engines of both markets share one list of books; each minute goes only to its market's books."""
|
| 98 |
+
|
| 99 |
+
def __init__(self, books: list | None = None, live: bool = False, market=None):
|
| 100 |
+
self.spec = market or markets.SOL; self.rh = self.spec.chain != "sol"
|
| 101 |
+
# how old a minute may be and still be traded: Solana's feed flushes within seconds; the RH stream's minutes land
|
| 102 |
+
# ~2–3 min late (confirmations, the index pass, the price marks), so on RH "the current minute" never arrives
|
| 103 |
+
self.trade_lag_s = float(config.RH_TRADE_LAG_S) if self.rh else 0.0
|
| 104 |
self.books = list(books or []); self.live = live
|
| 105 |
self.states: dict[str, MintState] = {}
|
| 106 |
self.meta_cache: dict[str, dict] = {}; self.last_sweep = time.time(); self.last_minute: float | None = None
|
|
|
|
| 109 |
def has_book(self, name: str) -> bool:
|
| 110 |
return any(b.name == name for b in self.books)
|
| 111 |
|
| 112 |
+
def tradeable(self, minute: float, m1: float) -> bool:
|
| 113 |
+
"""Whether minute ``minute`` (its end, epoch s) is recent enough to trade at wall-clock minute ``m1``."""
|
| 114 |
+
return int(m1) - self.trade_lag_s <= int(minute) <= int(m1)
|
| 115 |
+
|
| 116 |
+
def my_books(self) -> list:
|
| 117 |
+
return [b for b in self.books if getattr(b, "chain", "sol") == self.spec.chain]
|
| 118 |
+
|
| 119 |
# ---- lookups ----
|
| 120 |
def _meta(self, conn, mint: str) -> dict | None:
|
| 121 |
"""``corpus_meta`` row (graduation time + creation/creator features), cached for 10 minutes: rows appear at
|
| 122 |
graduation and the archive rebuild later replaces stream values, so the cache is refreshed."""
|
| 123 |
hit = self.meta_cache.get(mint)
|
| 124 |
if hit is None or time.time() - hit["at"] > META_TTL_S:
|
| 125 |
+
if self.rh: # Robinhood Chain: rh_meta, the same columns (rh/meta.py), plus the quote class
|
| 126 |
+
r = conn.execute("SELECT graduated_at, supply, quote_class, " + ", ".join(META_COLS) + " FROM rh_meta WHERE mint = %s", (mint,)).fetchone()
|
| 127 |
+
else:
|
| 128 |
+
r = conn.execute("SELECT graduated_at, supply, " + ", ".join(META_COLS) + " FROM corpus_meta WHERE mint = %s", (mint,)).fetchone()
|
| 129 |
hit = self.meta_cache[mint] = {"row": dict(r) if r else None, "at": time.time()}
|
| 130 |
return hit["row"]
|
| 131 |
|
| 132 |
def _state(self, conn, mint: str, pool: str | None, program_label: str | None) -> MintState:
|
| 133 |
s = self.states.get(mint)
|
| 134 |
if s is None:
|
| 135 |
+
if self.rh: # every Pons launch has 18 decimals
|
| 136 |
+
s = MintState(mint, 18, pool, program_label, None, None); s.meta.ec_size = 0.1 * self.spec.k()
|
| 137 |
+
else:
|
| 138 |
+
t = conn.execute("SELECT decimals FROM tokens WHERE mint = %s", (mint,)).fetchone()
|
| 139 |
+
s = MintState(mint, int((t or {}).get("decimals") or 6), pool, program_label, None, None)
|
| 140 |
self.states[mint] = s
|
| 141 |
row = self._meta(conn, mint)
|
| 142 |
s.meta_row = row
|
|
|
|
| 150 |
s = self.states.get(mint)
|
| 151 |
if s is not None and s.st.last_res_quote_sol:
|
| 152 |
return s.st.last_res_quote_sol
|
| 153 |
+
if self.rh:
|
| 154 |
+
r = conn.execute("SELECT resq_eth AS q FROM rh_minutes WHERE mint = %s AND resq_eth IS NOT NULL ORDER BY ts DESC LIMIT 1", (mint,)).fetchone()
|
| 155 |
+
else:
|
| 156 |
+
r = conn.execute("SELECT resq_sol AS q FROM pump_minutes WHERE mint = %s AND resq_sol IS NOT NULL ORDER BY ts DESC LIMIT 1", (mint,)).fetchone()
|
| 157 |
+
return float(r["q"]) if r else None
|
| 158 |
|
| 159 |
def _mcap(self, mint: str, price: float | None) -> float | None:
|
| 160 |
"""Market cap in SOL (price × supply), which sets the pool fee tier; None without a price."""
|
|
|
|
| 163 |
|
| 164 |
# ---- one minute ----
|
| 165 |
def _aggregate(self, conn, m0: datetime, m1: datetime) -> dict[str, dict]:
|
| 166 |
+
"""PumpAPI minutes: every SOL-quoted PumpSwap pump.fun token, the same fields as the archive. Robinhood Chain: the
|
| 167 |
+
rh_minutes of graduated Pons pools in ETH (rh/minutes.py), under the same keys."""
|
| 168 |
+
if self.rh:
|
| 169 |
+
rows = conn.execute("SELECT mint, pool_id, open, high, low, close, buy_eth, sell_eth, n_buys, n_sells, n_traders, resq_eth, fee_rate, "
|
| 170 |
+
"n_buyers, wash_eth, wash_buy_eth, top_sell_eth, insider_sell_eth, skill_buy FROM rh_minutes WHERE ts = %s", (m0,)).fetchall()
|
| 171 |
+
return {r["mint"]: {"open": r["open"], "high": r["high"], "low": r["low"], "close": r["close"], "buy": r["buy_eth"] or 0.0, "sell": r["sell_eth"] or 0.0,
|
| 172 |
+
"nb": r["n_buys"] or 0, "ns": r["n_sells"] or 0, "n_traders": int(r["n_traders"] or 0), "resq": r["resq_eth"],
|
| 173 |
+
"pool": r["pool_id"], "program_label": self.spec.program_label, "fee_rate": r["fee_rate"],
|
| 174 |
+
"n_buyers": r["n_buyers"], "wash_sol": r["wash_eth"], "wash_buy_sol": r["wash_buy_eth"], "top_sell_sol": r["top_sell_eth"],
|
| 175 |
+
"insider_sell_sol": r["insider_sell_eth"], "skill_buy": r["skill_buy"]} for r in rows if r["close"]}
|
| 176 |
rows = conn.execute("SELECT mint, pool_id, open, high, low, close, buy_sol, sell_sol, n_buys, n_sells, n_traders, resq_sol, fee_rate, "
|
| 177 |
"n_buyers, wash_sol, wash_buy_sol, top_sell_sol, insider_sell_sol, skill_buy FROM pump_minutes WHERE ts = %s", (m0,)).fetchall()
|
| 178 |
return {r["mint"]: {"open": r["open"], "high": r["high"], "low": r["low"], "close": r["close"], "buy": r["buy_sol"] or 0.0, "sell": r["sell_sol"] or 0.0,
|
|
|
|
| 185 |
"""The full feature vector (``X_COLS``) of a mint for the minute ending ``t_end``, and what trading needs."""
|
| 186 |
s = self._state(conn, mint, a["pool"], a["program_label"])
|
| 187 |
st, price, resq = s.st, float(a["close"]), a["resq"]
|
| 188 |
+
if self.rh:
|
| 189 |
+
s.meta.pool_fee = a.get("fee_rate") # the Pons hook fee this minute charged (the cost model's fee)
|
| 190 |
+
if (s.prev_close and not (1 / 50 <= price / s.prev_close <= 50)) or (resq is not None and resq > 1e5 * self.spec.k()):
|
| 191 |
s.broken = True # train/decisions.py: a scale break makes the mint ineligible from that minute on
|
| 192 |
s.prev_close = price
|
| 193 |
if a["buy"] > 0:
|
|
|
|
| 211 |
"meta_known": 1.0 if meta.get("ttg_min") is not None else 0.0, **s.flow.features(t_end), "mkt_vol_1h": self.market.value(t_end)}
|
| 212 |
by_name = {**{n: float(f[i]) for i, n in enumerate(FEATURES)}, **extra,
|
| 213 |
**{c: float(meta.get(c)) if meta.get(c) is not None else 0.0 for c in META_COLS}}
|
| 214 |
+
if self.rh: # the chain inputs (train/decisions.CHAIN_COLS)
|
| 215 |
+
qc = meta.get("quote_class")
|
| 216 |
+
by_name.update(chain_rh=1.0, qc_stable=float(qc == "stable"), qc_btc=float(qc == "btc"), qc_stock=float(qc == "stock"))
|
| 217 |
x = np.nan_to_num(np.asarray([by_name.get(c, 0.0) for c in X_COLS], dtype=np.float32), nan=0.0, posinf=0.0, neginf=0.0)
|
| 218 |
info = {"price": price, "open": float(a["open"]) if a.get("open") else price, "resq": resq, "logvol_15m": f[FIDX["logvol_15m"]],
|
| 219 |
# "ec" is the training label's own cost model (market/exit_cost.cost_at_size), at the size the book trades
|
| 220 |
+
"ec": float(cost_at_size(f[FIDX["exit_cost_0p1"]], resq, market=self.spec if self.rh else None)),
|
| 221 |
+
"age_h": (t_end - s.graduated_at) / 3600 if s.graduated_at else None,
|
| 222 |
"mcap": price * s.meta.supply, "fee_rate": a.get("fee_rate"), "decimals": s.decimals, "pool": s.pool, "program_label": s.program_label, "broken": s.broken,
|
| 223 |
+
"skill_missing": a.get("skill_buy") is None, "curve_known": bool(meta.get("curve_known")), "chain": self.spec.chain,
|
| 224 |
+
"quote_class": meta.get("quote_class")}
|
| 225 |
return x, info
|
| 226 |
|
| 227 |
+
def eligible(self, info: dict) -> bool:
|
| 228 |
+
if self.rh: # the same gates in the chain's unit (x K), as training (train/decisions)
|
| 229 |
+
return not info["broken"] and info["resq"] is not None and info["resq"] >= self.spec.min_resq() and info["logvol_15m"] >= math.log1p(self.spec.min_vol_15m())
|
| 230 |
return not info["broken"] and info["resq"] is not None and info["resq"] >= MIN_RESQ_SOL and info["logvol_15m"] >= math.log1p(MIN_VOL_15M_SOL)
|
| 231 |
|
| 232 |
def _stream_through(self) -> float | None:
|
| 233 |
with transaction() as conn:
|
| 234 |
+
r = conn.execute("SELECT value->>'flushed_through' AS ft FROM ui_settings WHERE key = %s", (self.spec.stream_status_key,)).fetchone()
|
| 235 |
return datetime.fromisoformat(r["ft"]).timestamp() if r and r["ft"] else None
|
| 236 |
|
| 237 |
def ready_through(self, m1_epoch: float) -> float:
|
|
|
|
| 266 |
for mint, a in agg.items():
|
| 267 |
_x, info = self._features(conn, mint, a, t1); n += 1
|
| 268 |
bars[mint] = (t1 - 60.0, info["open"], info["price"], info["ec"])
|
| 269 |
+
for b in self.my_books():
|
| 270 |
b.on_bars(t1 - 60.0, bars)
|
| 271 |
log.info("warm-up: %d token-minutes over the last %d minutes (%d tokens)", n, minutes, len(self.states))
|
| 272 |
return n
|
|
|
|
| 289 |
X, infos, mints, bars = self._minute_rows(conn, agg, m1_epoch)
|
| 290 |
ctx = Minute(conn=conn, m0=m0, m1=m1, m1_epoch=m1_epoch, agg=agg, X=X, infos=infos, mints=mints, bars=bars,
|
| 291 |
prices={m: float(a["close"]) for m, a in agg.items()}, resqs={m: a["resq"] for m, a in agg.items()},
|
| 292 |
+
fees={m: a.get("fee_rate") for m, a in agg.items()}, fresh=trade and self.stream_fresh(m1_epoch - 60), trade=trade, engine=self,
|
| 293 |
+
market=self.spec)
|
| 294 |
out.update(mints_traded=len(agg), eligible=len(mints)); held: set[str] = set()
|
| 295 |
+
for b in list(self.my_books()):
|
| 296 |
try:
|
| 297 |
with conn.transaction(): # a savepoint per book
|
| 298 |
b.on_bars(m1_epoch - 60.0, bars)
|
|
|
|
| 302 |
log.exception("%s: minute %s failed", b.name, m1.isoformat())
|
| 303 |
record_event("error", b.name, f"minute failed: {m1.isoformat()}")
|
| 304 |
self._sweep(m1_epoch, held)
|
| 305 |
+
for b in [b for b in self.my_books() if b.done]:
|
| 306 |
b.finish(); self.books.remove(b)
|
| 307 |
return out
|
| 308 |
|
|
|
|
| 331 |
from . import fly_session, handover
|
| 332 |
from .selector_session import NoModel, SelectorBook
|
| 333 |
new = []
|
| 334 |
+
for m in markets.enabled(): # Solana first: another chain's fly shares the Solana fly's brain
|
| 335 |
+
fly_name = "fly" if m.chain == "sol" else f"fly_{m.chain}"; sel_name = "selector" if m.chain == "sol" else f"selector_{m.chain}"
|
| 336 |
+
if not engine.has_book(fly_name) and not any(b.name == fly_name for b in new):
|
| 337 |
+
brain = next((b for b in engine.books + new if b.name == "fly"), None) if m.chain != "sol" else None
|
| 338 |
+
fb, why = fly_session.try_start(live, chain=m.chain, brain=brain)
|
| 339 |
+
if fb is not None:
|
| 340 |
+
new.append(fb)
|
| 341 |
+
else:
|
| 342 |
+
_status(m.fly_status_key, {"stage": "not trading", "detail": why, "chain": m.chain})
|
| 343 |
+
fly_on = engine.has_book(fly_name) or any(b.name == fly_name for b in new)
|
| 344 |
+
with transaction() as conn:
|
| 345 |
+
handed = handover.state(conn, m.chain) is not None
|
| 346 |
+
if not engine.has_book(sel_name) and not (handed and fly_on):
|
| 347 |
+
try:
|
| 348 |
+
new.append(SelectorBook(m.chain))
|
| 349 |
+
except NoModel as e:
|
| 350 |
+
_status(m.selector_status_key, {"stage": "waiting for a model", "detail": str(e), "chain": m.chain})
|
| 351 |
return new
|
| 352 |
|
| 353 |
|
| 354 |
def main(stop_event: threading.Event | None = None, live: bool = False) -> None:
|
| 355 |
+
"""One engine per enabled chain (markets.enabled()), all sharing one list of books: each engine feeds its minutes, as
|
| 356 |
+
its own stream writes them, to its chain's books only — a lagging chain never holds another chain's minutes back."""
|
| 357 |
+
engines = [MinuteEngine(live=live, market=m) for m in markets.enabled()]
|
| 358 |
+
lead = engines[0]; books: list = []
|
| 359 |
+
for e in engines:
|
| 360 |
+
e.books = books # the same list object: admission and removal are seen by every engine
|
| 361 |
while not (stop_event is not None and stop_event.is_set()):
|
| 362 |
+
books += admit_books(lead, live)
|
| 363 |
+
if books:
|
| 364 |
break
|
| 365 |
log.info("trading engine waiting: no book qualifies yet")
|
| 366 |
(stop_event or threading.Event()).wait(MODEL_CHECK_S)
|
| 367 |
else:
|
| 368 |
return
|
| 369 |
m_start = math.floor(time.time() / 60) * 60
|
| 370 |
+
for e in engines:
|
| 371 |
+
e.warm_up(m_start); e.last_minute = m_start - 60
|
| 372 |
+
last_check = time.time(); n_min = 0
|
| 373 |
last_release, last_stamp_check = release_stamp(), 0.0
|
| 374 |
try:
|
| 375 |
while not (stop_event is not None and stop_event.is_set()):
|
|
|
|
| 380 |
last_release, last_check = stamp, 0.0
|
| 381 |
if now - last_check >= MODEL_CHECK_S:
|
| 382 |
last_check = now
|
| 383 |
+
for b in books:
|
| 384 |
try:
|
| 385 |
b.maybe_reload()
|
| 386 |
except Exception:
|
| 387 |
log.exception("%s: model reload check failed", b.name)
|
| 388 |
try:
|
| 389 |
+
books += admit_books(lead, live)
|
| 390 |
except Exception:
|
| 391 |
log.exception("book admission failed")
|
| 392 |
+
for engine in engines:
|
| 393 |
+
if not engine.my_books() or m1 <= engine.last_minute or now - m1 < 4.0:
|
| 394 |
+
continue
|
| 395 |
+
if (upto := engine.ready_through(m1)) <= engine.last_minute: # only minutes this chain's stream has written
|
| 396 |
+
continue
|
| 397 |
for minute in range(int(engine.last_minute) + 60, int(upto) + 1, 60):
|
| 398 |
try:
|
| 399 |
+
st = engine.run_minute(float(minute), trade=engine.tradeable(minute, m1)) # older minutes update features (and the fly's learning) only
|
| 400 |
n_min += 1
|
| 401 |
+
if engine.tradeable(minute, m1) and n_min % 10 == 0:
|
| 402 |
+
log.info("%s minute %s: traded %d eligible %d | %s", engine.spec.chain, st["minute"], st.get("mints_traded", 0), st.get("eligible", 0),
|
| 403 |
" | ".join(f"{b.name}: picks {(st.get(b.name) or {}).get('picks')} open {(st.get(b.name) or {}).get('open')} wealth {(st.get(b.name) or {}).get('wealth')}"
|
| 404 |
+
for b in engine.my_books()))
|
| 405 |
except Exception:
|
| 406 |
+
log.exception("%s minute %s failed", engine.spec.chain, minute); record_event("error", "engine", f"{engine.spec.chain} minute failed: {minute}")
|
| 407 |
engine.last_minute = upto
|
| 408 |
time.sleep(0.5)
|
| 409 |
finally:
|
| 410 |
+
for b in books:
|
| 411 |
try:
|
| 412 |
b.finish()
|
| 413 |
except Exception:
|
| 414 |
+
log.exception("%s: finish failed", b.name)
|
fly_trader/agent/paper_trading.py
CHANGED
|
@@ -20,10 +20,11 @@ from ..market.exit_cost import exit_cost_fraction
|
|
| 20 |
|
| 21 |
def trade_minute(ctx, *, book: str, run_id: str, beat_no: int, broker, kind: str, mints: list, infos: list, scores, threshold,
|
| 22 |
table: list | None, horizon_s: float, block: str | None = None, holds=None, strategies=None, allow=None, reasons=None,
|
| 23 |
-
tables: list | None = None) -> dict:
|
| 24 |
"""Optional per-row arguments (the selector's strategy stack, train/strategies.decide): ``threshold`` may be per row,
|
| 25 |
``holds`` (s), ``strategies``, ``tables`` (sizing) and ``allow`` / ``reasons`` (a blocked row is recorded with its reason).
|
| 26 |
-
Left out, the call behaves exactly as the single-strategy engine.
|
|
|
|
| 27 |
conn, m1 = ctx.conn, ctx.m1
|
| 28 |
beat = conn.execute("INSERT INTO beats (run_id, ts, beat_no, n_slots_active, notes) VALUES (%s,%s,%s,%s,%s) RETURNING id",
|
| 29 |
(run_id, m1, beat_no, len(mints), json.dumps({"minute": m1.isoformat(), "mints_traded": len(ctx.agg), "book": book}))).fetchone()["id"]
|
|
@@ -41,7 +42,9 @@ def trade_minute(ctx, *, book: str, run_id: str, beat_no: int, broker, kind: str
|
|
| 41 |
opens_now = ledger.open_positions(conn, book); open_mints = {p["mint"] for p in opens_now}
|
| 42 |
cash = ledger.paper_cash(conn, book); n_enter = 0; picks = []; entries = []
|
| 43 |
bankroll = cash + sum(float(p["cost_sol"]) for p in opens_now) # wealth at cost: the sizing base (agent/sizing.py)
|
| 44 |
-
|
|
|
|
|
|
|
| 45 |
blocked = (block or ("kill switch" if circuit and circuit["kill_switch"] else None) or ("paused" if circuit and circuit["entries_paused"] else None)
|
| 46 |
or ("book halted" if rails.book_halted(conn, book) else None))
|
| 47 |
thr_v = np.broadcast_to(np.asarray(threshold, dtype=np.float64), (len(mints),)) if len(mints) else np.array([])
|
|
@@ -51,7 +54,7 @@ def trade_minute(ctx, *, book: str, run_id: str, beat_no: int, broker, kind: str
|
|
| 51 |
continue
|
| 52 |
picks.append((m, float(sc)))
|
| 53 |
tab_k = tables[k] if tables is not None else table
|
| 54 |
-
size, why = sizing.size_position(float(sc), thr_k, tab_k or [], bankroll, cash, i["resq"])
|
| 55 |
denied = allow is not None and not bool(allow[k])
|
| 56 |
if m in open_mints or blocked or denied or size <= 0:
|
| 57 |
rail = blocked or ("held" if m in open_mints else (str(reasons[k]) if denied and reasons is not None else "filtered") if denied else "sizing")
|
|
@@ -84,6 +87,6 @@ def trade_minute(ctx, *, book: str, run_id: str, beat_no: int, broker, kind: str
|
|
| 84 |
peak_row = conn.execute("SELECT max(wealth) AS pk FROM wealth_marks WHERE book = %s", (book,)).fetchone(); peak = max(float(peak_row["pk"] or 0.0), wealth)
|
| 85 |
conn.execute("INSERT INTO wealth_marks (beat_id, book, ts, sol_free, positions_value, exit_cost, wealth, peak, drawdown, exposure, n_open) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)",
|
| 86 |
(beat, book, m1, cash, positions_net, exit_cost, wealth, peak, (1.0 - wealth / peak) if peak > 0 else 0.0, exposure, len(opens)))
|
| 87 |
-
rails.check_book_drawdown(conn, book, wealth, peak)
|
| 88 |
return {"beat_id": beat, "entries": entries, "entered": n_enter, "exited": n_exit, "open": len(opens), "cash": cash, "wealth": wealth, "picks": len(picks), "blocked": blocked,
|
| 89 |
"top_scores": sorted(picks, key=lambda x: -x[1])[:5], "open_mints": sorted({p["mint"] for p in opens})}
|
|
|
|
| 20 |
|
| 21 |
def trade_minute(ctx, *, book: str, run_id: str, beat_no: int, broker, kind: str, mints: list, infos: list, scores, threshold,
|
| 22 |
table: list | None, horizon_s: float, block: str | None = None, holds=None, strategies=None, allow=None, reasons=None,
|
| 23 |
+
tables: list | None = None, flat: bool = False) -> dict:
|
| 24 |
"""Optional per-row arguments (the selector's strategy stack, train/strategies.decide): ``threshold`` may be per row,
|
| 25 |
``holds`` (s), ``strategies``, ``tables`` (sizing) and ``allow`` / ``reasons`` (a blocked row is recorded with its reason).
|
| 26 |
+
Left out, the call behaves exactly as the single-strategy engine. ``flat``: every buy the same share of the bankroll
|
| 27 |
+
(agent/sizing.size_position), the fly's rule (``config.FLY_SIZING``)."""
|
| 28 |
conn, m1 = ctx.conn, ctx.m1
|
| 29 |
beat = conn.execute("INSERT INTO beats (run_id, ts, beat_no, n_slots_active, notes) VALUES (%s,%s,%s,%s,%s) RETURNING id",
|
| 30 |
(run_id, m1, beat_no, len(mints), json.dumps({"minute": m1.isoformat(), "mints_traded": len(ctx.agg), "book": book}))).fetchone()["id"]
|
|
|
|
| 42 |
opens_now = ledger.open_positions(conn, book); open_mints = {p["mint"] for p in opens_now}
|
| 43 |
cash = ledger.paper_cash(conn, book); n_enter = 0; picks = []; entries = []
|
| 44 |
bankroll = cash + sum(float(p["cost_sol"]) for p in opens_now) # wealth at cost: the sizing base (agent/sizing.py)
|
| 45 |
+
from ..markets import for_book
|
| 46 |
+
market = for_book(book) # the book's market: its circuit, money settings and sizes
|
| 47 |
+
circuit = conn.execute("SELECT kill_switch, entries_paused FROM circuit_state WHERE id = %s", (market.circuit_id,)).fetchone()
|
| 48 |
blocked = (block or ("kill switch" if circuit and circuit["kill_switch"] else None) or ("paused" if circuit and circuit["entries_paused"] else None)
|
| 49 |
or ("book halted" if rails.book_halted(conn, book) else None))
|
| 50 |
thr_v = np.broadcast_to(np.asarray(threshold, dtype=np.float64), (len(mints),)) if len(mints) else np.array([])
|
|
|
|
| 54 |
continue
|
| 55 |
picks.append((m, float(sc)))
|
| 56 |
tab_k = tables[k] if tables is not None else table
|
| 57 |
+
size, why = sizing.size_position(float(sc), thr_k, tab_k or [], bankroll, cash, i["resq"], flat=flat, market=market)
|
| 58 |
denied = allow is not None and not bool(allow[k])
|
| 59 |
if m in open_mints or blocked or denied or size <= 0:
|
| 60 |
rail = blocked or ("held" if m in open_mints else (str(reasons[k]) if denied and reasons is not None else "filtered") if denied else "sizing")
|
|
|
|
| 87 |
peak_row = conn.execute("SELECT max(wealth) AS pk FROM wealth_marks WHERE book = %s", (book,)).fetchone(); peak = max(float(peak_row["pk"] or 0.0), wealth)
|
| 88 |
conn.execute("INSERT INTO wealth_marks (beat_id, book, ts, sol_free, positions_value, exit_cost, wealth, peak, drawdown, exposure, n_open) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)",
|
| 89 |
(beat, book, m1, cash, positions_net, exit_cost, wealth, peak, (1.0 - wealth / peak) if peak > 0 else 0.0, exposure, len(opens)))
|
| 90 |
+
rails.check_book_drawdown(conn, book, wealth, peak, drawdown=market.kill_drawdown(), unit=market.unit) # the book's own kill switch
|
| 91 |
return {"beat_id": beat, "entries": entries, "entered": n_enter, "exited": n_exit, "open": len(opens), "cash": cash, "wealth": wealth, "picks": len(picks), "blocked": blocked,
|
| 92 |
"top_scores": sorted(picks, key=lambda x: -x[1])[:5], "open_mints": sorted({p["mint"] for p in opens})}
|
fly_trader/agent/rails.py
CHANGED
|
@@ -1,6 +1,9 @@
|
|
| 1 |
"""Risk rails outside the model. KillSwitch (−KILL_SWITCH_DRAWDOWN from peak wealth → no entries), CircuitBreaker
|
| 2 |
(CIRCUIT_THRESHOLD consecutive execution failures), the rolling 24 h notional ledger, and PauseEntries (console
|
| 3 |
toggle). State lives in circuit_state / circuit_events / notional_ledger; the trading engine reads it before entries.
|
|
|
|
|
|
|
|
|
|
| 4 |
"""
|
| 5 |
from __future__ import annotations
|
| 6 |
|
|
@@ -13,6 +16,7 @@ from ..db.apilog import record_event
|
|
| 13 |
from ..db.connection import transaction
|
| 14 |
|
| 15 |
log = logging.getLogger(__name__)
|
|
|
|
| 16 |
|
| 17 |
|
| 18 |
@dataclass
|
|
@@ -25,63 +29,80 @@ class CircuitState:
|
|
| 25 |
entries_paused: bool = False
|
| 26 |
|
| 27 |
|
| 28 |
-
def load_circuit(conn) -> CircuitState:
|
| 29 |
-
r = conn.execute("SELECT * FROM circuit_state WHERE id =
|
| 30 |
if not r:
|
| 31 |
return CircuitState()
|
| 32 |
return CircuitState(r["fail_count"], r["tripped"], r["kill_switch"], r["kill_reason"], r["peak_wealth"], r["entries_paused"])
|
| 33 |
|
| 34 |
|
| 35 |
-
def _event(conn, kind: str, detail: dict | None = None) -> None:
|
| 36 |
-
conn.execute("INSERT INTO circuit_events (kind, detail) VALUES (%s, %s)", (kind, json.dumps(detail or {}, default=str)))
|
| 37 |
|
| 38 |
|
| 39 |
-
def record_failure(conn, reason: str) -> CircuitState:
|
| 40 |
r = conn.execute(
|
| 41 |
-
"UPDATE circuit_state SET fail_count = fail_count + 1, last_failure_ts = now(), updated_at = now() WHERE id =
|
| 42 |
).fetchone()
|
| 43 |
tripped = r["tripped"]
|
| 44 |
if r["fail_count"] >= config.CIRCUIT_THRESHOLD and not tripped:
|
| 45 |
-
conn.execute("UPDATE circuit_state SET tripped = true, updated_at = now() WHERE id =
|
| 46 |
-
_event(conn, "tripped", {"reason": reason, "fail_count": r["fail_count"]})
|
| 47 |
-
record_event("error", "rails", "circuit breaker tripped", {"reason": reason})
|
| 48 |
-
return load_circuit(conn)
|
| 49 |
|
| 50 |
|
| 51 |
-
def record_success(conn) -> None:
|
| 52 |
-
conn.execute("UPDATE circuit_state SET fail_count = 0, updated_at = now() WHERE id =
|
| 53 |
|
| 54 |
|
| 55 |
-
def check_drawdown(conn, wealth: float, peak: float) -> bool:
|
| 56 |
-
"""Trip the kill switch when ``wealth`` has fallen KILL_SWITCH_DRAWDOWN below ``peak`` (the book's
|
| 57 |
-
trading engine passes the highest wealth mark of the current run). Returns whether the kill switch is on."""
|
| 58 |
-
st = load_circuit(conn)
|
| 59 |
if st.kill_switch:
|
| 60 |
return True
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 66 |
return True
|
| 67 |
return False
|
| 68 |
|
| 69 |
|
| 70 |
-
def kill_rebase_at(conn, circuit_id: int =
|
| 71 |
-
"""When the kill switch was last cleared
|
| 72 |
-
|
| 73 |
-
|
|
|
|
| 74 |
"AND COALESCE((detail->>'circuit')::int, 1) = %s", (circuit_id,)).fetchone()
|
| 75 |
return r["ts"] if r else None
|
| 76 |
|
| 77 |
|
| 78 |
-
def
|
| 79 |
-
"""
|
| 80 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 81 |
if book_halted(conn, book):
|
| 82 |
return True
|
| 83 |
-
|
| 84 |
-
|
|
|
|
| 85 |
conn.execute("INSERT INTO book_state (book, halted, reason, peak, updated_at) VALUES (%s, true, %s, %s, now()) "
|
| 86 |
"ON CONFLICT (book) DO UPDATE SET halted = true, reason = EXCLUDED.reason, peak = EXCLUDED.peak, updated_at = now()",
|
| 87 |
(book, reason, peak))
|
|
@@ -103,28 +124,29 @@ def clear_book_halt(book: str) -> None:
|
|
| 103 |
record_event("info", "rails", f"{book}: drawdown halt cleared")
|
| 104 |
|
| 105 |
|
| 106 |
-
def notional_24h(conn) -> float:
|
| 107 |
-
|
|
|
|
| 108 |
return float(r["s"])
|
| 109 |
|
| 110 |
|
| 111 |
-
def record_notional(conn, sol: float, kind: str = "entry") -> None:
|
| 112 |
-
conn.execute("INSERT INTO notional_ledger (sol, kind) VALUES (%s, %s)", (sol, kind))
|
| 113 |
|
| 114 |
|
| 115 |
-
def reset_circuit(kill: bool = False) -> None:
|
| 116 |
with transaction() as conn:
|
| 117 |
-
conn.execute("UPDATE circuit_state SET fail_count = 0, tripped = false, updated_at = now() WHERE id =
|
| 118 |
-
_event(conn, "reset", {"kill": kill})
|
| 119 |
if kill:
|
| 120 |
-
conn.execute("UPDATE circuit_state SET kill_switch = false, kill_reason = NULL, peak_wealth = NULL, updated_at = now() WHERE id =
|
| 121 |
-
record_event("info", "rails", "circuit reset" + (" incl. kill switch (peak re-based)" if kill else ""))
|
| 122 |
print("circuit reset" + (" + kill switch cleared" if kill else ""))
|
| 123 |
|
| 124 |
|
| 125 |
-
def set_entries_paused(paused: bool) -> None:
|
| 126 |
with transaction() as conn:
|
| 127 |
-
conn.execute("UPDATE circuit_state SET entries_paused = %s, updated_at = now() WHERE id =
|
| 128 |
-
_event(conn, "paused" if paused else "resumed", {})
|
| 129 |
-
record_event("info", "rails", "entries paused" if paused else "entries resumed")
|
| 130 |
print("entries paused" if paused else "entries resumed")
|
|
|
|
| 1 |
"""Risk rails outside the model. KillSwitch (−KILL_SWITCH_DRAWDOWN from peak wealth → no entries), CircuitBreaker
|
| 2 |
(CIRCUIT_THRESHOLD consecutive execution failures), the rolling 24 h notional ledger, and PauseEntries (console
|
| 3 |
toggle). State lives in circuit_state / circuit_events / notional_ledger; the trading engine reads it before entries.
|
| 4 |
+
One circuit per trading type: ``circuit_id`` 1 is the memecoin live book (the default everywhere), 2 the Kalshi live
|
| 5 |
+
books (``KALSHI_CIRCUIT``, fly_trader/kalshi), 3 the Robinhood Chain memecoin live book (``RH_CIRCUIT``,
|
| 6 |
+
fly_trader/rh); ``check_drawdown`` takes the drawdown limit of the caller's type.
|
| 7 |
"""
|
| 8 |
from __future__ import annotations
|
| 9 |
|
|
|
|
| 16 |
from ..db.connection import transaction
|
| 17 |
|
| 18 |
log = logging.getLogger(__name__)
|
| 19 |
+
SOLANA_CIRCUIT, KALSHI_CIRCUIT, RH_CIRCUIT = 1, 2, 3
|
| 20 |
|
| 21 |
|
| 22 |
@dataclass
|
|
|
|
| 29 |
entries_paused: bool = False
|
| 30 |
|
| 31 |
|
| 32 |
+
def load_circuit(conn, circuit_id: int = SOLANA_CIRCUIT) -> CircuitState:
|
| 33 |
+
r = conn.execute("SELECT * FROM circuit_state WHERE id = %s", (circuit_id,)).fetchone()
|
| 34 |
if not r:
|
| 35 |
return CircuitState()
|
| 36 |
return CircuitState(r["fail_count"], r["tripped"], r["kill_switch"], r["kill_reason"], r["peak_wealth"], r["entries_paused"])
|
| 37 |
|
| 38 |
|
| 39 |
+
def _event(conn, kind: str, detail: dict | None = None, circuit_id: int = SOLANA_CIRCUIT) -> None:
|
| 40 |
+
conn.execute("INSERT INTO circuit_events (kind, detail) VALUES (%s, %s)", (kind, json.dumps({**(detail or {}), "circuit": circuit_id}, default=str)))
|
| 41 |
|
| 42 |
|
| 43 |
+
def record_failure(conn, reason: str, circuit_id: int = SOLANA_CIRCUIT) -> CircuitState:
|
| 44 |
r = conn.execute(
|
| 45 |
+
"UPDATE circuit_state SET fail_count = fail_count + 1, last_failure_ts = now(), updated_at = now() WHERE id = %s RETURNING fail_count, tripped", (circuit_id,)
|
| 46 |
).fetchone()
|
| 47 |
tripped = r["tripped"]
|
| 48 |
if r["fail_count"] >= config.CIRCUIT_THRESHOLD and not tripped:
|
| 49 |
+
conn.execute("UPDATE circuit_state SET tripped = true, updated_at = now() WHERE id = %s", (circuit_id,))
|
| 50 |
+
_event(conn, "tripped", {"reason": reason, "fail_count": r["fail_count"]}, circuit_id)
|
| 51 |
+
record_event("error", "rails", "circuit breaker tripped", {"reason": reason, "circuit": circuit_id})
|
| 52 |
+
return load_circuit(conn, circuit_id)
|
| 53 |
|
| 54 |
|
| 55 |
+
def record_success(conn, circuit_id: int = SOLANA_CIRCUIT) -> None:
|
| 56 |
+
conn.execute("UPDATE circuit_state SET fail_count = 0, updated_at = now() WHERE id = %s", (circuit_id,))
|
| 57 |
|
| 58 |
|
| 59 |
+
def check_drawdown(conn, wealth: float, peak: float, circuit_id: int = SOLANA_CIRCUIT, drawdown: float | None = None, unit: str = "SOL") -> bool:
|
| 60 |
+
"""Trip the kill switch when ``wealth`` has fallen ``drawdown`` (default KILL_SWITCH_DRAWDOWN) below ``peak`` (the book's
|
| 61 |
+
own peak: the trading engine passes the highest wealth mark of the current run). Returns whether the kill switch is on."""
|
| 62 |
+
st = load_circuit(conn, circuit_id)
|
| 63 |
if st.kill_switch:
|
| 64 |
return True
|
| 65 |
+
dd = config.KILL_SWITCH_DRAWDOWN if drawdown is None else float(drawdown)
|
| 66 |
+
if peak > 0 and wealth <= peak * (1.0 - dd):
|
| 67 |
+
conn.execute("UPDATE circuit_state SET kill_switch = true, kill_reason = %s, peak_wealth = %s, updated_at = now() WHERE id = %s",
|
| 68 |
+
(f"wealth {wealth:.4f} {unit} <= {1 - dd:.2f} x peak {peak:.4f} {unit}", peak, circuit_id))
|
| 69 |
+
_event(conn, "kill_switch", {"wealth": wealth, "peak": peak}, circuit_id)
|
| 70 |
+
record_event("error", "rails", "kill switch tripped: entries blocked", {"wealth": wealth, "peak": peak, "circuit": circuit_id})
|
| 71 |
+
if config.VAULT_ENABLED: # the hosted fly: tell the operator at once
|
| 72 |
+
try:
|
| 73 |
+
from ..vault import alerts
|
| 74 |
+
alerts.send(f"KILL SWITCH tripped: {wealth:.4f} {unit} <= {1 - dd:.2f} x peak {peak:.4f} {unit}. Entries blocked"
|
| 75 |
+
+ (", open positions are being liquidated" if config.KILL_SWITCH_LIQUIDATE else "")
|
| 76 |
+
+ ". Clear it with `fly-trader reset-circuit --kill` once you know why.")
|
| 77 |
+
except Exception:
|
| 78 |
+
pass
|
| 79 |
return True
|
| 80 |
return False
|
| 81 |
|
| 82 |
|
| 83 |
+
def kill_rebase_at(conn, circuit_id: int = SOLANA_CIRCUIT):
|
| 84 |
+
"""When the kill switch was last cleared, or money last withdrawn from the live wallet: peaks are measured from marks
|
| 85 |
+
after it, so clearing really re-bases and a withdrawal is never a drawdown. (Before this, the live book recomputed its
|
| 86 |
+
peak from every wealth mark ever and re-tripped the next minute.)"""
|
| 87 |
+
r = conn.execute("SELECT max(ts) AS ts FROM circuit_events WHERE ((kind = 'reset' AND (detail->>'kill')::boolean) OR kind = 'withdrawal') "
|
| 88 |
"AND COALESCE((detail->>'circuit')::int, 1) = %s", (circuit_id,)).fetchone()
|
| 89 |
return r["ts"] if r else None
|
| 90 |
|
| 91 |
|
| 92 |
+
def record_withdrawal(conn, detail: dict, circuit_id: int = SOLANA_CIRCUIT) -> None:
|
| 93 |
+
"""The operator took money out of a live wallet (chain/withdraw.py, rh/withdraw.py): ``kill_rebase_at`` re-bases the
|
| 94 |
+
live book's peak at it."""
|
| 95 |
+
_event(conn, "withdrawal", detail, circuit_id)
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
def check_book_drawdown(conn, book: str, wealth: float, peak: float, drawdown: float | None = None, unit: str = "SOL") -> bool:
|
| 99 |
+
"""A paper race book's own kill switch: ``drawdown`` (default KILL_SWITCH_DRAWDOWN) below the book's peak halts that
|
| 100 |
+
book's entries only, so one book's drawdown never stops the other during the race. Returns whether the book is halted."""
|
| 101 |
if book_halted(conn, book):
|
| 102 |
return True
|
| 103 |
+
dd = config.KILL_SWITCH_DRAWDOWN if drawdown is None else float(drawdown)
|
| 104 |
+
if peak > 0 and wealth <= peak * (1.0 - dd):
|
| 105 |
+
reason = f"wealth {wealth:.4f} {unit} <= {1 - dd:.2f} x peak {peak:.4f} {unit}"
|
| 106 |
conn.execute("INSERT INTO book_state (book, halted, reason, peak, updated_at) VALUES (%s, true, %s, %s, now()) "
|
| 107 |
"ON CONFLICT (book) DO UPDATE SET halted = true, reason = EXCLUDED.reason, peak = EXCLUDED.peak, updated_at = now()",
|
| 108 |
(book, reason, peak))
|
|
|
|
| 124 |
record_event("info", "rails", f"{book}: drawdown halt cleared")
|
| 125 |
|
| 126 |
|
| 127 |
+
def notional_24h(conn, market: str = "sol") -> float:
|
| 128 |
+
"""The last 24 h of entries in ``market``'s unit (``sol`` holds ETH on the ``rh`` market's rows)."""
|
| 129 |
+
r = conn.execute("SELECT COALESCE(sum(sol), 0) AS s FROM notional_ledger WHERE ts > now() - interval '24 hours' AND market = %s", (market,)).fetchone()
|
| 130 |
return float(r["s"])
|
| 131 |
|
| 132 |
|
| 133 |
+
def record_notional(conn, sol: float, kind: str = "entry", market: str = "sol") -> None:
|
| 134 |
+
conn.execute("INSERT INTO notional_ledger (sol, kind, market) VALUES (%s, %s, %s)", (sol, kind, market))
|
| 135 |
|
| 136 |
|
| 137 |
+
def reset_circuit(kill: bool = False, circuit_id: int = SOLANA_CIRCUIT) -> None:
|
| 138 |
with transaction() as conn:
|
| 139 |
+
conn.execute("UPDATE circuit_state SET fail_count = 0, tripped = false, updated_at = now() WHERE id = %s", (circuit_id,))
|
| 140 |
+
_event(conn, "reset", {"kill": kill}, circuit_id)
|
| 141 |
if kill:
|
| 142 |
+
conn.execute("UPDATE circuit_state SET kill_switch = false, kill_reason = NULL, peak_wealth = NULL, updated_at = now() WHERE id = %s", (circuit_id,))
|
| 143 |
+
record_event("info", "rails", "circuit reset" + (" incl. kill switch (peak re-based)" if kill else ""), {"circuit": circuit_id})
|
| 144 |
print("circuit reset" + (" + kill switch cleared" if kill else ""))
|
| 145 |
|
| 146 |
|
| 147 |
+
def set_entries_paused(paused: bool, circuit_id: int = SOLANA_CIRCUIT) -> None:
|
| 148 |
with transaction() as conn:
|
| 149 |
+
conn.execute("UPDATE circuit_state SET entries_paused = %s, updated_at = now() WHERE id = %s", (paused, circuit_id))
|
| 150 |
+
_event(conn, "paused" if paused else "resumed", {}, circuit_id)
|
| 151 |
+
record_event("info", "rails", "entries paused" if paused else "entries resumed", {"circuit": circuit_id})
|
| 152 |
print("entries paused" if paused else "entries resumed")
|
fly_trader/agent/selector_session.py
CHANGED
|
@@ -50,23 +50,53 @@ def pinned_snapshot() -> int | None:
|
|
| 50 |
return int(v) if v is not None else None
|
| 51 |
|
| 52 |
|
| 53 |
-
|
| 54 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 55 |
|
| 56 |
-
|
|
|
|
|
|
|
| 57 |
from ..train import selector as sel
|
|
|
|
|
|
|
|
|
|
| 58 |
pin = pinned_snapshot()
|
| 59 |
self.model = sel.load_snapshot(pin) if pin is not None else sel.load_latest()
|
| 60 |
if self.model is None:
|
| 61 |
raise NoModel("no model has qualified to trade yet (trained on the current data, with a backtest that made money after costs and beat random picks)")
|
|
|
|
|
|
|
|
|
|
| 62 |
self.idx = _columns(self.model)
|
| 63 |
self.horizon_s = self.model.horizon_min * 60
|
| 64 |
-
self.broker = PaperBroker(BOOK); self.done = False
|
| 65 |
self.run_id = str(uuid.uuid4()); self.beat_no = 0
|
| 66 |
with transaction() as conn:
|
| 67 |
self.snapshot_id = pin if pin is not None else sel.latest_current(conn)["id"] # the snapshot actually loaded
|
| 68 |
conn.execute("INSERT INTO runs (run_id, kind, config, brain_snapshot_id, status) VALUES (%s,%s,%s,%s,'running')",
|
| 69 |
-
(self.run_id, KIND, json.dumps({"threshold": self.model.threshold, "horizon_min": self.model.horizon_min, "book": BOOK
|
|
|
|
| 70 |
record_event("info", "selector", "selector session started", {"run_id": self.run_id, "snapshot": self.snapshot_id, "threshold": self.model.threshold, "horizon_min": self.model.horizon_min})
|
| 71 |
log.info("selector session: snapshot %d threshold %.4f horizon %d min", self.snapshot_id, self.model.threshold, self.model.horizon_min)
|
| 72 |
|
|
@@ -74,8 +104,7 @@ class SelectorBook:
|
|
| 74 |
"""Switch to a newer deployable model without a restart (the paper book and open positions carry on). While a
|
| 75 |
snapshot is pinned (``ui_settings['pinned_selector_snapshot']``) the book stays on it, so a forward test measures
|
| 76 |
one model instead of a blend of every retrain that lands during it."""
|
| 77 |
-
import
|
| 78 |
-
from ..train import selector as sel
|
| 79 |
pin = pinned_snapshot()
|
| 80 |
with transaction() as conn:
|
| 81 |
if pin is not None: # a pin freezes the book, but a NEW pin (a release) switches to it
|
|
@@ -86,7 +115,7 @@ class SelectorBook:
|
|
| 86 |
r = sel.latest_current(conn)
|
| 87 |
if not r or r["id"] == self.snapshot_id:
|
| 88 |
return False
|
| 89 |
-
m =
|
| 90 |
try:
|
| 91 |
idx = _columns(m)
|
| 92 |
except RuntimeError as e:
|
|
@@ -95,7 +124,7 @@ class SelectorBook:
|
|
| 95 |
old = self.snapshot_id; self.model, self.idx, self.snapshot_id, self.horizon_s = m, idx, r["id"], m.horizon_min * 60
|
| 96 |
with transaction() as conn:
|
| 97 |
conn.execute("UPDATE runs SET brain_snapshot_id = %s, config = %s WHERE run_id = %s",
|
| 98 |
-
(r["id"], json.dumps({"threshold": m.threshold, "horizon_min": m.horizon_min, "book": BOOK}, default=str), self.run_id))
|
| 99 |
record_event("info", "selector", f"switched to model #{r['id']}", {"from": old, "to": r["id"], "threshold": m.threshold})
|
| 100 |
log.info("switched from model #%s to #%d (threshold %.4f)", old, r["id"], m.threshold)
|
| 101 |
return True
|
|
@@ -104,7 +133,7 @@ class SelectorBook:
|
|
| 104 |
pass
|
| 105 |
|
| 106 |
def open_mints(self, conn) -> set[str]:
|
| 107 |
-
return {p["mint"] for p in ledger.open_positions(conn, BOOK)}
|
| 108 |
|
| 109 |
def _decide(self, ctx) -> dict | None:
|
| 110 |
"""The strategy stack's per-row decision (train/strategies.decide), with the live fail-closed rules: rows whose
|
|
@@ -113,15 +142,7 @@ class SelectorBook:
|
|
| 113 |
if not (getattr(self.model, "stack", None) or {}).get("strategies") or not len(ctx.mints):
|
| 114 |
return None
|
| 115 |
d = self.model.decide(ctx.X, X_COLS, ctx.t_start)
|
| 116 |
-
|
| 117 |
-
for k, i in enumerate(ctx.infos):
|
| 118 |
-
if not d["allow"][k]:
|
| 119 |
-
continue
|
| 120 |
-
if "skill" in groups and i.get("skill_missing"):
|
| 121 |
-
d["allow"][k] = False; d["reason"][k] = "fail closed: no wallet-skill table for today"
|
| 122 |
-
elif "rug" in groups and i.get("age_h") is not None and not i.get("curve_known"):
|
| 123 |
-
d["allow"][k] = False; d["reason"][k] = "fail closed: curve facts not known yet"
|
| 124 |
-
return d
|
| 125 |
|
| 126 |
def on_minute(self, ctx) -> dict:
|
| 127 |
conn = ctx.conn
|
|
@@ -141,10 +162,10 @@ class SelectorBook:
|
|
| 141 |
status = {**summary, "stage": "stream stale: holding (no entries or exits)", "updated_at": datetime.now(timezone.utc).isoformat()}
|
| 142 |
self._write_status(conn, status)
|
| 143 |
return status
|
| 144 |
-
handed = handover.state(conn) is not None and ctx.engine.has_book("fly")
|
| 145 |
self.beat_no += 1
|
| 146 |
extra = {} if d is None else {"holds": d["hold_s"], "strategies": d["strategy"], "allow": d["allow"], "reasons": d["reason"], "tables": d["tables"]}
|
| 147 |
-
st = paper_trading.trade_minute(ctx, book=BOOK, run_id=self.run_id, beat_no=self.beat_no, broker=self.broker, kind=KIND, mints=ctx.mints, infos=ctx.infos,
|
| 148 |
scores=scores, threshold=thr, table=getattr(self.model, "sizing", None), horizon_s=self.horizon_s,
|
| 149 |
block="handed over to the fly" if handed else None, **extra)
|
| 150 |
if handed and st["open"] == 0:
|
|
@@ -155,10 +176,9 @@ class SelectorBook:
|
|
| 155 |
self._write_status(conn, status)
|
| 156 |
return status
|
| 157 |
|
| 158 |
-
|
| 159 |
-
|
| 160 |
-
|
| 161 |
-
(json.dumps(status, default=str),))
|
| 162 |
|
| 163 |
def finish(self) -> None:
|
| 164 |
with transaction() as conn:
|
|
|
|
| 50 |
return int(v) if v is not None else None
|
| 51 |
|
| 52 |
|
| 53 |
+
def chain_deployable(model, chain: str) -> tuple[bool, str]:
|
| 54 |
+
"""Whether the selector may trade ``chain``: Solana by its own deployability (as before); another chain only when the
|
| 55 |
+
combined stack's holdout on that chain's rows showed an edge (train/strategies.score_holdout_by_chain)."""
|
| 56 |
+
if chain == "sol":
|
| 57 |
+
return True, "ok"
|
| 58 |
+
c = ((getattr(model, "metrics", None) or {}).get("chains") or {}).get(chain)
|
| 59 |
+
if not c:
|
| 60 |
+
return False, f"the selector was not judged on {chain} rows"
|
| 61 |
+
return bool(c.get("deployable")), f"{chain} holdout: {c.get('n', 0)} trades, mean {(c.get('mean') or 0) * 100:+.2f}% (random {(c.get('random_mean') or 0) * 100:+.2f}%)"
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def fail_closed(groups: set, infos: list, d: dict) -> dict:
|
| 65 |
+
"""The selector's live fail-closed rules on a ``strategies.decide`` result (in place): rows whose inputs training had
|
| 66 |
+
but live lacks are blocked. Shared by the fly's teacher gate (agent/fly_session.py), so both books obey the same rules."""
|
| 67 |
+
for k, i in enumerate(infos):
|
| 68 |
+
if not d["allow"][k]:
|
| 69 |
+
continue
|
| 70 |
+
if "skill" in groups and i.get("skill_missing"):
|
| 71 |
+
d["allow"][k] = False; d["reason"][k] = "fail closed: no wallet-skill table for today"
|
| 72 |
+
elif "rug" in groups and i.get("age_h") is not None and not i.get("curve_known"):
|
| 73 |
+
d["allow"][k] = False; d["reason"][k] = "fail closed: curve facts not known yet"
|
| 74 |
+
return d
|
| 75 |
+
|
| 76 |
|
| 77 |
+
class SelectorBook:
|
| 78 |
+
def __init__(self, chain: str = "sol"):
|
| 79 |
+
from .. import markets
|
| 80 |
from ..train import selector as sel
|
| 81 |
+
self.chain = chain; self.market = markets.for_chain(chain)
|
| 82 |
+
self.name = "selector" if chain == "sol" else f"selector_{chain}"
|
| 83 |
+
self.BOOK = self.market.selector_book; self.KIND = self.name; self.status_key = self.market.selector_status_key
|
| 84 |
pin = pinned_snapshot()
|
| 85 |
self.model = sel.load_snapshot(pin) if pin is not None else sel.load_latest()
|
| 86 |
if self.model is None:
|
| 87 |
raise NoModel("no model has qualified to trade yet (trained on the current data, with a backtest that made money after costs and beat random picks)")
|
| 88 |
+
ok, why = chain_deployable(self.model, chain)
|
| 89 |
+
if not ok:
|
| 90 |
+
raise NoModel(f"the selector may not trade {self.market.name}: {why}")
|
| 91 |
self.idx = _columns(self.model)
|
| 92 |
self.horizon_s = self.model.horizon_min * 60
|
| 93 |
+
self.broker = PaperBroker(self.BOOK); self.done = False
|
| 94 |
self.run_id = str(uuid.uuid4()); self.beat_no = 0
|
| 95 |
with transaction() as conn:
|
| 96 |
self.snapshot_id = pin if pin is not None else sel.latest_current(conn)["id"] # the snapshot actually loaded
|
| 97 |
conn.execute("INSERT INTO runs (run_id, kind, config, brain_snapshot_id, status) VALUES (%s,%s,%s,%s,'running')",
|
| 98 |
+
(self.run_id, self.KIND, json.dumps({"threshold": self.model.threshold, "horizon_min": self.model.horizon_min, "book": self.BOOK, "chain": chain},
|
| 99 |
+
default=str), self.snapshot_id))
|
| 100 |
record_event("info", "selector", "selector session started", {"run_id": self.run_id, "snapshot": self.snapshot_id, "threshold": self.model.threshold, "horizon_min": self.model.horizon_min})
|
| 101 |
log.info("selector session: snapshot %d threshold %.4f horizon %d min", self.snapshot_id, self.model.threshold, self.model.horizon_min)
|
| 102 |
|
|
|
|
| 104 |
"""Switch to a newer deployable model without a restart (the paper book and open positions carry on). While a
|
| 105 |
snapshot is pinned (``ui_settings['pinned_selector_snapshot']``) the book stays on it, so a forward test measures
|
| 106 |
one model instead of a blend of every retrain that lands during it."""
|
| 107 |
+
from ..train import model_io, selector as sel
|
|
|
|
| 108 |
pin = pinned_snapshot()
|
| 109 |
with transaction() as conn:
|
| 110 |
if pin is not None: # a pin freezes the book, but a NEW pin (a release) switches to it
|
|
|
|
| 115 |
r = sel.latest_current(conn)
|
| 116 |
if not r or r["id"] == self.snapshot_id:
|
| 117 |
return False
|
| 118 |
+
m = model_io.load_selector(r["path"])
|
| 119 |
try:
|
| 120 |
idx = _columns(m)
|
| 121 |
except RuntimeError as e:
|
|
|
|
| 124 |
old = self.snapshot_id; self.model, self.idx, self.snapshot_id, self.horizon_s = m, idx, r["id"], m.horizon_min * 60
|
| 125 |
with transaction() as conn:
|
| 126 |
conn.execute("UPDATE runs SET brain_snapshot_id = %s, config = %s WHERE run_id = %s",
|
| 127 |
+
(r["id"], json.dumps({"threshold": m.threshold, "horizon_min": m.horizon_min, "book": self.BOOK}, default=str), self.run_id))
|
| 128 |
record_event("info", "selector", f"switched to model #{r['id']}", {"from": old, "to": r["id"], "threshold": m.threshold})
|
| 129 |
log.info("switched from model #%s to #%d (threshold %.4f)", old, r["id"], m.threshold)
|
| 130 |
return True
|
|
|
|
| 133 |
pass
|
| 134 |
|
| 135 |
def open_mints(self, conn) -> set[str]:
|
| 136 |
+
return {p["mint"] for p in ledger.open_positions(conn, self.BOOK)}
|
| 137 |
|
| 138 |
def _decide(self, ctx) -> dict | None:
|
| 139 |
"""The strategy stack's per-row decision (train/strategies.decide), with the live fail-closed rules: rows whose
|
|
|
|
| 142 |
if not (getattr(self.model, "stack", None) or {}).get("strategies") or not len(ctx.mints):
|
| 143 |
return None
|
| 144 |
d = self.model.decide(ctx.X, X_COLS, ctx.t_start)
|
| 145 |
+
return fail_closed(set((getattr(self.model, "metrics", None) or {}).get("groups") or []), ctx.infos, d)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 146 |
|
| 147 |
def on_minute(self, ctx) -> dict:
|
| 148 |
conn = ctx.conn
|
|
|
|
| 162 |
status = {**summary, "stage": "stream stale: holding (no entries or exits)", "updated_at": datetime.now(timezone.utc).isoformat()}
|
| 163 |
self._write_status(conn, status)
|
| 164 |
return status
|
| 165 |
+
handed = handover.state(conn, self.chain) is not None and ctx.engine.has_book("fly" if self.chain == "sol" else f"fly_{self.chain}")
|
| 166 |
self.beat_no += 1
|
| 167 |
extra = {} if d is None else {"holds": d["hold_s"], "strategies": d["strategy"], "allow": d["allow"], "reasons": d["reason"], "tables": d["tables"]}
|
| 168 |
+
st = paper_trading.trade_minute(ctx, book=self.BOOK, run_id=self.run_id, beat_no=self.beat_no, broker=self.broker, kind=self.KIND, mints=ctx.mints, infos=ctx.infos,
|
| 169 |
scores=scores, threshold=thr, table=getattr(self.model, "sizing", None), horizon_s=self.horizon_s,
|
| 170 |
block="handed over to the fly" if handed else None, **extra)
|
| 171 |
if handed and st["open"] == 0:
|
|
|
|
| 176 |
self._write_status(conn, status)
|
| 177 |
return status
|
| 178 |
|
| 179 |
+
def _write_status(self, conn, status: dict) -> None:
|
| 180 |
+
conn.execute("INSERT INTO ui_settings (key, value) VALUES (%s, %s) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()",
|
| 181 |
+
(self.status_key, json.dumps({**status, "chain": self.chain}, default=str)))
|
|
|
|
| 182 |
|
| 183 |
def finish(self) -> None:
|
| 184 |
with transaction() as conn:
|
fly_trader/agent/sizing.py
CHANGED
|
@@ -4,16 +4,20 @@ Certainty is measured, not assumed. During training the backtest's out-of-sample
|
|
| 4 |
score cleared the buy line (``build_table``: equal-count bands of ``score − threshold``). For each band the growth-optimal
|
| 5 |
(Kelly) fraction is found on the band's actual net returns — the bet f that maximises the average of log(1 + f·r), so the
|
| 6 |
rare −100 % rugs count in full; a band with no edge gets 0. Live (``size_position``): the current score's band →
|
| 7 |
-
``KELLY_FRACTION`` of that fraction of the deployable bankroll (wealth at cost minus the gas reserve)
|
| 8 |
-
``
|
| 9 |
-
|
| 10 |
-
|
|
|
|
|
|
|
|
|
|
| 11 |
"""
|
| 12 |
from __future__ import annotations
|
| 13 |
|
| 14 |
import numpy as np
|
| 15 |
|
| 16 |
from .. import config
|
|
|
|
| 17 |
|
| 18 |
N_BANDS = 4
|
| 19 |
MIN_BAND_TRADES = 50
|
|
@@ -57,26 +61,31 @@ def band_for(table: list[dict], margin: float) -> dict | None:
|
|
| 57 |
|
| 58 |
|
| 59 |
def size_position(score: float, threshold: float, table: list[dict] | None, bankroll_sol: float, cash_sol: float,
|
| 60 |
-
res_quote_sol: float | None) -> tuple[float, str]:
|
| 61 |
-
"""(size in
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
if b is None or b["kelly"] <= 0:
|
| 68 |
-
return 0.0, "no edge in this score band"
|
| 69 |
deployable = max(0.0, bankroll_sol - reserve)
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 74 |
limit = min(caps, key=caps.get)
|
| 75 |
-
why = f"band from +{b['lo']:.3f} over the line: Kelly {b['kelly']:.2f} × {config.KELLY_FRACTION:g} of {deployable:.2f} SOL"
|
| 76 |
if caps[limit] < size:
|
| 77 |
size = caps[limit]; why += f", capped by {limit}"
|
| 78 |
-
if size <
|
| 79 |
-
return 0.0, why + f" → {size:.
|
| 80 |
return float(size), why + f" → {size:.3f} SOL"
|
| 81 |
|
| 82 |
|
|
|
|
| 4 |
score cleared the buy line (``build_table``: equal-count bands of ``score − threshold``). For each band the growth-optimal
|
| 5 |
(Kelly) fraction is found on the band's actual net returns — the bet f that maximises the average of log(1 + f·r), so the
|
| 6 |
rare −100 % rugs count in full; a band with no edge gets 0. Live (``size_position``): the current score's band →
|
| 7 |
+
``KELLY_FRACTION`` of that fraction of the deployable bankroll (wealth at cost minus the gas reserve); ``flat`` (the
|
| 8 |
+
fly, ``config.FLY_SIZING``): ``MAX_POSITION_FRACTION`` of it on every buy that cleared the line, no bands. Either way
|
| 9 |
+
capped at ``MAX_POSITION_FRACTION`` of the deployable bankroll, the free cash above the reserve, and the label size
|
| 10 |
+
(``market/exit_cost.label_size``: ``LABEL_SIZE_SOL`` or ``MAX_POOL_SHARE`` of the pool) -- the size every edge was
|
| 11 |
+
measured at, so the book behaves the same with 5 SOL or 500: a bigger bankroll buys no bigger than the backtest did,
|
| 12 |
+
it only runs out of cash later. Below ``MIN_POSITION_SOL`` the buy is skipped. A model without a table (trained
|
| 13 |
+
before sizing) trades the fixed ``MAX_POSITION_SOL``, under the same caps.
|
| 14 |
"""
|
| 15 |
from __future__ import annotations
|
| 16 |
|
| 17 |
import numpy as np
|
| 18 |
|
| 19 |
from .. import config
|
| 20 |
+
from ..market.exit_cost import label_size
|
| 21 |
|
| 22 |
N_BANDS = 4
|
| 23 |
MIN_BAND_TRADES = 50
|
|
|
|
| 61 |
|
| 62 |
|
| 63 |
def size_position(score: float, threshold: float, table: list[dict] | None, bankroll_sol: float, cash_sol: float,
|
| 64 |
+
res_quote_sol: float | None, flat: bool = False, market=None) -> tuple[float, str]:
|
| 65 |
+
"""(size in the book's unit, why). Size 0 means skip the buy. ``flat``: ``MAX_POSITION_FRACTION`` of the deployable
|
| 66 |
+
bankroll on every buy that cleared the line (the fly); otherwise the Kelly band of ``table``. ``market``
|
| 67 |
+
(fly_trader/markets; default Solana): the gas reserve, minimum position and label size of the book's own market."""
|
| 68 |
+
rh = market is not None and market.chain != "sol"
|
| 69 |
+
reserve = market.gas_reserve() if rh else config.GAS_RESERVE_SOL; free = cash_sol - reserve
|
| 70 |
+
min_pos = market.min_position() if rh else config.MIN_POSITION_SOL
|
|
|
|
|
|
|
| 71 |
deployable = max(0.0, bankroll_sol - reserve)
|
| 72 |
+
if flat:
|
| 73 |
+
size = config.MAX_POSITION_FRACTION * deployable; why = f"flat {config.MAX_POSITION_FRACTION:g} of {deployable:.2f} SOL"
|
| 74 |
+
elif not table:
|
| 75 |
+
size = config.MAX_POSITION_SOL * (market.k() if rh else 1.0); why = f"fixed {size:g} {'ETH' if rh else 'SOL'} (model has no sizing table)"
|
| 76 |
+
else:
|
| 77 |
+
b = band_for(table, score - threshold)
|
| 78 |
+
if b is None or b["kelly"] <= 0:
|
| 79 |
+
return 0.0, "no edge in this score band"
|
| 80 |
+
size = config.KELLY_FRACTION * b["kelly"] * deployable
|
| 81 |
+
why = f"band from +{b['lo']:.3f} over the line: Kelly {b['kelly']:.2f} × {config.KELLY_FRACTION:g} of {deployable:.2f} SOL"
|
| 82 |
+
# the label size is the largest buy any backtest measured: past it the edge is an extrapolation that impact eats
|
| 83 |
+
caps = {"bankroll cap": config.MAX_POSITION_FRACTION * deployable, "free cash": free, "label size": label_size(res_quote_sol, market if rh else None)}
|
| 84 |
limit = min(caps, key=caps.get)
|
|
|
|
| 85 |
if caps[limit] < size:
|
| 86 |
size = caps[limit]; why += f", capped by {limit}"
|
| 87 |
+
if size < min_pos:
|
| 88 |
+
return 0.0, why + f" → {max(size, 0.0):.6f} is below the {min_pos:g} minimum"
|
| 89 |
return float(size), why + f" → {size:.3f} SOL"
|
| 90 |
|
| 91 |
|
fly_trader/brain/connectome.py
CHANGED
|
@@ -115,6 +115,9 @@ class Connectome:
|
|
| 115 |
self.pop_ranges, self.pathway_gains)
|
| 116 |
self.W_KM0 = torch.from_numpy(np.ascontiguousarray(z["W_KM0"])).to(torch.float32)
|
| 117 |
self.M_KM = torch.from_numpy(np.ascontiguousarray(z["M_KM"])).to(torch.bool)
|
|
|
|
|
|
|
|
|
|
| 118 |
self.spectral_radius_raw = float(z["spectral_radius_raw"])
|
| 119 |
self.content_sha256 = str(z["content_sha256"])
|
| 120 |
self.s = float(scale["s"]) if scale and "s" in scale else float(z["s0"])
|
|
@@ -159,3 +162,27 @@ class SubConnectome:
|
|
| 159 |
self.pathway_gains = getattr(c, "pathway_gains", {})
|
| 160 |
self.excluded = exclude
|
| 161 |
self.keep = keep
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 115 |
self.pop_ranges, self.pathway_gains)
|
| 116 |
self.W_KM0 = torch.from_numpy(np.ascontiguousarray(z["W_KM0"])).to(torch.float32)
|
| 117 |
self.M_KM = torch.from_numpy(np.ascontiguousarray(z["M_KM"])).to(torch.bool)
|
| 118 |
+
# FlyBrain's finer annotation per neuron (VIS_R1R6, VIS_ME, LPTC, ...): the Kalshi fly finds its photoreceptors by it
|
| 119 |
+
self.flybrain_group = np.asarray(z["flybrain_group"]) if "flybrain_group" in z.files else None
|
| 120 |
+
self.flybrain_group_names = [str(x) for x in z["flybrain_group_names"]] if "flybrain_group_names" in z.files else []
|
| 121 |
self.spectral_radius_raw = float(z["spectral_radius_raw"])
|
| 122 |
self.content_sha256 = str(z["content_sha256"])
|
| 123 |
self.s = float(scale["s"]) if scale and "s" in scale else float(z["s0"])
|
|
|
|
| 162 |
self.pathway_gains = getattr(c, "pathway_gains", {})
|
| 163 |
self.excluded = exclude
|
| 164 |
self.keep = keep
|
| 165 |
+
fg = getattr(c, "flybrain_group", None)
|
| 166 |
+
self.flybrain_group = fg[keep.numpy()] if fg is not None else None
|
| 167 |
+
self.flybrain_group_names = list(getattr(c, "flybrain_group_names", []))
|
| 168 |
+
|
| 169 |
+
|
| 170 |
+
# ---------------------------------------------------------------- the Kalshi fly's graph: the optic lobes kept, the memecoin senses dropped
|
| 171 |
+
VISUAL_SUB = tuple(AFFERENT_POPS) # exclude these and VISUAL stays: the second fly's sub-connectome
|
| 172 |
+
PHOTORECEPTOR_GROUPS = ("VIS_R1R6", "VIS_R7R8") # FlyBrain groups of the retina's photoreceptors (R7R8 is absent in the v783 join)
|
| 173 |
+
|
| 174 |
+
|
| 175 |
+
def photoreceptor_rows(graph) -> torch.Tensor:
|
| 176 |
+
"""Rows of ``graph`` (a Connectome or SubConnectome) that are photoreceptors: VISUAL neurons whose FlyBrain group is
|
| 177 |
+
R1–R6 or R7–R8. The Kalshi fly's features enter here and travel the optic lobes into the central brain."""
|
| 178 |
+
if "VISUAL" not in graph.pop_ranges:
|
| 179 |
+
raise ValueError("this graph has no VISUAL population: the Kalshi fly needs the optic lobes")
|
| 180 |
+
fg = getattr(graph, "flybrain_group", None); names = list(getattr(graph, "flybrain_group_names", []) or [])
|
| 181 |
+
lo, hi = graph.pop_ranges["VISUAL"]
|
| 182 |
+
if fg is None or not names:
|
| 183 |
+
raise ValueError("the connectome artifact carries no flybrain_group annotation; rebuild it (brain/build.py)")
|
| 184 |
+
want = [i for i, n in enumerate(names) if n in PHOTORECEPTOR_GROUPS]
|
| 185 |
+
rows = np.flatnonzero(np.isin(fg[lo:hi], want)) + lo
|
| 186 |
+
if not len(rows):
|
| 187 |
+
raise ValueError(f"no photoreceptor rows: VISUAL groups present are {sorted(set(names[int(g)] for g in set(fg[lo:hi].tolist()) if int(g) < len(names)))}")
|
| 188 |
+
return torch.from_numpy(rows.astype(np.int64))
|
fly_trader/brain/geometry.py
CHANGED
|
@@ -25,6 +25,8 @@ EXCLUDE = ("VISUAL",)
|
|
| 25 |
POS_COLS = ["pos_x", "pos_y", "pos_z"]
|
| 26 |
SOMA_COLS = ["soma_x", "soma_y", "soma_z"]
|
| 27 |
BIN, META = "geometry.bin", "geometry.json"
|
|
|
|
|
|
|
| 28 |
|
| 29 |
|
| 30 |
def keep_mask(pop_ranges: dict, n: int, exclude: tuple[str, ...] = EXCLUDE) -> np.ndarray:
|
|
@@ -60,10 +62,11 @@ def read_meta(out_dir: Path) -> dict | None:
|
|
| 60 |
|
| 61 |
|
| 62 |
def build_geometry(out_dir: Path, connectome_path: str | Path | None = None, annotations: str | Path | None = None,
|
| 63 |
-
exclude: tuple[str, ...] = EXCLUDE) -> dict:
|
| 64 |
"""Write (once per connectome) and return the geometry metadata. Soma centroid where known, else the neuron's
|
| 65 |
-
annotation point, else its population's centroid (a handful of neurons); scaled to micrometres and centred.
|
| 66 |
-
|
|
|
|
| 67 |
path = Path(connectome_path) if connectome_path else current_connectome_path()
|
| 68 |
with np.load(path, allow_pickle=False) as z:
|
| 69 |
sha = str(z["content_sha256"]); n = int(z["N"]); pop_ranges = {k: tuple(v) for k, v in json.loads(str(z["pop_ranges"])).items()}
|
|
@@ -97,3 +100,39 @@ def build_geometry(out_dir: Path, connectome_path: str | Path | None = None, ann
|
|
| 97 |
out_dir.mkdir(parents=True, exist_ok=True)
|
| 98 |
_atomic(bpath, np.ascontiguousarray(xyz).tobytes()); _atomic(jpath, json.dumps(meta).encode())
|
| 99 |
return meta
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 25 |
POS_COLS = ["pos_x", "pos_y", "pos_z"]
|
| 26 |
SOMA_COLS = ["soma_x", "soma_y", "soma_z"]
|
| 27 |
BIN, META = "geometry.bin", "geometry.json"
|
| 28 |
+
FULL_BIN, FULL_META = "geometry_full.bin", "geometry_full.json" # every neuron: the scene both flies are drawn into (ui/brain3d)
|
| 29 |
+
FLY_EXCLUDES = {"memecoin": EXCLUDE, "kalshi": None} # None: brain/connectome.VISUAL_SUB (the memecoin afferents), resolved lazily
|
| 30 |
|
| 31 |
|
| 32 |
def keep_mask(pop_ranges: dict, n: int, exclude: tuple[str, ...] = EXCLUDE) -> np.ndarray:
|
|
|
|
| 62 |
|
| 63 |
|
| 64 |
def build_geometry(out_dir: Path, connectome_path: str | Path | None = None, annotations: str | Path | None = None,
|
| 65 |
+
exclude: tuple[str, ...] = EXCLUDE, names: tuple[str, str] = (BIN, META)) -> dict:
|
| 66 |
"""Write (once per connectome) and return the geometry metadata. Soma centroid where known, else the neuron's
|
| 67 |
+
annotation point, else its population's centroid (a handful of neurons); scaled to micrometres and centred.
|
| 68 |
+
``exclude=()`` with ``names=(FULL_BIN, FULL_META)`` is the whole brain (``build_full_geometry``)."""
|
| 69 |
+
out_dir = Path(out_dir); bpath, jpath = out_dir / names[0], out_dir / names[1]
|
| 70 |
path = Path(connectome_path) if connectome_path else current_connectome_path()
|
| 71 |
with np.load(path, allow_pickle=False) as z:
|
| 72 |
sha = str(z["content_sha256"]); n = int(z["N"]); pop_ranges = {k: tuple(v) for k, v in json.loads(str(z["pop_ranges"])).items()}
|
|
|
|
| 100 |
out_dir.mkdir(parents=True, exist_ok=True)
|
| 101 |
_atomic(bpath, np.ascontiguousarray(xyz).tobytes()); _atomic(jpath, json.dumps(meta).encode())
|
| 102 |
return meta
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
def fly_index(pop_ranges: dict, n: int, exclude: tuple[str, ...]) -> np.ndarray:
|
| 106 |
+
"""int32 map from a fly's sub-graph index to the full-graph index (the rows ``SubConnectome`` keeps, in order)."""
|
| 107 |
+
return np.flatnonzero(keep_mask(pop_ranges, n, exclude)).astype(np.int32)
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
def build_full_geometry(out_dir: Path, connectome_path: str | Path | None = None, annotations: str | Path | None = None) -> dict:
|
| 111 |
+
"""The whole brain's geometry plus, per fly, the index map from its sub-graph into it (``index_<fly>.bin``): the
|
| 112 |
+
scene the console draws both flies into. ``geometry_full.json`` carries ``flies`` = {name: {"exclude", "n", "index"}}."""
|
| 113 |
+
from .connectome import VISUAL_SUB
|
| 114 |
+
out_dir = Path(out_dir); jpath = out_dir / FULL_META
|
| 115 |
+
meta = read_meta_named(out_dir, FULL_META)
|
| 116 |
+
path = Path(connectome_path) if connectome_path else current_connectome_path()
|
| 117 |
+
with np.load(path, allow_pickle=False) as z:
|
| 118 |
+
sha = str(z["content_sha256"]); n = int(z["N"]); pop_ranges = {k: tuple(v) for k, v in json.loads(str(z["pop_ranges"])).items()}
|
| 119 |
+
if meta and meta.get("connectome_sha256") == sha and meta.get("version") == VERSION and meta.get("flies") and (out_dir / FULL_BIN).exists() \
|
| 120 |
+
and all((out_dir / f["index"]).exists() for f in meta["flies"].values()):
|
| 121 |
+
return meta
|
| 122 |
+
meta = build_geometry(out_dir, path, annotations, exclude=(), names=(FULL_BIN, FULL_META))
|
| 123 |
+
flies = {}
|
| 124 |
+
for name, exc in FLY_EXCLUDES.items():
|
| 125 |
+
exc = tuple(exc) if exc is not None else tuple(VISUAL_SUB)
|
| 126 |
+
idx = fly_index(pop_ranges, n, exc); fname = f"index_{name}.bin"
|
| 127 |
+
_atomic(out_dir / fname, np.ascontiguousarray(idx).tobytes())
|
| 128 |
+
flies[name] = {"exclude": list(exc), "n": int(len(idx)), "index": fname}
|
| 129 |
+
meta = {**meta, "flies": flies}
|
| 130 |
+
_atomic(jpath, json.dumps(meta).encode())
|
| 131 |
+
return meta
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
def read_meta_named(out_dir: Path, name: str) -> dict | None:
|
| 135 |
+
try:
|
| 136 |
+
return json.loads((Path(out_dir) / name).read_text())
|
| 137 |
+
except (FileNotFoundError, ValueError):
|
| 138 |
+
return None
|
fly_trader/brain/pathways.py
CHANGED
|
@@ -13,7 +13,8 @@ from datetime import datetime
|
|
| 13 |
from pathlib import Path
|
| 14 |
|
| 15 |
import numpy as np
|
| 16 |
-
|
|
|
|
| 17 |
|
| 18 |
TOP_K = 200
|
| 19 |
|
|
@@ -27,7 +28,7 @@ def km_pairs(connectome_path: str | Path) -> tuple[np.ndarray, np.ndarray, int,
|
|
| 27 |
|
| 28 |
def bootstrap_weights(boot_path: str | Path) -> tuple[np.ndarray, np.ndarray, list[str]]:
|
| 29 |
"""(base weight per pair, learn [S, J], strategy names) from a fly bootstrap checkpoint (``fly_selector.save``)."""
|
| 30 |
-
d =
|
| 31 |
w0 = np.logaddexp(0.0, sd["theta_km"].detach().float().numpy()).astype(np.float32) # softplus
|
| 32 |
n_mbon = int(sd["c_sign"].numel()) if "c_sign" in sd else int(sd["mask"].shape[1])
|
| 33 |
learn = sd["learn"].numpy().astype(bool) if "learn" in sd else np.ones((1, n_mbon), dtype=bool)
|
|
@@ -37,7 +38,7 @@ def bootstrap_weights(boot_path: str | Path) -> tuple[np.ndarray, np.ndarray, li
|
|
| 37 |
|
| 38 |
def bank_D(path: str | Path) -> tuple[np.ndarray, int | None]:
|
| 39 |
"""The learned change per pair from ``state.pt`` or a ``snap_*.pt`` (both carry ``bank`` and ``bootstrap_id``)."""
|
| 40 |
-
s =
|
| 41 |
D = s["bank"]["D"]; D = D[0] if D.ndim == 2 else D
|
| 42 |
bid = s.get("bootstrap_id")
|
| 43 |
return D.detach().float().numpy(), (int(bid) if bid is not None else None)
|
|
|
|
| 13 |
from pathlib import Path
|
| 14 |
|
| 15 |
import numpy as np
|
| 16 |
+
|
| 17 |
+
from ..train import model_io
|
| 18 |
|
| 19 |
TOP_K = 200
|
| 20 |
|
|
|
|
| 28 |
|
| 29 |
def bootstrap_weights(boot_path: str | Path) -> tuple[np.ndarray, np.ndarray, list[str]]:
|
| 30 |
"""(base weight per pair, learn [S, J], strategy names) from a fly bootstrap checkpoint (``fly_selector.save``)."""
|
| 31 |
+
d = model_io.torch_load(boot_path); sd = d["state_dict"]
|
| 32 |
w0 = np.logaddexp(0.0, sd["theta_km"].detach().float().numpy()).astype(np.float32) # softplus
|
| 33 |
n_mbon = int(sd["c_sign"].numel()) if "c_sign" in sd else int(sd["mask"].shape[1])
|
| 34 |
learn = sd["learn"].numpy().astype(bool) if "learn" in sd else np.ones((1, n_mbon), dtype=bool)
|
|
|
|
| 38 |
|
| 39 |
def bank_D(path: str | Path) -> tuple[np.ndarray, int | None]:
|
| 40 |
"""The learned change per pair from ``state.pt`` or a ``snap_*.pt`` (both carry ``bank`` and ``bootstrap_id``)."""
|
| 41 |
+
s = model_io.torch_load(path)
|
| 42 |
D = s["bank"]["D"]; D = D[0] if D.ndim == 2 else D
|
| 43 |
bid = s.get("bootstrap_id")
|
| 44 |
return D.detach().float().numpy(), (int(bid) if bid is not None else None)
|
fly_trader/brain/plastic.py
CHANGED
|
@@ -140,7 +140,7 @@ def row_weights(scores: torch.Tensor, lines: torch.Tensor) -> torch.Tensor:
|
|
| 140 |
even at 10x the losing population outweighed the tradeable rows by more than 100:1 and dragged every prediction
|
| 141 |
toward the candidate mean. Learning then made the fly worse than the same network frozen (PF 1.012 vs 1.098). A row
|
| 142 |
below the line is not a decision, so it carries no weight."""
|
| 143 |
-
return torch.where(scores >= lines[:, None], 1.0, 0.0)
|
| 144 |
|
| 145 |
|
| 146 |
class PlasticBank:
|
|
@@ -187,6 +187,12 @@ class PlasticBank:
|
|
| 187 |
m = torch.tanh(u)
|
| 188 |
return (y_dn[None] + (((m - self.mu) / self.sigma) * self._read(s, len(y_dn))) @ self.c) / self.scale, m
|
| 189 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 190 |
def frozen(self, y_dn: torch.Tensor, u0: torch.Tensor, s=None) -> torch.Tensor:
|
| 191 |
"""The shadow: the bootstrap fly's score (D = 0) [B]."""
|
| 192 |
return (y_dn + (((torch.tanh(u0) - self.mu) / self.sigma) * self._read(s, len(y_dn))) @ self.c) / self.scale
|
|
|
|
| 140 |
even at 10x the losing population outweighed the tradeable rows by more than 100:1 and dragged every prediction
|
| 141 |
toward the candidate mean. Learning then made the fly worse than the same network frozen (PF 1.012 vs 1.098). A row
|
| 142 |
below the line is not a decision, so it carries no weight."""
|
| 143 |
+
return torch.where(scores >= (lines if lines.dim() == 2 else lines[:, None]), 1.0, 0.0) # lines [C] or per row [C, B] (each row's chain)
|
| 144 |
|
| 145 |
|
| 146 |
class PlasticBank:
|
|
|
|
| 187 |
m = torch.tanh(u)
|
| 188 |
return (y_dn[None] + (((m - self.mu) / self.sigma) * self._read(s, len(y_dn))) @ self.c) / self.scale, m
|
| 189 |
|
| 190 |
+
def head_for(self, target: torch.Tensor, u0: torch.Tensor, s=None) -> torch.Tensor:
|
| 191 |
+
"""The head output under which the frozen score (D = 0) is exactly ``target`` (net-return units) [B]: with it,
|
| 192 |
+
``predict`` is ``target`` plus the plastic change of the MBON readout, and ``update`` learns the error of that
|
| 193 |
+
sum -- the fly that scores as its teacher plus a learned correction (config.FLY_SCORE = teacher)."""
|
| 194 |
+
return target * self.scale - (((torch.tanh(u0) - self.mu) / self.sigma) * self._read(s, len(target))) @ self.c
|
| 195 |
+
|
| 196 |
def frozen(self, y_dn: torch.Tensor, u0: torch.Tensor, s=None) -> torch.Tensor:
|
| 197 |
"""The shadow: the bootstrap fly's score (D = 0) [B]."""
|
| 198 |
return (y_dn + (((torch.tanh(u0) - self.mu) / self.sigma) * self._read(s, len(y_dn))) @ self.c) / self.scale
|
fly_trader/chain/cluster_guard.py
CHANGED
|
@@ -6,6 +6,9 @@ quotes, signature statuses) never do. Signing is allowed only when ALL of these
|
|
| 6 |
exists), ``LIVE_ENABLED=1``, and ``config.live_prerequisites_missing()`` is empty (capital, position
|
| 7 |
cap, gas reserve, bot key and API keys all set). ``config`` attributes are read at call time so a
|
| 8 |
test can monkeypatch them.
|
|
|
|
|
|
|
|
|
|
| 9 |
"""
|
| 10 |
from __future__ import annotations
|
| 11 |
|
|
@@ -35,6 +38,18 @@ def signing_allowed() -> bool:
|
|
| 35 |
return False
|
| 36 |
|
| 37 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 38 |
def assert_vault_signing_allowed() -> None:
|
| 39 |
"""The vault's own signing (claim payouts, principal withdrawals, token-account closes). On mainnet it is the same
|
| 40 |
gate as trading. The devnet rehearsal (``VAULT_CLUSTER=devnet``) may sign with a throwaway key that is not a
|
|
|
|
| 6 |
exists), ``LIVE_ENABLED=1``, and ``config.live_prerequisites_missing()`` is empty (capital, position
|
| 7 |
cap, gas reserve, bot key and API keys all set). ``config`` attributes are read at call time so a
|
| 8 |
test can monkeypatch them.
|
| 9 |
+
|
| 10 |
+
The one exception is a withdrawal to the operator's own wallet (``assert_withdraw_allowed``,
|
| 11 |
+
chain/withdraw.py): it takes money out of the fly's reach and never trades, so it needs no LIVE_ENABLED.
|
| 12 |
"""
|
| 13 |
from __future__ import annotations
|
| 14 |
|
|
|
|
| 38 |
return False
|
| 39 |
|
| 40 |
|
| 41 |
+
def assert_withdraw_allowed(dest: str) -> None:
|
| 42 |
+
"""A withdrawal of the bot's SOL (chain/withdraw.py): mainnet, and only to a wallet listed in FUNDING_ADDRESSES — the
|
| 43 |
+
list lives in .env and is never typed into the console, so whoever reaches the console can at most send the bot's SOL
|
| 44 |
+
to its owner. Never on the hosted vault fly: its SOL leaves through the treasury (``fly-trader vault withdraw``)."""
|
| 45 |
+
if config.VAULT_ENABLED:
|
| 46 |
+
raise SigningRefused("refusing: the vault fly's SOL leaves through the treasury (fly-trader vault withdraw), not from the trading wallet")
|
| 47 |
+
if config.SOLANA_CLUSTER != "mainnet-beta":
|
| 48 |
+
raise SigningRefused(f"refusing to withdraw on cluster {config.SOLANA_CLUSTER!r}: the bot wallet is on mainnet-beta")
|
| 49 |
+
if dest not in config.FUNDING_ADDRESSES:
|
| 50 |
+
raise SigningRefused(f"refusing: {dest} is not one of your FUNDING_ADDRESSES (.env)")
|
| 51 |
+
|
| 52 |
+
|
| 53 |
def assert_vault_signing_allowed() -> None:
|
| 54 |
"""The vault's own signing (claim payouts, principal withdrawals, token-account closes). On mainnet it is the same
|
| 55 |
gate as trading. The devnet rehearsal (``VAULT_CLUSTER=devnet``) may sign with a throwaway key that is not a
|
fly_trader/chain/jupiter_swap.py
CHANGED
|
@@ -58,7 +58,7 @@ JUPITER_EXECUTE_RPS = config.JUPITER_RPS # separate bucket, same rate
|
|
| 58 |
JUPITER_LABELS_URL = "https://api.jup.ag/swap/v2/program-id-to-label"
|
| 59 |
JUPITER_LABELS_CACHE = config.DATA_DIR / "jupiter_program_labels.json"
|
| 60 |
JUPITER_LABELS_MAX_AGE_S = 86400.0
|
| 61 |
-
DEFAULT_EXCLUDE_ROUTERS = "jupiterz"
|
| 62 |
|
| 63 |
|
| 64 |
class JupiterError(RuntimeError):
|
|
|
|
| 58 |
JUPITER_LABELS_URL = "https://api.jup.ag/swap/v2/program-id-to-label"
|
| 59 |
JUPITER_LABELS_CACHE = config.DATA_DIR / "jupiter_program_labels.json"
|
| 60 |
JUPITER_LABELS_MAX_AGE_S = 86400.0
|
| 61 |
+
DEFAULT_EXCLUDE_ROUTERS = "jupiterz,dflow,okx" # Metis only: the signer refuses other routers' programs (signer/policy.py)
|
| 62 |
|
| 63 |
|
| 64 |
class JupiterError(RuntimeError):
|
fly_trader/chain/rpc.py
CHANGED
|
@@ -224,3 +224,9 @@ class HttpSolanaRpc:
|
|
| 224 |
sig = self.call("sendTransaction", [tx_b64, {"encoding": "base64", "skipPreflight": bool(skip_preflight),
|
| 225 |
"preflightCommitment": COMMITMENT, "maxRetries": int(max_retries)}])
|
| 226 |
return str(sig)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 224 |
sig = self.call("sendTransaction", [tx_b64, {"encoding": "base64", "skipPreflight": bool(skip_preflight),
|
| 225 |
"preflightCommitment": COMMITMENT, "maxRetries": int(max_retries)}])
|
| 226 |
return str(sig)
|
| 227 |
+
|
| 228 |
+
|
| 229 |
+
def check_rpc() -> "HttpSolanaRpc | None":
|
| 230 |
+
"""A client for the second, independent provider (``SOLANA_CHECK_RPC_URL``) that money-moving reads must agree with,
|
| 231 |
+
or None when none is configured."""
|
| 232 |
+
return HttpSolanaRpc(config.SOLANA_CHECK_RPC_URL) if config.SOLANA_CHECK_RPC_URL else None
|
fly_trader/chain/withdraw.py
ADDED
|
@@ -0,0 +1,129 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Withdraw SOL from the bot wallet to one of the operator's own wallets (System → Safety & wallet in the console).
|
| 2 |
+
|
| 3 |
+
- Only to a wallet listed in FUNDING_ADDRESSES and only on mainnet; never on the hosted vault fly, whose SOL leaves
|
| 4 |
+
through the treasury (cluster_guard.assert_withdraw_allowed). No LIVE_ENABLED: a withdrawal takes money out of the
|
| 5 |
+
fly's reach and never trades.
|
| 6 |
+
- What is left is 0 or at least the rent-exempt minimum, and while live positions are open at least the gas reserve
|
| 7 |
+
(their exits pay fees from it). An empty destination must receive at least the rent-exempt minimum.
|
| 8 |
+
- It holds the live broker's in-flight lock, so a swap's balance delta never contains a withdrawal.
|
| 9 |
+
- The signed transaction is recorded (wallet_events 'withdrawal', status 'signed') before it is broadcast; its outcome is
|
| 10 |
+
added to that row. A confirmed withdrawal re-bases the live book's kill-switch peak (agent/rails.record_withdrawal):
|
| 11 |
+
money taken out is not a drawdown.
|
| 12 |
+
"""
|
| 13 |
+
from __future__ import annotations
|
| 14 |
+
|
| 15 |
+
import base64
|
| 16 |
+
import logging
|
| 17 |
+
from decimal import Decimal
|
| 18 |
+
|
| 19 |
+
from psycopg.types.json import Jsonb
|
| 20 |
+
from solders.hash import Hash
|
| 21 |
+
from solders.message import MessageV0
|
| 22 |
+
from solders.pubkey import Pubkey
|
| 23 |
+
from solders.system_program import TransferParams, transfer
|
| 24 |
+
from solders.transaction import VersionedTransaction
|
| 25 |
+
|
| 26 |
+
from .. import config
|
| 27 |
+
from ..agent import rails
|
| 28 |
+
from ..db.apilog import record_event
|
| 29 |
+
from ..db.connection import transaction
|
| 30 |
+
from ..logging_setup import scrub
|
| 31 |
+
from ..markets import SOL
|
| 32 |
+
from .cluster_guard import assert_withdraw_allowed
|
| 33 |
+
from .rpc import RpcError, RpcTransportError
|
| 34 |
+
|
| 35 |
+
log = logging.getLogger(__name__)
|
| 36 |
+
LAMPORTS = config.LAMPORTS_PER_SOL
|
| 37 |
+
FEE_LAMPORTS = 5_000 # one signature, no priority fee
|
| 38 |
+
RENT_EXEMPT_MIN = 890_880 # a system account without data: a balance below it (other than 0) is refused
|
| 39 |
+
LOCK_WAIT_S = 120.0 # a swap holds the broker's lock until it confirms or expires
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
class WithdrawRefused(RuntimeError):
|
| 43 |
+
"""Nothing was signed; the message says why."""
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def destinations() -> list[str]:
|
| 47 |
+
"""The wallets SOL may go to: the valid Solana addresses in FUNDING_ADDRESSES."""
|
| 48 |
+
out = []
|
| 49 |
+
for a in config.FUNDING_ADDRESSES:
|
| 50 |
+
try:
|
| 51 |
+
Pubkey.from_string(a)
|
| 52 |
+
out.append(a)
|
| 53 |
+
except ValueError:
|
| 54 |
+
log.warning("FUNDING_ADDRESSES: %r is not a Solana address", a)
|
| 55 |
+
return out
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def limits(rpc, me: str, to: str | None = None) -> dict:
|
| 59 |
+
"""Fresh reads: the wallet's lamports, what must stay (the gas reserve while live positions are open), the most that
|
| 60 |
+
can go out, and the destination's lamports."""
|
| 61 |
+
balance = int(rpc.get_balance(me))
|
| 62 |
+
with transaction() as conn:
|
| 63 |
+
n_open = int(conn.execute("SELECT count(*) AS n FROM positions WHERE book = %s AND status = 'open'", (SOL.live_book,)).fetchone()["n"])
|
| 64 |
+
keep = max(int(config.GAS_RESERVE_SOL * LAMPORTS), RENT_EXEMPT_MIN) if n_open else 0
|
| 65 |
+
return {"balance": balance, "fee": FEE_LAMPORTS, "keep": keep, "n_open": n_open, "max": max(0, balance - FEE_LAMPORTS - keep),
|
| 66 |
+
"to_balance": int(rpc.get_balance(to)) if to else None}
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
def plan(lim: dict, sol: float | None) -> tuple[int, str | None]:
|
| 70 |
+
"""(lamports to send, why not or None) for ``sol`` SOL, or everything withdrawable when ``sol`` is None."""
|
| 71 |
+
lam = lim["max"] if sol is None else int(Decimal(str(sol)) * LAMPORTS)
|
| 72 |
+
if lam <= 0:
|
| 73 |
+
return 0, "nothing to withdraw" if sol is None else "enter an amount"
|
| 74 |
+
if lam > lim["max"]:
|
| 75 |
+
return lam, (f"at most {lim['max'] / LAMPORTS:.6f} SOL can go out"
|
| 76 |
+
+ (f": {lim['keep'] / LAMPORTS:g} SOL stays for the fees of {lim['n_open']} open live position(s)" if lim["keep"] else " after the fee"))
|
| 77 |
+
left = lim["balance"] - lam - lim["fee"]
|
| 78 |
+
if 0 < left < RENT_EXEMPT_MIN:
|
| 79 |
+
return lam, f"Solana refuses to leave {left / LAMPORTS:.6f} SOL behind: leave at least {RENT_EXEMPT_MIN / LAMPORTS:g} SOL or withdraw everything"
|
| 80 |
+
if lim.get("to_balance") == 0 and lam < RENT_EXEMPT_MIN:
|
| 81 |
+
return lam, f"an empty wallet must receive at least {RENT_EXEMPT_MIN / LAMPORTS:g} SOL"
|
| 82 |
+
return lam, None
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
def withdraw(sol: float | None, to: str, rpc=None, keypair=None) -> dict:
|
| 86 |
+
"""Send ``sol`` SOL (None: everything withdrawable) from the bot wallet to ``to``. Returns {status, signature, lamports,
|
| 87 |
+
to, slot}; status 'confirmed', 'failed_on_chain' (only the fee moved) or 'expired' (nothing moved). Raises
|
| 88 |
+
WithdrawRefused, cluster_guard.SigningRefused or keys.WalletKeyError when nothing was signed."""
|
| 89 |
+
from ..execution.broker_live import _INFLIGHT, await_confirmation
|
| 90 |
+
assert_withdraw_allowed(to)
|
| 91 |
+
if keypair is None:
|
| 92 |
+
from .keys import load_keypair
|
| 93 |
+
keypair = load_keypair()
|
| 94 |
+
if rpc is None:
|
| 95 |
+
from .rpc import HttpSolanaRpc
|
| 96 |
+
rpc = HttpSolanaRpc()
|
| 97 |
+
me = str(keypair.pubkey())
|
| 98 |
+
if not _INFLIGHT.acquire(timeout=LOCK_WAIT_S):
|
| 99 |
+
raise WithdrawRefused("a swap is still in flight: try again in a minute")
|
| 100 |
+
try:
|
| 101 |
+
lam, why = plan(limits(rpc, me, to), sol)
|
| 102 |
+
if why:
|
| 103 |
+
raise WithdrawRefused(why)
|
| 104 |
+
bh = rpc.get_latest_blockhash()
|
| 105 |
+
ix = transfer(TransferParams(from_pubkey=keypair.pubkey(), to_pubkey=Pubkey.from_string(to), lamports=lam))
|
| 106 |
+
tx = VersionedTransaction(MessageV0.try_compile(keypair.pubkey(), [ix], [], Hash.from_string(bh["blockhash"])), [keypair])
|
| 107 |
+
sig = str(tx.signatures[0])
|
| 108 |
+
detail = {"to": to, "lamports": lam, "sol": lam / LAMPORTS, "signature": sig, "last_valid_block_height": bh["last_valid_block_height"], "status": "signed"}
|
| 109 |
+
with transaction() as conn: # recorded before the network ever sees it
|
| 110 |
+
eid = conn.execute("INSERT INTO wallet_events (kind, pubkey, detail) VALUES ('withdrawal', %s, %s) RETURNING id", (me, Jsonb(detail))).fetchone()["id"]
|
| 111 |
+
try:
|
| 112 |
+
rpc.send_transaction(base64.b64encode(bytes(tx)).decode())
|
| 113 |
+
except RpcError as e: # refused by the node (preflight): never broadcast
|
| 114 |
+
with transaction() as conn:
|
| 115 |
+
conn.execute("UPDATE wallet_events SET detail = detail || %s WHERE id = %s", (Jsonb({"status": "rejected", "error": scrub(str(e))[:300]}), eid))
|
| 116 |
+
raise WithdrawRefused(f"the network refused it: {scrub(str(e))[:200]}") from None
|
| 117 |
+
except RpcTransportError as e: # it may have gone out: its status decides
|
| 118 |
+
log.warning("withdrawal %s: send failed in transport (%s); polling its status", sig, e)
|
| 119 |
+
status, st = await_confirmation(rpc, sig, bh["last_valid_block_height"])
|
| 120 |
+
slot = int((st or {}).get("slot") or 0)
|
| 121 |
+
with transaction() as conn:
|
| 122 |
+
conn.execute("UPDATE wallet_events SET detail = detail || %s WHERE id = %s", (Jsonb({"status": status, "slot": slot}), eid))
|
| 123 |
+
if status == "confirmed":
|
| 124 |
+
rails.record_withdrawal(conn, {"sol": lam / LAMPORTS, "to": to, "signature": sig})
|
| 125 |
+
finally:
|
| 126 |
+
_INFLIGHT.release()
|
| 127 |
+
record_event("info" if status == "confirmed" else "warning", "wallet", f"withdrawal of {lam / LAMPORTS:.6f} SOL to {to}: {status}",
|
| 128 |
+
{"signature": sig, "slot": slot})
|
| 129 |
+
return {"status": status, "signature": sig, "lamports": lam, "to": to, "slot": slot}
|
fly_trader/config.py
CHANGED
|
@@ -14,8 +14,10 @@ from dotenv import load_dotenv
|
|
| 14 |
REPO_ROOT = Path(__file__).resolve().parent.parent
|
| 15 |
load_dotenv(REPO_ROOT / ".env", override=False)
|
| 16 |
|
| 17 |
-
SECRET_ENV_NAMES = ("HELIUS_API_KEY", "JUPITER_API_KEY", "BOT_PRIVATE_KEY",
|
| 18 |
-
"RELAY_SECRET", "TELEGRAM_BOT_TOKEN", "VAULT_BACKUP_HF_TOKEN", "VAULT_SOLANA_RPC_URL"
|
|
|
|
|
|
|
| 19 |
|
| 20 |
|
| 21 |
def utcnow() -> datetime:
|
|
@@ -76,9 +78,12 @@ KELLY_FRACTION = env_float("KELLY_FRACTION", 0.25) # quarter Kel
|
|
| 76 |
MAX_POSITION_FRACTION = env_float("MAX_POSITION_FRACTION", 0.10) # of the deployable bankroll (wealth minus the gas reserve)
|
| 77 |
MAX_POOL_SHARE = env_float("MAX_POOL_SHARE", 0.02) # of the pool's quote reserve (bounds price impact)
|
| 78 |
# The training label charges what an exit really costs at the size the book takes. market/features.py prices
|
| 79 |
-
# exit_cost_0p1 for a fixed 0.1 SOL, but
|
| 80 |
-
#
|
| 81 |
-
|
|
|
|
|
|
|
|
|
|
| 82 |
MIN_POSITION_SOL = env_float("MIN_POSITION_SOL", 0.02) # smaller sizes are skipped
|
| 83 |
KILL_SWITCH_DRAWDOWN = env_float("KILL_SWITCH_DRAWDOWN", 0.30)
|
| 84 |
KILL_SWITCH_LIQUIDATE = env_bool("KILL_SWITCH_LIQUIDATE", False) # a tripped kill switch also sells every open live position (agent/fly_live.py)
|
|
@@ -103,6 +108,25 @@ LAMPORTS_PER_SOL = 1_000_000_000
|
|
| 103 |
STATS_REFRESH_S = env_float("STATS_REFRESH_S", 600.0)
|
| 104 |
|
| 105 |
# ---- connectome (the fly) ----
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 106 |
DEVICE = env_str("DEVICE", "auto") # auto (CUDA, else MPS, else CPU) | cpu | cuda | cuda:N | mps — brain/device.py
|
| 107 |
NT_SIGN_MODE = env_str("NT_SIGN_MODE", "shiu") # shiu | flybrain
|
| 108 |
CELL_TYPES_SOURCE = env_str("CELL_TYPES_SOURCE", "annotations") # annotations | flybrain
|
|
@@ -158,8 +182,8 @@ def live_prerequisites_missing() -> list[str]:
|
|
| 158 |
missing.append("MAX_POSITION_SOL")
|
| 159 |
if GAS_RESERVE_SOL <= 0:
|
| 160 |
missing.append("GAS_RESERVE_SOL")
|
| 161 |
-
if not env_str("BOT_PRIVATE_KEY") and not env_str("BOT_PRIVATE_KEY_FILE"):
|
| 162 |
-
missing.append("BOT_PRIVATE_KEY")
|
| 163 |
if not JUPITER_API_KEY:
|
| 164 |
missing.append("JUPITER_API_KEY")
|
| 165 |
if not HELIUS_API_KEY:
|
|
@@ -168,9 +192,11 @@ def live_prerequisites_missing() -> list[str]:
|
|
| 168 |
|
| 169 |
# ---- the $FLY vault (fly_trader/vault, docs/vault/SPEC.md): only the hosted vault fly sets VAULT_ENABLED ----
|
| 170 |
VAULT_ENABLED = env_bool("VAULT_ENABLED", False)
|
|
|
|
|
|
|
| 171 |
VAULT_CLUSTER = env_str("VAULT_CLUSTER", "mainnet-beta") # devnet only for the rehearsal (trading off, claims on devnet)
|
| 172 |
VAULT_SOLANA_RPC_URL = env_str("VAULT_SOLANA_RPC_URL") # overrides Helius for the vault's reads/claims (the devnet rehearsal)
|
| 173 |
-
FUNDING_ADDRESSES = env_list("FUNDING_ADDRESSES", []) # SOL from these is a deposit;
|
| 174 |
RH_CHAIN_ID = env_int("RH_CHAIN_ID", 4663)
|
| 175 |
RH_RPC_URL = env_str("RH_RPC_URL", "https://rpc.mainnet.chain.robinhood.com")
|
| 176 |
VAULT_ADDRESS = env_str("VAULT_ADDRESS") # the FlyVault proxy
|
|
@@ -180,6 +206,8 @@ VAULT_IMPL_CODEHASHES = env_list("VAULT_IMPL_CODEHASHES", []) # implementa
|
|
| 180 |
VAULT_PERIOD_S = env_int("VAULT_PERIOD_S", 7 * 86400) # one settlement period; the rehearsal uses 900
|
| 181 |
VAULT_EPOCH = env_int("VAULT_EPOCH", 345_600) # boundaries at EPOCH + k*PERIOD; 345600 = Monday 1970-01-05 00:00 UTC
|
| 182 |
CLAIM_MIN_LAMPORTS = env_int("CLAIM_MIN_LAMPORTS", 2_000_000) # 0.002 SOL: above the rent-exempt minimum of a new wallet
|
|
|
|
|
|
|
| 183 |
RELAY_URL = env_str("RELAY_URL") # the site origin; the fly calls {RELAY_URL}/api/...
|
| 184 |
RELAY_KEY_ID = env_str("RELAY_KEY_ID", "k1")
|
| 185 |
RELAY_SECRET = env_str("RELAY_SECRET")
|
|
@@ -193,11 +221,161 @@ TELEGRAM_CHAT_ID = env_str("TELEGRAM_CHAT_ID")
|
|
| 193 |
VAULT_BACKUP_RECIPIENT = env_str("VAULT_BACKUP_RECIPIENT") # age / ssh-ed25519 public key the backups are encrypted to
|
| 194 |
VAULT_BACKUP_BUCKET = env_str("VAULT_BACKUP_BUCKET") # private HF storage bucket, e.g. bryceweiner/fly-vault-backups
|
| 195 |
VAULT_BACKUP_HF_TOKEN = env_str("VAULT_BACKUP_HF_TOKEN") # scoped to that bucket only
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 196 |
|
| 197 |
|
| 198 |
def vault_solana_rpc_url() -> str:
|
| 199 |
return VAULT_SOLANA_RPC_URL or helius_http_url()
|
| 200 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 201 |
# ---- historical corpus (every pump.fun graduation, pulled from the migration wallet + pump.fun's swap-api) ----
|
| 202 |
CORPUS_DIR = DATA_DIR / "corpus"
|
| 203 |
CORPUS_DAYS = env_int("CORPUS_DAYS", 240) # enumerate graduations this far back (must reach before CORPUS_PULL_BEFORE)
|
|
|
|
| 14 |
REPO_ROOT = Path(__file__).resolve().parent.parent
|
| 15 |
load_dotenv(REPO_ROOT / ".env", override=False)
|
| 16 |
|
| 17 |
+
SECRET_ENV_NAMES = ("HELIUS_API_KEY", "JUPITER_API_KEY", "BOT_PRIVATE_KEY", "KALSHI_API_KEY_ID", "KALSHI_PRIVATE_KEY_PATH",
|
| 18 |
+
"RELAY_SECRET", "TELEGRAM_BOT_TOKEN", "VAULT_BACKUP_HF_TOKEN", "VAULT_SOLANA_RPC_URL", "SIGNER_RPC_URL",
|
| 19 |
+
"SOLANA_CHECK_RPC_URL", "HEALTHCHECK_URL", "RH_BOT_PRIVATE_KEY", "RH_RPC_URL_LOGS", "ENVIO_API_TOKEN", "QUICKNODE_API_KEY",
|
| 20 |
+
"QUICKNODE_RH_API_KEY", "QUICKNODE_RH_API_URL", "RH_RPC_URL_TX")
|
| 21 |
|
| 22 |
|
| 23 |
def utcnow() -> datetime:
|
|
|
|
| 78 |
MAX_POSITION_FRACTION = env_float("MAX_POSITION_FRACTION", 0.10) # of the deployable bankroll (wealth minus the gas reserve)
|
| 79 |
MAX_POOL_SHARE = env_float("MAX_POOL_SHARE", 0.02) # of the pool's quote reserve (bounds price impact)
|
| 80 |
# The training label charges what an exit really costs at the size the book takes. market/features.py prices
|
| 81 |
+
# exit_cost_0p1 for a fixed 0.1 SOL, but impact = value/(value + reserves) grows with size. Labels are priced at
|
| 82 |
+
# min(LABEL_SIZE_SOL, MAX_POOL_SHARE of the pool) -- the "label size" -- and no position is ever bigger (agent/sizing.py):
|
| 83 |
+
# every edge the models show was measured at that size. A fixed reference, never a share of the bankroll: derived from
|
| 84 |
+
# CAPITAL_SOL it made the labels, and the sizes, grow with the bankroll, and in the 2026-09-26 replay a 500 SOL book
|
| 85 |
+
# sized past it lost 426 SOL over 60 days where one capped at it made the same +13 SOL as a 5 or 50 SOL book.
|
| 86 |
+
LABEL_SIZE_SOL = env_float("LABEL_SIZE_SOL", 0.5)
|
| 87 |
MIN_POSITION_SOL = env_float("MIN_POSITION_SOL", 0.02) # smaller sizes are skipped
|
| 88 |
KILL_SWITCH_DRAWDOWN = env_float("KILL_SWITCH_DRAWDOWN", 0.30)
|
| 89 |
KILL_SWITCH_LIQUIDATE = env_bool("KILL_SWITCH_LIQUIDATE", False) # a tripped kill switch also sells every open live position (agent/fly_live.py)
|
|
|
|
| 108 |
STATS_REFRESH_S = env_float("STATS_REFRESH_S", 600.0)
|
| 109 |
|
| 110 |
# ---- connectome (the fly) ----
|
| 111 |
+
# How the fly picks its buy line (train/fly_calibrate.py): total | selector | match. match = its teacher's selectivity:
|
| 112 |
+
# in the 2026-09-26 replays it took 92 trades at +10.1 % (58 % winners, PF 1.92) where total took 1,607 at +2.0 %.
|
| 113 |
+
FLY_CALIB_RULE = env_str("FLY_CALIB_RULE", "match")
|
| 114 |
+
# How the fly sizes a buy (agent/sizing.py): kelly = the selector's own Kelly band for the row (its score, line and sizing
|
| 115 |
+
# table; with FLY_TEACHER_GATE every fly buy is a selector buy, sized exactly as the selector sizes it) | flat =
|
| 116 |
+
# MAX_POSITION_FRACTION of the deployable bankroll on every buy. (Flat was the default while the fly sized from its own
|
| 117 |
+
# calibrated bands, which flipped between Kelly 0.97, 0 and 0.15 on its few trades a week; the selector's bands are fitted
|
| 118 |
+
# on its walk-forward and do not.)
|
| 119 |
+
FLY_SIZING = env_str("FLY_SIZING", "kelly")
|
| 120 |
+
# The fly buys only what its teacher (the deployed selector stack) would buy: the same trigger, line, win filter, gates,
|
| 121 |
+
# dump veto and fail-closed rules. Its own learning (plastic score below its bootstrap score by FLY_LEARNED_VETO) can
|
| 122 |
+
# only drop a teacher trade, never add one (2026-10-08: ungated, the fly bought rows the selector rejected and lost).
|
| 123 |
+
FLY_TEACHER_GATE = env_bool("FLY_TEACHER_GATE", True)
|
| 124 |
+
FLY_LEARNED_VETO = env_float("FLY_LEARNED_VETO", 0.02)
|
| 125 |
+
# How the fly scores a row. teacher: its teacher's own score plus what its mushroom body has learned (the plastic change of
|
| 126 |
+
# its KC->MBON synapses, zero at bootstrap), so it starts exactly as the selector and its learning only corrects the
|
| 127 |
+
# selector. own: its distilled copy of the selector (until 2026-10-11): that copy ranked the selector's top 1 % of
|
| 128 |
+
# candidates only ~0.6-correlated, so its own judgement at the line was half noise (fly #157 and #251 alike).
|
| 129 |
+
FLY_SCORE = env_str("FLY_SCORE", "teacher")
|
| 130 |
DEVICE = env_str("DEVICE", "auto") # auto (CUDA, else MPS, else CPU) | cpu | cuda | cuda:N | mps — brain/device.py
|
| 131 |
NT_SIGN_MODE = env_str("NT_SIGN_MODE", "shiu") # shiu | flybrain
|
| 132 |
CELL_TYPES_SOURCE = env_str("CELL_TYPES_SOURCE", "annotations") # annotations | flybrain
|
|
|
|
| 182 |
missing.append("MAX_POSITION_SOL")
|
| 183 |
if GAS_RESERVE_SOL <= 0:
|
| 184 |
missing.append("GAS_RESERVE_SOL")
|
| 185 |
+
if not SIGNER_SOCKET and not env_str("BOT_PRIVATE_KEY") and not env_str("BOT_PRIVATE_KEY_FILE"):
|
| 186 |
+
missing.append("BOT_PRIVATE_KEY") # the vault server's brain has no key: its signer does
|
| 187 |
if not JUPITER_API_KEY:
|
| 188 |
missing.append("JUPITER_API_KEY")
|
| 189 |
if not HELIUS_API_KEY:
|
|
|
|
| 192 |
|
| 193 |
# ---- the $FLY vault (fly_trader/vault, docs/vault/SPEC.md): only the hosted vault fly sets VAULT_ENABLED ----
|
| 194 |
VAULT_ENABLED = env_bool("VAULT_ENABLED", False)
|
| 195 |
+
PAYOUT_PRIVATE_KEY_FILE = env_str("PAYOUT_PRIVATE_KEY_FILE") # the key that pays claims from the treasury (spending limit L2); read by the signer only
|
| 196 |
+
VAULT_BOOK = env_str("VAULT_BOOK", "live") # the book the vault shares: live; a paper book (e.g. paper_fly) for a dry run on real decisions without real SOL
|
| 197 |
VAULT_CLUSTER = env_str("VAULT_CLUSTER", "mainnet-beta") # devnet only for the rehearsal (trading off, claims on devnet)
|
| 198 |
VAULT_SOLANA_RPC_URL = env_str("VAULT_SOLANA_RPC_URL") # overrides Helius for the vault's reads/claims (the devnet rehearsal)
|
| 199 |
+
FUNDING_ADDRESSES = env_list("FUNDING_ADDRESSES", []) # your own Solana wallets: SOL from these is a deposit (vault); the console withdraws only to these
|
| 200 |
RH_CHAIN_ID = env_int("RH_CHAIN_ID", 4663)
|
| 201 |
RH_RPC_URL = env_str("RH_RPC_URL", "https://rpc.mainnet.chain.robinhood.com")
|
| 202 |
VAULT_ADDRESS = env_str("VAULT_ADDRESS") # the FlyVault proxy
|
|
|
|
| 206 |
VAULT_PERIOD_S = env_int("VAULT_PERIOD_S", 7 * 86400) # one settlement period; the rehearsal uses 900
|
| 207 |
VAULT_EPOCH = env_int("VAULT_EPOCH", 345_600) # boundaries at EPOCH + k*PERIOD; 345600 = Monday 1970-01-05 00:00 UTC
|
| 208 |
CLAIM_MIN_LAMPORTS = env_int("CLAIM_MIN_LAMPORTS", 2_000_000) # 0.002 SOL: above the rent-exempt minimum of a new wallet
|
| 209 |
+
CLAIM_MIN_WEI = env_int("CLAIM_MIN_WEI", 100_000_000_000_000) # 0.0001 ETH: smaller ETH owed waits for the next claim
|
| 210 |
+
VAULT_RH_GAS_RESERVE_WEI = env_int("VAULT_RH_GAS_RESERVE_WEI", 200_000_000_000_000) # ETH the settlement never allocates (the RH wallet pays payout gas)
|
| 211 |
RELAY_URL = env_str("RELAY_URL") # the site origin; the fly calls {RELAY_URL}/api/...
|
| 212 |
RELAY_KEY_ID = env_str("RELAY_KEY_ID", "k1")
|
| 213 |
RELAY_SECRET = env_str("RELAY_SECRET")
|
|
|
|
| 221 |
VAULT_BACKUP_RECIPIENT = env_str("VAULT_BACKUP_RECIPIENT") # age / ssh-ed25519 public key the backups are encrypted to
|
| 222 |
VAULT_BACKUP_BUCKET = env_str("VAULT_BACKUP_BUCKET") # private HF storage bucket, e.g. bryceweiner/fly-vault-backups
|
| 223 |
VAULT_BACKUP_HF_TOKEN = env_str("VAULT_BACKUP_HF_TOKEN") # scoped to that bucket only
|
| 224 |
+
# custody (docs/vault/SPEC.md "Custody"): a Squads v4 treasury the hot keys draw on only through its spending limits
|
| 225 |
+
VAULT_MULTISIG = env_str("VAULT_MULTISIG") # the Squads multisig; the treasury is its vault 0
|
| 226 |
+
VAULT_LIMIT_TRADING = env_str("VAULT_LIMIT_TRADING") # L1: trading key -> the trading wallet only, per day
|
| 227 |
+
VAULT_LIMIT_PAYOUT = env_str("VAULT_LIMIT_PAYOUT") # L2: payout key -> any holder, per week
|
| 228 |
+
TREASURY_FLOAT_SOL = env_float("TREASURY_FLOAT_SOL", 2.0) # the most the trading wallet keeps idle; the rest goes back
|
| 229 |
+
TOPUP_MIN_SOL = env_float("TOPUP_MIN_SOL", 0.1) # smaller refills and returns are skipped
|
| 230 |
+
SIGNER_SOCKET = env_str("SIGNER_SOCKET") # the signer container's socket; unset: an in-process signer
|
| 231 |
+
SIGNER_RPC_URL = env_str("SIGNER_RPC_URL") # the signer's own RPC (a second provider)
|
| 232 |
+
SOLANA_CHECK_RPC_URL = env_str("SOLANA_CHECK_RPC_URL") # a second provider money-moving reads must agree with
|
| 233 |
+
TELEGRAM_ADMIN_USER_ID = env_str("TELEGRAM_ADMIN_USER_ID") # the only Telegram user whose /panic and /status count
|
| 234 |
+
HEALTHCHECK_URL = env_str("HEALTHCHECK_URL") # dead-man ping (healthchecks.io) every 5 min
|
| 235 |
|
| 236 |
|
| 237 |
def vault_solana_rpc_url() -> str:
|
| 238 |
return VAULT_SOLANA_RPC_URL or helius_http_url()
|
| 239 |
|
| 240 |
+
|
| 241 |
+
def signer_env() -> dict:
|
| 242 |
+
"""What an in-process signer reads (fly_trader/signer; the signer container has its own environment)."""
|
| 243 |
+
return {k: str(v) for k, v in {"VAULT_MULTISIG": VAULT_MULTISIG, "VAULT_LIMIT_TRADING": VAULT_LIMIT_TRADING,
|
| 244 |
+
"VAULT_LIMIT_PAYOUT": VAULT_LIMIT_PAYOUT,
|
| 245 |
+
"TREASURY_FLOAT_SOL": TREASURY_FLOAT_SOL, "VAULT_BACKUP_RECIPIENT": VAULT_BACKUP_RECIPIENT}.items() if v}
|
| 246 |
+
|
| 247 |
+
# ---- Robinhood Chain memecoins (fly_trader/rh; Pons launches graduated into Uniswap v4; branch rh-memecoins) ----
|
| 248 |
+
# The same selector and fly as Solana, trained on both chains' rows (a chain input tells them apart); RH books keep their
|
| 249 |
+
# money in native ETH. Amounts on RH rows are ETH (non-ETH-quoted pools converted at the minute's base/ETH price).
|
| 250 |
+
RH_ENABLED = env_bool("RH_ENABLED", True) # index RH and paper-trade it
|
| 251 |
+
RH_LIVE_ENABLED = env_bool("RH_LIVE_ENABLED", False) # sign and send real RH transactions (live_rh)
|
| 252 |
+
RH_TESTNET = env_bool("RH_TESTNET", False) # chain 46630 (the rehearsal); the guard refuses a mismatch
|
| 253 |
+
RH_EXPECTED_CHAIN_ID = 46630 if RH_TESTNET else RH_CHAIN_ID
|
| 254 |
+
RH_RPC_URL_LOGS = env_str("RH_RPC_URL_LOGS") # optional better endpoint for eth_getLogs / tx lookups (else RH_RPC_URL)
|
| 255 |
+
|
| 256 |
+
|
| 257 |
+
def quicknode_rh_url() -> str | None:
|
| 258 |
+
"""The QuickNode RH endpoint: QUICKNODE_RH_API_URL, with QUICKNODE_RH_API_KEY appended when the URL does not carry it."""
|
| 259 |
+
url, key = env_str("QUICKNODE_RH_API_URL"), env_str("QUICKNODE_RH_API_KEY")
|
| 260 |
+
if not url:
|
| 261 |
+
return None
|
| 262 |
+
return url if not key or key in url else url.rstrip("/") + "/" + key
|
| 263 |
+
|
| 264 |
+
|
| 265 |
+
# sending and reading transactions (sends, receipts, nonces, balances, eth_call, gas): every RhRpc() without an explicit URL.
|
| 266 |
+
# Never logs: QuickNode's free Discover plan caps eth_getLogs at 5 blocks (measured 2026-10-09), so logs stay on HyperSync /
|
| 267 |
+
# RH_RPC_URL_LOGS / RH_RPC_URL; its 10M credits a month cover the trading path, not a log tail. QuickNode serves mainnet
|
| 268 |
+
# only while RH_RPC_URL is the default: a run that points RH_RPC_URL elsewhere (anvil, the testnet rehearsal) sends there.
|
| 269 |
+
RH_RPC_URL_TX = env_str("RH_RPC_URL_TX") or (quicknode_rh_url() if env_str("RH_RPC_URL") is None and not RH_TESTNET else None)
|
| 270 |
+
# block headers for timing the index (rh/blocktime.py): PublicNode serves header batches fast but refuses eth_getLogs (403),
|
| 271 |
+
# the official RPC serves logs but rate-limits at ~20 requests/min (measured 2026-09-28) — so each does what it does well
|
| 272 |
+
RH_RPC_URL_BLOCKS = env_str("RH_RPC_URL_BLOCKS", "https://robinhood-rpc.publicnode.com")
|
| 273 |
+
RH_HYPERSYNC_URL = env_str("RH_HYPERSYNC_URL", "https://robinhood.hypersync.xyz") # log history in bulk when ENVIO_API_TOKEN is set (rh/hypersync.py)
|
| 274 |
+
RH_HYPERSYNC_RPM = env_float("RH_HYPERSYNC_RPM", 40) # HyperSync queries per minute (free tier: fair use, refused ~6 % at 60; Starter plan: 100)
|
| 275 |
+
RH_BOT_ADDRESS = env_str("RH_BOT_ADDRESS") # the address RH_BOT_PRIVATE_KEY must derive to (a guard, not a secret)
|
| 276 |
+
RH_FUNDING_ADDRESSES = env_list("RH_FUNDING_ADDRESSES", []) # your own EVM wallets: the console withdraws the RH bot's ETH only to these
|
| 277 |
+
RH_CONFIRMATIONS = env_int("RH_CONFIRMATIONS", 3) # blocks before a live receipt or a live log is applied
|
| 278 |
+
RH_START_BLOCK = env_int("RH_START_BLOCK", 0) # the indexer's first block (0: the Pons V2 factory's deployment block)
|
| 279 |
+
RH_START_DAY = env_str("RH_START_DAY", "2026-08-04") # Pons V2 went live: the first RH corpus day
|
| 280 |
+
# Fixed at build time by tools/rh_constants.py (2026-09-28 13:12 UTC: SOL/USD 119.51, ETH/USD 2682.05): the Solana sizes in ETH.
|
| 281 |
+
# Never derived live: they set the size every RH label is priced at (see LABEL_SIZE_SOL).
|
| 282 |
+
RH_ETH_PER_SOL = env_float("RH_ETH_PER_SOL", 0.0445583) # K: converts fixed-unit thresholds (eligibility, triggers) for RH rows
|
| 283 |
+
RH_CAPITAL_ETH = env_float("RH_CAPITAL_ETH", 0.222791) # the RH paper books' bankroll (5 SOL at build time)
|
| 284 |
+
RH_LABEL_SIZE_ETH = env_float("RH_LABEL_SIZE_ETH", 0.0222791) # 0.5 SOL at build time: RH labels are priced at it; no RH buy is bigger
|
| 285 |
+
RH_MIN_POSITION_ETH = env_float("RH_MIN_POSITION_ETH", 0.000891166) # 0.02 SOL at build time
|
| 286 |
+
RH_GAS_RESERVE_ETH = env_float("RH_GAS_RESERVE_ETH", 0.0029) # 200 round trips at 2x the gas rh-probe measured (356k gas, 0.02 gwei, 2026-09-28)
|
| 287 |
+
RH_MAX_FEE_GWEI = env_float("RH_MAX_FEE_GWEI", 5.0) # a transaction never offers more per gas
|
| 288 |
+
RH_TX_FEE_ETH = env_float("RH_TX_FEE_ETH", 7.2e-6) # one swap's gas in ETH (rh-probe 2026-09-28: 356k gas × 0.02 gwei); fixed like the SOL fee
|
| 289 |
+
RH_HOOK_FEE = env_float("RH_HOOK_FEE", 0.0099) # Pons hook fee + creator tax per side when a minute has none: the median of 52k indexed swaps (1 % of the net = 0.99 % of gross; high-tax launches reach 5.7 %)
|
| 290 |
+
RH_KILL_SWITCH_DRAWDOWN = env_float("RH_KILL_SWITCH_DRAWDOWN", 0.30)
|
| 291 |
+
RH_MIN_SWEEP_ETH = env_float("RH_MIN_SWEEP_ETH", 0.0002) # base-asset or dead-bag balances worth less are dust
|
| 292 |
+
RH_STREAM_POLL_S = env_float("RH_STREAM_POLL_S", 2.0)
|
| 293 |
+
RH_TRADE_LAG_S = env_float("RH_TRADE_LAG_S", 240.0) # an RH minute this old (s) is still traded: its stream lands ~2–3 min late
|
| 294 |
+
RH_MINUTES_KEEP_DAYS = env_int("RH_MINUTES_KEEP_DAYS", 14)
|
| 295 |
+
RH_DIR_NAME = "rh" # data/corpus/rh/... (the RH corpus beside the Solana one)
|
| 296 |
+
# contracts on Robinhood Chain (mainnet; verified to carry code 2026-09-28)
|
| 297 |
+
PONS_FACTORY = env_str("PONS_FACTORY", "0x7ed598bcef8bd9edd8c97a195c6d13f40801ec7e")
|
| 298 |
+
PONS_ROUTER = env_str("PONS_ROUTER", "0xe33e9e479df8802cb0866d5d05258bec4cf62948")
|
| 299 |
+
PONS_HOOK = env_str("PONS_HOOK", "0xe5e702641ea86f4ae6cc3cdaed2b886f976be044")
|
| 300 |
+
PONS_LOCKER = env_str("PONS_LOCKER", "0x267444d099b10fb5ed7c3cc7b7c767adca574952")
|
| 301 |
+
PONS_GRAD_EXECUTOR = env_str("PONS_GRAD_EXECUTOR", "0xc7819b64a1daecd7ec19856d026cb14efbd89046")
|
| 302 |
+
V4_POOL_MANAGER = env_str("V4_POOL_MANAGER", "0x8366a39cc670b4001a1121b8f6a443a643e40951")
|
| 303 |
+
V4_QUOTER = env_str("V4_QUOTER", "0x8dc178efb8111bb0973dd9d722ebeff267c98f94")
|
| 304 |
+
V4_STATE_VIEW = env_str("V4_STATE_VIEW", "0xf3334192d15450cdd385c8b70e03f9a6bd9e673b")
|
| 305 |
+
UNIVERSAL_ROUTER = env_str("UNIVERSAL_ROUTER", "0x8876789976decbfcbbbe364623c63652db8c0904")
|
| 306 |
+
PERMIT2 = env_str("PERMIT2", "0x000000000022D473030F116dDEE9F6B43aC78BA3")
|
| 307 |
+
V3_FACTORY = env_str("V3_FACTORY", "0x1f7d7550b1b028f7571e69a784071f0205fd2efa")
|
| 308 |
+
V3_QUOTER_V2 = env_str("V3_QUOTER_V2", "0x33e885ed0ec9bf04ecfb19341582aadcb4c8a9e7")
|
| 309 |
+
RH_WETH = env_str("RH_WETH", "0x0bd7d308f8e1639fab988df18a8011f41eacad73") # SwapRouter02.WETH9()
|
| 310 |
+
RH_USDG = env_str("RH_USDG", "0x5fc5360D0400a0Fd4f2af552ADD042D716F1d168")
|
| 311 |
+
KYBER_API = env_str("KYBER_API", "https://aggregator-api.kyberswap.com/robinhood/api/v1")
|
| 312 |
+
KYBER_ROUTER = env_str("KYBER_ROUTER", "0x6131B5fae19EA4f9D964eAc0408E4408b66337b5") # MetaAggregationRouterV2: the only router a built Kyber tx may call
|
| 313 |
+
KYBER_CLIENT_ID = env_str("KYBER_CLIENT_ID", "fly-trader")
|
| 314 |
+
|
| 315 |
+
|
| 316 |
+
def rh_live_prerequisites_missing() -> list[str]:
|
| 317 |
+
missing = []
|
| 318 |
+
if not RH_LIVE_ENABLED:
|
| 319 |
+
missing.append("RH_LIVE_ENABLED=1")
|
| 320 |
+
if not env_str("RH_BOT_PRIVATE_KEY"):
|
| 321 |
+
missing.append("RH_BOT_PRIVATE_KEY")
|
| 322 |
+
if not RH_BOT_ADDRESS:
|
| 323 |
+
missing.append("RH_BOT_ADDRESS")
|
| 324 |
+
for name in ("RH_CAPITAL_ETH", "RH_LABEL_SIZE_ETH", "RH_MIN_POSITION_ETH", "RH_ETH_PER_SOL", "RH_GAS_RESERVE_ETH"):
|
| 325 |
+
if globals()[name] <= 0:
|
| 326 |
+
missing.append(name)
|
| 327 |
+
return missing
|
| 328 |
+
|
| 329 |
+
|
| 330 |
+
# ---- Kalshi prediction markets (fly_trader/kalshi; the same key names as better_bot, whose client is vendored) ----
|
| 331 |
+
KALSHI_API_KEY_ID = env_str("KALSHI_API_KEY_ID")
|
| 332 |
+
KALSHI_PRIVATE_KEY_PATH = env_str("KALSHI_PRIVATE_KEY_PATH")
|
| 333 |
+
KALSHI_API_BASE_URL = env_str("KALSHI_API_BASE_URL", "https://api.elections.kalshi.com/trade-api/v2")
|
| 334 |
+
KALSHI_WS_URL = env_str("KALSHI_WS_URL", "wss://api.elections.kalshi.com/trade-api/ws/v2")
|
| 335 |
+
KALSHI_SUBACCOUNT = env_int("KALSHI_SUBACCOUNT", 0) # 0 = the primary account: only a dedicated subaccount may trade live
|
| 336 |
+
KALSHI_FEE_FACTOR = env_float("KALSHI_FEE_FACTOR", 0.07) # taker fee per contract = factor x multiplier x P(1-P)
|
| 337 |
+
KALSHI_MAKER_FEE_FACTOR = env_float("KALSHI_MAKER_FEE_FACTOR", 0.0175)
|
| 338 |
+
KALSHI_CAPITAL_USD = env_float("KALSHI_CAPITAL_USD", 100.0) # the cap both Kalshi arms share (paper start; live never sizes above it)
|
| 339 |
+
KALSHI_CASH_FLOOR_USD = env_float("KALSHI_CASH_FLOOR_USD", 5.0) # the gas reserve's analogue: never spent
|
| 340 |
+
KALSHI_MIN_PAYOUT_CENTS = env_int("KALSHI_MIN_PAYOUT_CENTS", 25) # a fill whose winning profit is below this is skipped
|
| 341 |
+
KALSHI_LIVE_ENABLED = env_bool("KALSHI_LIVE_ENABLED", False)
|
| 342 |
+
KALSHI_TAKER_LIVE = env_bool("KALSHI_TAKER_LIVE", True)
|
| 343 |
+
KALSHI_MAKER_LIVE = env_bool("KALSHI_MAKER_LIVE", True)
|
| 344 |
+
KALSHI_KILL_SWITCH_DRAWDOWN = env_float("KALSHI_KILL_SWITCH_DRAWDOWN", 0.30)
|
| 345 |
+
KALSHI_MAX_SLATE_FRACTION = env_float("KALSHI_MAX_SLATE_FRACTION", 0.50) # of the cap deployed at once across every open position
|
| 346 |
+
KALSHI_MAX_DAYS_TO_CLOSE = env_float("KALSHI_MAX_DAYS_TO_CLOSE", 10.0) # the evidence base (Whelan 2026) covers the last 10 days
|
| 347 |
+
KALSHI_MIN_MINUTES_TO_CLOSE = env_float("KALSHI_MIN_MINUTES_TO_CLOSE", 5.0)
|
| 348 |
+
KALSHI_MAX_SPREAD_CENTS = env_int("KALSHI_MAX_SPREAD_CENTS", 10)
|
| 349 |
+
KALSHI_MIN_OPEN_INTEREST = env_float("KALSHI_MIN_OPEN_INTEREST", 100.0)
|
| 350 |
+
KALSHI_MIN_VOLUME_24H = env_float("KALSHI_MIN_VOLUME_24H", 50.0)
|
| 351 |
+
KALSHI_MAKER_QUIET_MIN = env_float("KALSHI_MAKER_QUIET_MIN", 30.0) # no resting orders this close to the market's close
|
| 352 |
+
KALSHI_MAKER_TTL_H = env_float("KALSHI_MAKER_TTL_H", 6.0)
|
| 353 |
+
KALSHI_MAKER_MAX_RESTING = env_int("KALSHI_MAKER_MAX_RESTING", 40)
|
| 354 |
+
KALSHI_HISTORY_START = env_str("KALSHI_HISTORY_START", "2025-01-01")
|
| 355 |
+
KALSHI_DATASET_DIR = env_str("KALSHI_DATASET_DIR") # jon-becker/prediction-market-analysis checkout (data/kalshi/{markets,trades})
|
| 356 |
+
KALSHI_MINUTES_KEEP_DAYS = env_int("KALSHI_MINUTES_KEEP_DAYS", 7)
|
| 357 |
+
KALSHI_MARKETS_PER_DAY = env_int("KALSHI_MARKETS_PER_DAY", 150) # the event sample's target: about this many markets per category per day (kalshi/history.sample_rates);
|
| 358 |
+
# events are kept by md5(event ticker) < the category's rate, blind to outcome and volume
|
| 359 |
+
KALSHI_DIR = DATA_DIR / "kalshi"
|
| 360 |
+
KALSHI_TRAIN_MAX_ROWS = env_int("KALSHI_TRAIN_MAX_ROWS", 8_000_000) # training rows per walk-forward block / final fit (uniform random subset)
|
| 361 |
+
KALSHI_FILL_THREADS = env_int("KALSHI_FILL_THREADS", 8) # markets filled at once by the history worker, all under the one KALSHI_RPS bucket
|
| 362 |
+
KALSHI_RPS = env_float("KALSHI_RPS", 16.0) # per process; the account's Advanced grant reads 200 tokens/s at 10 per call = 20 calls/s, the feed uses ~3
|
| 363 |
+
|
| 364 |
+
|
| 365 |
+
def kalshi_live_prerequisites_missing() -> list[str]:
|
| 366 |
+
missing = []
|
| 367 |
+
if not KALSHI_LIVE_ENABLED:
|
| 368 |
+
missing.append("KALSHI_LIVE_ENABLED=1")
|
| 369 |
+
if not KALSHI_API_KEY_ID:
|
| 370 |
+
missing.append("KALSHI_API_KEY_ID")
|
| 371 |
+
if not KALSHI_PRIVATE_KEY_PATH or not Path(KALSHI_PRIVATE_KEY_PATH).expanduser().exists():
|
| 372 |
+
missing.append("KALSHI_PRIVATE_KEY_PATH (readable PEM file)")
|
| 373 |
+
if KALSHI_SUBACCOUNT <= 0:
|
| 374 |
+
missing.append("KALSHI_SUBACCOUNT (a dedicated subaccount, never the primary)")
|
| 375 |
+
if KALSHI_CAPITAL_USD <= 0:
|
| 376 |
+
missing.append("KALSHI_CAPITAL_USD")
|
| 377 |
+
return missing
|
| 378 |
+
|
| 379 |
# ---- historical corpus (every pump.fun graduation, pulled from the migration wallet + pump.fun's swap-api) ----
|
| 380 |
CORPUS_DIR = DATA_DIR / "corpus"
|
| 381 |
CORPUS_DAYS = env_int("CORPUS_DAYS", 240) # enumerate graduations this far back (must reach before CORPUS_PULL_BEFORE)
|
fly_trader/db/schema.py
CHANGED
|
@@ -6,7 +6,9 @@ appending a version-gated block AND bumping SCHEMA_VERSION, or it never runs.
|
|
| 6 |
"""
|
| 7 |
from __future__ import annotations
|
| 8 |
|
|
|
|
| 9 |
import logging
|
|
|
|
| 10 |
from datetime import datetime, timedelta, timezone
|
| 11 |
|
| 12 |
import psycopg
|
|
@@ -17,8 +19,15 @@ from .connection import connect, database_name, database_url
|
|
| 17 |
|
| 18 |
log = logging.getLogger(__name__)
|
| 19 |
|
| 20 |
-
SCHEMA_VERSION =
|
| 21 |
_LOCK_KEY = 0x666C795F6D6967 # "fly_mig"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 22 |
|
| 23 |
BASE_DDL: list[str] = [
|
| 24 |
"""CREATE TABLE IF NOT EXISTS schema_version (
|
|
@@ -150,11 +159,11 @@ BASE_DDL: list[str] = [
|
|
| 150 |
running_mean real[], running_var real[], n bigint NOT NULL DEFAULT 0,
|
| 151 |
updated_at timestamptz NOT NULL DEFAULT now())""",
|
| 152 |
"""CREATE TABLE IF NOT EXISTS circuit_state (
|
| 153 |
-
id int PRIMARY KEY
|
| 154 |
tripped boolean NOT NULL DEFAULT false, kill_switch boolean NOT NULL DEFAULT false, kill_reason text,
|
| 155 |
peak_wealth double precision, entries_paused boolean NOT NULL DEFAULT false,
|
| 156 |
updated_at timestamptz NOT NULL DEFAULT now())""",
|
| 157 |
-
"INSERT INTO circuit_state (id) VALUES (1) ON CONFLICT DO NOTHING",
|
| 158 |
"""CREATE TABLE IF NOT EXISTS circuit_events (
|
| 159 |
id bigserial PRIMARY KEY, ts timestamptz NOT NULL DEFAULT now(), kind text NOT NULL, detail jsonb)""",
|
| 160 |
"""CREATE TABLE IF NOT EXISTS notional_ledger (
|
|
@@ -223,6 +232,8 @@ BASE_DDL: list[str] = [
|
|
| 223 |
label real, state text NOT NULL DEFAULT 'pending', resolved_at timestamptz, bootstrap_id bigint,
|
| 224 |
PRIMARY KEY (ts, mint))""",
|
| 225 |
"CREATE INDEX IF NOT EXISTS fly_scored_state_ts_idx ON fly_scored (state, ts)",
|
|
|
|
|
|
|
| 226 |
"""CREATE TABLE IF NOT EXISTS fly_calibrations (
|
| 227 |
day date NOT NULL, arm text NOT NULL, line real, sizing jsonb, trades int, total real, mean real, window_days int,
|
| 228 |
created_at timestamptz NOT NULL DEFAULT now(), PRIMARY KEY (day, arm))""",
|
|
@@ -261,6 +272,105 @@ MIGRATIONS: dict[int, list[str]] = {
|
|
| 261 |
"ALTER TABLE fly_scored ADD COLUMN IF NOT EXISTS hold_min int",
|
| 262 |
"ALTER TABLE fly_scored DROP CONSTRAINT IF EXISTS fly_scored_pkey",
|
| 263 |
"ALTER TABLE fly_scored ADD PRIMARY KEY (ts, mint, strategy)"],
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 264 |
}
|
| 265 |
|
| 266 |
|
|
@@ -278,41 +388,93 @@ def ensure_database(url: str | None = None) -> bool:
|
|
| 278 |
return True
|
| 279 |
|
| 280 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 281 |
def apply_schema(url: str | None = None) -> int:
|
| 282 |
-
"""Apply the ladder under an advisory lock. Returns the resulting version.
|
|
|
|
|
|
|
| 283 |
with connect(url) as conn:
|
| 284 |
-
|
| 285 |
-
|
| 286 |
-
|
| 287 |
-
|
| 288 |
-
|
| 289 |
-
|
| 290 |
-
|
| 291 |
-
|
| 292 |
-
|
| 293 |
-
|
| 294 |
-
|
| 295 |
-
|
| 296 |
-
|
| 297 |
-
|
| 298 |
-
|
| 299 |
-
|
| 300 |
-
|
| 301 |
-
|
| 302 |
-
|
| 303 |
-
|
| 304 |
-
|
| 305 |
-
|
| 306 |
-
|
| 307 |
-
|
| 308 |
-
|
| 309 |
-
|
| 310 |
-
except Exception:
|
| 311 |
-
conn.rollback()
|
| 312 |
-
raise
|
| 313 |
-
finally:
|
| 314 |
-
cur.execute("SELECT pg_advisory_unlock(%s)", (_LOCK_KEY,))
|
| 315 |
-
conn.commit()
|
| 316 |
|
| 317 |
|
| 318 |
def _partition_name(base: str, start: datetime, granularity: str) -> str:
|
|
|
|
| 6 |
"""
|
| 7 |
from __future__ import annotations
|
| 8 |
|
| 9 |
+
import hashlib
|
| 10 |
import logging
|
| 11 |
+
import time
|
| 12 |
from datetime import datetime, timedelta, timezone
|
| 13 |
|
| 14 |
import psycopg
|
|
|
|
| 19 |
|
| 20 |
log = logging.getLogger(__name__)
|
| 21 |
|
| 22 |
+
SCHEMA_VERSION = 12 # 5: the plastic fly's tables and per-book halts (BASE_DDL); 6: the selector's strategy stack (MIGRATIONS[6]); 7: Kalshi (MIGRATIONS[7]); 8: Kalshi minute extremes
|
| 23 |
_LOCK_KEY = 0x666C795F6D6967 # "fly_mig"
|
| 24 |
+
# Applying the DDL takes ACCESS EXCLUSIVE on the tables it touches (Postgres takes it for ALTER TABLE ... ADD COLUMN IF NOT
|
| 25 |
+
# EXISTS even when the column exists) and holds it to the end of the transaction, while live workers write those tables.
|
| 26 |
+
# 2026-10-07: the RH stream's start-up apply deadlocked with the trading engine on positions/decisions and crashed. So:
|
| 27 |
+
# unchanged DDL is not re-run at all (its fingerprint is stored), and a run never waits more than LOCK_TIMEOUT for a
|
| 28 |
+
# lock: it rolls back, releases everything it holds, and retries.
|
| 29 |
+
LOCK_TIMEOUT = "5s"
|
| 30 |
+
APPLY_TRIES = 30
|
| 31 |
|
| 32 |
BASE_DDL: list[str] = [
|
| 33 |
"""CREATE TABLE IF NOT EXISTS schema_version (
|
|
|
|
| 159 |
running_mean real[], running_var real[], n bigint NOT NULL DEFAULT 0,
|
| 160 |
updated_at timestamptz NOT NULL DEFAULT now())""",
|
| 161 |
"""CREATE TABLE IF NOT EXISTS circuit_state (
|
| 162 |
+
id int PRIMARY KEY, fail_count int NOT NULL DEFAULT 0, last_failure_ts timestamptz,
|
| 163 |
tripped boolean NOT NULL DEFAULT false, kill_switch boolean NOT NULL DEFAULT false, kill_reason text,
|
| 164 |
peak_wealth double precision, entries_paused boolean NOT NULL DEFAULT false,
|
| 165 |
updated_at timestamptz NOT NULL DEFAULT now())""",
|
| 166 |
+
"INSERT INTO circuit_state (id) VALUES (1) ON CONFLICT DO NOTHING", # id 2 (the Kalshi rails) is inserted by MIGRATIONS[7], after the CHECK is dropped
|
| 167 |
"""CREATE TABLE IF NOT EXISTS circuit_events (
|
| 168 |
id bigserial PRIMARY KEY, ts timestamptz NOT NULL DEFAULT now(), kind text NOT NULL, detail jsonb)""",
|
| 169 |
"""CREATE TABLE IF NOT EXISTS notional_ledger (
|
|
|
|
| 232 |
label real, state text NOT NULL DEFAULT 'pending', resolved_at timestamptz, bootstrap_id bigint,
|
| 233 |
PRIMARY KEY (ts, mint))""",
|
| 234 |
"CREATE INDEX IF NOT EXISTS fly_scored_state_ts_idx ON fly_scored (state, ts)",
|
| 235 |
+
# would the fly's teacher have traded this row (its final decision)? the match calibration rule's budget (train/fly_calibrate.py)
|
| 236 |
+
"ALTER TABLE fly_scored ADD COLUMN IF NOT EXISTS teacher_allow boolean",
|
| 237 |
"""CREATE TABLE IF NOT EXISTS fly_calibrations (
|
| 238 |
day date NOT NULL, arm text NOT NULL, line real, sizing jsonb, trades int, total real, mean real, window_days int,
|
| 239 |
created_at timestamptz NOT NULL DEFAULT now(), PRIMARY KEY (day, arm))""",
|
|
|
|
| 272 |
"ALTER TABLE fly_scored ADD COLUMN IF NOT EXISTS hold_min int",
|
| 273 |
"ALTER TABLE fly_scored DROP CONSTRAINT IF EXISTS fly_scored_pkey",
|
| 274 |
"ALTER TABLE fly_scored ADD PRIMARY KEY (ts, mint, strategy)"],
|
| 275 |
+
# 7: Kalshi prediction markets (fly_trader/kalshi): the exchange catalogue, live minutes and quotes, the corpus registry,
|
| 276 |
+
# the four Kalshi books' positions/orders/fills, the visual fly's tables, and a second circuit (id 2) for the Kalshi rails
|
| 277 |
+
7: ["ALTER TABLE circuit_state DROP CONSTRAINT IF EXISTS circuit_state_id_check",
|
| 278 |
+
"INSERT INTO circuit_state (id) VALUES (2) ON CONFLICT DO NOTHING",
|
| 279 |
+
"""CREATE TABLE IF NOT EXISTS kalshi_series (
|
| 280 |
+
ticker text PRIMARY KEY, title text, category text, categories text[], tags text[], frequency text, fee_type text,
|
| 281 |
+
fee_multiplier double precision, settlement_sources jsonb, updated_at timestamptz NOT NULL DEFAULT now())""",
|
| 282 |
+
"""CREATE TABLE IF NOT EXISTS kalshi_events (
|
| 283 |
+
event_ticker text PRIMARY KEY, series_ticker text, title text, sub_title text, category text, mutually_exclusive boolean,
|
| 284 |
+
strike_date timestamptz, strike_period text, collateral_return_type text, exchange_index int, raw jsonb,
|
| 285 |
+
updated_at timestamptz NOT NULL DEFAULT now())""",
|
| 286 |
+
"CREATE INDEX IF NOT EXISTS kalshi_events_series_idx ON kalshi_events (series_ticker)",
|
| 287 |
+
"""CREATE TABLE IF NOT EXISTS kalshi_markets (
|
| 288 |
+
ticker text PRIMARY KEY, event_ticker text, market_type text, title text, yes_sub_title text, no_sub_title text, status text,
|
| 289 |
+
open_time timestamptz, close_time timestamptz, expected_expiration_time timestamptz, latest_expiration_time timestamptz,
|
| 290 |
+
settlement_ts timestamptz, result text, settlement_value double precision, strike_type text, floor_strike double precision,
|
| 291 |
+
cap_strike double precision, exchange_index int, price_level_structure text, is_provisional boolean, can_close_early boolean,
|
| 292 |
+
created_time timestamptz, source text, raw jsonb, updated_at timestamptz NOT NULL DEFAULT now())""",
|
| 293 |
+
"CREATE INDEX IF NOT EXISTS kalshi_markets_status_close_idx ON kalshi_markets (status, close_time)",
|
| 294 |
+
"CREATE INDEX IF NOT EXISTS kalshi_markets_event_idx ON kalshi_markets (event_ticker)",
|
| 295 |
+
"CREATE INDEX IF NOT EXISTS kalshi_markets_settled_idx ON kalshi_markets (settlement_ts)",
|
| 296 |
+
# the pump_minutes analogue: one row per (market, UTC minute) from the ticker and trade channels
|
| 297 |
+
"""CREATE TABLE IF NOT EXISTS kalshi_minutes (
|
| 298 |
+
ticker text NOT NULL, ts timestamptz NOT NULL, yes_bid real, yes_ask real, last real, bid_size real, ask_size real,
|
| 299 |
+
volume_fp double precision, open_interest_fp double precision, dollar_volume double precision,
|
| 300 |
+
taker_buy_yes double precision, taker_buy_no double precision, n_trades int, max_trade double precision, block_contracts double precision,
|
| 301 |
+
PRIMARY KEY (ticker, ts))""",
|
| 302 |
+
"CREATE INDEX IF NOT EXISTS kalshi_minutes_ts_idx ON kalshi_minutes (ts)",
|
| 303 |
+
# the live top of book (every ticker message), read by the maker arm and the vendored venue's spread tiers
|
| 304 |
+
"""CREATE TABLE IF NOT EXISTS kalshi_quotes (
|
| 305 |
+
ticker text PRIMARY KEY, ts timestamptz, yes_bid real, yes_ask real, mid real, bid_size real, ask_size real, last real,
|
| 306 |
+
updated_at timestamptz NOT NULL DEFAULT now())""",
|
| 307 |
+
"""CREATE OR REPLACE VIEW venue_market_cache AS
|
| 308 |
+
SELECT 'kalshi'::text AS venue, ticker AS market_id, yes_ask AS yes_ask_cents, mid AS market_prob FROM kalshi_quotes""",
|
| 309 |
+
# the corpus registry (kalshi/history.py) and the daily feature builds (kalshi/mature.py)
|
| 310 |
+
"""CREATE TABLE IF NOT EXISTS kalshi_corpus (
|
| 311 |
+
ticker text PRIMARY KEY, status text NOT NULL DEFAULT 'pending', settled_ts timestamptz, result text, open_time timestamptz,
|
| 312 |
+
close_time timestamptz, candles_1m int, trades int, candle_path text, trade_path text, seeded_from text, last_error text,
|
| 313 |
+
updated_at timestamptz NOT NULL DEFAULT now())""",
|
| 314 |
+
"CREATE INDEX IF NOT EXISTS kalshi_corpus_status_idx ON kalshi_corpus (status, settled_ts DESC)",
|
| 315 |
+
"""CREATE TABLE IF NOT EXISTS kalshi_days (
|
| 316 |
+
day date PRIMARY KEY, markets int, rows int, took_s real, built_at timestamptz NOT NULL DEFAULT now())""",
|
| 317 |
+
# the four Kalshi books (paper_kalshi_taker, paper_kalshi_maker, live_kalshi_taker, live_kalshi_maker); money in cents
|
| 318 |
+
"""CREATE TABLE IF NOT EXISTS kalshi_positions (
|
| 319 |
+
id bigserial PRIMARY KEY, book text NOT NULL, ticker text NOT NULL, side text NOT NULL, contracts double precision NOT NULL,
|
| 320 |
+
cost_cents double precision NOT NULL, fee_cents double precision NOT NULL DEFAULT 0, avg_price_cents double precision,
|
| 321 |
+
opened_at timestamptz NOT NULL DEFAULT now(), closed_at timestamptz, entry_decision_id bigint, exit_decision_id bigint,
|
| 322 |
+
order_id text, status text NOT NULL DEFAULT 'open', result text, payout_cents double precision, realized_cents double precision,
|
| 323 |
+
strategy text, arm text, score real, line real, last_mark_cents real, last_mark_ts timestamptz)""",
|
| 324 |
+
"CREATE INDEX IF NOT EXISTS kalshi_positions_book_status_idx ON kalshi_positions (book, status)",
|
| 325 |
+
"CREATE INDEX IF NOT EXISTS kalshi_positions_ticker_idx ON kalshi_positions (ticker)",
|
| 326 |
+
"""CREATE TABLE IF NOT EXISTS kalshi_orders (
|
| 327 |
+
id bigserial PRIMARY KEY, book text NOT NULL, order_id text, client_order_id text, decision_id bigint, ticker text NOT NULL,
|
| 328 |
+
side text NOT NULL, action text NOT NULL DEFAULT 'buy', price_cents int NOT NULL, count double precision NOT NULL, tif text NOT NULL,
|
| 329 |
+
post_only boolean NOT NULL DEFAULT false, expiration_ts timestamptz, status text NOT NULL, fill_count double precision NOT NULL DEFAULT 0,
|
| 330 |
+
remaining double precision, avg_fill_cents double precision, fee_cents double precision, exchange_index int, subaccount int,
|
| 331 |
+
fair_cents real, margin_cents real, strategy text, request jsonb, response jsonb, error text,
|
| 332 |
+
ts timestamptz NOT NULL DEFAULT now(), updated_at timestamptz NOT NULL DEFAULT now())""",
|
| 333 |
+
"CREATE INDEX IF NOT EXISTS kalshi_orders_book_status_idx ON kalshi_orders (book, status)",
|
| 334 |
+
"CREATE UNIQUE INDEX IF NOT EXISTS kalshi_orders_order_id_uniq ON kalshi_orders (order_id) WHERE order_id IS NOT NULL",
|
| 335 |
+
"""CREATE TABLE IF NOT EXISTS kalshi_fills (
|
| 336 |
+
trade_id text PRIMARY KEY, order_id text, book text, ticker text NOT NULL, side text NOT NULL, price_cents double precision NOT NULL,
|
| 337 |
+
count double precision NOT NULL, fee_cents double precision, is_taker boolean, action text, ts timestamptz NOT NULL,
|
| 338 |
+
post_position double precision, raw jsonb)""",
|
| 339 |
+
"CREATE INDEX IF NOT EXISTS kalshi_fills_order_idx ON kalshi_fills (order_id)",
|
| 340 |
+
# the visual fly (kalshi/fly_session.py): scored (market, minute, side) rows waiting for settlement, calibrations, learning, rollbacks
|
| 341 |
+
"""CREATE TABLE IF NOT EXISTS kalshi_fly_scored (
|
| 342 |
+
ts timestamptz NOT NULL, ticker text NOT NULL, side text NOT NULL, strategy text NOT NULL, x real[], score real, frozen_score real,
|
| 343 |
+
line real, frozen_line real, label real, state text NOT NULL DEFAULT 'pending', due_ts timestamptz, resolved_at timestamptz,
|
| 344 |
+
bootstrap_id bigint, PRIMARY KEY (ts, ticker, side, strategy))""",
|
| 345 |
+
"CREATE INDEX IF NOT EXISTS kalshi_fly_scored_state_idx ON kalshi_fly_scored (state, due_ts)",
|
| 346 |
+
"""CREATE TABLE IF NOT EXISTS kalshi_fly_calibrations (
|
| 347 |
+
day date NOT NULL, arm text NOT NULL, line real, sizing jsonb, trades int, total real, mean real, window_days int,
|
| 348 |
+
created_at timestamptz NOT NULL DEFAULT now(), PRIMARY KEY (day, arm))""",
|
| 349 |
+
"""CREATE TABLE IF NOT EXISTS kalshi_fly_updates (
|
| 350 |
+
hour timestamptz PRIMARY KEY, n int, mean_delta real, mean_abs_delta real, step real, capped int, drift real, ic real, detail jsonb)""",
|
| 351 |
+
"""CREATE TABLE IF NOT EXISTS kalshi_fly_rollbacks (
|
| 352 |
+
id bigserial PRIMARY KEY, ts timestamptz NOT NULL DEFAULT now(), reason text, checks jsonb, from_snapshot bigint, to_snapshot bigint)"""],
|
| 353 |
+
# the minute's lowest YES ask and highest YES bid: whether a resting order would have filled (kalshi/maker.py) — the
|
| 354 |
+
# candle archive's yes_ask_low / yes_bid_high, which the label rule reads (kalshi/mature.py)
|
| 355 |
+
8: ["ALTER TABLE kalshi_minutes ADD COLUMN IF NOT EXISTS yes_ask_low real",
|
| 356 |
+
"ALTER TABLE kalshi_minutes ADD COLUMN IF NOT EXISTS yes_bid_high real"],
|
| 357 |
+
# 9: a market's lifetime volume on its corpus row, so the corpus can keep the most traded markets per day (kalshi/history.prune_pending)
|
| 358 |
+
9: ["ALTER TABLE kalshi_corpus ADD COLUMN IF NOT EXISTS volume double precision"],
|
| 359 |
+
# 10: every settled single market Kalshi listed (the dataset's and the exchange walk's), from which the corpus samples whole
|
| 360 |
+
# events outcome-blind (kalshi/history.select_corpus)
|
| 361 |
+
10: ["""CREATE TABLE IF NOT EXISTS kalshi_universe (
|
| 362 |
+
ticker text PRIMARY KEY, event_ticker text, series_ticker text, title text, open_time timestamptz, close_time timestamptz,
|
| 363 |
+
settled_ts timestamptz, result text, volume double precision, source text, selected boolean NOT NULL DEFAULT false,
|
| 364 |
+
updated_at timestamptz NOT NULL DEFAULT now())""",
|
| 365 |
+
"CREATE INDEX IF NOT EXISTS kalshi_universe_close_idx ON kalshi_universe (close_time)",
|
| 366 |
+
"CREATE INDEX IF NOT EXISTS kalshi_universe_selected_idx ON kalshi_universe (selected) WHERE selected"],
|
| 367 |
+
# 11: a market's early-close condition: such a market may close before its listed close when its event happens, so its
|
| 368 |
+
# realized close carries the outcome; its time anchor is the scheduled expected expiration (kalshi/mature.load_meta)
|
| 369 |
+
11: ["ALTER TABLE kalshi_markets ADD COLUMN IF NOT EXISTS early_close_condition text"],
|
| 370 |
+
# 12: the minute's taker sales at their extremes: the lowest YES price a taker sold YES at (bought NO) and the highest a
|
| 371 |
+
# taker bought YES at (sold NO at 100 − it): a resting bid fills when a taker sells into it (kalshi/decisions.maker_filled)
|
| 372 |
+
12: ["ALTER TABLE kalshi_minutes ADD COLUMN IF NOT EXISTS yes_sold_low real",
|
| 373 |
+
"ALTER TABLE kalshi_minutes ADD COLUMN IF NOT EXISTS yes_bought_high real"],
|
| 374 |
}
|
| 375 |
|
| 376 |
|
|
|
|
| 388 |
return True
|
| 389 |
|
| 390 |
|
| 391 |
+
def ddl_fingerprint() -> str:
|
| 392 |
+
"""sha256 of every statement apply_schema runs (the ladder, the idempotent vault/RH/ETH DDL) and SCHEMA_VERSION."""
|
| 393 |
+
from ..rh.schema import RH_DDL
|
| 394 |
+
from ..vault.eth_schema import ETH_DDL
|
| 395 |
+
from ..vault.schema import VAULT_DDL
|
| 396 |
+
h = hashlib.sha256()
|
| 397 |
+
for stmt in [str(SCHEMA_VERSION), *BASE_DDL, *(x for v in sorted(MIGRATIONS) for x in MIGRATIONS[v]), *VAULT_DDL, *RH_DDL, *ETH_DDL]:
|
| 398 |
+
h.update(stmt.encode()); h.update(b"\0")
|
| 399 |
+
return h.hexdigest()
|
| 400 |
+
|
| 401 |
+
|
| 402 |
+
def _applied(conn, fp: str) -> int | None:
|
| 403 |
+
"""The schema version when exactly this DDL is already applied (nothing to run), else None."""
|
| 404 |
+
try:
|
| 405 |
+
row = conn.execute("SELECT version, ddl_sha256 FROM schema_version WHERE singleton").fetchone()
|
| 406 |
+
except (psycopg.errors.UndefinedTable, psycopg.errors.UndefinedColumn):
|
| 407 |
+
conn.rollback()
|
| 408 |
+
return None
|
| 409 |
+
conn.commit()
|
| 410 |
+
if row and row["ddl_sha256"] == fp and int(row["version"]) >= SCHEMA_VERSION:
|
| 411 |
+
return int(row["version"])
|
| 412 |
+
return None
|
| 413 |
+
|
| 414 |
+
|
| 415 |
+
def _apply_all(conn, fp: str) -> int:
|
| 416 |
+
with conn.cursor() as cur:
|
| 417 |
+
cur.execute(f"SET LOCAL lock_timeout = '{LOCK_TIMEOUT}'")
|
| 418 |
+
for stmt in BASE_DDL:
|
| 419 |
+
cur.execute(stmt)
|
| 420 |
+
cur.execute("ALTER TABLE schema_version ADD COLUMN IF NOT EXISTS ddl_sha256 text")
|
| 421 |
+
cur.execute("SELECT version FROM schema_version WHERE singleton")
|
| 422 |
+
row = cur.fetchone()
|
| 423 |
+
current = int(row["version"]) if row else 0
|
| 424 |
+
for v in sorted(MIGRATIONS):
|
| 425 |
+
if current < v:
|
| 426 |
+
for stmt in MIGRATIONS[v]:
|
| 427 |
+
cur.execute(stmt)
|
| 428 |
+
current = v
|
| 429 |
+
from ..vault.schema import VAULT_DDL # idempotent, outside the ladder: same DDL on master and distribution
|
| 430 |
+
for stmt in VAULT_DDL:
|
| 431 |
+
cur.execute(stmt)
|
| 432 |
+
from ..rh.schema import RH_DDL # idempotent, outside the ladder: the Robinhood Chain market (fly_trader/rh)
|
| 433 |
+
for stmt in RH_DDL:
|
| 434 |
+
cur.execute(stmt)
|
| 435 |
+
from ..vault.eth_schema import ETH_DDL # the vault's ETH pot (Robinhood Chain profits paid in ETH)
|
| 436 |
+
for stmt in ETH_DDL:
|
| 437 |
+
cur.execute(stmt)
|
| 438 |
+
target = max(current, SCHEMA_VERSION)
|
| 439 |
+
cur.execute(
|
| 440 |
+
"INSERT INTO schema_version (singleton, version, ddl_sha256) VALUES (true, %s, %s) "
|
| 441 |
+
"ON CONFLICT (singleton) DO UPDATE SET version = EXCLUDED.version, ddl_sha256 = EXCLUDED.ddl_sha256, updated_at = now()",
|
| 442 |
+
(target, fp),
|
| 443 |
+
)
|
| 444 |
+
return target
|
| 445 |
+
|
| 446 |
+
|
| 447 |
def apply_schema(url: str | None = None) -> int:
|
| 448 |
+
"""Apply the ladder under an advisory lock. Returns the resulting version. A no-op (no table locks) when this exact
|
| 449 |
+
DDL was applied before; otherwise every lock wait is bounded by LOCK_TIMEOUT and the whole run is retried."""
|
| 450 |
+
fp = ddl_fingerprint()
|
| 451 |
with connect(url) as conn:
|
| 452 |
+
conn.execute("SELECT pg_advisory_lock(%s)", (_LOCK_KEY,))
|
| 453 |
+
conn.commit() # the advisory lock is the session's; no transaction stays open
|
| 454 |
+
try:
|
| 455 |
+
target = _applied(conn, fp)
|
| 456 |
+
if target is None:
|
| 457 |
+
for attempt in range(APPLY_TRIES):
|
| 458 |
+
try:
|
| 459 |
+
target = _apply_all(conn, fp)
|
| 460 |
+
conn.commit()
|
| 461 |
+
break
|
| 462 |
+
except (psycopg.errors.LockNotAvailable, psycopg.errors.DeadlockDetected) as e:
|
| 463 |
+
conn.rollback() # releases every table lock this run took
|
| 464 |
+
if attempt == APPLY_TRIES - 1:
|
| 465 |
+
raise
|
| 466 |
+
wait = min(30.0, 1.0 + attempt)
|
| 467 |
+
log.warning("schema: %s while applying the DDL (live workers hold the tables); retrying in %.0f s", type(e).__name__, wait)
|
| 468 |
+
time.sleep(wait)
|
| 469 |
+
ensure_partitions(conn)
|
| 470 |
+
conn.commit()
|
| 471 |
+
return target
|
| 472 |
+
except Exception:
|
| 473 |
+
conn.rollback()
|
| 474 |
+
raise
|
| 475 |
+
finally:
|
| 476 |
+
conn.execute("SELECT pg_advisory_unlock(%s)", (_LOCK_KEY,))
|
| 477 |
+
conn.commit()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 478 |
|
| 479 |
|
| 480 |
def _partition_name(base: str, start: datetime, granularity: str) -> str:
|
fly_trader/execution/broker_live.py
CHANGED
|
@@ -23,7 +23,6 @@ Every write is committed immediately so a crash mid-poll leaves an auditable row
|
|
| 23 |
"""
|
| 24 |
from __future__ import annotations
|
| 25 |
|
| 26 |
-
import base64
|
| 27 |
import logging
|
| 28 |
import re
|
| 29 |
import threading
|
|
@@ -31,20 +30,16 @@ import time
|
|
| 31 |
from dataclasses import dataclass
|
| 32 |
|
| 33 |
from psycopg.types.json import Jsonb
|
| 34 |
-
from solders.hash import Hash
|
| 35 |
-
from solders.instruction import AccountMeta, Instruction
|
| 36 |
-
from solders.message import MessageV0
|
| 37 |
-
from solders.pubkey import Pubkey
|
| 38 |
-
from solders.transaction import VersionedTransaction
|
| 39 |
-
|
| 40 |
from .. import config
|
| 41 |
from ..chain.balances import tx_deltas, Snapshot, compute_delta, snapshot_balances
|
| 42 |
from ..chain.cluster_guard import assert_signing_allowed
|
| 43 |
from ..chain.jupiter_swap import JupiterError, JupiterSwap
|
| 44 |
from ..db.connection import transaction
|
| 45 |
-
from ..chain.keys import load_keypair
|
| 46 |
from ..chain.rpc import TOKEN_2022_PROGRAM_ID, TOKEN_PROGRAM_ID, HttpSolanaRpc
|
| 47 |
-
from ..chain.signing import
|
|
|
|
|
|
|
|
|
|
| 48 |
from ..db.apilog import record_event
|
| 49 |
from ..db.connection import connect, transaction
|
| 50 |
from ..logging_setup import scrub
|
|
@@ -110,10 +105,14 @@ def await_confirmation(rpc, signature: str | None, last_valid_block_height: int
|
|
| 110 |
|
| 111 |
|
| 112 |
class LiveBroker:
|
| 113 |
-
|
|
|
|
|
|
|
|
|
|
| 114 |
self._rpc = rpc
|
| 115 |
self._jup = jup
|
| 116 |
-
self.
|
|
|
|
| 117 |
self._labels: dict[str, str] | None = None
|
| 118 |
|
| 119 |
@property
|
|
@@ -129,14 +128,16 @@ class LiveBroker:
|
|
| 129 |
return self._jup
|
| 130 |
|
| 131 |
@property
|
| 132 |
-
def
|
| 133 |
-
if self.
|
| 134 |
-
self.
|
| 135 |
-
return self.
|
| 136 |
|
| 137 |
@property
|
| 138 |
def pubkey(self) -> str:
|
| 139 |
-
|
|
|
|
|
|
|
| 140 |
|
| 141 |
@property
|
| 142 |
def labels(self) -> dict[str, str]:
|
|
@@ -185,7 +186,7 @@ class LiveBroker:
|
|
| 185 |
taker = self.pubkey
|
| 186 |
pre = snapshot_balances(self.rpc, taker)
|
| 187 |
request = {"inputMint": input_mint, "outputMint": output_mint, "amount": str(amount_in), "taker": taker,
|
| 188 |
-
"slippageBps": slippage_bps, "excludeRouters":
|
| 189 |
base = dict(decision_id=decision_id, book=book, attempt=attempt, side=side, input_mint=input_mint,
|
| 190 |
output_mint=output_mint, amount_in=amount_in, slippage_bps=slippage_bps, request=request)
|
| 191 |
|
|
@@ -200,7 +201,7 @@ class LiveBroker:
|
|
| 200 |
|
| 201 |
# 1. order
|
| 202 |
try:
|
| 203 |
-
order = self.jup.order(input_mint, output_mint, amount_in, taker=taker, slippage_bps=slippage_bps)
|
| 204 |
except JupiterError as e:
|
| 205 |
return fail("order_error", scrub(str(e)))
|
| 206 |
err_code = _int_or_none(order.get("errorCode"))
|
|
@@ -223,14 +224,18 @@ class LiveBroker:
|
|
| 223 |
conn, **base, order_response=order, request_id=order.get("requestId"), router=order.get("router"),
|
| 224 |
fee_bps=_int_or_none(order.get("feeBps")), program_ids=program_ids, lvbh=lvbh, status="ordered")
|
| 225 |
|
| 226 |
-
# 2. sign (
|
| 227 |
try:
|
| 228 |
-
|
| 229 |
-
|
| 230 |
-
|
| 231 |
-
|
| 232 |
-
|
| 233 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 234 |
|
| 235 |
# 3. execute
|
| 236 |
exec_req = {"requestId": order.get("requestId"), "lastValidBlockHeight": lvbh, "signedTransaction": signed_b64}
|
|
@@ -419,14 +424,13 @@ def swap_smoke(sol: float) -> None:
|
|
| 419 |
print(f"net lamports delta: {net} ({net / config.LAMPORTS_PER_SOL:.9f} SOL)")
|
| 420 |
|
| 421 |
|
| 422 |
-
def close_empty_atas(rpc=None,
|
| 423 |
-
"""Close every zero-balance token account owned by the bot (rent back to the wallet).
|
| 424 |
-
|
| 425 |
-
|
| 426 |
assert_signing_allowed()
|
| 427 |
-
|
| 428 |
-
|
| 429 |
-
owner_s = str(owner)
|
| 430 |
rpc = rpc or HttpSolanaRpc()
|
| 431 |
skip = set(skip_mints or ())
|
| 432 |
empties = [a for a in rpc.get_token_accounts_by_owner(owner_s) if int(a["amount"]) == 0 and a["mint"] not in skip and not a.get("is_native")]
|
|
@@ -436,16 +440,17 @@ def close_empty_atas(rpc=None, keypair=None, url: str | None = None, batch: int
|
|
| 436 |
for i in range(0, len(empties), batch):
|
| 437 |
chunk = empties[i:i + batch]
|
| 438 |
accounts = [a["address"] for a in chunk]
|
| 439 |
-
ixs = [Instruction(Pubkey.from_string(a["program"]), bytes([9]),
|
| 440 |
-
[AccountMeta(Pubkey.from_string(a["address"]), False, True),
|
| 441 |
-
AccountMeta(owner, False, True),
|
| 442 |
-
AccountMeta(owner, True, False)]) for a in chunk]
|
| 443 |
-
bh = rpc.get_latest_blockhash()
|
| 444 |
-
msg = MessageV0.try_compile(owner, ixs, [], Hash.from_string(bh["blockhash"]))
|
| 445 |
-
tx = VersionedTransaction(msg, [kp])
|
| 446 |
-
tx_b64 = base64.b64encode(bytes(tx)).decode()
|
| 447 |
try:
|
| 448 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 449 |
except Exception as e:
|
| 450 |
err = f"{type(e).__name__}: {scrub(str(e))}"
|
| 451 |
result["failed"].append({"accounts": accounts, "error": err})
|
|
|
|
| 23 |
"""
|
| 24 |
from __future__ import annotations
|
| 25 |
|
|
|
|
| 26 |
import logging
|
| 27 |
import re
|
| 28 |
import threading
|
|
|
|
| 30 |
from dataclasses import dataclass
|
| 31 |
|
| 32 |
from psycopg.types.json import Jsonb
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 33 |
from .. import config
|
| 34 |
from ..chain.balances import tx_deltas, Snapshot, compute_delta, snapshot_balances
|
| 35 |
from ..chain.cluster_guard import assert_signing_allowed
|
| 36 |
from ..chain.jupiter_swap import JupiterError, JupiterSwap
|
| 37 |
from ..db.connection import transaction
|
|
|
|
| 38 |
from ..chain.rpc import TOKEN_2022_PROGRAM_ID, TOKEN_PROGRAM_ID, HttpSolanaRpc
|
| 39 |
+
from ..chain.signing import n_lookup_programs, static_program_ids
|
| 40 |
+
from ..signer import client as signer_client
|
| 41 |
+
from ..signer.client import SignerUnavailable
|
| 42 |
+
from ..signer.policy import EXCLUDE_ROUTERS, PolicyError
|
| 43 |
from ..db.apilog import record_event
|
| 44 |
from ..db.connection import connect, transaction
|
| 45 |
from ..logging_setup import scrub
|
|
|
|
| 105 |
|
| 106 |
|
| 107 |
class LiveBroker:
|
| 108 |
+
"""Swaps through Jupiter; the trading key never enters this process: ``signer`` (fly_trader/signer, the signer
|
| 109 |
+
container's socket on the vault server) checks and signs each transaction."""
|
| 110 |
+
|
| 111 |
+
def __init__(self, rpc=None, jup=None, signer=None):
|
| 112 |
self._rpc = rpc
|
| 113 |
self._jup = jup
|
| 114 |
+
self._signer = signer
|
| 115 |
+
self._pubkey: str | None = None
|
| 116 |
self._labels: dict[str, str] | None = None
|
| 117 |
|
| 118 |
@property
|
|
|
|
| 128 |
return self._jup
|
| 129 |
|
| 130 |
@property
|
| 131 |
+
def signer(self):
|
| 132 |
+
if self._signer is None:
|
| 133 |
+
self._signer = signer_client.get()
|
| 134 |
+
return self._signer
|
| 135 |
|
| 136 |
@property
|
| 137 |
def pubkey(self) -> str:
|
| 138 |
+
if self._pubkey is None:
|
| 139 |
+
self._pubkey = self.signer.call("pubkeys")["trading"]
|
| 140 |
+
return self._pubkey
|
| 141 |
|
| 142 |
@property
|
| 143 |
def labels(self) -> dict[str, str]:
|
|
|
|
| 186 |
taker = self.pubkey
|
| 187 |
pre = snapshot_balances(self.rpc, taker)
|
| 188 |
request = {"inputMint": input_mint, "outputMint": output_mint, "amount": str(amount_in), "taker": taker,
|
| 189 |
+
"slippageBps": slippage_bps, "excludeRouters": EXCLUDE_ROUTERS}
|
| 190 |
base = dict(decision_id=decision_id, book=book, attempt=attempt, side=side, input_mint=input_mint,
|
| 191 |
output_mint=output_mint, amount_in=amount_in, slippage_bps=slippage_bps, request=request)
|
| 192 |
|
|
|
|
| 201 |
|
| 202 |
# 1. order
|
| 203 |
try:
|
| 204 |
+
order = self.jup.order(input_mint, output_mint, amount_in, taker=taker, slippage_bps=slippage_bps, exclude_routers=EXCLUDE_ROUTERS)
|
| 205 |
except JupiterError as e:
|
| 206 |
return fail("order_error", scrub(str(e)))
|
| 207 |
err_code = _int_or_none(order.get("errorCode"))
|
|
|
|
| 224 |
conn, **base, order_response=order, request_id=order.get("requestId"), router=order.get("router"),
|
| 225 |
fee_bps=_int_or_none(order.get("feeBps")), program_ids=program_ids, lvbh=lvbh, status="ordered")
|
| 226 |
|
| 227 |
+
# 2. sign: the signer checks the transaction (programs, roles, a simulation of our balances) and signs our slot
|
| 228 |
try:
|
| 229 |
+
signed = self.signer.call("sign_swap", tx=tx_b64, in_mint=input_mint, out_mint=output_mint, in_amount=amount_in,
|
| 230 |
+
min_out=max(1, int(_int_or_none(order.get("otherAmountThreshold")) or 1)))
|
| 231 |
+
except PolicyError as e:
|
| 232 |
+
r = fail("sign_refused", f"{e.code}: {scrub(str(e))}", None, order_id=order_id)
|
| 233 |
+
if e.code == "policy":
|
| 234 |
+
record_event("error", "signer", f"swap refused: {e}", {"order_id": order_id, "side": side, "mint": mint})
|
| 235 |
+
return r[0], e.code == "in_flight" # a simulation that failed (slippage, stale route): retry with a new order
|
| 236 |
+
except SignerUnavailable as e:
|
| 237 |
+
return fail("sign_error", scrub(str(e)), None, order_id=order_id)
|
| 238 |
+
signed_b64, tx_sig = signed["tx"], signed["signature"]
|
| 239 |
|
| 240 |
# 3. execute
|
| 241 |
exec_req = {"requestId": order.get("requestId"), "lastValidBlockHeight": lvbh, "signedTransaction": signed_b64}
|
|
|
|
| 424 |
print(f"net lamports delta: {net} ({net / config.LAMPORTS_PER_SOL:.9f} SOL)")
|
| 425 |
|
| 426 |
|
| 427 |
+
def close_empty_atas(rpc=None, signer=None, url: str | None = None, batch: int = CLOSE_ATA_BATCH, skip_mints: set | None = None) -> dict:
|
| 428 |
+
"""Close every zero-balance token account owned by the bot (rent back to the wallet). The signer builds and signs
|
| 429 |
+
the CloseAccount transactions (it re-checks that each account is ours and empty); each is recorded as
|
| 430 |
+
wallet_events(kind='ata_closed')."""
|
| 431 |
assert_signing_allowed()
|
| 432 |
+
signer = signer or signer_client.get()
|
| 433 |
+
owner_s = signer.call("pubkeys")["trading"]
|
|
|
|
| 434 |
rpc = rpc or HttpSolanaRpc()
|
| 435 |
skip = set(skip_mints or ())
|
| 436 |
empties = [a for a in rpc.get_token_accounts_by_owner(owner_s) if int(a["amount"]) == 0 and a["mint"] not in skip and not a.get("is_native")]
|
|
|
|
| 440 |
for i in range(0, len(empties), batch):
|
| 441 |
chunk = empties[i:i + batch]
|
| 442 |
accounts = [a["address"] for a in chunk]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 443 |
try:
|
| 444 |
+
built = signer.call("close_atas", accounts=accounts)
|
| 445 |
+
except (PolicyError, SignerUnavailable) as e:
|
| 446 |
+
result["failed"].append({"accounts": accounts, "error": str(e)})
|
| 447 |
+
record_event("error", "close_empty_atas", "the signer refused", {"accounts": accounts, "error": str(e)})
|
| 448 |
+
continue
|
| 449 |
+
if not built:
|
| 450 |
+
continue
|
| 451 |
+
bh = {"last_valid_block_height": built[0]["last_valid_block_height"]}
|
| 452 |
+
try:
|
| 453 |
+
sig = rpc.send_transaction(built[0]["tx"])
|
| 454 |
except Exception as e:
|
| 455 |
err = f"{type(e).__name__}: {scrub(str(e))}"
|
| 456 |
result["failed"].append({"accounts": accounts, "error": err})
|
fly_trader/execution/broker_paper.py
CHANGED
|
@@ -32,7 +32,9 @@ class PaperFill:
|
|
| 32 |
|
| 33 |
class PaperBroker:
|
| 34 |
def __init__(self, book: str):
|
| 35 |
-
|
|
|
|
|
|
|
| 36 |
self.book = book
|
| 37 |
|
| 38 |
def buy(self, conn, *, decision_id: int | None, mint: str, pool: str | None, size_sol: float, price: float | None,
|
|
@@ -45,7 +47,7 @@ class PaperBroker:
|
|
| 45 |
return PaperFill(False, None, None, 0.0, 0.0, 0, 0.0, "no price")
|
| 46 |
if res_quote_sol is None or res_quote_sol <= 0:
|
| 47 |
return PaperFill(False, None, None, price, 0.0, 0, 0.0, "no liquidity estimate")
|
| 48 |
-
fee_frac = fee_fraction(size_sol, mcap_sol, pool_fee)
|
| 49 |
impact = impact_fraction(size_sol, res_quote_sol, program_label)
|
| 50 |
if impact > config.EXECUTABILITY_MAX_IMPACT:
|
| 51 |
# a router would not fill this either; refuse like the live executability check
|
|
@@ -74,7 +76,7 @@ class PaperBroker:
|
|
| 74 |
gross = qty_raw / (10 ** decimals) * price
|
| 75 |
c = exit_cost_fraction(gross, res_quote_sol, mcap_sol, program_label, pool_fee)
|
| 76 |
proceeds = gross * (1.0 - c)
|
| 77 |
-
fee_sol = gross * fee_fraction(gross, mcap_sol, pool_fee)
|
| 78 |
if fraction < 0.999: # partial: shrink the position, realise the sold part
|
| 79 |
realized = ledger.realize_partial(conn, position_id=int(position["id"]), qty_raw=qty_raw,
|
| 80 |
cost_part=float(position["cost_sol"]) * fraction, proceeds_sol=proceeds,
|
|
|
|
| 32 |
|
| 33 |
class PaperBroker:
|
| 34 |
def __init__(self, book: str):
|
| 35 |
+
from ..markets import MARKETS
|
| 36 |
+
paper = {b for m in MARKETS.values() for b in m.paper_books}
|
| 37 |
+
assert book in paper or book.startswith("replay_"), book # the race books of every market; replay_* for tests
|
| 38 |
self.book = book
|
| 39 |
|
| 40 |
def buy(self, conn, *, decision_id: int | None, mint: str, pool: str | None, size_sol: float, price: float | None,
|
|
|
|
| 47 |
return PaperFill(False, None, None, 0.0, 0.0, 0, 0.0, "no price")
|
| 48 |
if res_quote_sol is None or res_quote_sol <= 0:
|
| 49 |
return PaperFill(False, None, None, price, 0.0, 0, 0.0, "no liquidity estimate")
|
| 50 |
+
fee_frac = fee_fraction(size_sol, mcap_sol, pool_fee, program_label)
|
| 51 |
impact = impact_fraction(size_sol, res_quote_sol, program_label)
|
| 52 |
if impact > config.EXECUTABILITY_MAX_IMPACT:
|
| 53 |
# a router would not fill this either; refuse like the live executability check
|
|
|
|
| 76 |
gross = qty_raw / (10 ** decimals) * price
|
| 77 |
c = exit_cost_fraction(gross, res_quote_sol, mcap_sol, program_label, pool_fee)
|
| 78 |
proceeds = gross * (1.0 - c)
|
| 79 |
+
fee_sol = gross * fee_fraction(gross, mcap_sol, pool_fee, program_label)
|
| 80 |
if fraction < 0.999: # partial: shrink the position, realise the sold part
|
| 81 |
realized = ledger.realize_partial(conn, position_id=int(position["id"]), qty_raw=qty_raw,
|
| 82 |
cost_part=float(position["cost_sol"]) * fraction, proceeds_sol=proceeds,
|
fly_trader/execution/ledger.py
CHANGED
|
@@ -20,7 +20,8 @@ log = logging.getLogger(__name__)
|
|
| 20 |
|
| 21 |
def open_positions(conn, book: str) -> list[dict]:
|
| 22 |
rows = conn.execute(
|
| 23 |
-
"""SELECT p.*, t.decimals,
|
|
|
|
| 24 |
COALESCE(p.hold_s, (d.detail->>'hold_s')::float) AS hold_s, COALESCE(p.strategy, d.detail->>'strategy') AS strategy
|
| 25 |
FROM positions p LEFT JOIN tokens t ON t.mint = p.mint
|
| 26 |
LEFT JOIN watch_pools wp ON wp.pool = p.pool
|
|
@@ -38,7 +39,8 @@ def paper_cash(conn, book: str) -> float:
|
|
| 38 |
FROM positions WHERE book = %s""",
|
| 39 |
(book,),
|
| 40 |
).fetchone()
|
| 41 |
-
|
|
|
|
| 42 |
|
| 43 |
|
| 44 |
def record_fill(conn, *, order_id: int | None, book: str, mint: str, side: str, token_delta: int,
|
|
@@ -55,6 +57,11 @@ def record_fill(conn, *, order_id: int | None, book: str, mint: str, side: str,
|
|
| 55 |
return int(row["id"])
|
| 56 |
|
| 57 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 58 |
def open_position(conn, *, book: str, mint: str, pool: str | None, qty_raw: int, cost_sol: float,
|
| 59 |
entry_price: float, decision_id: int | None, fees_sol: float, ts: datetime | None = None,
|
| 60 |
hold_s: float | None = None, strategy: str | None = None) -> int:
|
|
@@ -70,9 +77,9 @@ def open_position(conn, *, book: str, mint: str, pool: str | None, qty_raw: int,
|
|
| 70 |
return int(existing["id"])
|
| 71 |
row = conn.execute(
|
| 72 |
"""INSERT INTO positions (book, mint, pool, opened_at, entry_decision_id, qty, cost_sol, entry_price, peak_price,
|
| 73 |
-
last_mark_price, last_mark_ts, fees_sol, status, last_swap_ts, hold_s, strategy)
|
| 74 |
-
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,'open',%s,%s,%s) RETURNING id""",
|
| 75 |
-
(book, mint, pool, ts, decision_id, qty_raw, cost_sol, entry_price, entry_price, entry_price, ts, fees_sol, ts, hold_s, strategy),
|
| 76 |
).fetchone()
|
| 77 |
return int(row["id"])
|
| 78 |
|
|
@@ -127,11 +134,11 @@ def mark_positions(conn, book: str, prices: dict[str, float], last_swaps: dict[s
|
|
| 127 |
|
| 128 |
def verify_fills() -> None:
|
| 129 |
"""Reconcile live positions against on-chain token balances; prints mismatches, writes wallet_events."""
|
| 130 |
-
from ..chain import keys
|
| 131 |
from ..chain.rpc import HttpSolanaRpc
|
| 132 |
from ..db.connection import transaction
|
|
|
|
| 133 |
rpc = HttpSolanaRpc()
|
| 134 |
-
pub =
|
| 135 |
accounts = rpc.get_token_accounts_by_owner(pub)
|
| 136 |
onchain = {a["mint"]: int(a["amount"]) for a in accounts}
|
| 137 |
with transaction() as conn:
|
|
|
|
| 20 |
|
| 21 |
def open_positions(conn, book: str) -> list[dict]:
|
| 22 |
rows = conn.execute(
|
| 23 |
+
"""SELECT p.*, COALESCE(t.decimals, CASE WHEN p.chain = 'rh' THEN 18 END) AS decimals, t.graduated_at,
|
| 24 |
+
COALESCE(wp.program_label, CASE WHEN p.chain = 'rh' THEN 'Pons v4' END) AS program_label,
|
| 25 |
COALESCE(p.hold_s, (d.detail->>'hold_s')::float) AS hold_s, COALESCE(p.strategy, d.detail->>'strategy') AS strategy
|
| 26 |
FROM positions p LEFT JOIN tokens t ON t.mint = p.mint
|
| 27 |
LEFT JOIN watch_pools wp ON wp.pool = p.pool
|
|
|
|
| 39 |
FROM positions WHERE book = %s""",
|
| 40 |
(book,),
|
| 41 |
).fetchone()
|
| 42 |
+
from ..markets import for_book
|
| 43 |
+
return for_book(book).capital() + float(r["realized"]) - float(r["open_cost"]) # the book's own market's capital (SOL / ETH)
|
| 44 |
|
| 45 |
|
| 46 |
def record_fill(conn, *, order_id: int | None, book: str, mint: str, side: str, token_delta: int,
|
|
|
|
| 57 |
return int(row["id"])
|
| 58 |
|
| 59 |
|
| 60 |
+
def _chain_of(book: str) -> str:
|
| 61 |
+
from ..markets import for_book
|
| 62 |
+
return for_book(book).chain
|
| 63 |
+
|
| 64 |
+
|
| 65 |
def open_position(conn, *, book: str, mint: str, pool: str | None, qty_raw: int, cost_sol: float,
|
| 66 |
entry_price: float, decision_id: int | None, fees_sol: float, ts: datetime | None = None,
|
| 67 |
hold_s: float | None = None, strategy: str | None = None) -> int:
|
|
|
|
| 77 |
return int(existing["id"])
|
| 78 |
row = conn.execute(
|
| 79 |
"""INSERT INTO positions (book, mint, pool, opened_at, entry_decision_id, qty, cost_sol, entry_price, peak_price,
|
| 80 |
+
last_mark_price, last_mark_ts, fees_sol, status, last_swap_ts, hold_s, strategy, chain)
|
| 81 |
+
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,'open',%s,%s,%s,%s) RETURNING id""",
|
| 82 |
+
(book, mint, pool, ts, decision_id, qty_raw, cost_sol, entry_price, entry_price, entry_price, ts, fees_sol, ts, hold_s, strategy, _chain_of(book)),
|
| 83 |
).fetchone()
|
| 84 |
return int(row["id"])
|
| 85 |
|
|
|
|
| 134 |
|
| 135 |
def verify_fills() -> None:
|
| 136 |
"""Reconcile live positions against on-chain token balances; prints mismatches, writes wallet_events."""
|
|
|
|
| 137 |
from ..chain.rpc import HttpSolanaRpc
|
| 138 |
from ..db.connection import transaction
|
| 139 |
+
from ..signer import client
|
| 140 |
rpc = HttpSolanaRpc()
|
| 141 |
+
pub = client.get().call("pubkeys")["trading"]
|
| 142 |
accounts = rpc.get_token_accounts_by_owner(pub)
|
| 143 |
onchain = {a["mint"]: int(a["amount"]) for a in accounts}
|
| 144 |
with transaction() as conn:
|
fly_trader/kalshi/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
"""Kalshi prediction markets: the fly's second trading type, on the optic lobes (see the plan of 2026-09-22)."""
|
fly_trader/kalshi/client.py
ADDED
|
@@ -0,0 +1,342 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""fly-trader's Kalshi REST client: one persistent, authenticated SDK client on a private event-loop thread, paced under
|
| 2 |
+
the account's rate tier, every call logged to ``api_calls`` (service 'kalshi'; the key never appears).
|
| 3 |
+
|
| 4 |
+
The vendored better_bot client (``kalshi/vendor/kalshi_client.py``) opens and closes an SDK client per call, which is
|
| 5 |
+
right for a daily batch and wrong for a history pull of tens of thousands of candlestick requests or a trading loop.
|
| 6 |
+
This wrapper keeps the same authentication (``Configuration.api_key_id`` / ``private_key_pem``, RSA-PSS by the SDK) and
|
| 7 |
+
the same ``call_api`` transport, adds the endpoints better_bot never needed — subaccounts, resting (GTC, post-only)
|
| 8 |
+
orders, cancel/amend, the historical tier, series and events — and passes ``KALSHI_SUBACCOUNT`` on every portfolio and
|
| 9 |
+
order call. Prices cross this boundary in cents (ints) and contracts as floats; the exchange's fixed-point dollar
|
| 10 |
+
strings are converted here and nowhere else. Paths and body fields verified against kalshi-python-async 3.10 on
|
| 11 |
+
2026-09-22 (``docs.kalshi.com``): V2 orders are ``POST /portfolio/orders`` with ``yes_price``/``no_price`` in cents.
|
| 12 |
+
"""
|
| 13 |
+
from __future__ import annotations
|
| 14 |
+
|
| 15 |
+
import asyncio
|
| 16 |
+
import json
|
| 17 |
+
import logging
|
| 18 |
+
import threading
|
| 19 |
+
import time
|
| 20 |
+
import uuid
|
| 21 |
+
from urllib.parse import urlencode
|
| 22 |
+
|
| 23 |
+
from .. import config
|
| 24 |
+
from ..chain.jupiter_tokens import TokenBucket
|
| 25 |
+
from ..db.apilog import record_api_call
|
| 26 |
+
from ..logging_setup import scrub
|
| 27 |
+
|
| 28 |
+
log = logging.getLogger(__name__)
|
| 29 |
+
RETRIES_429 = 5
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
class KalshiApiError(RuntimeError):
|
| 33 |
+
def __init__(self, method: str, path: str, status: int | None, body):
|
| 34 |
+
self.method, self.path, self.status, self.body = method, path, status, body
|
| 35 |
+
err = (body or {}).get("error") if isinstance(body, dict) else None
|
| 36 |
+
msg = (err.get("message") if isinstance(err, dict) else None) or (body.get("message") if isinstance(body, dict) else None) or str(body)[:200]
|
| 37 |
+
super().__init__(f"kalshi {method} {path}: HTTP {status}: {scrub(str(msg))}")
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def _f(x) -> float | None:
|
| 41 |
+
try:
|
| 42 |
+
return float(x) if x not in (None, "") else None
|
| 43 |
+
except (TypeError, ValueError):
|
| 44 |
+
return None
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def cents(dollars) -> int | None:
|
| 48 |
+
"""A fixed-point dollar string ('0.5600') as whole cents; None when absent."""
|
| 49 |
+
v = _f(dollars)
|
| 50 |
+
return int(round(v * 100)) if v is not None else None
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def _summary(params: dict | None, body: dict | None) -> dict | None:
|
| 54 |
+
out = {}
|
| 55 |
+
if params:
|
| 56 |
+
out["params"] = {k: v for k, v in params.items()}
|
| 57 |
+
if body:
|
| 58 |
+
out["body"] = {k: v for k, v in body.items() if k not in ("signedTransaction",)}
|
| 59 |
+
return out or None
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
class KalshiRest:
|
| 63 |
+
"""Synchronous facade over the async SDK client (one per process is enough: MPS and the minute loop are single-threaded)."""
|
| 64 |
+
|
| 65 |
+
def __init__(self, rps: float | None = None, subaccount: int | None = None, base_url: str | None = None):
|
| 66 |
+
self.base = (base_url or config.KALSHI_API_BASE_URL).rstrip("/")
|
| 67 |
+
self.subaccount = config.KALSHI_SUBACCOUNT if subaccount is None else int(subaccount)
|
| 68 |
+
self.bucket = TokenBucket(rps or config.KALSHI_RPS)
|
| 69 |
+
self._loop: asyncio.AbstractEventLoop | None = None
|
| 70 |
+
self._thread: threading.Thread | None = None
|
| 71 |
+
self._client = None
|
| 72 |
+
self._lock = threading.Lock()
|
| 73 |
+
|
| 74 |
+
# ---- lifecycle ----
|
| 75 |
+
def _ensure(self):
|
| 76 |
+
with self._lock:
|
| 77 |
+
if self._client is not None:
|
| 78 |
+
return self._client
|
| 79 |
+
if not config.KALSHI_API_KEY_ID or not config.KALSHI_PRIVATE_KEY_PATH:
|
| 80 |
+
raise RuntimeError("KALSHI_API_KEY_ID and KALSHI_PRIVATE_KEY_PATH are not set")
|
| 81 |
+
from kalshi_python_async import Configuration, KalshiClient
|
| 82 |
+
cfg = Configuration(); cfg.host = self.base
|
| 83 |
+
cfg.api_key_id = config.KALSHI_API_KEY_ID
|
| 84 |
+
with open(config.KALSHI_PRIVATE_KEY_PATH) as f:
|
| 85 |
+
cfg.private_key_pem = f.read()
|
| 86 |
+
try:
|
| 87 |
+
import certifi
|
| 88 |
+
cfg.ssl_ca_cert = certifi.where()
|
| 89 |
+
except ImportError:
|
| 90 |
+
pass
|
| 91 |
+
self._loop = asyncio.new_event_loop()
|
| 92 |
+
self._thread = threading.Thread(target=self._loop.run_forever, name=f"{threading.current_thread().name}-kalshi-io", daemon=True)
|
| 93 |
+
self._thread.start()
|
| 94 |
+
self._client = asyncio.run_coroutine_threadsafe(self._make(cfg, KalshiClient), self._loop).result(30)
|
| 95 |
+
return self._client
|
| 96 |
+
|
| 97 |
+
@staticmethod
|
| 98 |
+
async def _make(cfg, KalshiClient):
|
| 99 |
+
return KalshiClient(configuration=cfg)
|
| 100 |
+
|
| 101 |
+
def close(self) -> None:
|
| 102 |
+
with self._lock:
|
| 103 |
+
if self._client is not None and self._loop is not None:
|
| 104 |
+
try:
|
| 105 |
+
asyncio.run_coroutine_threadsafe(self._client.close(), self._loop).result(10)
|
| 106 |
+
except Exception:
|
| 107 |
+
pass
|
| 108 |
+
self._loop.call_soon_threadsafe(self._loop.stop)
|
| 109 |
+
self._client = None
|
| 110 |
+
|
| 111 |
+
# ---- transport ----
|
| 112 |
+
async def _call(self, method: str, url: str, body: dict | None):
|
| 113 |
+
raw = await self._client.call_api(method, url, body=body, header_params={"Content-Type": "application/json"} if body is not None else None)
|
| 114 |
+
data = await raw.read()
|
| 115 |
+
text = data.decode("utf-8", "replace") if isinstance(data, (bytes, bytearray)) else str(data)
|
| 116 |
+
start = text.find("{")
|
| 117 |
+
try:
|
| 118 |
+
parsed = json.loads(text[start:]) if start >= 0 else {}
|
| 119 |
+
except ValueError:
|
| 120 |
+
parsed = {"raw": text[:300]}
|
| 121 |
+
return int(raw.status), parsed
|
| 122 |
+
|
| 123 |
+
def request(self, method: str, path: str, params: dict | None = None, body: dict | None = None, timeout: float = 60.0) -> dict:
|
| 124 |
+
"""One paced, logged call. Raises ``KalshiApiError`` on HTTP >= 400 (429 is retried with backoff first)."""
|
| 125 |
+
client = self._ensure()
|
| 126 |
+
q = {k: v for k, v in (params or {}).items() if v is not None}
|
| 127 |
+
url = self.base + path + (("?" + urlencode(q)) if q else "")
|
| 128 |
+
t0 = time.monotonic(); status = None; err = None; parsed = None
|
| 129 |
+
try:
|
| 130 |
+
for attempt in range(RETRIES_429 + 1):
|
| 131 |
+
self.bucket.acquire()
|
| 132 |
+
status, parsed = asyncio.run_coroutine_threadsafe(self._call(method, url, body), self._loop).result(timeout)
|
| 133 |
+
if status == 429 and attempt < RETRIES_429:
|
| 134 |
+
time.sleep(1.0 + attempt)
|
| 135 |
+
continue
|
| 136 |
+
break
|
| 137 |
+
if status >= 400:
|
| 138 |
+
raise KalshiApiError(method, path, status, parsed)
|
| 139 |
+
return parsed
|
| 140 |
+
except KalshiApiError as e:
|
| 141 |
+
err = str(e); raise
|
| 142 |
+
except Exception as e:
|
| 143 |
+
err = f"{type(e).__name__}: {scrub(str(e))}"; raise
|
| 144 |
+
finally:
|
| 145 |
+
record_api_call("kalshi", path, method, status, int((time.monotonic() - t0) * 1000), err is None, err,
|
| 146 |
+
request=_summary(q, body), response_bytes=len(json.dumps(parsed)) if parsed is not None else None)
|
| 147 |
+
|
| 148 |
+
def get(self, path: str, **params) -> dict:
|
| 149 |
+
return self.request("GET", path, params)
|
| 150 |
+
|
| 151 |
+
def pages(self, path: str, key: str, params: dict | None = None, limit: int = 1000, max_pages: int = 10_000, cursor: str | None = None, with_cursor: bool = False):
|
| 152 |
+
"""Cursor walk: yields each page's ``key`` list until the cursor ends or a page is empty. ``cursor`` resumes a walk
|
| 153 |
+
from a page cursor seen before; ``with_cursor`` yields ``(items, next_cursor)`` so the caller can persist its place."""
|
| 154 |
+
for _ in range(max_pages):
|
| 155 |
+
data = self.request("GET", path, {**(params or {}), "limit": limit, "cursor": cursor})
|
| 156 |
+
items = data.get(key) or []; nxt = data.get("cursor") or None
|
| 157 |
+
if items:
|
| 158 |
+
yield (items, nxt) if with_cursor else items
|
| 159 |
+
cursor = nxt
|
| 160 |
+
if not cursor or not items:
|
| 161 |
+
break
|
| 162 |
+
|
| 163 |
+
# ---- exchange / catalogue ----
|
| 164 |
+
def exchange_status(self) -> dict:
|
| 165 |
+
return self.get("/exchange/status")
|
| 166 |
+
|
| 167 |
+
def markets(self, cursor: str | None = None, with_cursor: bool = False, **params):
|
| 168 |
+
"""Pages of market rows (``status`` one of unopened|open|closed|settled|...; ``min_settled_ts`` etc.)."""
|
| 169 |
+
yield from self.pages("/markets", "markets", params, cursor=cursor, with_cursor=with_cursor)
|
| 170 |
+
|
| 171 |
+
def historical_markets(self, cursor: str | None = None, with_cursor: bool = False, **params):
|
| 172 |
+
yield from self.pages("/historical/markets", "markets", params, cursor=cursor, with_cursor=with_cursor)
|
| 173 |
+
|
| 174 |
+
def historical_cutoff(self) -> dict:
|
| 175 |
+
return self.get("/historical/cutoff")
|
| 176 |
+
|
| 177 |
+
def market(self, ticker: str) -> dict:
|
| 178 |
+
return (self.get(f"/markets/{ticker}") or {}).get("market") or {}
|
| 179 |
+
|
| 180 |
+
def historical_market(self, ticker: str) -> dict:
|
| 181 |
+
return (self.get(f"/historical/markets/{ticker}") or {}).get("market") or {}
|
| 182 |
+
|
| 183 |
+
def events(self, **params):
|
| 184 |
+
"""Pages of events (``with_nested_markets=true`` nests their markets); limit is capped at 200 by the exchange."""
|
| 185 |
+
yield from self.pages("/events", "events", params, limit=200)
|
| 186 |
+
|
| 187 |
+
def event(self, event_ticker: str, with_nested_markets: bool = True) -> dict:
|
| 188 |
+
return self.get(f"/events/{event_ticker}", with_nested_markets=str(with_nested_markets).lower())
|
| 189 |
+
|
| 190 |
+
def series_list(self, category: str | None = None, **params) -> list[dict]:
|
| 191 |
+
return (self.get("/series", category=category, **params) or {}).get("series") or []
|
| 192 |
+
|
| 193 |
+
def series(self, series_ticker: str) -> dict:
|
| 194 |
+
return (self.get(f"/series/{series_ticker}") or {}).get("series") or {}
|
| 195 |
+
|
| 196 |
+
def orderbook(self, ticker: str, depth: int | None = None) -> dict:
|
| 197 |
+
return self.get(f"/markets/{ticker}/orderbook", depth=depth)
|
| 198 |
+
|
| 199 |
+
def candlesticks(self, series_ticker: str, ticker: str, start_ts: int, end_ts: int, period_interval: int = 1) -> list[dict]:
|
| 200 |
+
"""1/60/1440-minute candles; a market archived past the historical cutoff answers on the historical path."""
|
| 201 |
+
params = {"start_ts": int(start_ts), "end_ts": int(end_ts), "period_interval": int(period_interval)}
|
| 202 |
+
try:
|
| 203 |
+
return (self.request("GET", f"/series/{series_ticker}/markets/{ticker}/candlesticks", params) or {}).get("candlesticks") or []
|
| 204 |
+
except KalshiApiError as e:
|
| 205 |
+
if e.status != 404:
|
| 206 |
+
raise
|
| 207 |
+
cs = (self.request("GET", f"/historical/markets/{ticker}/candlesticks", params) or {}).get("candlesticks") or []
|
| 208 |
+
return [{**c, "_units": "dollars"} for c in cs] # the archive sends prices as dollar strings under plain keys ("close": "0.0100")
|
| 209 |
+
|
| 210 |
+
def trades(self, ticker: str, min_ts: int | None = None, max_ts: int | None = None, historical: bool = False):
|
| 211 |
+
yield from self.pages("/historical/trades" if historical else "/markets/trades", "trades", {"ticker": ticker, "min_ts": min_ts, "max_ts": max_ts})
|
| 212 |
+
|
| 213 |
+
# ---- portfolio (always on this process's subaccount unless told otherwise) ----
|
| 214 |
+
def _sub(self, subaccount) -> int:
|
| 215 |
+
return self.subaccount if subaccount is None else int(subaccount)
|
| 216 |
+
|
| 217 |
+
def balance(self, subaccount: int | None = None) -> dict:
|
| 218 |
+
"""``{"balance_cents", "portfolio_value_cents"}`` of one subaccount (0 = primary)."""
|
| 219 |
+
d = self.get("/portfolio/balance", subaccount=self._sub(subaccount))
|
| 220 |
+
return {"balance_cents": int(d.get("balance") or 0), "portfolio_value_cents": int(d.get("portfolio_value") or 0), "updated_ts": d.get("updated_ts")}
|
| 221 |
+
|
| 222 |
+
def subaccount_balances(self) -> list[dict]:
|
| 223 |
+
return (self.get("/portfolio/subaccounts/balances") or {}).get("subaccount_balances") or []
|
| 224 |
+
|
| 225 |
+
def create_subaccount(self) -> int:
|
| 226 |
+
d = self.request("POST", "/portfolio/subaccounts", body={})
|
| 227 |
+
return int(d["subaccount_number"])
|
| 228 |
+
|
| 229 |
+
def transfer_between_subaccounts(self, from_subaccount: int, to_subaccount: int, amount_cents: int) -> dict:
|
| 230 |
+
body = {"client_transfer_id": str(uuid.uuid4()), "from_subaccount": int(from_subaccount), "to_subaccount": int(to_subaccount), "amount_cents": int(amount_cents)}
|
| 231 |
+
return self.request("POST", "/portfolio/subaccounts/transfer", body=body)
|
| 232 |
+
|
| 233 |
+
def positions(self, subaccount: int | None = None, **params) -> list[dict]:
|
| 234 |
+
out = []
|
| 235 |
+
for page in self.pages("/portfolio/positions", "market_positions", {**params, "subaccount": self._sub(subaccount)}, limit=1000):
|
| 236 |
+
out.extend(page)
|
| 237 |
+
return out
|
| 238 |
+
|
| 239 |
+
def orders(self, status: str | None = None, subaccount: int | None = None, **params) -> list[dict]:
|
| 240 |
+
out = []
|
| 241 |
+
for page in self.pages("/portfolio/orders", "orders", {**params, "status": status, "subaccount": self._sub(subaccount)}, limit=1000):
|
| 242 |
+
out.extend(page)
|
| 243 |
+
return out
|
| 244 |
+
|
| 245 |
+
def order(self, order_id: str) -> dict:
|
| 246 |
+
return (self.get(f"/portfolio/orders/{order_id}") or {}).get("order") or {}
|
| 247 |
+
|
| 248 |
+
def fills(self, subaccount: int | None = None, **params) -> list[dict]:
|
| 249 |
+
out = []
|
| 250 |
+
for page in self.pages("/portfolio/fills", "fills", {**params, "subaccount": self._sub(subaccount)}, limit=1000):
|
| 251 |
+
out.extend(page)
|
| 252 |
+
return out
|
| 253 |
+
|
| 254 |
+
def settlements(self, subaccount: int | None = None, **params) -> list[dict]:
|
| 255 |
+
out = []
|
| 256 |
+
for page in self.pages("/portfolio/settlements", "settlements", {**params, "subaccount": self._sub(subaccount)}, limit=1000):
|
| 257 |
+
out.extend(page)
|
| 258 |
+
return out
|
| 259 |
+
|
| 260 |
+
# ---- orders ----
|
| 261 |
+
def create_order(self, ticker: str, side: str, price_cents: int, count: float, *, tif: str = "immediate_or_cancel", post_only: bool = False,
|
| 262 |
+
expiration_ts: int | None = None, client_order_id: str | None = None, subaccount: int | None = None, action: str = "buy") -> dict:
|
| 263 |
+
"""Buy (or sell) ``count`` contracts of ``side`` ('yes'|'no') at ``price_cents`` each. ``tif``: immediate_or_cancel |
|
| 264 |
+
good_till_canceled | fill_or_kill; a resting order is GTC (+ ``post_only`` never crosses, ``expiration_ts`` ends it).
|
| 265 |
+
Returns the exchange's order object (``order_id``, ``status``, ``fill_count``/``remaining_count`` ...)."""
|
| 266 |
+
if side not in ("yes", "no"):
|
| 267 |
+
raise ValueError(f"side must be yes|no, got {side!r}")
|
| 268 |
+
if not 1 <= int(price_cents) <= 99:
|
| 269 |
+
raise ValueError(f"price {price_cents} outside 1..99 cents")
|
| 270 |
+
body = {"ticker": ticker, "client_order_id": client_order_id or f"fly-{uuid.uuid4().hex[:16]}", "side": side, "action": action,
|
| 271 |
+
"count_fp": f"{float(count):.2f}", ("yes_price" if side == "yes" else "no_price"): int(price_cents), "time_in_force": tif,
|
| 272 |
+
"post_only": bool(post_only), "self_trade_prevention_type": "maker" if post_only else "taker_at_cross",
|
| 273 |
+
"subaccount": self._sub(subaccount)}
|
| 274 |
+
if expiration_ts is not None:
|
| 275 |
+
body["expiration_ts"] = int(expiration_ts)
|
| 276 |
+
d = self.request("POST", "/portfolio/orders", body=body)
|
| 277 |
+
return d.get("order") or d
|
| 278 |
+
|
| 279 |
+
def cancel_order(self, order_id: str, subaccount: int | None = None) -> dict:
|
| 280 |
+
d = self.request("DELETE", f"/portfolio/orders/{order_id}", {"subaccount": self._sub(subaccount)})
|
| 281 |
+
return d.get("order") or d
|
| 282 |
+
|
| 283 |
+
def amend_order(self, order_id: str, ticker: str, side: str, price_cents: int, count: float, client_order_id: str, subaccount: int | None = None,
|
| 284 |
+
action: str = "buy") -> dict:
|
| 285 |
+
body = {"ticker": ticker, "side": side, "action": action, "client_order_id": client_order_id, "updated_client_order_id": f"fly-{uuid.uuid4().hex[:16]}",
|
| 286 |
+
("yes_price" if side == "yes" else "no_price"): int(price_cents), "count_fp": f"{float(count):.2f}", "subaccount": self._sub(subaccount)}
|
| 287 |
+
d = self.request("POST", f"/portfolio/orders/{order_id}/amend", body=body)
|
| 288 |
+
return d.get("order") or d
|
| 289 |
+
|
| 290 |
+
|
| 291 |
+
_REST: KalshiRest | None = None
|
| 292 |
+
_REST_LOCK = threading.Lock()
|
| 293 |
+
|
| 294 |
+
|
| 295 |
+
def rest() -> KalshiRest:
|
| 296 |
+
"""The process-wide client (built on first use)."""
|
| 297 |
+
global _REST
|
| 298 |
+
with _REST_LOCK:
|
| 299 |
+
if _REST is None:
|
| 300 |
+
_REST = KalshiRest()
|
| 301 |
+
return _REST
|
| 302 |
+
|
| 303 |
+
|
| 304 |
+
# ---- operator commands (fly-trader kalshi-subaccount / kalshi-fund / kalshi-status) ----
|
| 305 |
+
def subaccount_create(write_env: bool = True) -> int:
|
| 306 |
+
"""Create the dedicated subaccount and record it in .env (``KALSHI_SUBACCOUNT``); refuses when one is already set."""
|
| 307 |
+
import re
|
| 308 |
+
from ..db.apilog import record_event
|
| 309 |
+
if config.KALSHI_SUBACCOUNT > 0:
|
| 310 |
+
raise RuntimeError(f"KALSHI_SUBACCOUNT={config.KALSHI_SUBACCOUNT} is already set; refusing to create another")
|
| 311 |
+
n = KalshiRest(subaccount=0).create_subaccount()
|
| 312 |
+
if write_env:
|
| 313 |
+
path = config.REPO_ROOT / ".env"; text = path.read_text() if path.exists() else ""
|
| 314 |
+
text = re.sub(r"^KALSHI_SUBACCOUNT=.*$", f"KALSHI_SUBACCOUNT={n}", text, flags=re.M) if re.search(r"^KALSHI_SUBACCOUNT=", text, re.M) else text.rstrip("\n") + f"\nKALSHI_SUBACCOUNT={n}\n"
|
| 315 |
+
path.write_text(text)
|
| 316 |
+
record_event("info", "kalshi", f"subaccount {n} created", {"subaccount": n})
|
| 317 |
+
return n
|
| 318 |
+
|
| 319 |
+
|
| 320 |
+
def fund(usd: float, to_subaccount: int | None = None) -> dict:
|
| 321 |
+
"""Move ``usd`` from the primary account into the fly's subaccount."""
|
| 322 |
+
from ..db.apilog import record_event
|
| 323 |
+
to = config.KALSHI_SUBACCOUNT if to_subaccount is None else int(to_subaccount)
|
| 324 |
+
if to <= 0:
|
| 325 |
+
raise RuntimeError("no subaccount: run fly-trader kalshi-subaccount create first")
|
| 326 |
+
out = KalshiRest(subaccount=0).transfer_between_subaccounts(0, to, int(round(usd * 100)))
|
| 327 |
+
record_event("info", "kalshi", f"funded subaccount {to} with ${usd:.2f}", {"transfer": out})
|
| 328 |
+
return out
|
| 329 |
+
|
| 330 |
+
|
| 331 |
+
def status() -> dict:
|
| 332 |
+
r = rest()
|
| 333 |
+
out = {"exchange": r.exchange_status(), "subaccount": r.subaccount, "balances": r.subaccount_balances(),
|
| 334 |
+
"prerequisites_missing": config.kalshi_live_prerequisites_missing()}
|
| 335 |
+
try:
|
| 336 |
+
out["balance"] = r.balance()
|
| 337 |
+
out["resting_orders"] = r.orders(status="resting")
|
| 338 |
+
out["positions"] = [p for p in r.positions() if _f(p.get("position_fp") or p.get("position")) not in (None, 0.0)]
|
| 339 |
+
out["settlements_24h"] = r.settlements(min_ts=int(time.time()) - 86400)
|
| 340 |
+
except KalshiApiError as e:
|
| 341 |
+
out["error"] = str(e)
|
| 342 |
+
return out
|
fly_trader/kalshi/data.py
ADDED
|
@@ -0,0 +1,164 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""The Kalshi corpus on disk and the catalogue rows both the history puller and the live stream write.
|
| 2 |
+
|
| 3 |
+
Layout (``config.KALSHI_DIR``):
|
| 4 |
+
- ``candles/<ticker>.parquet`` — 1-minute candles (``CANDLE_SCHEMA``: period end, YES bid/ask OHLC in cents, trade price
|
| 5 |
+
OHLC when traded, contracts traded in the period, open interest at its end);
|
| 6 |
+
- ``trades/<ticker>.parquet`` — public trades (``TRADE_SCHEMA``: time, YES price in cents, contracts, the taker's side);
|
| 7 |
+
- ``features/<day>/part.parquet`` — the daily feature rows (kalshi/mature.py).
|
| 8 |
+
Rows of ``kalshi_markets`` / ``kalshi_events`` / ``kalshi_series`` are upserted from the exchange's JSON by
|
| 9 |
+
``upsert_market`` and friends; prices are kept in cents, contracts as floats, times as UTC timestamps.
|
| 10 |
+
"""
|
| 11 |
+
from __future__ import annotations
|
| 12 |
+
|
| 13 |
+
import json
|
| 14 |
+
from datetime import datetime, timezone
|
| 15 |
+
from pathlib import Path
|
| 16 |
+
|
| 17 |
+
import pyarrow as pa
|
| 18 |
+
|
| 19 |
+
from .. import config
|
| 20 |
+
|
| 21 |
+
CANDLES_DIR = config.KALSHI_DIR / "candles"
|
| 22 |
+
TRADES_DIR = config.KALSHI_DIR / "trades"
|
| 23 |
+
FEATURES_DIR = config.KALSHI_DIR / "features"
|
| 24 |
+
CANDLE_SCHEMA = pa.schema([("end_ts", pa.int64()), ("yes_bid_open", pa.float32()), ("yes_bid_high", pa.float32()), ("yes_bid_low", pa.float32()),
|
| 25 |
+
("yes_bid_close", pa.float32()), ("yes_ask_open", pa.float32()), ("yes_ask_high", pa.float32()), ("yes_ask_low", pa.float32()),
|
| 26 |
+
("yes_ask_close", pa.float32()), ("price_open", pa.float32()), ("price_high", pa.float32()), ("price_low", pa.float32()),
|
| 27 |
+
("price_close", pa.float32()), ("price_mean", pa.float32()), ("volume", pa.float64()), ("open_interest", pa.float64())])
|
| 28 |
+
TRADE_SCHEMA = pa.schema([("ts", pa.timestamp("ms", tz="UTC")), ("yes_price", pa.int16()), ("count", pa.float64()), ("taker_side", pa.string()), ("is_block", pa.bool_())])
|
| 29 |
+
# discovery categories the exchange uses, folded into the fixed set the features one-hot (kalshi/features.CATEGORIES)
|
| 30 |
+
CATEGORY_KEYS = (("politic", "politics"), ("election", "politics"), ("sport", "sports"), ("crypto", "crypto"), ("bitcoin", "crypto"), ("econom", "economics"),
|
| 31 |
+
("financ", "economics"), ("weather", "weather"), ("climate", "weather"), ("entertain", "entertainment"), ("culture", "entertainment"),
|
| 32 |
+
("science", "science"), ("tech", "science"), ("compan", "companies"), ("world", "world"), ("health", "health"))
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def category_key(raw: str | None) -> str:
|
| 36 |
+
s = raw.lower() if isinstance(raw, str) else "" # None or NaN (a market whose series is not catalogued yet): 'other'
|
| 37 |
+
for needle, key in CATEGORY_KEYS:
|
| 38 |
+
if needle in s:
|
| 39 |
+
return key
|
| 40 |
+
return "other"
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def ts(v) -> datetime | None:
|
| 44 |
+
"""An ISO-8601 string (or epoch seconds) from the exchange as an aware UTC datetime."""
|
| 45 |
+
if v in (None, ""):
|
| 46 |
+
return None
|
| 47 |
+
if isinstance(v, (int, float)):
|
| 48 |
+
return datetime.fromtimestamp(float(v), timezone.utc)
|
| 49 |
+
s = str(v).replace("Z", "+00:00")
|
| 50 |
+
try:
|
| 51 |
+
d = datetime.fromisoformat(s)
|
| 52 |
+
except ValueError:
|
| 53 |
+
return None
|
| 54 |
+
return d if d.tzinfo else d.replace(tzinfo=timezone.utc)
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
def _f(v) -> float | None:
|
| 58 |
+
try:
|
| 59 |
+
return float(v) if v not in (None, "") else None
|
| 60 |
+
except (TypeError, ValueError):
|
| 61 |
+
return None
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def price_cents(row: dict, key: str) -> float | None:
|
| 65 |
+
"""``<key>_dollars`` (fixed point) or ``<key>`` (cents) of a market/ticker row, as cents."""
|
| 66 |
+
d = _f(row.get(f"{key}_dollars"))
|
| 67 |
+
if d is not None:
|
| 68 |
+
return d * 100.0
|
| 69 |
+
return _f(row.get(key))
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def series_ticker_of(ticker: str | None) -> str | None:
|
| 73 |
+
return str(ticker).split("-")[0] if ticker else None
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
MARKET_COLS = ("ticker", "event_ticker", "market_type", "title", "yes_sub_title", "no_sub_title", "status", "open_time", "close_time",
|
| 77 |
+
"expected_expiration_time", "latest_expiration_time", "settlement_ts", "result", "settlement_value", "strike_type", "floor_strike",
|
| 78 |
+
"cap_strike", "exchange_index", "price_level_structure", "is_provisional", "can_close_early", "early_close_condition", "created_time", "source", "raw")
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
def market_row(m: dict, source: str) -> dict:
|
| 82 |
+
return {"ticker": m.get("ticker"), "event_ticker": m.get("event_ticker"), "market_type": m.get("market_type"), "title": m.get("title"),
|
| 83 |
+
"yes_sub_title": m.get("yes_sub_title"), "no_sub_title": m.get("no_sub_title"), "status": m.get("status"), "open_time": ts(m.get("open_time")),
|
| 84 |
+
"close_time": ts(m.get("close_time")), "expected_expiration_time": ts(m.get("expected_expiration_time")),
|
| 85 |
+
"latest_expiration_time": ts(m.get("latest_expiration_time")), "settlement_ts": ts(m.get("settlement_ts") or m.get("settled_time")),
|
| 86 |
+
"result": m.get("result") or None, "settlement_value": _f(m.get("settlement_value_dollars")) if m.get("settlement_value_dollars") is not None else _f(m.get("settlement_value")),
|
| 87 |
+
"strike_type": m.get("strike_type"), "floor_strike": _f(m.get("floor_strike")), "cap_strike": _f(m.get("cap_strike")),
|
| 88 |
+
"exchange_index": m.get("exchange_index"), "price_level_structure": m.get("price_level_structure"), "is_provisional": m.get("is_provisional"),
|
| 89 |
+
"can_close_early": m.get("can_close_early"), "early_close_condition": m.get("early_close_condition") or None, "created_time": ts(m.get("created_time")), "source": source,
|
| 90 |
+
"raw": json.dumps({k: v for k, v in m.items() if k not in ("rules_primary", "rules_secondary")}, default=str)}
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
def upsert_markets(conn, markets: list[dict], source: str) -> int:
|
| 94 |
+
rows = [market_row(m, source) for m in markets if m.get("ticker")]
|
| 95 |
+
if not rows:
|
| 96 |
+
return 0
|
| 97 |
+
cols = MARKET_COLS
|
| 98 |
+
sets = ", ".join(f"{c} = COALESCE(EXCLUDED.{c}, kalshi_markets.{c})" for c in cols if c not in ("ticker", "source", "raw"))
|
| 99 |
+
conn.cursor().executemany(
|
| 100 |
+
f"INSERT INTO kalshi_markets ({', '.join(cols)}) VALUES ({', '.join(['%s'] * len(cols))}) ON CONFLICT (ticker) DO UPDATE SET {sets}, "
|
| 101 |
+
"source = EXCLUDED.source, raw = EXCLUDED.raw, updated_at = now()", [tuple(r[c] for c in cols) for r in rows])
|
| 102 |
+
return len(rows)
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
def upsert_event(conn, e: dict) -> None:
|
| 106 |
+
if not e.get("event_ticker"):
|
| 107 |
+
return
|
| 108 |
+
conn.execute("INSERT INTO kalshi_events (event_ticker, series_ticker, title, sub_title, category, mutually_exclusive, strike_date, strike_period, "
|
| 109 |
+
"collateral_return_type, exchange_index, raw) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT (event_ticker) DO UPDATE SET "
|
| 110 |
+
"series_ticker = COALESCE(EXCLUDED.series_ticker, kalshi_events.series_ticker), title = COALESCE(EXCLUDED.title, kalshi_events.title), "
|
| 111 |
+
"sub_title = COALESCE(EXCLUDED.sub_title, kalshi_events.sub_title), category = COALESCE(EXCLUDED.category, kalshi_events.category), "
|
| 112 |
+
"mutually_exclusive = COALESCE(EXCLUDED.mutually_exclusive, kalshi_events.mutually_exclusive), strike_date = COALESCE(EXCLUDED.strike_date, kalshi_events.strike_date), "
|
| 113 |
+
"strike_period = COALESCE(EXCLUDED.strike_period, kalshi_events.strike_period), collateral_return_type = COALESCE(EXCLUDED.collateral_return_type, kalshi_events.collateral_return_type), "
|
| 114 |
+
"exchange_index = COALESCE(EXCLUDED.exchange_index, kalshi_events.exchange_index), raw = EXCLUDED.raw, updated_at = now()",
|
| 115 |
+
(e["event_ticker"], e.get("series_ticker") or series_ticker_of(e["event_ticker"]), e.get("title"), e.get("sub_title"), e.get("category"),
|
| 116 |
+
e.get("mutually_exclusive"), ts(e.get("strike_date")), e.get("strike_period"), e.get("collateral_return_type"), e.get("exchange_index"),
|
| 117 |
+
json.dumps({k: v for k, v in e.items() if k != "markets"}, default=str)))
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
def upsert_series(conn, s: dict) -> None:
|
| 121 |
+
if not s.get("ticker"):
|
| 122 |
+
return
|
| 123 |
+
conn.execute("INSERT INTO kalshi_series (ticker, title, category, categories, tags, frequency, fee_type, fee_multiplier, settlement_sources) "
|
| 124 |
+
"VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT (ticker) DO UPDATE SET title = EXCLUDED.title, category = EXCLUDED.category, "
|
| 125 |
+
"categories = EXCLUDED.categories, tags = EXCLUDED.tags, frequency = EXCLUDED.frequency, fee_type = EXCLUDED.fee_type, "
|
| 126 |
+
"fee_multiplier = EXCLUDED.fee_multiplier, settlement_sources = EXCLUDED.settlement_sources, updated_at = now()",
|
| 127 |
+
(s["ticker"], s.get("title"), s.get("category"), list(s.get("categories") or []), list(s.get("tags") or []), s.get("frequency"),
|
| 128 |
+
s.get("fee_type"), _f(s.get("fee_multiplier")), json.dumps(s.get("settlement_sources") or [], default=str)))
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
def candle_rows(candles: list[dict]) -> list[dict]:
|
| 132 |
+
"""Exchange candlesticks → ``CANDLE_SCHEMA`` rows (cents; missing trade prices stay NaN)."""
|
| 133 |
+
out = []
|
| 134 |
+
for c in candles:
|
| 135 |
+
yb, ya, pr = c.get("yes_bid") or {}, c.get("yes_ask") or {}, c.get("price") or {}
|
| 136 |
+
plain = 100.0 if c.get("_units") == "dollars" else 1.0 # live: <k>_dollars or <k> in cents; archive: <k> in dollars
|
| 137 |
+
|
| 138 |
+
def q(d, k, plain=plain):
|
| 139 |
+
v = _f(d.get(f"{k}_dollars"))
|
| 140 |
+
return v * 100.0 if v is not None else (_f(d.get(k)) * plain if d.get(k) is not None and _f(d.get(k)) is not None else float("nan"))
|
| 141 |
+
out.append({"end_ts": int(c.get("end_period_ts") or 0),
|
| 142 |
+
"yes_bid_open": q(yb, "open"), "yes_bid_high": q(yb, "high"), "yes_bid_low": q(yb, "low"), "yes_bid_close": q(yb, "close"),
|
| 143 |
+
"yes_ask_open": q(ya, "open"), "yes_ask_high": q(ya, "high"), "yes_ask_low": q(ya, "low"), "yes_ask_close": q(ya, "close"),
|
| 144 |
+
"price_open": q(pr, "open"), "price_high": q(pr, "high"), "price_low": q(pr, "low"), "price_close": q(pr, "close"), "price_mean": q(pr, "mean"),
|
| 145 |
+
"volume": _f(c.get("volume_fp") or c.get("volume")) or 0.0, "open_interest": _f(c.get("open_interest_fp") or c.get("open_interest")) or 0.0})
|
| 146 |
+
return out
|
| 147 |
+
|
| 148 |
+
|
| 149 |
+
def trade_rows(trades: list[dict]) -> list[dict]:
|
| 150 |
+
out = []
|
| 151 |
+
for t in trades:
|
| 152 |
+
p = price_cents(t, "yes_price")
|
| 153 |
+
if p is None:
|
| 154 |
+
continue
|
| 155 |
+
out.append({"ts": ts(t.get("created_time")), "yes_price": int(round(p)), "count": _f(t.get("count_fp") or t.get("count")) or 0.0,
|
| 156 |
+
"taker_side": t.get("taker_outcome_side") or t.get("taker_side") or "", "is_block": bool(t.get("is_block_trade"))})
|
| 157 |
+
return [r for r in out if r["ts"] is not None]
|
| 158 |
+
|
| 159 |
+
|
| 160 |
+
def write_parquet(rows: list[dict], schema: pa.Schema, path: Path) -> None:
|
| 161 |
+
import pyarrow.parquet as pq
|
| 162 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 163 |
+
tmp = path.with_name(path.name + ".tmp")
|
| 164 |
+
pq.write_table(pa.Table.from_pylist(rows, schema=schema), tmp, compression="zstd"); tmp.replace(path)
|
fly_trader/kalshi/decisions.py
ADDED
|
@@ -0,0 +1,149 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Decision points for the Kalshi selector and fly: one row per (market, minute, side) of the feature parts
|
| 2 |
+
(kalshi/mature.py), with the settlement labels both arms trade on.
|
| 3 |
+
|
| 4 |
+
Reuses ``train.decisions.DecisionSet`` unchanged: ``mint`` = ``"<ticker>:<side>"``, ``ts`` = minute start, ``day`` =
|
| 5 |
+
the minute's date, and — because every position is held to settlement — a per-row hold ``hold_s`` = settlement − ts
|
| 6 |
+
(``taken_idx`` / ``evaluate`` already take per-row holds). Labels, per dollar staked at the fee-inclusive price:
|
| 7 |
+
- ``fwd_pess`` (taker): ``(100·y − eff) / eff`` with ``eff = effective_price_cents(side ask)`` and ``y`` the settlement
|
| 8 |
+
of the side (1 when it paid);
|
| 9 |
+
- ``fwd`` (reference): the same at the side's mid;
|
| 10 |
+
- ``fwd_h["maker"]``: the maker arm's order (kalshi/maker.py): a bid resting one tick above the side's bid, never at or
|
| 11 |
+
through its ask (``maker_rest``), for the order's lifetime (``KALSHI_MAKER_TTL_H``, ending ``KALSHI_MAKER_QUIET_MIN``
|
| 12 |
+
before close); filled (``maker_filled``) when the side's ask came down to it — every bid at that price, ours included,
|
| 13 |
+
was taken — or a taker sold the side below it, or at it when our bid set a new best price and so stood first in its
|
| 14 |
+
queue; the maker fee where the series charges one; NaN when unfilled. Until feature version 5 the order rested one tick
|
| 15 |
+
inside the ask, for the market's whole life, and filled only when the ask later fell strictly below it: it paid away
|
| 16 |
+
the spread makers earn and counted only the fills that came with an adverse move (2026-10-09).
|
| 17 |
+
Eligibility (the live engine applies the same, ``kalshi/engine.KalshiMinuteEngine.eligible``): both quotes present,
|
| 18 |
+
spread ≤ KALSHI_MAX_SPREAD_CENTS, side ask in 1..99, open interest ≥ KALSHI_MIN_OPEN_INTEREST, 24 h volume ≥
|
| 19 |
+
KALSHI_MIN_VOLUME_24H, settlement yes/no, time to close inside the entry window.
|
| 20 |
+
"""
|
| 21 |
+
from __future__ import annotations
|
| 22 |
+
|
| 23 |
+
import glob
|
| 24 |
+
import math
|
| 25 |
+
from datetime import date
|
| 26 |
+
|
| 27 |
+
import numpy as np
|
| 28 |
+
import pandas as pd
|
| 29 |
+
import pyarrow.parquet as pq
|
| 30 |
+
|
| 31 |
+
from .. import config
|
| 32 |
+
from ..train.corpus_features import _epoch_s
|
| 33 |
+
from ..train.decisions import DecisionSet
|
| 34 |
+
from . import data as D
|
| 35 |
+
from .features import K_COLS, KALSHI_FEATURE_VERSION, KIDX
|
| 36 |
+
from .mature import part_current
|
| 37 |
+
from .vendor import kalshi_client as kc
|
| 38 |
+
|
| 39 |
+
X_COLS = list(K_COLS)
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def maker_rest(side_bid, side_ask) -> np.ndarray:
|
| 43 |
+
"""The maker arm's bid (cents): one tick above the side's best bid, at most one tick below its ask (``post_only``
|
| 44 |
+
never crosses); with a one-tick spread it joins the bid's queue."""
|
| 45 |
+
return np.clip(np.minimum(np.round(np.asarray(side_bid, dtype=float)) + 1.0, np.round(np.asarray(side_ask, dtype=float)) - 1.0), 1.0, 99.0)
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def maker_filled(rest, side_bid, ask_low, sold_low) -> np.ndarray:
|
| 49 |
+
"""Would a bid at ``rest``, posted when the side's bid was ``side_bid``, have filled, given the lowest ask the side
|
| 50 |
+
showed (``ask_low``) and the lowest price a taker sold the side at (``sold_low``) while it rested (NaN: none). The ask
|
| 51 |
+
reaching ``rest`` means every bid at ``rest`` was taken; a taker sale below ``rest`` means the same; a sale at
|
| 52 |
+
``rest`` reaches us only when we were first there — our bid improved on the best bid, so nobody was ahead of us."""
|
| 53 |
+
rest = np.asarray(rest, dtype=float); ask_low = np.asarray(ask_low, dtype=float); sold_low = np.asarray(sold_low, dtype=float)
|
| 54 |
+
first = rest > np.round(np.asarray(side_bid, dtype=float))
|
| 55 |
+
with np.errstate(invalid="ignore"):
|
| 56 |
+
return (np.isfinite(ask_low) & (ask_low <= rest)) | (np.isfinite(sold_low) & ((sold_low < rest) | (first & (sold_low <= rest))))
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def maker_fee_cents(price_cents, fee_mult, charged) -> np.ndarray:
|
| 60 |
+
"""Per-contract maker fee in cents where the series charges makers (KALSHI_MAKER_FEE_FACTOR·mult·p(1−p)·100), else 0."""
|
| 61 |
+
p = np.asarray(price_cents, dtype=float) / 100.0
|
| 62 |
+
return np.where(np.asarray(charged, dtype=float) > 0, config.KALSHI_MAKER_FEE_FACTOR * np.asarray(fee_mult, dtype=float) * p * (1 - p) * 100.0, 0.0)
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def eligible_mask(X: np.ndarray, cols: list[str], to_close_s: np.ndarray) -> np.ndarray:
|
| 66 |
+
c = {n: i for i, n in enumerate(cols)}
|
| 67 |
+
return ((X[:, c["spread"]] <= config.KALSHI_MAX_SPREAD_CENTS) & (X[:, c["side_ask"]] >= 1.0) & (X[:, c["side_ask"]] <= 99.0)
|
| 68 |
+
& (X[:, c["side_bid"]] >= 1.0) & (np.expm1(X[:, c["log_oi"]]) >= config.KALSHI_MIN_OPEN_INTEREST)
|
| 69 |
+
& (np.expm1(X[:, c["log_vol_24h"]]) >= config.KALSHI_MIN_VOLUME_24H)
|
| 70 |
+
& (to_close_s >= config.KALSHI_MIN_MINUTES_TO_CLOSE * 60.0) & (to_close_s <= config.KALSHI_MAX_DAYS_TO_CLOSE * 86400.0))
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def build(days: int | None = None, feature_dir=None, label_thr: float = 0.0) -> DecisionSet:
|
| 74 |
+
"""The decision rows of the most recent ``days`` feature days (all of them when None). Each part is reduced to its
|
| 75 |
+
eligible rows as it is read — float32 inputs, labels, integer market ids — so the whole corpus (tens of millions of
|
| 76 |
+
rows) fits: reading every part into one pandas frame with its string columns peaked near three times the final size."""
|
| 77 |
+
root = feature_dir or D.FEATURES_DIR
|
| 78 |
+
dirs = sorted(glob.glob(str(root / "*")))
|
| 79 |
+
if days:
|
| 80 |
+
dirs = dirs[-days:]
|
| 81 |
+
files = [f for d in dirs for f in sorted(glob.glob(str(root / d.split("/")[-1] / "part-*.parquet")))]
|
| 82 |
+
if not files:
|
| 83 |
+
raise RuntimeError("no Kalshi feature parts; run kalshi-build")
|
| 84 |
+
stale = [f for f in files if not part_current(f)]
|
| 85 |
+
if stale:
|
| 86 |
+
raise RuntimeError(f"{len(stale)} Kalshi feature part(s) from another version (e.g. {stale[0]}); run kalshi-build")
|
| 87 |
+
need = list(dict.fromkeys([*X_COLS, "ticker", "side", "ts", "category", "close_ts", "end_ts", "settled_ts", "result", "fill_ask_low", "fill_sold_low"]))
|
| 88 |
+
tick_id: dict[str, int] = {}; tick_names: list[str] = []; cat_id: dict[str, int] = {}; cat_names: list[str] = []
|
| 89 |
+
acc: dict[str, list] = {k: [] for k in ("X", "ts", "settled", "y", "fwd", "fwd_pess", "maker", "tick", "side", "cat", "ord")}
|
| 90 |
+
for f in files:
|
| 91 |
+
t = pq.read_table(f, columns=need)
|
| 92 |
+
if not t.num_rows:
|
| 93 |
+
continue
|
| 94 |
+
res = np.asarray(t["result"].to_numpy(zero_copy_only=False), dtype=object); side = np.asarray(t["side"].to_numpy(zero_copy_only=False), dtype=object)
|
| 95 |
+
ok = (res == "yes") | (res == "no")
|
| 96 |
+
if not ok.any():
|
| 97 |
+
continue
|
| 98 |
+
X = np.column_stack([t[c].to_numpy(zero_copy_only=False).astype(np.float32) for c in X_COLS])
|
| 99 |
+
X[~np.isfinite(X)] = 0.0
|
| 100 |
+
ts = t["ts"].cast("int64").to_numpy().astype(np.float64) / 1000.0
|
| 101 |
+
close = t["close_ts"].to_numpy(zero_copy_only=False).astype(np.float64); st_ = t["settled_ts"].to_numpy(zero_copy_only=False).astype(np.float64)
|
| 102 |
+
end = t["end_ts"].to_numpy(zero_copy_only=False).astype(np.float64)
|
| 103 |
+
settled = np.where(np.isfinite(st_), st_, end); to_close = close - ts # window on the anchor; a label's timing on the realized end
|
| 104 |
+
y = (res == side).astype(np.float64)
|
| 105 |
+
eff = X[:, KIDX["eff_price"]].astype(np.float64); mid = np.clip(X[:, KIDX["side_mid"]].astype(np.float64), 1.0, 99.0)
|
| 106 |
+
fwd_pess = (100.0 * y - eff) / eff; fwd = (100.0 * y - mid) / mid
|
| 107 |
+
rest = maker_rest(X[:, KIDX["side_bid"]], X[:, KIDX["side_ask"]])
|
| 108 |
+
filled = maker_filled(rest, X[:, KIDX["side_bid"]], t["fill_ask_low"].to_numpy(zero_copy_only=False).astype(np.float64),
|
| 109 |
+
t["fill_sold_low"].to_numpy(zero_copy_only=False).astype(np.float64))
|
| 110 |
+
m_eff = rest + maker_fee_cents(rest, X[:, KIDX["fee_mult"]], X[:, KIDX["maker_fee"]])
|
| 111 |
+
maker = np.where(filled, (100.0 * y - m_eff) / m_eff, np.nan)
|
| 112 |
+
elig = ok & eligible_mask(X, X_COLS, to_close) & np.isfinite(fwd_pess) & (settled > ts)
|
| 113 |
+
if not elig.any():
|
| 114 |
+
continue
|
| 115 |
+
tk = t["ticker"].to_numpy(zero_copy_only=False)[elig]; cats = t["category"].to_numpy(zero_copy_only=False)[elig]
|
| 116 |
+
acc["X"].append(X[elig]); acc["ts"].append(ts[elig]); acc["settled"].append(settled[elig]); acc["y"].append(y[elig])
|
| 117 |
+
acc["fwd"].append(fwd[elig].astype(np.float32)); acc["fwd_pess"].append(fwd_pess[elig].astype(np.float32)); acc["maker"].append(maker[elig].astype(np.float32))
|
| 118 |
+
acc["tick"].append(np.fromiter((tick_id.setdefault(x, len(tick_id)) for x in tk), dtype=np.int64, count=len(tk)))
|
| 119 |
+
acc["side"].append((side[elig] == "yes").astype(np.int8))
|
| 120 |
+
acc["cat"].append(np.fromiter((cat_id.setdefault(x, len(cat_id)) for x in cats), dtype=np.int16, count=len(cats)))
|
| 121 |
+
acc["ord"].append((np.floor(ts[elig] / 86400.0) + 719163).astype(np.int32)) # date.fromordinal(719163) is 1970-01-01
|
| 122 |
+
del X, t
|
| 123 |
+
if not acc["X"]:
|
| 124 |
+
raise RuntimeError("no eligible Kalshi decision rows")
|
| 125 |
+
cat = lambda k: np.concatenate(acc.pop(k))
|
| 126 |
+
X = cat("X"); ts = cat("ts"); settled = cat("settled"); y = cat("y"); fwd = cat("fwd"); fwd_pess = cat("fwd_pess"); maker = cat("maker")
|
| 127 |
+
tick = cat("tick"); side = cat("side"); catc = cat("cat"); ords = cat("ord")
|
| 128 |
+
tick_names = np.array([None] * len(tick_id), dtype=object)
|
| 129 |
+
for k_, v in tick_id.items():
|
| 130 |
+
tick_names[v] = k_
|
| 131 |
+
cat_names = np.array([None] * len(cat_id), dtype=object)
|
| 132 |
+
for k_, v in cat_id.items():
|
| 133 |
+
cat_names[v] = k_
|
| 134 |
+
day_objs = {o: date.fromordinal(int(o)) for o in np.unique(ords)}
|
| 135 |
+
day_arr = np.array([day_objs[o] for o in sorted(day_objs)], dtype=object)[np.searchsorted(np.array(sorted(day_objs)), ords)]
|
| 136 |
+
ds = DecisionSet(X=X, y=(fwd_pess > label_thr).astype(np.int8), fwd=fwd, fwd_pess=fwd_pess, day=day_arr, ts=ts, mint=tick * 2 + side.astype(np.int64),
|
| 137 |
+
cols=list(X_COLS), horizon_s=float(np.median(settled - ts)), fwd_h={"maker": maker})
|
| 138 |
+
ds._day_ord = ords
|
| 139 |
+
ds.train_max_rows = int(config.KALSHI_TRAIN_MAX_ROWS) or None # training rows per fit: a random subset (train/strategies.cap_rows)
|
| 140 |
+
ds.hold_s = (settled - ts).astype(np.float64) # per-row: every position is held to its own settlement
|
| 141 |
+
ds.data_version = f"kalshi-features-{KALSHI_FEATURE_VERSION}" # in the walk-forward cache key: another feature version never reuses a fit
|
| 142 |
+
ds.label_ts = (settled + 60.0).astype(np.float64) # when the outcome is known: training for a period may use only rows known before it
|
| 143 |
+
ds.outcome = y.astype(np.float32)
|
| 144 |
+
ds.side = np.array(["no", "yes"], dtype=object)[side]; ds.ticker = tick_names[tick]; ds.category = cat_names[catc]
|
| 145 |
+
# the stack's bars (train/strategies.Score): a positive mean with a weekly sign test, not the memecoin PF ≥ 1.3 and 65 %
|
| 146 |
+
# winners — a contract bought at 95c that pays 5c when it wins needs a 1.3 % edge for PF 1.3, about all the edge the
|
| 147 |
+
# literature finds left on Kalshi, and its win rate is set by its price (operator decision, 2026-10-09)
|
| 148 |
+
ds.gate = "significance"
|
| 149 |
+
return ds
|