diff --git a/.env.example b/.env.example index 25d53c470a3564d59fcde2312e206a3de808bbba..762a607df075677a810010bed2cc57c497635fd9 100644 --- a/.env.example +++ b/.env.example @@ -2,10 +2,14 @@ # Keep values bare: Docker Compose keeps a trailing "# comment" as part of an empty value, so comments live on their own lines. # --- keys (never logged) --- -# Helius: https://dashboard.helius.dev (RPC for balances and transactions) +# Helius: https://dashboard.helius.dev (RPC for balances and transactions; also the signer's, unless SIGNER_RPC_URL is set) HELIUS_API_KEY= -# Jupiter: https://portal.jup.ag (swaps and token stats) +# Jupiter: https://portal.jup.ag (swaps, through Jupiter's own Metis routes only, and token stats) JUPITER_API_KEY= +# The bot key: `docker compose exec fly fly-trader wallet new` appends BOT_PRIVATE_KEY=... to this file. Do not add an +# empty BOT_PRIVATE_KEY= line yourself: wallet new refuses when the name is already here. +# Optional: a second Solana RPC from another provider for the signer, which simulates every swap before signing it. +# SIGNER_RPC_URL= # --- database (docker-compose.yml sets the container's URL; this one is for running outside Docker) --- DATABASE_URL=postgresql:///fly_trader @@ -15,7 +19,7 @@ DATABASE_URL=postgresql:///fly_trader LIVE_ENABLED=1 # What you fund the wallet with; also the paper warm-up bankroll. CAPITAL_SOL=0.5 -# Hard cap per position. +# The fixed size for a model without a sizing table (the shipped fly has one); must be above 0 to trade live. MAX_POSITION_SOL=0.05 # Per position, of the bankroll minus the gas reserve. MAX_POSITION_FRACTION=0.10 @@ -26,6 +30,13 @@ MIN_POSITION_SOL=0.02 # Never spent (fees and token-account rent). GAS_RESERVE_SOL=0.05 KELLY_FRACTION=0.25 +# The signer's own limits (defaults shown): SOL per buy, SOL of buys per day, swaps per minute (buys and sells), priority +# fee per swap, and the least value a swap may return for what it costs, at the signer's own Jupiter prices. +# SIGNER_MAX_BUY_SOL=1 +# SIGNER_DAILY_BUY_SOL=10 +# SIGNER_SWAPS_PER_MIN=6 +# SIGNER_MAX_PRIORITY_SOL=0.005 +# SIGNER_MIN_VALUE_RATIO=0.6 # --- rails --- # 20 % below the wallet's peak: stop entering ... @@ -41,18 +52,42 @@ HANDOVER_BEAT_SELECTOR=0 # A restart keeps the paper book, so the warm-up count survives it. RESET_ON_START=0 +# --- Robinhood Chain memecoins (Pons launches graduated into Uniswap v4; money in ETH) --- +# Off by default. 1 = index Robinhood Chain (the rh_stream worker) and paper-trade it in ETH beside Solana. Indexing needs +# an Envio HyperSync token (ENVIO_API_TOKEN below; the free tier works, slower) -- the public RPC is far too slow for +# the backfill -- and disk for the history: ~30 GB from 2026-08-04. RH_START_DAY a ~40 days back indexes much less +# (the RH models need about a month of history per wallet); the RH books trade once the stream reaches the present. +RH_ENABLED=0 +# ENVIO_API_TOKEN= +# RH_START_DAY=2026-08-04 +# Real RH transactions stay off: the RH books trade paper until you set 1 and add the key and address below. +RH_LIVE_ENABLED=0 +# 1 = the rehearsal chain (46630); the guard refuses any other chain id than the one selected here. +RH_TESTNET=0 +# RPCs (defaults shown): logs from HyperSync (the official RPC when HyperSync fails), transactions from the official RPC, +# block headers from PublicNode. A QuickNode Robinhood Chain endpoint, if you have one, serves the transactions. +# RH_RPC_URL=https://rpc.mainnet.chain.robinhood.com +# RH_RPC_URL_BLOCKS=https://robinhood-rpc.publicnode.com +# QUICKNODE_RH_API_URL= +# The address RH_BOT_PRIVATE_KEY must derive to (a guard, not a secret). Needed only for live. +RH_BOT_ADDRESS= +# RH_BOT_PRIVATE_KEY (hex) goes in this .env only, on its own line, when you enable live. It is read from here at run +# time and never baked into the image. Do not leave an empty RH_BOT_PRIVATE_KEY= line. + # --- runtime --- SOLANA_CLUSTER=mainnet-beta # auto: an NVIDIA GPU when the container has one (docker-compose.cuda.yml), else the CPU. Or cpu | cuda | cuda:1. DEVICE=auto # Batch sizes are derived from the device's free memory; on a shared GPU cap what the fly may use, in GB. # DEVICE_MEMORY_GB= +# docker-compose.yml sets these to /app/data and /app/logs, your ./data and ./logs DATA_DIR=data LOG_DIR=logs # --- staying current --- -# Every UPDATE_INTERVAL_S the container asks Hugging Face whether HF_REPO has a new release; if so it downloads the new -# code and model, restarts itself (about a minute; open positions are untouched) and keeps the previous release for a +# Every UPDATE_INTERVAL_S the container asks Hugging Face whether HF_REPO has a new release. It installs one only if it is +# signed by a key in release/allowed_signers and newer than the installed one, and every file matches the signed +# manifest; then it restarts itself (about a minute; open positions are untouched) and keeps the previous release for a # rollback if the new one fails to start. AUTO_UPDATE=0 pins this install to what it was built from. AUTO_UPDATE=1 HF_REPO=bryceweiner/fly-trader diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..1aa6a2b8ff5470563d741ca2a906744c629f3bc8 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +models/selectors/selector_20261003T201413Z.skops filter=lfs diff=lfs merge=lfs -text diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000000000000000000000000000000000000..db4a575593626b1209d1454e6d912cd73225980c --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,131 @@ +# Changelog + +Releases are identified by date. The Docker distribution (branch `distribution`, Hugging Face `bryceweiner/fly-trader`) +ships signed releases built from this development line. + +## 2026-10-11: Robinhood Chain, and a fly that starts as its selector + +Not yet published to Hugging Face. Ships fly #157 (network) with selector #154, both trained on Solana and Robinhood +Chain rows together. + +### Upgrading + +- **Docker installs from before October 2026 must rebuild once:** `git pull && docker compose up -d --build`. The old + image cannot verify signed releases, so it stops updating after this release. Data, the fly's learned state and + open positions are kept. +- **Robinhood Chain is off by default** in the distribution. To turn it on, set `RH_ENABLED=1` and `ENVIO_API_TOKEN` (an + Envio HyperSync token; the free tier works) in `.env`. + - Indexing the full history from 2026-08-04 takes about 30 GB. + - Setting `RH_START_DAY` about 40 days back is enough to trade. +- **New settings:** + - `FLY_SCORE` (default `teacher`), `FLY_TEACHER_GATE` (default on), `FLY_LEARNED_VETO` (default 0.02) and `FLY_SIZING` + (default `kelly`). + - Robinhood Chain: `ENVIO_API_TOKEN`, `RH_*`, and optionally `QUICKNODE_RH_API_URL` for transactions. + +### The fly + +- **It scores as its selector plus what it has learned** (`FLY_SCORE=teacher`). + - On the first day its score equals the selector's exactly, verified live to 3e-8 on every row. + - Its learnable mushroom-body pathway (the plastic KC→MBON synapses) adds a correction, learned from realized returns, + that starts at zero. + - It therefore learns only where the selector is wrong. + - The earlier fly was a distilled copy of the selector, accurate overall but not at the buy line: it ranked the + selector's top 1 % of candidates only about 0.6-correlated. +- **It buys only what the selector buys** (`FLY_TEACHER_GATE`). + - The selector's trigger, buy line, win filter, dump veto and fail-closed rules decide every row. + - The fly's learning can drop a selector trade (a learned veto: its score more than 0.02 below its starting score) but + can never add one. + - Ungated, the fly had bought rows the selector rejected; those rows averaged −25.9 % per trade. +- **It sizes each buy exactly as the selector sizes it** (`FLY_SIZING=kelly`): the selector's score, buy line and Kelly + table for that row. +- **It learns from the selector's trades,** live and in the replay, after a restart too. It no longer learns from rows + above its own buy line. +- **It never trades without its teacher.** A fly whose selector cannot be loaded enters nothing. A release install now + points the fly at the shipped selector's new database id. Before that fix, the Docker boot test found a fresh install + running the fly ungated. +- **Replay verdict** (2026-09-24 to 10-09, the fly against the selector itself): + - Solana +3.78 % per trade over 308 trades. + - Robinhood Chain +2.66 % over 1,477 trades (selector +2.63 %). + - Random picks lost 4.4–4.6 %. +- **Smaller changes:** + - The fly calibrates on days its teacher never saw. + - A NaN mean can no longer pass the deploy gate. + - A second chain's teacher gets 28 days of its own data before the replay starts. + - `fly-replay --chains` re-judges one chain and keeps the others' verdicts. + +### Robinhood Chain (new) + +- **Pons memecoins traded beside pump.fun.** It trades tokens launched on Pons V2 after they graduate into Uniswap v4 pools + with the Pons hook. + - Each chain has its own books in its own unit (Robinhood Chain in ETH), with its own bankroll, kill switch, handover + and verdict. + - One selector and one fly cover both chains. A chain trades only if its own replay verdict passed. +- **Its own indexer.** It reads launches, graduations, swaps, the base assets' prices and 1-minute rows. + - Logs come from Envio HyperSync, in one query per pass. + - Minutes land about 15–25 s after they close (they took 35–95 s at first). + - If HyperSync fails, the official RPC carries the feed until it recovers. + - RH minutes up to 240 s old are still traded (`RH_TRADE_LAG_S`). +- **Live trading** (`RH_LIVE_ENABLED=0` by default): + - Swaps route through KyberSwap, with a Uniswap v4 fallback. + - One transaction is in flight at a time, booked from measured balances. + - It recovers after a crash and is reconciled against the chain. + - It has its own circuit breaker (circuit 3). + - `fly-trader rh-smoke` runs a tiny mainnet round trip. + - A QuickNode endpoint (`QUICKNODE_RH_API_URL`), if set, carries transactions. Its free plan caps log queries at 5 + blocks, so logs stay on HyperSync. +- **Costs:** the Pons hook fee and creator tax (0.99 % per side at the median), price impact and gas. +- **Console:** + - A chain picker switches the Overview and Trades pages. + - A Robinhood Chain page shows the indexer, quote-asset prices and the ETH wallet. + - Safety & wallet shows both chains' bot wallets in USD and withdraws only to your own wallets. + +### $FLY vault + +- Robinhood Chain profit is paid to lockers in ETH, beside the SOL pot. +- **Custody:** a Squads treasury with spending limits and a key-isolated signer container. Claims are paid from a + separate payout wallet, and code releases are delayed. +- **Nothing published helps trade ahead of the fly.** The trading wallet's address, open positions and settlement + timing are not shown. +- **Audits:** red-team tests, mutation testing, Echidna and Halmos proofs. + - Fixed: a claim double-pay on resume. + - Added: Kyber calldata and fee checks, and relay griefing limits. + +### Docker distribution + +- Runs the development line's code again, merged instead of hand-ported (it had fallen about 35 commits behind). +- **Signed releases only.** + - The container installs a release only if its manifest is signed by a key in `release/allowed_signers`, its number + is higher than the installed one, and every file matches its hash. + - Releases carry no `.git*` file and no model they do not name. + - Startup removes model files an earlier release left behind. +- **Models are accepted whether or not Robinhood Chain is on.** Before release, an install with RH off would have + rejected the combined models and crash-looped at boot. +- The Robinhood Chain reference table ships in the seed. +- Kalshi ships dormant: no worker starts, and its pages appear only once `KALSHI_API_KEY_ID` is set. + +### Kalshi prediction markets (development line) + +- Under development, not traded in the distribution. Covers: + - an outcome-blind event corpus + - a selector judged by a weekly sign test + - maker and taker arms + - a visual fly + +### Fixes + +- **Schema changes no longer deadlock live workers.** An unchanged schema is skipped, and a changed one waits at most + 5 s for a lock, then retries. +- **The RH paper books had made no trades.** A day's wallet-skill table did not exist yet when its minutes were scored. + Tables are now written ahead of their day, and the stream falls back to the newest one. +- **HyperSync could stall for 76 s.** A dead server address hung new connections; connections now time out in 5 s and + are kept alive. +- **The RH price scan ran about 270 queries per pass.** It now reads every price cursor at once. + +## 2026-09-24: Fly #107 taught by selector #100 + +- The selector was retrained on real fees, and the fly was bootstrapped from it. +- The container keeps itself current from Hugging Face. + +## 2026-09-22: First distribution + +- Fire-and-forget trading in Docker, with the fly, its teacher selector and their connectome. diff --git a/Dockerfile b/Dockerfile index 5157cd810a733df3bcf90815df117b7d6d28fc48..922055936e389d250caaeb4914acccd22860ab39 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,28 +1,56 @@ # fly-trader: the trading engine and its console, on the CPU (a Linux container has no Metal GPU; the fly infers fine on CPU). -FROM python:3.13-slim +# Two images from one file: +# default (last stage, "brain"): the engine, console and vault worker. It holds NO wallet key. +# --target signer: the only holder of the hot keys (fly_trader/signer): standard library + solders + httpx, nothing +# else of fly_trader, so none of the brain's dependencies ever run next to a key. +# Base images are pinned by digest and every Python dependency by hash (uv.lock, docker/signer-requirements.txt): a +# release builds exactly what was reviewed, whatever the registries serve later. +ARG PYTHON_IMAGE=python:3.13-slim@sha256:7c61056e61ac89e852de05f3dc6fa51a6dd2181797bceed46aa725dd7cb2cd3b +# ---------------------------------------------------------------- the signer +FROM ${PYTHON_IMAGE} AS signer +ENV PYTHONUNBUFFERED=1 PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=/app +RUN apt-get update && apt-get install -y --no-install-recommends age && rm -rf /var/lib/apt/lists/* \ + && groupadd -g 10000 fly && useradd -u 10002 -g 10000 -M -s /usr/sbin/nologin signer +COPY docker/signer-requirements.txt /tmp/req.txt +RUN pip install --no-cache-dir --require-hashes --only-binary=:all: -r /tmp/req.txt && rm /tmp/req.txt +WORKDIR /app +COPY fly_trader/__init__.py fly_trader/__init__.py +COPY fly_trader/signer fly_trader/signer +USER 10002:10000 +HEALTHCHECK --interval=30s --timeout=10s --start-period=20s --retries=3 CMD python -m fly_trader.signer ping || exit 1 +CMD ["python", "-m", "fly_trader.signer", "serve"] + +# ---------------------------------------------------------------- the brain (default) +FROM ${PYTHON_IMAGE} AS brain ENV PYTHONUNBUFFERED=1 \ UV_PROJECT_ENVIRONMENT=/app/.venv \ + UV_COMPILE_BYTECODE=1 \ + HOME=/tmp \ PATH="/app/.venv/bin:${PATH}" -RUN apt-get update && apt-get install -y --no-install-recommends postgresql-client ca-certificates curl age \ - && rm -rf /var/lib/apt/lists/* -COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv +# openssh-client: ssh-keygen verifies signed releases (the self-updater, fly_trader/ops/autoupdate.py) +RUN apt-get update && apt-get install -y --no-install-recommends postgresql-client ca-certificates curl age openssh-client \ + && rm -rf /var/lib/apt/lists/* \ + && groupadd -g 10000 fly && useradd -u 10001 -g 10000 -M -s /usr/sbin/nologin fly +COPY --from=ghcr.io/astral-sh/uv:0.12.19@sha256:04d046b13e60d6bcec73cbc5e1cad25d680dea90c8573340950a0ac2d1aef424 /uv /usr/local/bin/uv WORKDIR /app COPY . . # Which torch: TORCH_INDEX=cpu (default: any machine, ~200 MB) or cu126 / cu130 for an NVIDIA GPU (the CUDA runtime is # inside the wheel; the host needs only the driver and the NVIDIA Container Toolkit -- see docker-compose.cuda.yml). -# Installed first so the project's own pin is already satisfied and PyPI's 3 GB CUDA default is never pulled by accident. -# Editable install: the package IS /app/fly_trader, so config.REPO_ROOT is /app -- the mounted .env the wallet writes -# to, the console's static directory, config/. A plain install ran a copy under site-packages and looked there instead. +# It names the project extra that pins torch to that index. --frozen: exactly uv.lock with its hashes, never a fresh +# resolution. The project installs editable: the package IS /app/fly_trader, so config.REPO_ROOT is /app. ARG TORCH_INDEX=cpu -RUN uv venv /app/.venv \ - && uv pip install --index-url "https://download.pytorch.org/whl/${TORCH_INDEX}" "torch>=2.12" \ - && uv pip install -e . \ - && chmod +x /app/docker/entrypoint.sh +ENV TORCH_INDEX=${TORCH_INDEX} +RUN uv sync --frozen --no-dev --extra "${TORCH_INDEX}" \ + && chmod +x /app/docker/entrypoint.sh \ + && mkdir -p /app/fly_trader/ui/static/brain && chown -R 10001:10000 /app/fly_trader/ui/static/brain +# The vault server runs this unprivileged user and never updates itself (AUTO_UPDATE=0). The self-hosted compose file +# runs the container as root instead: its self-updater replaces /app, and it writes the bind-mounted data/, logs/, .env. +USER 10001:10000 EXPOSE 8501 -# healthy = the database answers and the console's worker threads run (the vault server's updater waits on this) +# healthy = the database answers, the console's worker threads run and (on the vault server) the signer answers HEALTHCHECK --interval=30s --timeout=20s --start-period=300s --retries=5 CMD fly-trader health || exit 1 ENTRYPOINT ["/app/docker/entrypoint.sh"] diff --git a/README.md b/README.md index 653e2f53a95705ff4fa412c17c8d0e0acb169356..82626163e7efb96df04984a8facb7a5564c83db1 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ tags: - solana - memecoin - pump.fun + - robinhood-chain - trading - connectome - drosophila @@ -14,12 +15,12 @@ tags: # fly-trader — distribution -**A fruit-fly connectome that trades Solana memecoins with your money.** +**A fruit-fly connectome that trades memecoins with your money: on Solana, and on Robinhood Chain if you turn it on.** > ## ⚠️ Read this first > -> This software signs real transactions from a wallet you fund. It buys tokens that graduated from pump.fun, holds them -> for about two hours, and sells. **It can lose everything in that wallet, quickly.** The model was trained on a few +> This software signs real transactions from a wallet you fund. It buys tokens that graduated from pump.fun (and, if +> you turn Robinhood Chain on, from Pons), holds them for two to four hours, and sells. **It can lose everything in that wallet, quickly.** The model was trained on a few > months of data; the market it trades is adversarial and changes. Nothing here is a claim of profit, and nothing here is > financial advice. Run it only with money you can lose entirely, on a wallet used for nothing else. MIT licence — no > warranty of any kind. See [LICENSE](LICENSE). @@ -32,10 +33,16 @@ connectome they run on. The training pipeline lives on [`master`](https://github - Every minute it scores every PumpSwap token that graduated from pump.fun at least six hours ago, from live 1-minute candles (a token needs that much history before its features mean anything). - The fly — the FlyWire *Drosophila* central brain (41,756 neurons) as a rate network on its real signed synapses — - predicts each token's net return over the next 120 minutes. It was taught by a gradient-boosted selector; it then keeps - learning from what its own trades earn, through plasticity at its mushroom body's KC→MBON synapses. -- It buys when the predicted return clears its own buy line, sizes the position from measured certainty (a quarter - of the growth-optimal bet, capped), holds 120 minutes, sells. + starts as the gradient-boosted selector that taught it: it scores every token with the selector's own prediction of + its net return. Its mushroom body's KC→MBON synapses then learn, from what the trades earn, a correction to that + prediction, and can veto a selector buy they have learned to distrust. +- It buys only what the selector buys (the selector's strategy, buy line and filters), sizes the position exactly as the + selector does (a quarter of the growth-optimal bet for its score band, capped), holds two to four hours (the + strategy's hold), sells. +- **Every transaction is checked before it is signed** (`fly_trader/signer`): swaps go only through Jupiter's own + routes, each one is simulated first and refused if it would spend more than ordered, deliver the token anywhere but + your wallet, touch another holding, or return far less value than it costs; buys are capped per trade, per day and per + minute. - **It starts on paper.** After 20 closed paper trades it switches itself to the wallet and stays there. Before that, and whenever the wallet is unfunded or a key is missing, it trades paper and says so on the console. - **Rails:** 20 % below the wallet's peak it stops entering and sells every open position (the kill switch); three @@ -46,13 +53,23 @@ connectome they run on. The training pipeline lives on [`master`](https://github ## Requirements -- Docker (Desktop on macOS/Windows, or Engine + Compose v2 on Linux). ~2 GB of disk for the image and database. +- Docker (Desktop on macOS/Windows, or Engine + Compose v2 on Linux). ~2 GB of disk for the image and database + (Robinhood Chain, if you turn it on, adds ~30 GB for its full history; see below). - A [Helius](https://dashboard.helius.dev) API key (free tier is enough) and a [Jupiter](https://portal.jup.ag) API key. + Optional: a second Solana RPC from another provider as `SIGNER_RPC_URL`, so the checks above do not rely on Helius + alone. - 0.5 SOL you are prepared to lose. Not more: the caps below are sized for 0.5. - Any 64-bit CPU: the fly scores about as fast on 16 CPU threads as on a GPU (its forward pass is a memory-bound scatter over a million synapses). An NVIDIA GPU is used when you start it with the CUDA override (see below). Apple GPUs are not reachable from inside Docker. +> ### ⚠️ Robinhood Chain needs an Envio HyperSync API token +> +> Robinhood Chain is off by default. To trade it, create a token in the [Envio dashboard](https://envio.dev/app/api-tokens) +> (personal accounts get one free), put it in `.env` as `ENVIO_API_TOKEN`, and set `RH_ENABLED=1`. Its feed reads the +> chain's logs from [Envio HyperSync](https://envio.dev/pricing/hypersync); the public Robinhood Chain RPC is far too slow +> to index its history. Without a token, leave Robinhood Chain off: Solana trades either way. + ## Start trading 1. Download this branch and open a terminal in it: @@ -67,13 +84,15 @@ connectome they run on. The training pipeline lives on [`master`](https://github ```bash docker compose up -d --build ``` - The first start builds the image (several minutes), creates the database, installs the models and starts the market - feed and the trading engine. + The first start builds the image (several minutes), creates the database, downloads the wallet-skill table (~300 MB, + checked against its pinned sha256), installs the models and starts the market feed and the trading engine. The + Processes page also lists a `vault` worker that stops at once: it runs only on the hosted $FLY vault server. 4. Create the bot wallet — it prints the public key and writes the secret into your `.env`: ```bash docker compose exec fly fly-trader wallet new ``` - **Back up `.env` now.** It is the only copy of that key. + **Back up `.env` now.** It is the only copy of that key. (`wallet new` refuses if `.env` already has a + `BOT_PRIVATE_KEY=` line, even an empty one.) 5. Send **0.5 SOL** to the printed address, then restart the engine so it picks up the key: ```bash docker compose restart fly @@ -92,7 +111,7 @@ scores anything); then it trades paper until 20 trades have closed; then it trad | Clear a tripped kill switch | Safety & wallet → **Clear kill switch**, or `docker compose exec fly fly-trader reset-circuit --kill` (re-bases the peak) | | Stop everything | `docker compose down` — **open positions stay open in the wallet**; start again and it sells them on schedule, or sell them yourself | | See the logs | `docker compose logs -f fly` | -| Start again from nothing | `docker compose down -v` (deletes the database) and delete `data/` | +| Start again from nothing | `docker compose down -v` (deletes the database) and `sudo rm -rf data/` (the container writes it as root; it also holds the signer's ledger) | The console listens on `127.0.0.1` only. Do not expose it: it can start and stop the engine and clear the rails. @@ -100,15 +119,17 @@ The console listens on `127.0.0.1` only. Do not expose it: it can start and stop | Path | What | |---|---| -| `models/policies/fly_*.pt` | the fly (`brain_snapshots` #62): taught by selector #59, two strategies (`ev`, `capitulation`), 120-minute hold, its own buy lines | -| `models/selectors/selector_*.joblib` | the selector that taught it (#59), kept so the console can show the paper selector alongside | +| `models/policies/fly_*.pt` | the fly (#107 on the training machine; a fresh install gives it its own id): taught by selector #100, one strategy (`capitulation`), 240-minute hold, its own buy line. Loaded weights-only: the file cannot run code | +| `models/selectors/selector_*.skops` | the selector that taught it (#100), kept so the console can show the paper selector alongside. Saved with skops, not pickle | | `models/connectome/` | the FlyWire FAFB v783 central brain as the fly runs on it (14 MB), with its calibrated scale | -| `models/wallet_skill/` | the fitted wallet-skill inputs: `config.json` (the horizon and lookback the model was trained with) and the name of the newest per-day table. The table itself (~300 MB) is on Hugging Face; the container fetches it once at first start | -| `seed/seed.sql` | the database rows that admit the fly to trade: the model records, the passed replay verdict, autostart | +| `models/wallet_skill/` | the fitted wallet-skill inputs: `config.json` (the horizon and lookback the model was trained with), the name of the newest per-day table (`TABLE`) and its sha256 (`TABLE.sha256`). The table itself (~300 MB) is on Hugging Face; the container downloads it once and uses it only if the hash matches | +| `seed/release_seed.json` (`seed/seed.sql` is the fallback) | the database rows that admit the fly to trade: the model records (by file hash), the passed replay verdict, autostart | +| `release/allowed_signers` | the release keys this install trusts for automatic updates (see Updates) | | `.env.example` | every knob, set for 0.5 SOL | -| `docker-compose.yml`, `Dockerfile`, `docker/entrypoint.sh` | one Postgres, one engine + console | +| `docker-compose.yml`, `Dockerfile`, `docker/` | one Postgres, one engine + console (the Dockerfile's default stage; its `signer` stage is for the vault server) | +| `deploy/` | the hosted $FLY vault server: a separate signer container, host hardening, the signed-release updater ([docs/vault/RUNBOOK.md](docs/vault/RUNBOOK.md)) | -The fly's replay verdict shipped here: over the 28 evaluation days it made +1.52 % per trade after real costs (profit +The replay verdict shipped here: over the 29 evaluation days it made +1.52 % per trade after real costs (profit factor 1.13) against −5.27 % for random picks of the same tokens. That is a backtest on one corpus, not a forecast. ## Tuning @@ -118,14 +139,40 @@ Edit `.env`, then `docker compose restart fly`. The ones that matter: | Variable | Shipped | Meaning | |---|---|---| | `CAPITAL_SOL` | 0.5 | what you funded; also the paper warm-up bankroll | -| `MAX_POSITION_SOL` / `MAX_POSITION_FRACTION` | 0.05 / 0.10 | per-position caps (the smaller wins) | +| `MAX_POSITION_FRACTION` | 0.10 | the most of the bankroll one position may take | +| `MAX_POSITION_SOL` | 0.05 | the fixed size for a model without a sizing table (the shipped fly has one); must be above 0 to trade live | | `GAS_RESERVE_SOL` | 0.05 | never spent | | `KILL_SWITCH_DRAWDOWN` / `KILL_SWITCH_LIQUIDATE` | 0.20 / 1 | drawdown from peak that halts and sells everything | | `HANDOVER_MIN_TRADES` | 20 | closed paper trades before it goes live | | `LIVE_ENABLED` | 1 | set 0 to run paper only | -Scaling the wallet up scales the position caps with it (`MAX_POSITION_FRACTION`), but `MAX_POSITION_SOL` is a hard -ceiling: raise it deliberately or not at all. +Scaling the wallet up scales the positions with it (`MAX_POSITION_FRACTION`). The signer has its own limits that do +not scale: at most `SIGNER_MAX_BUY_SOL` (1) per buy, `SIGNER_DAILY_BUY_SOL` (10) of buys a day and +`SIGNER_SWAPS_PER_MIN` (6) swaps a minute; raise them in `.env` deliberately. + +## Robinhood Chain (Pons) memecoins + +The same engine also trades Robinhood Chain: tokens launched on Pons that graduated into Uniswap v4 pools. + +- **Off by default; turn it on with `RH_ENABLED=1` and `ENVIO_API_TOKEN` in `.env`** (an Envio HyperSync token; the + free tier works). The `rh_stream` worker then starts with the others and indexes Pons launches, graduations and swaps + into 1-minute rows: a backfill from `RH_START_DAY` (default 2026-08-04, ~30 GB; ~40 days back is enough to trade), + then live, with RH minutes landing ~15-25 s after they close. If HyperSync fails, the official RPC carries the feed + until it is back. +- **Paper, in ETH.** The Robinhood Chain books + (`paper_rh_selector`, `paper_rh_fly`) keep their money in native ETH (`RH_CAPITAL_ETH`, 5 SOL's worth at build time) + and pay Pons' hook fee, price impact and gas. They have their own kill switch and circuit (Safety & wallet → + Robinhood Chain trading rails) and their own handover. The console's chain picker switches Overview and Trades + between the two chains, and the **Robinhood Chain** page shows the indexer, the quote-asset prices and the ETH wallet. +- **One model for both chains.** The shipped selector and fly were trained on both chains' rows together, and the + replay judged each chain on its own trades; a chain's books trade only where its verdict passed. With RH off the same + models trade Solana alone. +- **Live** needs `RH_LIVE_ENABLED=1`, `RH_BOT_PRIVATE_KEY` (in `.env` only, never in the image) and the address it + derives to as `RH_BOT_ADDRESS`, plus a funded wallet (ETH on Robinhood Chain, chain 4663). Then + `docker compose restart fly`. As on Solana, the fly trades the wallet only after it has won the RH paper race. +- `RH_ENABLED=0` turns all of it off. + +RH training data does not ship: an install that turns RH on builds its own from the stream, as Solana does. ## NVIDIA GPU @@ -139,31 +186,47 @@ docker compose -f docker-compose.yml -f docker-compose.cuda.yml up -d --build The image is built with the CUDA 12.6 torch wheel (`TORCH_INDEX: cu126` in `docker-compose.cuda.yml`; `cu130` for the newest drivers) and the container is given every GPU. `DEVICE=auto` in `.env` then resolves to the GPU; the console's -Model page shows which device the fly is on. Batch sizes are derived from the card's free memory, so an 8 GB card works; +Model & training page shows which device the fly is on. Batch sizes are derived from the card's free memory, so an 8 GB card works; on a shared card set `DEVICE_MEMORY_GB` in `.env` to what the fly may use. ## Updates -The container keeps itself current. Every hour it asks Hugging Face whether -[`bryceweiner/fly-trader`](https://huggingface.co/bryceweiner/fly-trader) has a new release; when it has, it downloads -the new code and model, restarts (about a minute; the paper books, the fly's learned state and any open positions are -untouched -- open positions are sold on schedule by the restarted engine) and keeps the previous release beside the new -one. If the new release does not come up within five minutes it goes back to the previous one and never tries that -release again. `docker compose logs -f fly` shows what it did. `AUTO_UPDATE=0` in `.env` turns this off; then the +The container keeps itself current, with **signed releases only**. Every hour it asks Hugging Face whether +[`bryceweiner/fly-trader`](https://huggingface.co/bryceweiner/fly-trader) has a new release. It installs one only if: +- its `release.json` is signed by a key in this install's `release/allowed_signers` +- its release number is higher than the installed one +- every file matches the size and sha256 the signed manifest lists + +Then it restarts (about a minute; the paper books, the fly's learned state and any open positions are untouched -- +open positions are sold on schedule by the restarted engine) and keeps the previous release beside the new one. If the +new release does not come up within five minutes, it goes back to the previous one and never tries that release again. +An unsigned or wrongly signed commit is refused and logged. While `release/allowed_signers` lists no key, nothing is +installed. `docker compose logs -f fly` shows what it did. `AUTO_UPDATE=0` in `.env` turns this off; then the "Refreshing the model" steps below are the manual way. +**Installed before October 2026? Rebuild once.** The updater in those images installed whatever Hugging Face held, +unverified; it may install this release by itself, but only a rebuilt image can check signatures (it needs `ssh-keygen`), +so it stops at this release. From your install directory: + +```bash +git pull && docker compose up -d --build +``` + +Your data, the fly's learned state and open positions are kept. Robinhood Chain stays off until you turn it on. + ## Refreshing the model The model is a snapshot of one training run. To ship a newer one, on a `master` checkout with its database, after a passed replay: ```bash -.venv/bin/python tools/make_seed.py --out dist -cp -R dist/models/* models/ && cp -R dist/seed/* seed/ +.venv/bin/python tools/make_seed.py --out ../fly-trader-dist ``` -Commit `models/` and `seed/` on this branch. The seed is idempotent: an install that already has these rows is not -changed by it, so pull the branch, `docker compose up -d --build`, and the new model is in place. +Then publish it as a signed release with `tools/publish_release.py` (docs/vault/RUNBOOK.md, section 4.1): installs +with automatic updates pick it up within the hour. For a manual install, commit `models/` and `seed/` on this branch; +every start runs `fly-trader apply-release`, which reuses rows it already has (same file hash) and registers new +models, so pull the branch, `docker compose up -d --build`, and the new model is in place. ## Also on Hugging Face @@ -174,6 +237,6 @@ changed by it, so pull the branch, `docker compose up -d --build`, and the new m FlyWire consortium, Dorkenwald et al., *Neuronal wiring diagram of an adult brain*, Nature 2024; annotations Schlegel et al. 2024; synapse signs Shiu et al. 2024. Connectome export: [snedea/flybrain](https://github.com/snedea/flybrain) -(MIT). Market data: [PumpAPI](https://pumpapi.io); routing: [Jupiter](https://jup.ag). +(MIT). Market data: [PumpAPI](https://pumpapi.io); routing: [Jupiter](https://jup.ag) (Metis routes only). [MIT](LICENSE). diff --git a/deploy/bootstrap-host.sh b/deploy/bootstrap-host.sh index 7260f1a6af62715a8c8debcde316a4eba83bf52e..c9d16bc5aac9fdaad7862f0bfc94dcad2b27f728 100644 --- a/deploy/bootstrap-host.sh +++ b/deploy/bootstrap-host.sh @@ -1,64 +1,459 @@ #!/usr/bin/env bash -# One-time setup of the vault fly's server (Ubuntu 24.04, e.g. Netcup RS 2000 G12). Run as root: -# bash bootstrap-host.sh "" -# It installs Docker, locks the box down (SSH keys only, firewall: SSH in, nothing else), installs the updater with -# YOUR release key as the only trusted signer, and lays out /srv/fly. It does not create the wallet or start the fly. +# One-time setup of the vault fly's server: a dedicated machine (e.g. a mini PC at home) on a fresh Ubuntu Server 24.04 +# with full-disk encryption, hardened along the lines of the CIS Ubuntu 24.04 benchmark for a single-purpose host that +# holds a hot wallet. Run with sudo, in two passes: +# +# 1. sudo TS_AUTHKEY=tskey-auth-... RECOVERY_PUBKEY="ssh-ed25519 AAAA... recovery" EXPECT_MANIFEST= \ +# TPM_UNLOCK=1 ADMIN_USER=fly-admin bash bootstrap-host.sh "" +# then, from your Mac, log in over Tailscale: ssh fly-admin@fly-vault (MagicDNS name or the 100.x address) +# 2. sudo CLOSE_PUBLIC_SSH=1 bash bootstrap-host.sh "" +# removes the SSH rule for the local network / internet: after it the machine accepts SSH only over Tailscale and +# has no other open port. +# +# TPM_UNLOCK=1 binds the root disk's LUKS encryption to the TPM (clevis, PCR 7 = the Secure Boot state), so an +# unattended security reboot comes back up without a passphrase; it asks once for the disk passphrase. Without it the +# machine waits at every boot for the passphrase on its console, and automatic reboots are turned off. +# +# ADMIN_USER (default fly-admin) becomes the only account that can log in over SSH (key only, sudo without a +# password; root login is switched off). If it has no ~/.ssh/authorized_keys yet, root's are copied to it first, and +# the script stops before touching SSH if neither exists. Keep this root session open until you have logged in as +# ADMIN_USER from a second terminal. deploy/check-host.sh verifies every setting afterwards. +# +# RELEASE_PUBKEY / RECOVERY_PUBKEY: the two keys whose signatures the updater accepts (principals bryce and +# bryce-recovery), valid for the release namespace only. EXPECT_MANIFEST pins the first release the updater may +# install (tools/publish_release.py prints it as the manifest sha256); until it is set the updater installs nothing. +# TS_AUTHKEY: a one-off, pre-authorised key tagged tag:fly from the Tailscale admin console; used once, not stored. +# +# Idempotent: safe to run again (keys and edited config files are never overwritten). set -euo pipefail -RELEASE_PUBKEY="${1:?usage: bootstrap-host.sh \"ssh-ed25519 AAAA... release-key\"}" +RELEASE_PUBKEY="${1:?usage: ADMIN_USER=fly-admin bootstrap-host.sh \"ecdsa-sha2-nistp256 AAAA... release-key\"}" +RECOVERY_PUBKEY="${RECOVERY_PUBKEY:-}" +EXPECT_MANIFEST="${EXPECT_MANIFEST:-}" +TS_AUTHKEY="${TS_AUTHKEY:-}" +CLOSE_PUBLIC_SSH="${CLOSE_PUBLIC_SSH:-0}" +TPM_UNLOCK="${TPM_UNLOCK:-0}" +ADMIN_USER="${ADMIN_USER:-fly-admin}" +# container users (the images' USER lines); with userns-remap each is REMAP_UID + id on the host +BRAIN_UID=10001; SIGNER_UID=10002; FLY_GID=10000 +HERE="$(cd "$(dirname "$0")" && pwd)" +export DEBIAN_FRONTEND=noninteractive +step() { printf '\n== %s\n' "$*"; } -apt-get update -apt-get install -y ca-certificates curl ufw unattended-upgrades python3 openssh-client +# ---------------------------------------------------------------- 1. packages +step "packages" +apt-get update -q +apt-get install -y -q --no-install-recommends ca-certificates curl ufw unattended-upgrades python3 openssh-client openssl \ + fail2ban python3-systemd auditd audispd-plugins aide aide-common lynis apparmor-utils systemd-timesyncd needrestart \ + libpam-tmpdir debsums apt-show-versions apt-listchanges acct sysstat mokutil install -m 0755 -d /etc/apt/keyrings curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu $(. /etc/os-release && echo "$VERSION_CODENAME") stable" > /etc/apt/sources.list.d/docker.list -apt-get update -apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin -dpkg-reconfigure -f noninteractive unattended-upgrades +apt-get update -q +apt-get install -y -q docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin +# Tailscale: SSH reaches this box only over your tailnet (WireGuard); its repo key is pinned like Docker's +curl -fsSL "https://pkgs.tailscale.com/stable/ubuntu/$(. /etc/os-release && echo "$VERSION_CODENAME").noarmor.gpg" -o /etc/apt/keyrings/tailscale.gpg +echo "deb [signed-by=/etc/apt/keyrings/tailscale.gpg] https://pkgs.tailscale.com/stable/ubuntu $(. /etc/os-release && echo "$VERSION_CODENAME") main" > /etc/apt/sources.list.d/tailscale.list +apt-get update -q +apt-get install -y -q tailscale +# nothing on this box needs snaps; fewer daemons, fewer holes +if dpkg -l snapd >/dev/null 2>&1; then apt-get purge -y -q snapd; fi +# no mail server: nothing here sends mail (alerts go to Telegram), and an MTA is one more listening daemon +if dpkg -l postfix 2>/dev/null | grep -q '^ii'; then apt-get purge -y -q postfix; fi + +# ---------------------------------------------------------------- 1b. the machine itself +step "machine: wireless off, DMA protection, disk encryption" +# wired Ethernet only: Wi-Fi and Bluetooth drivers can never load (a radio is a way in that bypasses the router) +printf 'install cfg80211 /bin/false\ninstall mac80211 /bin/false\ninstall bluetooth /bin/false\ninstall btusb /bin/false\n' \ + > /etc/modprobe.d/61-fly-no-wireless.conf +command -v rfkill >/dev/null && rfkill block all 2>/dev/null || true +systemctl disable --now bluetooth.service wpa_supplicant.service 2>/dev/null || true +# USB4 / Thunderbolt / OCuLink devices can read memory directly; the IOMMU confines them +if [ -z "$(ls -A /sys/kernel/iommu_groups 2>/dev/null)" ]; then + echo "WARNING: the IOMMU is off: enable it (AMD-Vi / IOMMU) in the firmware settings" +fi +# the root filesystem must sit on LUKS: the wallet keys are files on this disk +ROOT_LUKS="$(lsblk -rno NAME,TYPE -s "$(findmnt -no SOURCE /)" 2>/dev/null | awk '$2=="crypt"{found=1} found && $2=="part"{print "/dev/"$1; exit}')" +if [ -z "$ROOT_LUKS" ]; then + echo "WARNING: the root filesystem is not on an encrypted (LUKS) disk: reinstall with 'Encrypt the LVM group with LUKS'" +elif [ "$TPM_UNLOCK" = 1 ]; then + apt-get install -y -q --no-install-recommends clevis clevis-luks clevis-tpm2 clevis-initramfs + if ! clevis luks list -d "$ROOT_LUKS" 2>/dev/null | grep -q tpm2; then + echo "binding $ROOT_LUKS to the TPM (PCR 7); enter the disk passphrase when asked" + clevis luks bind -d "$ROOT_LUKS" tpm2 '{"pcr_ids":"7"}' + update-initramfs -u -k all + fi +fi + +# ---------------------------------------------------------------- 2. patching +step "automatic security updates (reboot 04:00 UTC when a kernel needs it)" +cat > /etc/apt/apt.conf.d/20auto-upgrades <<'EOF' +APT::Periodic::Update-Package-Lists "1"; +APT::Periodic::Unattended-Upgrade "1"; +APT::Periodic::AutocleanInterval "7"; +EOF +REBOOT=true # a disk that needs a typed passphrase at boot: no unattended reboots +if [ -n "${ROOT_LUKS:-}" ] && ! { command -v clevis >/dev/null && clevis luks list -d "$ROOT_LUKS" 2>/dev/null | grep -q tpm2; }; then + REBOOT=false +fi +cat > /etc/apt/apt.conf.d/52fly-unattended < 5 min off + +# ---------------------------------------------------------------- 3. admin account +step "admin account: $ADMIN_USER" +id "$ADMIN_USER" >/dev/null 2>&1 || useradd -m -s /bin/bash "$ADMIN_USER" +passwd -l "$ADMIN_USER" >/dev/null # no password at all: SSH key only +home="$(getent passwd "$ADMIN_USER" | cut -d: -f6)" +install -d -m 0700 -o "$ADMIN_USER" -g "$ADMIN_USER" "$home/.ssh" +if [ ! -s "$home/.ssh/authorized_keys" ]; then + if [ -s /root/.ssh/authorized_keys ]; then + install -m 0600 -o "$ADMIN_USER" -g "$ADMIN_USER" /root/.ssh/authorized_keys "$home/.ssh/authorized_keys" + else + echo "STOP: neither $ADMIN_USER nor root has ~/.ssh/authorized_keys; add your key first" >&2; exit 1 + fi +fi +echo "$ADMIN_USER ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/90-fly-admin +chmod 0440 /etc/sudoers.d/90-fly-admin +visudo -cf /etc/sudoers.d/90-fly-admin >/dev/null +# NOT in the docker group: that group is root without sudo's log line; docker runs through sudo +gpasswd -d "$ADMIN_USER" docker >/dev/null 2>&1 || true + +# ---------------------------------------------------------------- 4. SSH +step "SSH: $ADMIN_USER only, key only, modern crypto" +cat > /etc/ssh/sshd_config.d/10-fly-hardening.conf </dev/null || systemctl restart ssh -# SSH: keys only (make sure your key is in ~/.ssh/authorized_keys BEFORE running this) -sed -i 's/^#\?PasswordAuthentication .*/PasswordAuthentication no/; s/^#\?PermitRootLogin .*/PermitRootLogin prohibit-password/' /etc/ssh/sshd_config -systemctl reload ssh || systemctl reload sshd +step "fail2ban: SSH brute force" +cat > /etc/fail2ban/jail.d/fly-sshd.local <<'EOF' +[sshd] +enabled = true +backend = systemd +maxretry = 4 +findtime = 10m +bantime = 1h +bantime.increment = true +EOF +systemctl enable --now fail2ban +systemctl restart fail2ban + +# ---------------------------------------------------------------- 5. Tailscale + firewall +step "Tailscale" +systemctl enable --now tailscaled +if [ -n "$TS_AUTHKEY" ] && ! tailscale status >/dev/null 2>&1; then + tailscale up --auth-key="$TS_AUTHKEY" --hostname=fly-vault --advertise-tags=tag:fly --ssh=false --accept-dns=false +fi +unset TS_AUTHKEY +TS_UP=0; tailscale status >/dev/null 2>&1 && TS_UP=1 + +step "firewall: SSH over Tailscale only (plus LAN/public SSH until CLOSE_PUBLIC_SSH=1); out: HTTPS, HTTP (apt), DNS, NTP, WireGuard" +ufw --force reset >/dev/null ufw default deny incoming -ufw default allow outgoing -ufw allow OpenSSH +ufw default deny outgoing +ufw default deny routed +ufw allow in on tailscale0 to any port 22 proto tcp comment 'ssh over the tailnet' +if [ "$CLOSE_PUBLIC_SSH" = 1 ]; then + [ "$TS_UP" = 1 ] || { echo "STOP: Tailscale is not up; closing public SSH now would lock you out" >&2; exit 1; } + echo "public SSH closed: log in with ssh $ADMIN_USER@fly-vault (Tailscale)" +else + ufw limit in 22/tcp comment 'ssh (rate limited) -- removed by CLOSE_PUBLIC_SSH=1' +fi +ufw allow out 443/tcp comment 'https: RPCs, relay, HF, Telegram, apt, Tailscale control' +ufw allow out 80/tcp comment 'http: apt mirrors' +ufw allow out 53 comment 'dns' +ufw allow out 123/udp comment 'ntp' +ufw allow out 41641/udp comment 'tailscale wireguard' +ufw allow out 3478/udp comment 'tailscale stun' +ufw logging low ufw --force enable -# layout -install -d -m 0750 /srv/fly /srv/fly/data /srv/fly/logs /srv/fly/releases +# Docker writes its own iptables rules and bypasses ufw; the DOCKER-USER chain is where the host gets a say. Containers +# may only talk to each other, reach DNS/HTTPS out, and nothing on the internet may open a connection to them. +cat > /usr/local/sbin/fly-docker-firewall <<'EOF' +#!/bin/sh +set -e +iptables -N DOCKER-USER 2>/dev/null || true +iptables -F DOCKER-USER +iptables -A DOCKER-USER -m conntrack --ctstate ESTABLISHED,RELATED -j RETURN +iptables -A DOCKER-USER -i docker0 -o docker0 -j RETURN +iptables -A DOCKER-USER -i br-+ -o br-+ -j RETURN +for p in 443 80 53; do # 80: apt mirrors during image builds (apt checks its own signatures) + iptables -A DOCKER-USER -i br-+ -p tcp --dport $p -j RETURN + iptables -A DOCKER-USER -i docker0 -p tcp --dport $p -j RETURN +done +iptables -A DOCKER-USER -i br-+ -p udp --dport 53 -j RETURN +iptables -A DOCKER-USER -i docker0 -p udp --dport 53 -j RETURN +iptables -A DOCKER-USER -i br-+ -j DROP +iptables -A DOCKER-USER -i docker0 -j DROP +iptables -A DOCKER-USER -m conntrack --ctstate NEW -j DROP +EOF +chmod 0750 /usr/local/sbin/fly-docker-firewall +cat > /etc/systemd/system/fly-docker-firewall.service <<'EOF' +[Unit] +Description=fly-trader: restrict container traffic (DOCKER-USER chain) +After=docker.service +Requires=docker.service +PartOf=docker.service +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/usr/local/sbin/fly-docker-firewall +[Install] +WantedBy=docker.service +EOF + +# ---------------------------------------------------------------- 6. kernel +step "kernel and network settings" +cat > /etc/sysctl.d/60-fly-hardening.conf <<'EOF' +net.ipv4.conf.all.accept_redirects = 0 +net.ipv4.conf.default.accept_redirects = 0 +net.ipv4.conf.all.secure_redirects = 0 +net.ipv4.conf.default.secure_redirects = 0 +net.ipv4.conf.all.send_redirects = 0 +net.ipv4.conf.default.send_redirects = 0 +net.ipv4.conf.all.accept_source_route = 0 +net.ipv4.conf.default.accept_source_route = 0 +net.ipv4.conf.all.rp_filter = 1 +net.ipv4.conf.default.rp_filter = 1 +net.ipv4.conf.all.log_martians = 1 +net.ipv4.conf.default.log_martians = 1 +net.ipv4.icmp_echo_ignore_broadcasts = 1 +net.ipv4.icmp_ignore_bogus_error_responses = 1 +net.ipv4.tcp_syncookies = 1 +net.ipv6.conf.all.accept_redirects = 0 +net.ipv6.conf.default.accept_redirects = 0 +net.ipv6.conf.all.accept_source_route = 0 +net.ipv6.conf.default.accept_source_route = 0 +kernel.kptr_restrict = 2 +kernel.dmesg_restrict = 1 +kernel.unprivileged_bpf_disabled = 1 +net.core.bpf_jit_harden = 2 +kernel.yama.ptrace_scope = 2 +kernel.randomize_va_space = 2 +kernel.sysrq = 0 +fs.protected_hardlinks = 1 +fs.protected_symlinks = 1 +fs.protected_fifos = 2 +fs.protected_regular = 2 +fs.suid_dumpable = 0 +EOF +sysctl -q --system +# core dumps could write a key to disk +cat > /etc/security/limits.d/60-fly-nocore.conf <<'EOF' +* hard core 0 +EOF +mkdir -p /etc/systemd/coredump.conf.d +printf '[Coredump]\nStorage=none\nProcessSizeMax=0\n' > /etc/systemd/coredump.conf.d/60-fly.conf +aa-status --enabled # AppArmor must be on (Docker's default profile uses it) +# protocols, filesystems and devices this server never uses: not loadable at all +for m in dccp sctp rds tipc usb-storage firewire-core cramfs freevxfs hfs hfsplus jffs2 udf; do + echo "install $m /bin/false" +done > /etc/modprobe.d/60-fly-blacklist.conf +# new files readable by their group at most, never by everyone +sed -i 's/^UMASK.*/UMASK\t\t027/' /etc/login.defs +printf 'Authorized access only. Activity on this system is logged.\n' > /etc/issue +cp /etc/issue /etc/issue.net +sed -i 's/^ENABLED=.*/ENABLED="true"/' /etc/default/sysstat 2>/dev/null || true +systemctl enable --now acct sysstat >/dev/null 2>&1 || true + +# ---------------------------------------------------------------- 7. Docker +step "Docker: user namespaces, no new privileges, bounded logs" +install -d -m 0755 /etc/docker +[ -f /etc/docker/daemon.json ] || cat > /etc/docker/daemon.json <<'EOF' +{ + "userns-remap": "default", + "no-new-privileges": true, + "icc": false, + "live-restore": true, + "userland-proxy": false, + "log-driver": "json-file", + "log-opts": {"max-size": "50m", "max-file": "5"} +} +EOF +systemctl restart docker +systemctl daemon-reload +systemctl enable --now fly-docker-firewall.service +# with user namespaces, root inside a container is this unprivileged uid on the host +REMAP_UID="$(awk -F: '$1=="dockremap"{print $2; exit}' /etc/subuid)" + +# ---------------------------------------------------------------- 8. layout, keys, updater +step "fly layout and wallets" +install -d -m 0750 /srv/fly +install -d -m 0755 /srv/fly/releases # public release files (from HF); the remapped container root reads them +H_BRAIN=$((REMAP_UID + BRAIN_UID)); H_SIGNER=$((REMAP_UID + SIGNER_UID)); H_FLY=$((REMAP_UID + FLY_GID)) +install -d -m 0750 -o "$H_BRAIN" -g "$H_FLY" /srv/fly/data /srv/fly/logs +install -d -m 0700 -o "$H_SIGNER" -g "$H_SIGNER" /srv/fly/signer # the signer's own ledger (policy state) +install -d -m 0750 -o "$H_SIGNER" -g "$H_FLY" /srv/fly/run # its socket: the brain may connect, nothing else install -d -m 0700 /srv/fly/secrets install -d -m 0755 /etc/fly /usr/local/lib/fly -echo "bryce ${RELEASE_PUBKEY}" > /etc/fly/allowed_signers +# release signers: valid for the release namespace only, so these keys sign nothing else this box would accept. A re-run +# without RECOVERY_PUBKEY keeps the recovery signer already installed (so closing SSH or updating the scripts never drops it). +if [ -z "$RECOVERY_PUBKEY" ] && [ -f /etc/fly/allowed_signers ]; then + RECOVERY_PUBKEY="$(awk '$1=="bryce-recovery"{ $1=""; $2=""; sub(/^ */, ""); print; exit }' /etc/fly/allowed_signers)" +fi +{ + echo "bryce namespaces=\"fly-trader-release\" ${RELEASE_PUBKEY}" + if [ -n "$RECOVERY_PUBKEY" ]; then echo "bryce-recovery namespaces=\"fly-trader-release\" ${RECOVERY_PUBKEY}"; fi +} > /etc/fly/allowed_signers chmod 0644 /etc/fly/allowed_signers -# the trading wallet: an ed25519 seed made HERE, never copied off this box in plaintext (the fly's backup job writes an -# age-encrypted copy for you). fly_trader.chain.keys accepts a base58 32-byte seed. -if [ ! -e /srv/fly/secrets/bot_key ]; then - umask 077 - openssl genpkey -algorithm ed25519 -out /srv/fly/secrets/bot_key.pem - python3 - <<'PY' -import subprocess +[ -n "$RECOVERY_PUBKEY" ] || echo "WARNING: no RECOVERY_PUBKEY: losing the release key would mean re-bootstrapping the updater" +install -m 0755 "$HERE/fly" /usr/local/bin/fly # sudo fly : fly-trader inside the running container +# the database password: made here, root-only, handed to both database and brain by compose +if [ ! -s /srv/fly/db.env ]; then + pw="$(openssl rand -hex 24)" + ( umask 077; printf 'POSTGRES_PASSWORD=%s\nDATABASE_URL=postgresql://fly:%s@db:5432/fly_trader\n' "$pw" "$pw" > /srv/fly/db.env ) + unset pw +fi +chmod 0600 /srv/fly/db.env +# two hot keys, each an ed25519 seed made HERE and never copied off this box in plaintext (the signer writes +# age-encrypted backups for you): the trading key (bot_key: the trading wallet; refills itself from the treasury under +# L1) and the payout key (payout_key: holds nothing; pays claims from the treasury under L2). Only the signer container +# mounts them. Their public keys are printed below, once; later: sudo fly signer status. +make_key() { + local name="$1" + if [ ! -e "/srv/fly/secrets/$name" ]; then + umask 077 + openssl genpkey -algorithm ed25519 -out "/srv/fly/secrets/$name.pem" + python3 - "$name" <<'PY' +import subprocess, sys +name = sys.argv[1] A = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz" def b58(b): n = int.from_bytes(b, "big"); s = "" while n: n, r = divmod(n, 58); s = A[r] + s return "1" * (len(b) - len(b.lstrip(b"\0"))) + s -priv = subprocess.run(["openssl", "pkey", "-in", "/srv/fly/secrets/bot_key.pem", "-outform", "DER"], check=True, capture_output=True).stdout[-32:] -pub = subprocess.run(["openssl", "pkey", "-in", "/srv/fly/secrets/bot_key.pem", "-pubout", "-outform", "DER"], check=True, capture_output=True).stdout[-32:] -open("/srv/fly/secrets/bot_key", "x").write(b58(priv) + "\n") -print("vault wallet address:", b58(pub)) +pem = f"/srv/fly/secrets/{name}.pem" +priv = subprocess.run(["openssl", "pkey", "-in", pem, "-outform", "DER"], check=True, capture_output=True).stdout[-32:] +pub = subprocess.run(["openssl", "pkey", "-in", pem, "-pubout", "-outform", "DER"], check=True, capture_output=True).stdout[-32:] +open(f"/srv/fly/secrets/{name}", "x").write(b58(priv) + "\n") +print(f"{name}: {b58(pub)}") PY - shred -u /srv/fly/secrets/bot_key.pem - chmod 0400 /srv/fly/secrets/bot_key -fi -HERE="$(cd "$(dirname "$0")" && pwd)" + shred -u "/srv/fly/secrets/$name.pem" + fi + # readable only by the signer container's user (remapped), not by the brain container or any host account + chown "$H_SIGNER:$H_SIGNER" "/srv/fly/secrets/$name" + chmod 0400 "/srv/fly/secrets/$name" +} +make_key bot_key # the trading hot key: holds at most the float; refills itself from the Squads treasury (L1) +make_key payout_key # the payout key: holds nothing; pays claims from the treasury within its weekly cap (L2) install -m 0755 "$HERE/fly_update.py" /usr/local/lib/fly/fly_update.py +install -m 0755 "$HERE/check-host.sh" /usr/local/lib/fly/check-host.sh [ -f /etc/fly/update.json ] || install -m 0600 "$HERE/update.json.example" /etc/fly/update.json +if [ -n "$EXPECT_MANIFEST" ]; then # pin the first release: the updater refuses anything else until one is installed + python3 - "$EXPECT_MANIFEST" <<'PY' +import json, re, sys +pin = sys.argv[1].strip().lower() +if not re.fullmatch(r"[0-9a-f]{64}", pin): + raise SystemExit("EXPECT_MANIFEST must be a sha256 hex digest") +p = "/etc/fly/update.json"; c = json.load(open(p)); c["expect_manifest_sha256"] = pin +open(p, "w").write(json.dumps(c, indent=1) + "\n") +PY +fi [ -f /srv/fly/vault.env ] || install -m 0600 "$HERE/vault.env.example" /srv/fly/vault.env +[ -f /srv/fly/custody.env ] || install -m 0600 "$HERE/custody.env.example" /srv/fly/custody.env +[ -f /srv/fly/signer.env ] || install -m 0600 "$HERE/signer.env.example" /srv/fly/signer.env install -m 0644 "$HERE/docker-compose.vault.yml" /srv/fly/docker-compose.yml install -m 0644 "$HERE/fly-update.service" /etc/systemd/system/fly-update.service install -m 0644 "$HERE/fly-update.timer" /etc/systemd/system/fly-update.timer +install -m 0755 "$HERE/fly_audit_watch.py" /usr/local/lib/fly/fly_audit_watch.py +install -m 0644 "$HERE/fly-audit-watch.service" /etc/systemd/system/fly-audit-watch.service +install -m 0644 "$HERE/fly-audit-watch.timer" /etc/systemd/system/fly-audit-watch.timer +install -m 0644 "$HERE/fly-aide.service" /etc/systemd/system/fly-aide.service +install -m 0644 "$HERE/fly-aide.timer" /etc/systemd/system/fly-aide.timer systemctl daemon-reload -echo -echo "Next (see deploy/VAULT.md):" -echo " 1. edit /srv/fly/vault.env and /etc/fly/update.json (Telegram)" -echo " 2. systemctl enable --now fly-update.timer # pulls and builds the first release" -echo " 3. create the wallet key, then restart: see VAULT.md 'wallet'" +systemctl enable --now fly-audit-watch.timer # a key read by anything but the signer goes to Telegram within a minute +systemctl enable --now fly-aide.timer # the daily file-integrity check reports to Telegram (there is no mail here) + +# ---------------------------------------------------------------- 9. audit and integrity +step "audit log: key reads and config changes" +cat > /etc/audit/rules.d/60-fly.rules <<'EOF' +-w /srv/fly/secrets -p rwa -k fly_keys +-w /etc/fly -p wa -k fly_conf +-w /srv/fly/vault.env -p rwa -k fly_conf +-w /srv/fly/db.env -p rwa -k fly_conf +-w /srv/fly/custody.env -p wa -k fly_conf +-w /srv/fly/signer.env -p rwa -k fly_conf +-w /srv/fly/signer -p wa -k fly_signer +-w /usr/local/lib/fly -p wa -k fly_updater +-w /etc/ssh/sshd_config -p wa -k sshd +-w /etc/ssh/sshd_config.d -p wa -k sshd +-w /etc/sudoers -p wa -k sudoers +-w /etc/sudoers.d -p wa -k sudoers +-w /etc/docker -p wa -k docker_conf +-w /etc/systemd/system -p wa -k systemd_units +-w /etc/passwd -p wa -k identity +-w /etc/shadow -p wa -k identity +-w /etc/group -p wa -k identity +EOF +augenrules --load >/dev/null +systemctl enable --now auditd + +step "file integrity baseline (AIDE, checked daily; takes a few minutes)" +echo 'Checksums = sha512' > /etc/aide/aide.conf.d/10_fly_checksums +cat > /etc/aide/aide.conf.d/90_fly_ignore <<'EOF' +!/srv/fly/data +!/srv/fly/logs +!/srv/fly/releases +!/var/lib/docker +!/var/lib/containerd +EOF +[ -f /var/lib/aide/aide.db ] || { aideinit -y -f >/dev/null 2>&1 && cp /var/lib/aide/aide.db.new /var/lib/aide/aide.db; } + +# ---------------------------------------------------------------- 10. logs +step "persistent journal" +mkdir -p /etc/systemd/journald.conf.d +printf '[Journal]\nStorage=persistent\nSystemMaxUse=2G\n' > /etc/systemd/journald.conf.d/60-fly.conf +systemctl restart systemd-journald + +step "done" +if [ "$CLOSE_PUBLIC_SSH" = 1 ]; then + next="Verify: sudo /usr/local/lib/fly/check-host.sh (every control should pass now)" +else + next="From your Mac, BEFORE closing this session: ssh $ADMIN_USER@fly-vault (over Tailscale; root login is now off) +Then run this script again with CLOSE_PUBLIC_SSH=1 to remove public SSH." +fi +cat </dev/null 2>&1; then pass "$what"; else bad "$what"; fi; } +sshd_is() { sshd -T 2>/dev/null | grep -qix "$1 $2"; } +sysctl_is() { [ "$(sysctl -n "$1" 2>/dev/null)" = "$2" ]; } + +echo "SSH" +check "root login off" sshd_is permitrootlogin no +check "passwords off" sshd_is passwordauthentication no +check "keyboard-interactive off" sshd_is kbdinteractiveauthentication no +check "only the admin account" sh -c "sshd -T | grep -qi '^allowusers '" +check "max 3 auth tries" sshd_is maxauthtries 3 +check "X11 / agent forwarding off" sh -c "sshd -T | grep -qix 'x11forwarding no' && sshd -T | grep -qix 'allowagentforwarding no'" +check "only local TCP forwarding" sshd_is allowtcpforwarding local +check "no weak ciphers" sh -c "! sshd -T | grep -i '^ciphers' | grep -Eqi 'cbc|3des|arcfour'" +check "fail2ban guards sshd" sh -c "fail2ban-client status sshd" +check "client keys: ed25519/P-256/FIDO only" sh -c "sshd -T | grep -i '^pubkeyacceptedalgorithms' | grep -vqi rsa" +check "admin not in the docker group" sh -c "! id -nG \"\$(sshd -T | awk 'tolower(\$1)==\"allowusers\"{print \$2}')\" | grep -qw docker" + +echo "Machine" +root_luks() { lsblk -rno NAME,TYPE -s "$(findmnt -no SOURCE /)" | awk '$2=="crypt"{f=1} f && $2=="part"{print "/dev/"$1; exit}'; } +check "root filesystem encrypted (LUKS)" sh -c "[ -n \"\$(lsblk -rno TYPE -s \$(findmnt -no SOURCE /) | grep -x crypt)\" ]" +check "unattended reboots unlock (TPM) or are off" sh -c "grep -q 'Automatic-Reboot \"false\"' /etc/apt/apt.conf.d/52fly-unattended || clevis luks list -d $(root_luks) 2>/dev/null | grep -q tpm2" +check "IOMMU on (USB4/OCuLink DMA)" sh -c "[ -n \"\$(ls -A /sys/kernel/iommu_groups)\" ]" +check "Secure Boot on" sh -c "mokutil --sb-state 2>/dev/null | grep -qi 'SecureBoot enabled'" +check "no Wi-Fi or Bluetooth" sh -c "! ls /sys/class/net/*/wireless >/dev/null 2>&1 && [ -z \"\$(ls -A /sys/class/bluetooth 2>/dev/null)\" ]" + +echo "Tailscale" +check "tailscaled up" sh -c "tailscale status >/dev/null" +check "Tailnet Lock enabled" sh -c "tailscale lock status 2>/dev/null | grep -qi 'lock is enabled'" +check "no public SSH (tailnet only)" sh -c "! ufw status | grep -E '^22/tcp' | grep -v tailscale0 | grep -q ." + +echo "Firewall" +check "ufw active" sh -c "ufw status | grep -q 'Status: active'" +check "incoming denied by default" sh -c "ufw status verbose | grep -q 'deny (incoming)'" +check "outgoing denied by default" sh -c "ufw status verbose | grep -q 'deny (outgoing)'" +check "only SSH listens publicly" sh -c "! ss -Hltn | awk '{print \$4}' | grep -vE '^(127\\.|\\[::1\\]|\\[::ffff:127|100\\.|\\[fd7a:)' | grep -vE ':22\$' | grep -q ." +check "container traffic restricted" sh -c "iptables -S DOCKER-USER | grep -q -- '-j DROP'" +check "console bound to localhost" sh -c "! ss -Hltn | grep -E ':8501 ' | grep -vqE '127\\.0\\.0\\.1:8501'" + +echo "Kernel" +for kv in net.ipv4.conf.all.accept_redirects=0 net.ipv4.conf.all.send_redirects=0 net.ipv4.conf.all.accept_source_route=0 \ + net.ipv4.conf.all.rp_filter=1 net.ipv4.tcp_syncookies=1 kernel.kptr_restrict=2 kernel.dmesg_restrict=1 \ + kernel.unprivileged_bpf_disabled=1 kernel.yama.ptrace_scope=2 kernel.randomize_va_space=2 fs.suid_dumpable=0 \ + fs.protected_symlinks=1 fs.protected_hardlinks=1; do + check "${kv%%=*} = ${kv#*=}" sysctl_is "${kv%%=*}" "${kv#*=}" +done +check "core dumps off" sh -c "grep -qs 'Storage=none' /etc/systemd/coredump.conf.d/60-fly.conf" +check "AppArmor enforcing" aa-status --enabled + +echo "Docker" +check "user namespace remapping" sh -c "docker info 2>/dev/null | grep -qi 'userns'" +check "no-new-privileges by default" sh -c "grep -q '\"no-new-privileges\": true' /etc/docker/daemon.json" +check "log size bounded" sh -c "grep -q 'max-size' /etc/docker/daemon.json" +check "live restore" sh -c "docker info 2>/dev/null | grep -qi 'Live Restore Enabled: true'" + +echo "Secrets" +for k in bot_key payout_key; do + f=/srv/fly/secrets/$k + check "$k exists, mode 400" sh -c "[ -f $f ] && [ \"\$(stat -c %a $f)\" = 400 ]" + check "$k not owned by a login user" sh -c "[ \"\$(stat -c %U $f)\" != root ] || true; ! getent passwd \"\$(stat -c %U $f)\" | grep -qE ':/bin/(ba)?sh\$'" +done +check "secrets dir only root" sh -c "[ \"\$(stat -c %a /srv/fly/secrets)\" = 700 ]" +check "signer up (it refuses equal keys)" sh -c "docker inspect fly-fly-signer-1 --format '{{.State.Health.Status}}' | grep -q healthy" +check "release signer pinned" sh -c "[ -s /etc/fly/allowed_signers ] && [ \"\$(stat -c %U /etc/fly/allowed_signers)\" = root ]" +check "updater owned by root" sh -c "[ \"\$(stat -c %U /usr/local/lib/fly/fly_update.py)\" = root ] && ! [ -w /usr/local/lib/fly/fly_update.py -a \"\$(stat -c %a /usr/local/lib/fly/fly_update.py)\" != 755 ]" +for e in vault custody signer db; do + check "$e.env mode 600" sh -c "[ \"\$(stat -c %a /srv/fly/$e.env)\" = 600 ]" +done +check "no default database password" sh -c "! grep -qE 'POSTGRES_PASSWORD(:|=) *fly\$' /srv/fly/docker-compose.yml /srv/fly/db.env" +check "release signers namespaced" sh -c "! grep -v 'namespaces=\"fly-trader-release\"' /etc/fly/allowed_signers | grep -q ." +check "first release pinned (no TOFU)" sh -c "grep -Eq '\"expect_manifest_sha256\": *\"[0-9a-f]{64}\"' /etc/fly/update.json || [ -s /srv/fly/state.json ]" + +echo "Key isolation" +check "keys mounted into the signer only" sh -c "! docker inspect fly-fly-1 --format '{{range .Mounts}}{{.Source}} {{end}}' | grep -q /srv/fly/secrets" +check "signer has the keys" sh -c "docker inspect fly-fly-signer-1 --format '{{range .Mounts}}{{.Source}} {{end}}' | grep -q /srv/fly/secrets/bot_key" +check "signer filesystem read-only" sh -c "docker inspect fly-fly-signer-1 --format '{{.HostConfig.ReadonlyRootfs}}' | grep -q true" +check "brain runs as uid 10001" sh -c "docker inspect fly-fly-1 --format '{{.Config.User}}' | grep -q '^10001'" +check "database has no route out" sh -c "docker network inspect fly_internal --format '{{.Internal}}' | grep -q true" + +echo "Monitoring" +check "auditd running" systemctl is-active auditd +check "key reads audited" sh -c "auditctl -l | grep -q fly_keys" +check "key-read watcher running" systemctl is-active fly-audit-watch.timer +check "daily AIDE report to Telegram" systemctl is-active fly-aide.timer +check "AIDE baseline present" test -f /var/lib/aide/aide.db +check "journal persistent" test -d /var/log/journal +check "automatic security updates" sh -c "systemctl is-enabled unattended-upgrades && grep -q 'Automatic-Reboot' /etc/apt/apt.conf.d/52fly-unattended" +check "clock synchronised" sh -c "timedatectl show -p NTPSynchronized --value | grep -q yes" +check "no snapd" sh -c "! dpkg -l snapd 2>/dev/null | grep -q '^ii'" + +echo "Lynis" +if command -v lynis >/dev/null; then + lynis audit system --quick --no-colors >/tmp/lynis.out 2>&1 + printf ' hardening index: %s\n' "$(grep -o 'Hardening index : [0-9]*' /tmp/lynis.out | grep -o '[0-9]*$' || echo '?')" + grep -E '^\s+\* ' /var/log/lynis-report.dat >/dev/null 2>&1 || true +fi + +echo +[ "$fail" = 0 ] && echo "all controls pass" || echo "SOME CONTROLS FAILED" +exit $fail diff --git a/deploy/custody.env.example b/deploy/custody.env.example new file mode 100644 index 0000000000000000000000000000000000000000..82e2929d750c22a046d6ae8ee283b8c663453dde --- /dev/null +++ b/deploy/custody.env.example @@ -0,0 +1,26 @@ +# /srv/fly/custody.env (mode 600): the treasury setup and the caps, read by BOTH the signer and the brain. +# Nothing secret: all of it is visible on chain or harmless. Comments on their own lines only. +# Create the multisig and its limits on https://fly-trader.app/owner.html with your owner wallet (docs/vault/RUNBOOK.md +# section 4.3); the page prints the three lines below. After changing any value: sudo python3 /usr/local/lib/fly/fly_update.py restart + +# the Squads v4 multisig whose vault 0 is the treasury (members: your owner wallets only; never a server key) +VAULT_MULTISIG= +# L1: SOL, daily, member = the trading key, the ONLY destination = the trading wallet. A new limit has a new address. +VAULT_LIMIT_TRADING= +# L2: SOL, weekly, member = the payout key, any destination (claims) +VAULT_LIMIT_PAYOUT= +# the most SOL the trading wallet keeps (positions are sized on the whole book, spent from this float); also the largest +# single top-up the signer signs. Moves smaller than TOPUP_MIN_SOL are skipped. +TREASURY_FLOAT_SOL=2.0 +TOPUP_MIN_SOL=0.1 +# the fixed size for a model without a sizing table (sizing is otherwise MAX_POSITION_FRACTION of the book); must be > 0 +MAX_POSITION_SOL=0.5 +# the signer refuses any buy larger than this (a bound on a compromised brain, not sizing) +SIGNER_MAX_BUY_SOL=0.6 +# the signer's own limits: SOL bought per day, swaps per minute, priority fee per swap, the worst value a swap may return +SIGNER_DAILY_BUY_SOL=10 +SIGNER_SWAPS_PER_MIN=6 +SIGNER_MAX_PRIORITY_SOL=0.005 +SIGNER_MIN_VALUE_RATIO=0.6 +# your age1... public key (NOT the release key): the signer encrypts the two hot keys to it, the brain the nightly ledger +VAULT_BACKUP_RECIPIENT= diff --git a/deploy/docker-compose.vault.yml b/deploy/docker-compose.vault.yml index 09025c1d9c315d80a6c9bd9e5b278e452815ac27..5abf0251a942f79914a5fe74bc210f9f9c134330 100644 --- a/deploy/docker-compose.vault.yml +++ b/deploy/docker-compose.vault.yml @@ -1,47 +1,91 @@ # The hosted $FLY vault fly (docs/vault/SPEC.md). Installed by bootstrap-host.sh as /srv/fly/docker-compose.yml and -# driven by the updater (fly_update.py): FLY_RELEASE picks the image tag it built from a verified release. -# Nothing listens on a public interface: the console is on 127.0.0.1 (reach it with an SSH tunnel) and the fly only -# makes outbound connections (Solana RPC, Jupiter, pumpapi, Robinhood Chain RPC, the site's relay, Telegram). +# driven by the updater (fly_update.py): FLY_RELEASE picks the image tags it built from a verified release. +# Nothing listens on a public interface: the console is on 127.0.0.1 (reach it with an SSH tunnel over Tailscale) and +# the containers only make outbound connections. +# +# fly-signer the ONLY container that can read the two hot keys; it checks and signs, over a Unix socket in +# /srv/fly/run (a directory only it and the brain's group can enter). Read-only filesystem, no capabilities. +# fly the brain: engine, console, vault worker, data feeds. No key is mounted into it. +# db Postgres on an internal network (no route out), its password generated by bootstrap-host.sh. +# +# Settings: /srv/fly/custody.env (public: the treasury multisig, its limits, the float and caps; read by both), +# /srv/fly/signer.env (the signer's own RPC URL), /srv/fly/vault.env (the brain's API keys, relay, Telegram, backups), +# /srv/fly/db.env (the database password). All root-only (0600) on the host. services: db: - image: postgres:17 + image: postgres:17@sha256:d74eeac9a635390a49bc21bd49fccd973de707e2a53a76ac49b552b8712ec46f + env_file: /srv/fly/db.env environment: POSTGRES_USER: fly - POSTGRES_PASSWORD: fly POSTGRES_DB: fly_trader volumes: - pgdata:/var/lib/postgresql/data + networks: [internal] healthcheck: test: ["CMD-SHELL", "pg_isready -U fly -d fly_trader"] interval: 5s timeout: 3s retries: 30 restart: unless-stopped + security_opt: ["no-new-privileges:true"] + logging: {driver: json-file, options: {max-size: "20m", max-file: "5"}} + + fly-signer: + image: fly-signer:${FLY_RELEASE:-none} + env_file: [/srv/fly/custody.env, /srv/fly/signer.env] + environment: + BOT_PRIVATE_KEY_FILE: /run/secrets/bot_key + PAYOUT_PRIVATE_KEY_FILE: /run/secrets/payout_key + SIGNER_SOCKET: /run/fly-signer/sock + SIGNER_DB: /srv/signer/ledger.sqlite + volumes: + - /srv/fly/secrets/bot_key:/run/secrets/bot_key:ro + - /srv/fly/secrets/payout_key:/run/secrets/payout_key:ro + - /srv/fly/signer:/srv/signer + - /srv/fly/run:/run/fly-signer + networks: [egress] # its own RPC and Jupiter's price API; nothing reaches in + read_only: true + tmpfs: [/tmp] + restart: unless-stopped + security_opt: ["no-new-privileges:true"] + cap_drop: [ALL] + mem_limit: 512m + pids_limit: 64 logging: {driver: json-file, options: {max-size: "20m", max-file: "5"}} + fly: - image: fly-trader:${FLY_RELEASE:?set by the updater} - env_file: /srv/fly/vault.env + image: fly-trader:${FLY_RELEASE:-none} + env_file: [/srv/fly/custody.env, /srv/fly/vault.env, /srv/fly/db.env] environment: - DATABASE_URL: postgresql://fly:fly@db:5432/fly_trader DATA_DIR: /app/data LOG_DIR: /app/logs - BOT_PRIVATE_KEY_FILE: /run/secrets/bot_key + SIGNER_SOCKET: /run/fly-signer/sock VAULT_ENABLED: "1" + AUTO_UPDATE: "0" # releases come through the host updater (signed, 24 h hold on code), never from inside volumes: - /srv/fly/data:/app/data - /srv/fly/logs:/app/logs - - /srv/fly/secrets/bot_key:/run/secrets/bot_key:ro + - /srv/fly/run:/run/fly-signer - ${FLY_RELEASES:-/srv/fly/releases}:/releases:ro + networks: [internal, egress] ports: - "127.0.0.1:8501:8501" depends_on: db: condition: service_healthy + fly-signer: + condition: service_healthy restart: unless-stopped stop_grace_period: 150s security_opt: ["no-new-privileges:true"] cap_drop: [ALL] - mem_limit: 14g + mem_limit: 48g # a dedicated 128 GB box; lower it on a smaller one (16 GB: 14g) logging: {driver: json-file, options: {max-size: "50m", max-file: "5"}} + +networks: + internal: + internal: true # the database has no route to anything but its two clients + egress: {} + volumes: pgdata: diff --git a/deploy/fly b/deploy/fly new file mode 100644 index 0000000000000000000000000000000000000000..20f62d7c421f9e6934713206ca5aaca1f31b64ce --- /dev/null +++ b/deploy/fly @@ -0,0 +1,9 @@ +#!/bin/sh +# Run a fly-trader command inside the running brain container, e.g. sudo fly vault status +# sudo fly signer runs python -m fly_trader.signer in the signer container (status, clear-panic) +set -e +if [ "$1" = "signer" ]; then + shift + exec docker exec -i fly-fly-signer-1 python -m fly_trader.signer "$@" +fi +exec docker exec -i fly-fly-1 fly-trader "$@" diff --git a/deploy/fly-aide.service b/deploy/fly-aide.service new file mode 100644 index 0000000000000000000000000000000000000000..99cfbe0d94a903d122ce298ffd050e17aff0a29f --- /dev/null +++ b/deploy/fly-aide.service @@ -0,0 +1,8 @@ +[Unit] +Description=fly-trader vault: daily file-integrity check (AIDE), findings to Telegram + +[Service] +Type=oneshot +Nice=10 +ExecStart=/usr/bin/python3 /usr/local/lib/fly/fly_audit_watch.py aide +TimeoutStartSec=3600 diff --git a/deploy/fly-aide.timer b/deploy/fly-aide.timer new file mode 100644 index 0000000000000000000000000000000000000000..e57ff96257bc89bd928c07ecea5285d2ac8f98ec --- /dev/null +++ b/deploy/fly-aide.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Run the AIDE check daily + +[Timer] +OnCalendar=*-*-* 05:30:00 UTC +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/deploy/fly-audit-watch.service b/deploy/fly-audit-watch.service new file mode 100644 index 0000000000000000000000000000000000000000..736b8a42538d5f18cf418bd149991cc4c1010fe8 --- /dev/null +++ b/deploy/fly-audit-watch.service @@ -0,0 +1,8 @@ +[Unit] +Description=fly-trader vault: alert when anything but the signer reads a wallet key +After=auditd.service + +[Service] +Type=oneshot +ExecStart=/usr/bin/python3 /usr/local/lib/fly/fly_audit_watch.py run +TimeoutStartSec=120 diff --git a/deploy/fly-audit-watch.timer b/deploy/fly-audit-watch.timer new file mode 100644 index 0000000000000000000000000000000000000000..788a859c60d16206ed595165671391fed190010c --- /dev/null +++ b/deploy/fly-audit-watch.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Check the key-read audit log every minute + +[Timer] +OnBootSec=1min +OnUnitActiveSec=1min +AccuracySec=5s + +[Install] +WantedBy=timers.target diff --git a/deploy/fly_audit_watch.py b/deploy/fly_audit_watch.py new file mode 100644 index 0000000000000000000000000000000000000000..b3c4be6495200155d2fffdd9ea0490a280dd8e7c --- /dev/null +++ b/deploy/fly_audit_watch.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +"""Tell the operator when anything but the signer reads a wallet key (runs on the HOST, root, every minute). + +auditd logs every access to /srv/fly/secrets (key ``fly_keys``, bootstrap-host.sh). The signer container's Python is +the only reader there should ever be (plus AIDE's daily integrity scan); a read by anything else -- a shell, cat, a +process in the brain container -- is the earliest sign of a compromise, so it goes to Telegram at once, with the +program, user and time. ``ausearch --checkpoint`` makes each event reported once. + + fly_audit_watch.py run (the systemd timer, every minute) + fly_audit_watch.py aide (fly-aide.timer, daily: AIDE's integrity check; changes go to Telegram, since + this host has no mail. After reviewing expected changes: sudo aide --update and + sudo cp /var/lib/aide/aide.db.new /var/lib/aide/aide.db) + fly_audit_watch.py test (send a test message) +""" +from __future__ import annotations + +import json +import re +import subprocess +import sys +import urllib.request +from pathlib import Path + +CONF = Path("/etc/fly/update.json") # the updater's Telegram settings +CHECKPOINT = Path("/var/lib/fly/audit.checkpoint") +SIGNER_EXE = {"/usr/local/bin/python3.13", "/usr/local/bin/python3"} # the signer image's interpreter +ALLOWED_EXE = {"/usr/bin/aide", "/usr/sbin/aide"} # AIDE hashes every file daily +SIGNER_UID_OFFSET = 10002 # the signer container's user; on the host: dockremap base + this + +FIELD = re.compile(r'(\w+)=("[^"]*"|\S+)') + + +def remap_base() -> int: + for line in Path("/etc/subuid").read_text().splitlines(): + name, start, _ = line.split(":") + if name == "dockremap": + return int(start) + return -1 + + +def events(raw: str) -> list[dict]: + """SYSCALL records of ``ausearch --raw`` output as dicts (pure).""" + out = [] + for line in raw.splitlines(): + if not line.startswith("type=SYSCALL"): + continue + out.append({k: v.strip('"') for k, v in FIELD.findall(line)}) + return out + + +def suspicious(evs: list[dict], signer_uid: int) -> list[dict]: + """Reads that are not the signer's interpreter as the signer's user, nor AIDE (pure).""" + bad = [] + for e in evs: + exe, uid = e.get("exe", ""), int(e.get("uid", "-1")) if e.get("uid", "").lstrip("-").isdigit() else -1 + if exe in SIGNER_EXE and uid == signer_uid: + continue + if exe in ALLOWED_EXE and uid == 0: + continue + bad.append(e) + return bad + + +def alert(text: str) -> None: + c = json.loads(CONF.read_text()) if CONF.exists() else {} + tok, chat = c.get("telegram_bot_token"), c.get("telegram_chat_id") + print("ALERT", text, flush=True) + if tok and chat: + body = json.dumps({"chat_id": chat, "text": "[fly host] " + text}).encode() + urllib.request.urlopen(urllib.request.Request(f"https://api.telegram.org/bot{tok}/sendMessage", data=body, + headers={"Content-Type": "application/json"}), timeout=10) + + +def run() -> int: + CHECKPOINT.parent.mkdir(parents=True, exist_ok=True) + r = subprocess.run(["ausearch", "-k", "fly_keys", "--raw", "--checkpoint", str(CHECKPOINT), "--start", "checkpoint"], + capture_output=True, text=True) + if r.returncode not in (0, 1): # 1: no matches; 10/11/12: checkpoint problems -> start fresh next run + CHECKPOINT.unlink(missing_ok=True) + return 0 + bad = suspicious(events(r.stdout), remap_base() + SIGNER_UID_OFFSET) + if bad: + lines = [f"{e.get('exe')} (uid {e.get('uid')}, comm {e.get('comm')}, syscall {e.get('syscall')})" for e in bad[:10]] + alert(f"WALLET KEY READ by something other than the signer ({len(bad)} event(s)):\n" + "\n".join(lines) + + "\nIf this was not you: revoke L1 and L2 in the owner wallet now, then investigate.") + return len(bad) + + +def aide() -> int: + """AIDE's exit code: 0 no change, 1-7 added/removed/changed files, >= 14 an error.""" + r = subprocess.run(["aide", "--check", "--config", "/etc/aide/aide.conf"], capture_output=True, text=True) + if r.returncode == 0: + return 0 + summary = [l for l in r.stdout.splitlines() if l.strip() and (l.startswith(("Added", "Removed", "Changed", "f", "d", "l")) + or "entries" in l)][:25] + what = "an ERROR" if r.returncode >= 14 else "file changes" + alert(f"AIDE found {what} (exit {r.returncode}). If you did not make them (a release, apt upgrades), investigate:\n" + + "\n".join(summary) + "\nFull report: sudo aide --check") + return r.returncode + + +if __name__ == "__main__": + cmd = sys.argv[1] if len(sys.argv) > 1 else "run" + if cmd == "test": + alert("audit watch test message") + elif cmd == "aide": + aide() + else: + run() diff --git a/deploy/fly_update.py b/deploy/fly_update.py index f432e497ae992ab67f9f578a989b7afffeb0bc47..a401e3a894ae7f63407074d071fc5fefaf82dc59 100644 --- a/deploy/fly_update.py +++ b/deploy/fly_update.py @@ -3,7 +3,8 @@ Runs on the HOST (root, systemd timer, standard library only), never inside the container, because it rebuilds and restarts the container. Everything it trusts comes from files on this host that no release can change: -``/etc/fly/allowed_signers`` (Bryce's release key) and ``/etc/fly/update.json``. +``/etc/fly/allowed_signers`` (Bryce's release keys, restricted to the release namespace) and ``/etc/fly/update.json`` +(which also pins the very first release by its manifest hash: no trust on first use). One round: 1. poll the HF model repo's commit sha; nothing to do if it is the installed one @@ -12,13 +13,19 @@ One round: no forks) 3. download exactly the listed paths (strict path check), verify size + sha256 of each, into a fresh directory 4. compare file classes with the installed release: - only model/seed -> ``docker compose exec fly fly-trader apply-release /releases/`` (hot swap) - code -> wait for ``ready-for-restart``, build ``fly-trader:`` from that directory, recreate, - wait for healthy, else roll back to the previous tag - infra -> stop and ask: ``fly_update.py approve `` (compose/Dockerfile/entrypoint can grant - host access, so a human looks first) - - fly_update.py run | approve | status | verify + only model/seed -> ``docker compose exec fly fly-trader apply-release /releases/`` (hot swap; models + cannot run code, fly_trader/train/model_io.py); unhealthy after it -> the previous release's + models are re-applied + code -> held for ``code_delay_s`` (24 h) with a Telegram notice, so a stolen signing key cannot ship + code faster than Bryce can ``veto`` it; ``approve`` applies it at once. Then: wait for + ``ready-for-restart``, build ``fly-trader:`` from that directory, recreate, wait for + healthy, else roll back to the previous tag and compose file + infra -> held until ``approve`` (compose/Dockerfile/entrypoint can grant host access, so a human + looks first) + + fly_update.py run | approve | veto | status | restart | verify [principal ...] + +``restart`` recreates the containers on the installed release (after you edit /srv/fly/*.env). """ from __future__ import annotations @@ -37,13 +44,17 @@ from pathlib import Path CONF = Path(os.environ.get("FLY_UPDATE_CONF", "/etc/fly/update.json")) PATH_RE = re.compile(r"^(?!.*\.\.)(?!/)[A-Za-z0-9_.\-/]+$") INFRA = ("Dockerfile", "docker-compose.yml", "docker-compose.cuda.yml", ".dockerignore", "docker/", "deploy/") +# what models/ and seed/ may hold: nothing a loader could execute (fly_trader/train/model_io.SAFE_SUFFIXES) +MODEL_SUFFIXES = {".pt", ".skops", ".npz", ".json", ".bin", ".parquet", ".txt", ".sha256", ""} +SEED_SUFFIXES = {".json", ".sql"} def conf() -> dict: c = json.loads(CONF.read_text()) c.setdefault("repo", "bryceweiner/fly-trader") c.setdefault("allowed_signers", "/etc/fly/allowed_signers") - c.setdefault("principal", "bryce") + c.setdefault("principals", [c.get("principal") or "bryce", "bryce-recovery"]) + c.setdefault("code_delay_s", 86400) c.setdefault("namespace", "fly-trader-release") c.setdefault("root", "/srv/fly") c.setdefault("project", "fly") @@ -95,13 +106,19 @@ def classify(path: str) -> str: return "code" -def verify_signature(c: dict, manifest: bytes, sig: bytes) -> None: +def verify_signature(c: dict, manifest: bytes, sig: bytes) -> str: + """The principal whose key signed ``manifest`` (any of ``principals``), else SystemExit.""" + principals = c.get("principals") or [c["principal"]] + err = "" with tempfile.TemporaryDirectory() as d: sp = Path(d) / "release.json.sig"; sp.write_bytes(sig) - r = subprocess.run(["ssh-keygen", "-Y", "verify", "-f", c["allowed_signers"], "-I", c["principal"], "-n", c["namespace"], "-s", str(sp)], - input=manifest, capture_output=True) - if r.returncode != 0: - raise SystemExit(f"release signature does NOT verify: {r.stderr.decode(errors='replace').strip()[:300]}") + for who in principals: + r = subprocess.run(["ssh-keygen", "-Y", "verify", "-f", c["allowed_signers"], "-I", who, "-n", c["namespace"], "-s", str(sp)], + input=manifest, capture_output=True) + if r.returncode == 0: + return who + err = r.stderr.decode(errors="replace").strip() + raise SystemExit(f"release signature does NOT verify: {err[:300]}") def check_manifest(m: dict) -> None: @@ -113,6 +130,8 @@ def check_manifest(m: dict) -> None: raise SystemExit(f"refusing path {p!r}") if f.get("class") != classify(p): raise SystemExit(f"{p}: class {f.get('class')} does not match {classify(p)}") + if f["class"] == "model" and Path(p).suffix not in MODEL_SUFFIXES or f["class"] == "seed" and Path(p).suffix not in SEED_SUFFIXES: + raise SystemExit(f"{p}: file type not allowed in {f['class']}/ (a model or seed must not be able to run code)") if not re.fullmatch(r"[0-9a-f]{64}", f.get("sha256", "")) or not isinstance(f.get("size"), int): raise SystemExit(f"{p}: bad hash or size") @@ -146,19 +165,27 @@ def save_state(c: dict, st: dict) -> None: def compose(c: dict, *args: str, check: bool = True, env: dict | None = None) -> subprocess.CompletedProcess: - e = {**os.environ, **(env or {})} + """docker compose on the installed release (FLY_RELEASE = the image tag this host runs, unless ``env`` says otherwise).""" + installed_tag = load_state(c).get("image") + e = {**os.environ, "FLY_RELEASE": str(installed_tag) if installed_tag is not None else "none", **(env or {})} return subprocess.run(["docker", "compose", "-p", c["project"], "-f", str(Path(c["root"]) / "docker-compose.yml"), *args], cwd=c["root"], check=check, capture_output=True, text=True, env=e) -def fetch(c: dict, sha: str) -> tuple[dict, Path, bytes]: +def fetch(c: dict, sha: str) -> tuple[dict, Path, bytes, str]: base = f"https://huggingface.co/{c['repo']}/resolve/{sha}/" raw = http(base + "release.json"); sig = http(base + "release.json.sig") - verify_signature(c, raw, sig) + who = verify_signature(c, raw, sig) m = json.loads(raw) check_manifest(m) dest = Path(c["root"]) / "releases" / str(m["seq"]) if dest.exists(): + try: # already downloaded (a held or retried release): re-verify, reuse + if (dest / "release.json").read_bytes() == raw: + verify_dir(dest, m) + return m, dest, raw, who + except (SystemExit, OSError): + pass shutil.rmtree(dest) part = dest.with_name(dest.name + ".part") if part.exists(): @@ -170,7 +197,7 @@ def fetch(c: dict, sha: str) -> tuple[dict, Path, bytes]: (part / "release.json").write_bytes(raw); (part / "release.json.sig").write_bytes(sig) verify_dir(part, m) part.rename(dest) - return m, dest, raw + return m, dest, raw, who def installed_manifest(c: dict, st: dict) -> dict | None: @@ -212,82 +239,155 @@ def healthy(c: dict, limit_s: int = 900) -> bool: return False -def deploy_code(c: dict, st: dict, m: dict, d: Path, infra: bool) -> bool: +def deploy_code(c: dict, st: dict, m: dict, d: Path, infra: bool) -> str: + """'ok', 'busy' (not ready for a restart: retried next round) or 'failed' (rolled back).""" tag = f"fly-trader:{m['seq']}" - subprocess.run(["docker", "build", "-t", tag, str(d)], check=True, capture_output=True) - if infra: # approved: the release's compose file becomes the running one - shutil.copy2(d / "deploy" / "docker-compose.vault.yml", Path(c["root"]) / "docker-compose.yml") - if not wait_ready(c): + root = Path(c["root"]); live, backup = root / "docker-compose.yml", root / "docker-compose.yml.prev" + try: # the brain (default stage) and the key-holding signer, same release + subprocess.run(["docker", "build", "-t", tag, str(d)], check=True, capture_output=True) + subprocess.run(["docker", "build", "--target", "signer", "-t", f"fly-signer:{m['seq']}", str(d)], check=True, capture_output=True) + except subprocess.CalledProcessError as e: + alert(c, f"release {m['seq']}: image build failed: {(e.stderr or b'').decode(errors='replace')[-300:]}") + return "failed" + if st.get("image") is not None and not wait_ready(c): # the first install has nothing running to wait for alert(c, f"release {m['seq']}: the fly never became ready for a restart (orders/claims/settlement in flight); will retry") - return False + return "busy" + if infra: # approved: the release's compose file becomes the running one + shutil.copy2(live, backup) + shutil.copy2(d / "deploy" / "docker-compose.vault.yml", live) prev = st.get("image") - compose(c, "up", "-d", env={"FLY_RELEASE": str(m["seq"]), "FLY_RELEASES": str(Path(c["root"]) / "releases")}) + env = {"FLY_RELEASES": str(root / "releases")} + compose(c, "up", "-d", check=False, env={**env, "FLY_RELEASE": str(m["seq"])}) if healthy(c): - return True + return "ok" alert(c, f"release {m['seq']} did not become healthy; rolling back to {prev}") + if infra and backup.exists(): + shutil.copy2(backup, live) if prev: - compose(c, "up", "-d", env={"FLY_RELEASE": str(prev), "FLY_RELEASES": str(Path(c["root"]) / "releases")}) - return False + compose(c, "up", "-d", check=False, env={**env, "FLY_RELEASE": str(prev)}) + return "failed" + +def _when(ts: float) -> str: + return time.strftime("%Y-%m-%d %H:%M UTC", time.gmtime(ts)) -def apply(c: dict, st: dict, m: dict, d: Path, raw: bytes, hf_sha: str, approved: bool = False) -> None: + +def apply(c: dict, st: dict, m: dict, d: Path, raw: bytes, hf_sha: str, approved: bool = False, who: str = "?", + now: float | None = None) -> None: + now = time.time() if now is None else now + msha = hashlib.sha256(raw).hexdigest() old = installed_manifest(c, st) - if old is not None and m["seq"] <= old["seq"]: - raise SystemExit(f"release seq {m['seq']} is not newer than installed {old['seq']} (rollback refused)") - if old is not None and m.get("prev_sha256") != st.get("manifest_sha256"): - raise SystemExit("release does not chain from the installed one (prev_sha256 mismatch)") + if old is None: # the first release is pinned by hand at bootstrap (no TOFU) + pin = c.get("expect_manifest_sha256") + if not pin: + raise SystemExit("no release installed and update.json pins none (expect_manifest_sha256): refusing to trust the first one seen") + if msha != pin: + raise SystemExit(f"first release's manifest {msha[:12]} is not the pinned {str(pin)[:12]}") + else: + if m["seq"] <= old["seq"]: + raise SystemExit(f"release seq {m['seq']} is not newer than installed {old['seq']} (rollback refused)") + pend = st.get("pending") or {} + chain = {st.get("manifest_sha256"), pend.get("manifest_sha256"), *(st.get("vetoed") or [])} - {None} + if m.get("prev_sha256") not in chain: + raise SystemExit("release does not chain from the installed one (prev_sha256 mismatch)") classes = changed_classes(old, m) - if "infra" in classes and not approved: - st["pending"] = {"seq": m["seq"], "hf_sha": hf_sha} + if old is not None and classes & {"code", "infra"}: + pend = st.get("pending") or {} + if pend.get("manifest_sha256") != msha: # newly seen: the delay starts now (a replaced pending one restarts it) + pend = {"seq": m["seq"], "hf_sha": hf_sha, "manifest_sha256": msha, "classes": sorted(classes), "signer": who, + "first_seen": now, "due_at": now + float(c.get("code_delay_s", 86400)), "approved": False} + st["pending"] = pend + what = ("changes Docker/compose/entrypoint files: review it, then `sudo python3 /usr/local/lib/fly/fly_update.py approve " + f"{m['seq']}`" if "infra" in classes else + f"changes code: it applies at {_when(pend['due_at'])} unless you run `sudo python3 /usr/local/lib/fly/fly_update.py veto " + f"{m['seq']}` (`approve {m['seq']}` applies it now)") + alert(c, f"release {m['seq']} signed by {who} {what}") + if approved: + pend["approved"] = True save_state(c, st) - alert(c, f"release {m['seq']} changes Docker/compose/entrypoint files: review, then run `fly_update.py approve {m['seq']}`") - return + if "infra" in classes and not pend["approved"]: + return + if not (pend["approved"] or now >= pend["due_at"]): + return if classes & {"code", "infra"} or st.get("image") is None: - if not deploy_code(c, st, m, d, "infra" in classes): + res = deploy_code(c, st, m, d, "infra" in classes) + if res == "busy": return + if res == "failed": + raise SystemExit("the new image did not come up healthy (rolled back)") st["image"] = m["seq"] elif classes & {"model", "seed"}: r = compose(c, "exec", "-T", "fly", "fly-trader", "apply-release", f"/releases/{m['seq']}", check=False) if r.returncode != 0: - alert(c, f"release {m['seq']}: apply-release failed: {(r.stderr or r.stdout)[-300:]}") - return - st.update({"seq": m["seq"], "manifest_sha256": hashlib.sha256(raw).hexdigest(), "hf_sha": hf_sha, "pending": None}) + raise SystemExit(f"apply-release failed: {(r.stderr or r.stdout)[-300:]}") + if not healthy(c, 300): + if old is not None: + compose(c, "exec", "-T", "fly", "fly-trader", "apply-release", f"/releases/{old['seq']}", check=False) + raise SystemExit("unhealthy after the model release; the previous release's models were re-applied") + st.update({"seq": m["seq"], "manifest_sha256": msha, "hf_sha": hf_sha, "pending": None}) save_state(c, st) - alert(c, f"release {m['seq']} applied ({', '.join(sorted(classes))})") + alert(c, f"release {m['seq']} applied ({', '.join(sorted(classes))}), signed by {who}") -def run(approve_seq: int | None = None) -> None: +def run(approve_seq: int | None = None, now: float | None = None) -> None: c = conf() + now = time.time() if now is None else now Path(c["root"], "releases").mkdir(parents=True, exist_ok=True) st = load_state(c) - info = json.loads(http(f"https://huggingface.co/api/models/{c['repo']}")) - sha = info["sha"] + pend = st.get("pending") or {} if approve_seq is not None: - pend = st.get("pending") or {} if pend.get("seq") != approve_seq: raise SystemExit(f"no pending release {approve_seq}") sha = pend["hf_sha"] - elif sha in (st.get("hf_sha"), st.get("hf_sha_rejected")) or (st.get("pending") or {}).get("hf_sha") == sha: - return # installed, already rejected (alerted once), or waiting for approval + else: + head = json.loads(http(f"https://huggingface.co/api/models/{c['repo']}"))["sha"] + if head not in (st.get("hf_sha"), st.get("hf_sha_rejected"), pend.get("hf_sha")): + sha = head # a new release + elif pend and (pend.get("approved") or now >= pend.get("due_at", float("inf"))): + sha = pend["hf_sha"] # a held release whose time came (or a retry after 'busy') + else: + return # installed, rejected (alerted once), or still held try: - m, d, raw = fetch(c, sha) - apply(c, st, m, d, raw, sha, approved=approve_seq is not None) + m, d, raw, who = fetch(c, sha) + apply(c, st, m, d, raw, sha, approved=approve_seq is not None, who=who, now=now) except SystemExit as e: + st = load_state(c) + if (st.get("pending") or {}).get("hf_sha") == sha: + st["pending"] = None st["hf_sha_rejected"] = sha; save_state(c, st) alert(c, f"release at {sha[:10]} rejected: {e}") raise +def veto(seq: int) -> None: + c = conf(); st = load_state(c); pend = st.get("pending") or {} + if pend.get("seq") != seq: + raise SystemExit(f"no pending release {seq}") + st["vetoed"] = [*(st.get("vetoed") or []), pend["manifest_sha256"]][-20:] + st["hf_sha_rejected"] = pend["hf_sha"]; st["pending"] = None + save_state(c, st) + alert(c, f"release {seq} VETOED: it will never apply. If you did not publish it, your release key is compromised: " + "remove it from /etc/fly/allowed_signers and rotate.") + + def main(argv: list[str]) -> None: cmd = argv[1] if len(argv) > 1 else "run" if cmd == "run": run() elif cmd == "approve": run(int(argv[2])) + elif cmd == "veto": + veto(int(argv[2])) + elif cmd == "restart": + c = conf() + if load_state(c).get("image") is None: + raise SystemExit("nothing installed yet") + r = compose(c, "up", "-d", "--force-recreate", check=False, env={"FLY_RELEASES": str(Path(c["root"]) / "releases")}) + print((r.stdout or "") + (r.stderr or "")) elif cmd == "status": print(json.dumps(load_state(conf()), indent=1)) elif cmd == "verify": # used by tools/publish_release.py on the Mac before uploading - d = Path(argv[2]); c = {"allowed_signers": argv[3], "principal": argv[4], "namespace": "fly-trader-release"} + d = Path(argv[2]); c = {"allowed_signers": argv[3], "principals": argv[4:] or ["bryce"], "namespace": "fly-trader-release"} raw = (d / "release.json").read_bytes() verify_signature(c, raw, (d / "release.json.sig").read_bytes()) m = json.loads(raw); check_manifest(m); verify_dir(d, m) diff --git a/deploy/signer.env.example b/deploy/signer.env.example new file mode 100644 index 0000000000000000000000000000000000000000..af01b2273dfe08d5dea8f9fbd919886ce1c8cd12 --- /dev/null +++ b/deploy/signer.env.example @@ -0,0 +1,4 @@ +# /srv/fly/signer.env (mode 600): the signer's own settings. Only the signer container reads this file. +# REQUIRED: the signer does not start without it. Use a different provider from the brain's Helius: it simulates every +# swap and reads the treasury itself. (Its value check reads Jupiter's public price API, lite-api.jup.ag, no key.) +SIGNER_RPC_URL= diff --git a/deploy/update.json.example b/deploy/update.json.example index 232d086f650d56e2f43c8dba72599cc863e123ce..8be09966010c20259601254903ee31bbb5bf5e60 100644 --- a/deploy/update.json.example +++ b/deploy/update.json.example @@ -1,10 +1,15 @@ { "repo": "bryceweiner/fly-trader", "allowed_signers": "/etc/fly/allowed_signers", - "principal": "bryce", + "principals": [ + "bryce", + "bryce-recovery" + ], "namespace": "fly-trader-release", "root": "/srv/fly", "project": "fly", + "code_delay_s": 86400, + "expect_manifest_sha256": "", "telegram_bot_token": "", "telegram_chat_id": "" } diff --git a/deploy/vault.env.example b/deploy/vault.env.example index 41d20e1d9fd703df903a2eb9b4a8efea3d3f5bae..615ff3b9792471bb7d7013165b64b88a16e994e5 100644 --- a/deploy/vault.env.example +++ b/deploy/vault.env.example @@ -1,16 +1,18 @@ -# /srv/fly/vault.env for the hosted $FLY vault fly (mode 600). Comments on their own lines only (Compose keeps a -# trailing "# comment" as part of the value). The wallet key is NOT here: it is the file /srv/fly/secrets/bot_key. +# /srv/fly/vault.env for the hosted $FLY vault fly's brain (mode 600). Comments on their own lines only (Compose keeps +# a trailing "# comment" as part of the value). No key is here or anywhere the brain can read: the two hot keys are +# files only the signer container mounts. Treasury and cap settings both containers share are in custody.env. -# --- keys --- +# --- API keys --- HELIUS_API_KEY= JUPITER_API_KEY= +# a second, independent Solana RPC (Triton, QuickNode, Alchemy...): money-moving reads must agree with it +SOLANA_CHECK_RPC_URL= -# --- money: the vault wallet (5 SOL principal) --- +# --- money: the book = the trading wallet (a float, TREASURY_FLOAT_SOL in custody.env) + the Squads treasury --- LIVE_ENABLED=1 SOLANA_CLUSTER=mainnet-beta +# your principal (what you send from FUNDING_ADDRESSES) CAPITAL_SOL=5 -# only used when the fly has no sizing table yet; the Kelly caps below govern (~0.47 SOL per position on 5 SOL) -MAX_POSITION_SOL=0.5 MAX_POSITION_FRACTION=0.10 MAX_POOL_SHARE=0.02 MIN_POSITION_SOL=0.02 @@ -22,15 +24,21 @@ HANDOVER_DAYS=0 HANDOVER_MIN_TRADES=20 HANDOVER_BEAT_SELECTOR=0 RESET_ON_START=0 +# the CPU: the fly's scoring is memory-bound, and 16+ cores score as fast as a GPU; no GPU driver stack in the container DEVICE=cpu -# leave memory for Postgres and the wallet-skill table (16 GB host) -DEVICE_MEMORY_GB=6 +# what the fly's batches may use, leaving room for Postgres and the wallet-skill table (128 GB host; 16 GB host: 6) +DEVICE_MEMORY_GB=24 # --- the vault --- -# SOL from these addresses is your deposit (principal); from anyone else it is profit for the lockers +# comma-separated; SOL from these addresses (to the treasury or the trading wallet) is your deposit (principal); from +# anyone else it is profit for the lockers. At least your Solflare owner address. FUNDING_ADDRESSES= RH_CHAIN_ID=4663 RH_RPC_URL=https://rpc.mainnet.chain.robinhood.com +# Robinhood Chain memecoin trading and the vault's ETH pot stay off on the hosted server: its signer container holds no +# RH key, and the brain holds no wallet key at all. +RH_ENABLED=0 +RH_LIVE_ENABLED=0 # from contracts/deployments/4663.json after you deploy VAULT_ADDRESS= VAULT_TIMELOCK= @@ -47,9 +55,11 @@ VAULT_SITE_URI=https://fly-trader.app/vault.html # --- alerts --- TELEGRAM_BOT_TOKEN= TELEGRAM_CHAT_ID= +# your own Telegram user id: only its /panic and /status are obeyed (message @userinfobot to learn it) +TELEGRAM_ADMIN_USER_ID= +# dead-man switch: a healthchecks.io check URL (its Telegram integration tells you when pings stop) +HEALTHCHECK_URL= -# --- encrypted backups (key once, ledger nightly) --- -# your age1... or ssh-ed25519 public key -VAULT_BACKUP_RECIPIENT= +# --- encrypted backups (hot keys once, by the signer; ledger nightly). The recipient key is in custody.env --- VAULT_BACKUP_BUCKET=bryceweiner/fly-vault-backups VAULT_BACKUP_HF_TOKEN= diff --git a/docker-compose.yml b/docker-compose.yml index 250ec5ad49d3c4f946b69c0b50a879e9d404f4b2..d97b1642faab1c9fe865e98327a71069eecbf72e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,12 +18,19 @@ services: fly: build: . + # root inside the container: the self-updater (AUTO_UPDATE=1) replaces the code in /app, and data/, logs/ and .env + # are folders and a file of yours bind-mounted from here (the vault server's compose runs an unprivileged user and + # never self-updates) + user: "0:0" + # every key (BOT_PRIVATE_KEY, and RH_BOT_PRIVATE_KEY for Robinhood Chain live) comes from .env at run time, never the image env_file: .env environment: DATABASE_URL: postgresql://fly:fly@db:5432/fly_trader DATA_DIR: /app/data LOG_DIR: /app/logs DEVICE: auto # the CPU in this image; an NVIDIA GPU with docker-compose.cuda.yml + # Robinhood Chain is opt-in: set RH_ENABLED=1 in .env (it needs ENVIO_API_TOKEN and room for its history) + RH_ENABLED: ${RH_ENABLED:-0} volumes: - ./data:/app/data # models, the fly's plastic state, its hourly snapshots: survives restarts and rebuilds - ./logs:/app/logs diff --git a/docker/bootstrap.sh b/docker/bootstrap.sh index 3161897f7b10995e01a08ed574c298019e4b2afa..463d066d0a38cd68c546d8ea51c793716e571e59 100644 --- a/docker/bootstrap.sh +++ b/docker/bootstrap.sh @@ -1,24 +1,58 @@ #!/bin/sh # What every boot (and every self-update) does before the console starts: schema, the shipped models into the data -# volume, the wallet-skill table, the neuron geometry, the seed rows that admit the fly. Nothing here overwrites a -# running install: copies skip files that exist, the seed is ON CONFLICT DO NOTHING. +# volume, the wallet-skill table, the neuron geometry, the rows that admit the fly. Nothing here overwrites a running +# install: copies skip files that exist, and apply-release reuses rows it already has. set -e cd /app until pg_isready -d "$DATABASE_URL" >/dev/null 2>&1; do echo "fly-trader: waiting for postgres"; sleep 2; done fly-trader init-db +# models/ holds exactly this release's fly and selector: an earlier release's (the updater before signed releases never +# deleted a file) would be copied below, and a pickled .joblib makes apply-release refuse the whole release +if [ -f seed/release_seed.json ]; then + python - <<'PY' +import json, pathlib +keep = {s["file"] for s in json.loads(pathlib.Path("seed/release_seed.json").read_text())["snapshots"]} +for sub in ("policies", "selectors"): + for f in pathlib.Path("models", sub).glob("*"): + if f.is_file() and f.name not in keep: + f.unlink(); print(f"fly-trader: removed {f} (not in this release)") +PY +fi for sub in policies selectors connectome; do mkdir -p "data/brain/$sub" for f in models/$sub/*; do [ -e "$f" ] || continue; [ -e "data/brain/$sub/$(basename "$f")" ] || cp "$f" "data/brain/$sub/"; done done +# The wallet-skill inputs: the fitted config and the newest table. The table (~300 MB) is too big for git; it comes from +# a verified release (the vault server's /releases), or is downloaded from Hugging Face and used only if its sha256 is +# the one pinned in models/wallet_skill/TABLE.sha256. Nothing unverified reaches the engine. mkdir -p data/corpus/wallet_skill -for f in models/wallet_skill/*; do [ -e "$f" ] || continue; [ -e "data/corpus/wallet_skill/$(basename "$f")" ] || cp "$f" data/corpus/wallet_skill/; done +for f in models/wallet_skill/*; do + [ -e "$f" ] || continue + case "$(basename "$f")" in TABLE|TABLE.sha256) continue;; esac + [ -e "data/corpus/wallet_skill/$(basename "$f")" ] || cp "$f" data/corpus/wallet_skill/ +done T=$(cat models/wallet_skill/TABLE 2>/dev/null || true) +WANT=$(cat models/wallet_skill/TABLE.sha256 2>/dev/null || true) if [ -n "$T" ] && [ ! -e "data/corpus/wallet_skill/$T" ]; then - echo "fly-trader: fetching the wallet-skill table $T (~300 MB, once)" - curl -fsSL --retry 5 -o "data/corpus/wallet_skill/$T.part" "https://huggingface.co/${HF_REPO:-bryceweiner/fly-trader}/resolve/main/models/wallet_skill/$T" \ - && mv "data/corpus/wallet_skill/$T.part" "data/corpus/wallet_skill/$T" \ - || echo "fly-trader: WARNING could not fetch the wallet-skill table; the paper selector will not trade and the fly's skill inputs read as zero until it is present" + src=$(ls -d /releases/*/models/wallet_skill/"$T" 2>/dev/null | tail -1 || true) + if [ -n "$src" ]; then + cp "$src" "data/corpus/wallet_skill/$T" + elif [ -n "$WANT" ]; then + echo "fly-trader: fetching the wallet-skill table $T (~300 MB, once; checked against its pinned sha256)" + if curl -fsSL --retry 5 -o "data/corpus/wallet_skill/$T.part" "https://huggingface.co/${HF_REPO:-bryceweiner/fly-trader}/resolve/main/models/wallet_skill/$T" \ + && [ "$(sha256sum "data/corpus/wallet_skill/$T.part" | cut -d' ' -f1)" = "$WANT" ]; then + mv "data/corpus/wallet_skill/$T.part" "data/corpus/wallet_skill/$T" + else + rm -f "data/corpus/wallet_skill/$T.part" + fi + fi fi +[ -z "$T" ] || [ -e "data/corpus/wallet_skill/$T" ] || \ + echo "fly-trader: WARNING the wallet-skill table $T is missing or did not match its sha256; the paper selector will not trade and the fly's skill inputs read as zero until it is present (data/corpus/wallet_skill/$T)" mkdir -p fly_trader/ui/static/brain && cp models/geometry/* fly_trader/ui/static/brain/ 2>/dev/null || true -psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -q -f seed/seed.sql +if [ -f seed/release_seed.json ]; then + fly-trader apply-release /app --bootstrap # the shipped models, registered by file hash (fresh ids), idempotently +else + psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -q -f seed/seed.sql +fi echo "fly-trader: device $(python -c 'from fly_trader.brain import device; print(device.describe(device.resolve()))' 2>/dev/null || echo unknown)" diff --git a/docker/signer-requirements.txt b/docker/signer-requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..cda1d3166492654a50af04c434906e122c8332ed --- /dev/null +++ b/docker/signer-requirements.txt @@ -0,0 +1,51 @@ +anyio==4.15.1 \ + --hash=sha256:6152fdbbf9a77fdec97731721bebf7c4c44f7c29b424b0065826173efc7ed101 \ + --hash=sha256:9f28306018cbd6d329e64a36d58256edff76dd996fe423bc957326e578b82a94 + # via httpx +base58==2.1.1 \ + --hash=sha256:11a36f4d3ce51dfc1043f3218591ac4eb1ceb172919cebe05b52a5bcc8d245c2 \ + --hash=sha256:c5d0cb3f5b6e81e8e35da5754388ddcc6d0d14b6c6a132cb93d69ed580a7278c + # via -r /private/tmp/claude-501/-Users-bryce-Documents-code-fly-trader/28aad393-6f38-484a-a2b2-a06a8f112cf1/scratchpad/signer.in +certifi==2026.7.22 \ + --hash=sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775 \ + --hash=sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55 + # via + # httpcore + # httpx +h11==0.16.0 \ + --hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \ + --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 + # via httpcore +httpcore==1.0.9 \ + --hash=sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55 \ + --hash=sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8 + # via httpx +httpx==0.28.1 \ + --hash=sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc \ + --hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad + # via -r /private/tmp/claude-501/-Users-bryce-Documents-code-fly-trader/28aad393-6f38-484a-a2b2-a06a8f112cf1/scratchpad/signer.in +idna==3.20 \ + --hash=sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44 \ + --hash=sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c + # via + # anyio + # httpx +jsonalias==0.1.1 \ + --hash=sha256:64f04d935397d579fc94509e1fcb6212f2d081235d9d6395bd10baedf760a769 \ + --hash=sha256:a56d2888e6397812c606156504e861e8ec00e188005af149f003c787db3d3f18 + # via solders +solders==0.29.0 \ + --hash=sha256:00524baf575ae5edb2ba43cdb1d80491c59efdc16b48fec96a6f929c0401d0e4 \ + --hash=sha256:09e80c545c728f434f627d3b4714675a0f654c4d387f3339d54f7d09d59f3f1c \ + --hash=sha256:62e8a5bb4bc8628d5fe49faa56aca149dce284a9f8bf5f3c56c4721e561be84b \ + --hash=sha256:785841623bdd7b8dedc68e7c794b54327d3b03e1ab9697eda90a139ca3c98cfe \ + --hash=sha256:9d2e011ff1d5f82a7de81d6eb89f54d002189f50ee48d4a15d7b51849e04b1c3 \ + --hash=sha256:c17098cab72a1c202ec4cc0ce3ecbff4296c9de97000e05944337ba5ddb681a8 \ + --hash=sha256:f36db7b43088866081b5d37ff62668551e30dd136b9ee58a5fb524fcd828082d + # via -r /private/tmp/claude-501/-Users-bryce-Documents-code-fly-trader/28aad393-6f38-484a-a2b2-a06a8f112cf1/scratchpad/signer.in +typing-extensions==4.16.0 \ + --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \ + --hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5 + # via + # anyio + # solders diff --git a/docs/vault/RUNBOOK.md b/docs/vault/RUNBOOK.md index bada6d2e043366211cff9bc9aa12449657e3663f..85505c7c168bddf43a73c1c2d7274cb891e5817b 100644 --- a/docs/vault/RUNBOOK.md +++ b/docs/vault/RUNBOOK.md @@ -1,120 +1,435 @@ # $FLY vault: launch runbook -These are the steps only you can do, because they need your keys or accounts. They are in order. The agent never -handles your keys. - -## 0. Accounts and keys (once) -- **Reown project id.** Create it at dashboard.reown.com and allowlist `fly-trader.app`, plus `localhost:5173` for the - rehearsal. -- **Telegram bot.** Create one with @BotFather; you need its token and your chat id. -- **Release-signing key (Mac).** Run `ssh-keygen -t ed25519 -f ~/.ssh/fly_release`. The public key goes to the server's - allowed signers in step 3. -- **Backup key.** This is the age or ssh-ed25519 public key the backups are encrypted to. You can use - `~/.ssh/fly_release.pub` or an age key. -- **HF storage bucket.** Create a private bucket named `bryceweiner/fly-vault-backups`, then a fine-grained token that - can write only to that bucket. -- **Hardware-wallet EVM account.** This account owns the vault. - -## 1. Rehearsal (testnet 46630 + Solana devnet) -1. Deploy the contracts to the testnet: - ``` - cd contracts - forge script script/DeployTestnet.s.sol --rpc-url https://rpc.testnet.chain.robinhood.com --ledger --broadcast - ``` - The addresses land in `contracts/deployments/46630.json`. -2. Build the site against the testnet: - ``` - cd web - VITE_NETWORK=testnet VITE_REOWN_PROJECT_ID=… VITE_VAULT_ADDRESS=… VITE_TIMELOCK_ADDRESS=… VITE_FLY_ADDRESS= npm run dev - ``` - Run a local relay (see `web/relay/README.md`) with `domain: localhost:5173`, `chain_id: 46630`, `sol_chain: devnet`. -3. Run the fly (Docker or the Mac) with these settings: - - `VAULT_ENABLED=1`, `VAULT_CLUSTER=devnet`, `VAULT_SOLANA_RPC_URL=https://api.devnet.solana.com` - - `VAULT_PERIOD_S=900` - - `RH_CHAIN_ID=46630`, `RH_RPC_URL=https://rpc.testnet.chain.robinhood.com` - - `VAULT_SITE_DOMAIN=localhost:5173`, `VAULT_SITE_URI=http://localhost:5173/vault.html` - - `LIVE_ENABLED=0` - - a throwaway devnet key, funded from the devnet faucet -4. Work through the checklist: - - lock from two wallets at different times, then request and cancel a withdrawal - - send devnet SOL from an address that is not a funding address (it counts as profit) - - wait for a 15-minute settlement - - claim from both wallets, then replay a used nonce - - kill the fly in the middle of a claim and restart it - - `fly-trader vault status` - -## 2. Mainnet contracts -1. Deploy: - ``` - cd contracts - FLY_TOKEN=0x2fC7f9E2911f20b2C4660d2AEf808aa91bDdb3D3 OWNER= \ - forge script script/Deploy.s.sol --rpc-url https://rpc.mainnet.chain.robinhood.com --ledger --broadcast - ``` - The script refuses to run on chain 4663 unless the delays are 8 days (timelock) and 7 days (withdrawal). -2. Verify the contracts on Blockscout. `contracts/README.md` has a manual fallback if Cloudflare blocks forge. -3. Record the implementation code hash: - ``` - cast keccak $(cast code --rpc-url https://rpc.mainnet.chain.robinhood.com) - ``` - This value becomes `VAULT_IMPL_CODEHASHES` in step 3. - -## 3. Server (Netcup RS 2000 G12, Ubuntu 24.04) -1. Put your SSH key on the server. Then copy the `deploy/` folder from the distribution branch and run: - `sudo bash deploy/bootstrap-host.sh "$(cat ~/.ssh/fly_release.pub)"` - It prints the **vault wallet address**. The key is created on the server and stays there. -2. Edit `/srv/fly/vault.env`. Fill in: - - `HELIUS_API_KEY`, `JUPITER_API_KEY` - - `FUNDING_ADDRESSES` - - `VAULT_ADDRESS`, `VAULT_TIMELOCK`, `VAULT_START_BLOCK`, `VAULT_IMPL_CODEHASHES` - - `RELAY_SECRET` - - `TELEGRAM_*` - - `VAULT_BACKUP_*` -3. Edit `/etc/fly/update.json`. Only the Telegram fields need filling. -4. Publish the first release from the Mac. Keep a distribution worktree at `../fly-trader-dist`, then run: - `.venv/bin/python tools/publish_release.py --worktree ../fly-trader-dist --key ~/.ssh/fly_release --models` -5. On the server, start the updater: - ``` - sudo systemctl enable --now fly-update.timer - journalctl -u fly-update -f - ``` - It builds the image and starts the fly, which warms up on paper. -6. To open the console: - ``` - ssh -L 8501:127.0.0.1:8501 - ``` - Then browse to http://localhost:8501/vault. - -## 4. Site (Gandi) +The steps only you can do, because they need your keys, accounts or devices, in the order to do them. The agent never +handles your keys. How the system works is in [SPEC.md](SPEC.md); this file is what to type. + +**Who holds what** (until the Ledger is back; the "Ledger" section moves everything to it without redeploying): + +| Key | Where | Can do | +|---|---|---| +| Solana owner | Solflare on your phone | Everything with the treasury: create/remove its spending limits, take principal out | +| EVM owner | MetaMask | Pause new $FLY locks; propose upgrades (public for 8 days); hand the roles to the Ledger | +| SSH | Secretive on the Mac (Secure Enclave, Touch ID) | Log in to the server over Tailscale | +| Release signing | Secretive (Touch ID) + an offline recovery key | Sign releases; the server still holds code changes 24 h | +| Backup decryption | an age key, offline | Read the encrypted backups | +| Trading / payout | on the server, readable only by the signer container | Refill the float (L1, per day), pay claims (L2, per week) | + +A full takeover of the server can lose at most the trading float plus what is left of this day's L1 and this week's +L2, and only until you revoke them from your phone ("Emergency"). + +Conventions below: +- `$` lines run on the **Mac** in the fly-trader checkout (the repo root). Commands for a subfolder run in a subshell, + `$ (cd web && …)`, so you are always back at the root. +- `server$` lines run on the **server** as `fly-admin`. +- `sudo fly ` runs `fly-trader ` inside the running fly container; `sudo fly signer ` runs + in the signer container. + +## 0. Once: tools, repos, accounts, keys + +### 0.1 Mac tools +``` +$ brew install age node uv git +$ brew install --cask secretive tailscale +$ curl -L https://foundry.paradigm.xyz | bash && foundryup # forge, cast +$ sh -c "$(curl -sSfL https://release.anza.xyz/stable/install)" # solana CLI + solana-test-validator +$ uv sync --extra dev # the Python environment (.venv) +$ (cd web && npm ci) +$ (cd contracts && forge soldeer install) +$ git worktree add ../fly-trader-dist distribution # the branch the server runs +``` + +### 0.2 Accounts +- **2FA everywhere.** Use a hardware key or TOTP on Gandi, Hugging Face, GitHub, + your Tailscale identity provider and your Apple ID. Set a Telegram two-step password. +- **Phone wallets.** Write the Solflare and MetaMask seed phrases on paper and keep them offline. The phone needs a + passcode and biometrics. Keep a little SOL in Solflare (about 0.05 SOL pays the treasury's setup and any later + change) and a little ETH on Robinhood Chain in MetaMask (for `pause()`). +- **Reown project id.** Create it at dashboard.reown.com. Allowlist `fly-trader.app`, and `localhost:5173` for local + testing. +- **Telegram.** + 1. Create a bot with @BotFather and note its token. + 2. Send `/start` to your new bot; it cannot message you before that. + 3. Message @userinfobot to learn your user id. In a private chat with the bot, the chat id is that same number. + Only that user's `/panic` and `/status` count. +- **Hugging Face.** + 1. Create a **public** model repo `bryceweiner/fly-trader`: the server and self-hosted installs read releases + without logging in. + 2. Create a **private** storage bucket `bryceweiner/fly-vault-backups` for the encrypted backups. + 3. Create two tokens: one with write access to the model repo, for publishing from the Mac; one fine-grained, + write-only to the bucket, for the server's backups. + 4. `$ uv run hf auth login` with the first token. +- **GitHub.** Push access to the fly-trader repo: publishing pushes the `distribution` branch. +- **Second Solana RPC.** Get an API URL from a provider other than Helius (Triton, QuickNode or Alchemy). It is used + twice: + - `SOLANA_CHECK_RPC_URL`: settlement balances and halts must agree with it. + - `SIGNER_RPC_URL`: the signer's own reads and simulations. +- **healthchecks.io.** Create a check with period 5 minutes and grace 10 minutes, and connect its Telegram + integration. Its ping URL is `HEALTHCHECK_URL`. + +### 0.3 Tailscale +1. Install Tailscale on the Mac, log in, and turn on **MagicDNS** in the admin console, so `fly-vault` resolves. +2. In the admin console's access controls, add: + ``` + "tagOwners": {"tag:fly": ["autogroup:admin"]}, + "acls": [{"action": "accept", "src": [""], "dst": ["tag:fly:22"]}] + ``` + Add no rule whose source is `tag:fly`: the server needs to reach nothing on your tailnet. +3. Turn on **Tailnet Lock** from the Mac: + ``` + $ tailscale lock status # shows this Mac's tlpub:… key + $ tailscale lock init --gen-disablements 3 --confirm tlpub: + ``` + Store the printed disablement secrets offline, like a seed phrase. +4. Create a one-off, pre-authorised auth key tagged `tag:fly` in the admin console. Sign it so the server may join the + locked tailnet, and keep the output (this is `TS_AUTHKEY` in section 4.2): + ``` + $ tailscale lock sign tskey-auth-… + ``` + +### 0.4 Your keys (Secretive, recovery, backup) +1. In Secretive create two keys, `fly-ssh` and `fly-release`, each requiring Touch ID. Save their public keys as + `~/.ssh/fly-ssh.pub` and `~/.ssh/fly-release.pub`. +2. Put the path of Secretive's agent in your shell profile (`~/.zshrc`), since several steps use it: + `export SECRETIVE=~/Library/Containers/com.maxgoedjen.Secretive.SecretAgent/Data/socket.ssh`. + Then tell ssh to use Secretive for the server, in `~/.ssh/config`: + ``` + Host fly-vault + User fly-admin + IdentityAgent ~/Library/Containers/com.maxgoedjen.Secretive.SecretAgent/Data/socket.ssh + ``` +3. The offline recovery release signer: + ``` + $ mkdir -p ~/fly-keys + $ ssh-keygen -t ed25519 -f ~/fly-keys/fly-release-recovery -C fly-release-recovery + $ age -p ~/fly-keys/fly-release-recovery > ~/fly-keys/fly-release-recovery.age && rm ~/fly-keys/fly-release-recovery + ``` + Put `fly-release-recovery.age` on a USB stick and in your password manager. Keep `fly-release-recovery.pub`. +4. The backup key: `$ age-keygen -o ~/fly-keys/fly-backup.key`. Store it like the recovery key; its public line + (`age1…`, printed and at the top of the file) is `VAULT_BACKUP_RECIPIENT`. It is deliberately not the release key. +5. Trust both release keys in the distribution, so that self-hosted installs accept your releases: + ``` + $ echo "bryce namespaces=\"fly-trader-release\" $(cat ~/.ssh/fly-release.pub)" >> ../fly-trader-dist/release/allowed_signers + $ echo "bryce-recovery namespaces=\"fly-trader-release\" $(cat ~/fly-keys/fly-release-recovery.pub)" >> ../fly-trader-dist/release/allowed_signers + $ git -C ../fly-trader-dist status # must be clean apart from this file (no merge in progress) + $ git -C ../fly-trader-dist commit -m "Release keys" release/allowed_signers + ``` + +## 1. Rehearse locally (no real money) +- **Custody against the real Squads program:** + `$ tools/custody_e2e.sh `. It dumps the program from mainnet (read-only) and runs it on + a throwaway validator on ports 11xxx. It checks: + - the owner page's code creates the treasury and both limits, changes L2 and revokes them + - the signer refills through L1 and pays a claim through L2 up to its cap + - a stolen key cannot send L1 anywhere but the trading wallet (refused on chain) +- **The whole vault on this Mac:** `$ .venv/bin/python tools/vault_demo.py up`. It runs anvil, the contracts, the relay, + the vault worker, a treasury on a local validator and the site at http://localhost:5173. The file's docstring lists + the wallet settings and commands (`fund-evm`, `profit`, `status`, `down`). +- **Optional: RH testnet + Solana devnet.** + 1. Deploy the contracts: + ``` + $ cast wallet new ~/.foundry/keystores rehearsal # fund it from faucet.testnet.chain.robinhood.com + $ (cd contracts && forge script script/DeployTestnet.s.sol --rpc-url robinhood_testnet --account rehearsal \ + --sender $(cast wallet address --account rehearsal) --broadcast) + $ cp contracts/deployments/46630.json ~/fly-keys/testnet-deployment.json + ``` + The addresses land in `contracts/deployments/46630.json`: `vault`, `timelock`, `token` (MockFLY), `blockNumber`. + Keep the copy: `vault_demo.py` deploys to anvil, which uses the same chain id, and overwrites that file. + 2. Run the site against it: + Run a relay as in `web/relay/README.md` (e.g. `tools/vault_demo.py` shows how it starts one on port 8612), with + `"domain": "localhost:5173"`, `"uri": "http://localhost:5173/vault.html"`, `"chain_id": 46630`, + `"sol_chain": "devnet"`. Then: + ``` + $ (cd web && VITE_NETWORK=testnet VITE_REOWN_PROJECT_ID=… VITE_VAULT_ADDRESS= VITE_TIMELOCK_ADDRESS= \ + VITE_FLY_ADDRESS= VITE_SOLANA_RPC=https://api.devnet.solana.com VITE_RELAY_PROXY=http://127.0.0.1:8612 npm run dev) + ``` + 3. Make two throwaway devnet keys (`$ solana-keygen new -o /tmp/devnet-trading.json`, the same for payout; fund + the trading one at faucet.solana.com). Create the treasury: http://localhost:5173/owner.html with Solflare on + devnet, as in section 4.3, with those two public keys. + 4. Run the vault worker on the Mac with these settings in the environment: + - `VAULT_ENABLED=1`, `VAULT_CLUSTER=devnet`, `VAULT_SOLANA_RPC_URL=https://api.devnet.solana.com`, + `SIGNER_RPC_URL=https://api.devnet.solana.com`, `VAULT_PERIOD_S=900`, `LIVE_ENABLED=0` + - `BOT_PRIVATE_KEY_FILE=/tmp/devnet-trading.json`, `PAYOUT_PRIVATE_KEY_FILE=/tmp/devnet-payout.json` + - `VAULT_MULTISIG`, `VAULT_LIMIT_TRADING`, `VAULT_LIMIT_PAYOUT` (from the owner page) + - `RH_CHAIN_ID=46630`, `RH_RPC_URL=https://rpc.testnet.chain.robinhood.com`, `VAULT_ADDRESS`, + `VAULT_TIMELOCK`, `VAULT_START_BLOCK` + - `VAULT_SITE_DOMAIN=localhost:5173`, `VAULT_SITE_URI=http://localhost:5173/vault.html` + - `RELAY_URL=http://127.0.0.1:8612`, `RELAY_SECRET` (the relay's), and a separate `DATABASE_URL` + (`createdb fly_vault_rehearsal; DATABASE_URL=postgresql:///fly_vault_rehearsal uv run fly-trader init-db`) + + ``` + $ uv run python -c "from fly_trader.vault import worker; worker.main()" + ``` + +## 2. Mainnet contracts (EVM owner = MetaMask) +1. Deploy from a throwaway keystore. It pays the gas and holds no role afterwards: + ``` + $ cast wallet new ~/.foundry/keystores deployer # fund with a little ETH on Robinhood Chain + $ (cd contracts && FLY_TOKEN=0x2fC7f9E2911f20b2C4660d2AEf808aa91bDdb3D3 OWNER= \ + forge script script/Deploy.s.sol --rpc-url robinhood --account deployer \ + --sender $(cast wallet address --account deployer) --broadcast \ + --verify --verifier blockscout --verifier-url https://robinhoodchain.blockscout.com/api/) + ``` + The script refuses to run on chain 4663 unless the token is the real $FLY and the delays are 8 days (timelock) and + 7 days (withdrawal). It writes `contracts/deployments/4663.json`: `vault`, `implementation`, `timelock`, + `blockNumber`. You need all four below. +2. Check that the deployer holds nothing. Each of these must print `false`: + ``` + $ D=$(cast wallet address --account deployer) + $ R=https://rpc.mainnet.chain.robinhood.com + $ cast call "hasRole(bytes32,address)(bool)" $(cast keccak PROPOSER_ROLE) $D --rpc-url $R + $ cast call "hasRole(bytes32,address)(bool)" 0x0000000000000000000000000000000000000000000000000000000000000000 $D --rpc-url $R + $ cast call "hasRole(bytes32,address)(bool)" $(cast keccak PAUSER_ROLE) $D --rpc-url $R + ``` + Then delete `~/.foundry/keystores/deployer`. +3. Check the contracts show as verified on Blockscout; `contracts/README.md` has a manual fallback if Cloudflare + blocked forge's verification. +4. Record the implementation code hash; it becomes `VAULT_IMPL_CODEHASHES`: + `$ cast keccak $(cast code --rpc-url https://rpc.mainnet.chain.robinhood.com)` +5. From your phone (MetaMask), you use Blockscout's page for each contract, under "Write proxy" / "Write contract": + - **Pause new locks:** FlyVault, `pause()`; later `unpause()`. Withdrawals always stay open. + - **Cancel a timelock operation you did not schedule:** Timelock, `cancel()`. The Telegram alert gives the full + id. + +## 3. Site (Gandi) +The owner page is part of the site, and you need it for the treasury, so the site goes up before the server. 1. Build the upload bundle: ``` - cd web - VITE_REOWN_PROJECT_ID=… VITE_VAULT_ADDRESS=… VITE_TIMELOCK_ADDRESS=… bash scripts/bundle_gandi.sh + $ (cd web && VITE_REOWN_PROJECT_ID=… VITE_VAULT_ADDRESS= VITE_TIMELOCK_ADDRESS= bash scripts/bundle_gandi.sh) + ``` + The bundle lands in `build/gandi/` at the repo root: `wsgi.py`, `site-headers.json`, `relay/` and `site/`. + `owner.html` is included; it is not linked anywhere, it is noindex, and it can do nothing without your owner wallet. +2. Upload the contents of `build/gandi/` over Gandi SFTP (or git) into the instance's deploy directory (where + `wsgi.py` lives), replacing `site/` entirely. +3. Make the relay secret: `$ python3 -c "import secrets;print(secrets.token_hex(32))"`. The fly gets the same value as + `RELAY_SECRET`. +4. Upload `relay.json` next to `wsgi.py` (never into `site/`, never into git). The database goes on the instance's + persistent disk, outside the deploy directory: + ```json + {"keys": {"k1": ""}, "db_path": "/srv/data/home/relay/relay.sqlite3", "domain": "fly-trader.app", + "uri": "https://fly-trader.app/vault.html", "chain_id": 4663, "sol_chain": "mainnet", "claim_ttl_s": 900, + "min_lamports": 2000000, "client_ip_header": null} ``` -2. Upload the contents of `build/gandi/` so they replace `site/` entirely. -3. Put `relay.json` next to `wsgi.py`, using the same secret as `RELAY_SECRET`, then restart the instance. -4. Probe the relay from the Mac: + Restart the instance. If a later deploy replaces the directory, upload `relay.json` again. +5. Probe the relay and work through the 7-point checklist in `web/relay/README.md` (writable, WAL, client IP header, + no caching, clock): + `$ FLY_RELAY_SECRET= python3 web/relay/hmacauth.py https://fly-trader.app/api/fly/probe k1` + +## 4. Server (a dedicated machine at home, Ubuntu Server 24.04) + +### 4.1 Publish the first release +1. The distribution worktree must hold exactly what you want to ship. Code is ported there from master by hand (or with + a merge); publishing does not port it. Check it is clean: `$ git -C ../fly-trader-dist status`. +2. `--models` refreshes the models from this Mac; use it for the first release. That needs: + - a deployable fly (`fly-trader train-fly-selector`) that passed `fly-trader fly-replay` + - a deployed selector (pinned, or the newest that qualifies) + - fitted wallet skill (`fly-trader fit-wallet-skill`) with its tables, and a built connectome + Without `--models` the release carries the models already in the worktree, and publishing refuses if the + wallet-skill table named in `models/wallet_skill/TABLE` is not there. +3. Publish. `SSH_AUTH_SOCK` points `ssh-keygen` at Secretive, so Touch ID confirms the signature: ``` - FLY_RELAY_SECRET= python web/relay/hmacauth.py https://fly-trader.app/api/fly/probe k1 + $ SSH_AUTH_SOCK=$SECRETIVE .venv/bin/python tools/publish_release.py --worktree ../fly-trader-dist --key ~/.ssh/fly-release.pub --models ``` - Check the output against the checklist in `web/relay/README.md`. The things that must hold: the database path is - writable, WAL works, and nothing caches `/api`. + It runs the tests, commits, signs, verifies with the server's own verifier, pushes the `distribution` branch and + uploads one Hugging Face commit. It refuses unless the key is in the worktree's `release/allowed_signers` (0.4 step + 5). Note the printed **manifest sha256**; it pins this first release on the server. +4. **Do not publish again until the server has installed this release** (section 4.4). A new server accepts only the + pinned manifest, and it permanently refuses any other head it sees first. + +### 4.2 The machine and its bootstrap +A dedicated machine at home (the reference is a GMKtec EVO-X3: Ryzen AI Max+ 395, 128 GB, 2 TB NVMe, 2.5 GbE). It +runs nothing but the vault fly. The fly runs on its CPU; the Radeon GPU is not used (no GPU driver stack in reach of +the keys). + +1. **Firmware** (press Del or F7 at power-on for the setup screen): + - Update the BIOS from the vendor's support site first. + - Set a supervisor (admin) password. + - Turn on the firmware TPM (AMD fTPM / "Security Device Support"), Secure Boot (standard keys) and the IOMMU + (AMD-Vi / "IOMMU: Enabled"). + - Turn off Wi-Fi and Bluetooth if the menu offers it (the bootstrap also blocks their drivers). + - "Restore on AC power loss": Power On. + - Boot order: the internal disk first. After installing (step 2), turn off booting from USB. +2. **A clean Ubuntu Server 24.04 LTS install** from a USB stick made from ubuntu.com's image. Erase the preinstalled + Windows; some mini-PC vendors have shipped malware in theirs. + - Storage: "Use an entire disk", "Set up this disk as an LVM group", **"Encrypt the LVM group with LUKS"**. Keep + the passphrase in your password manager. + - Profile: server name `fly-vault`, user **`fly-admin`**, any password (the bootstrap locks it). + - SSH: "Install OpenSSH server", with password authentication (only until the bootstrap). + - No featured snaps. +3. **Network:** plug it into the router by cable, on its own VLAN or guest port if the router can. Forward no ports to + it, and turn off UPnP on the router. Put it on a UPS: during a power or internet outage the fly cannot sell, + refill or pay, and healthchecks.io tells you it went quiet. +4. From the Mac, on the same network (the machine's address is on its console, `ip -4 addr`): + ``` + $ ssh-copy-id -o IdentityAgent=$SECRETIVE -f -i ~/.ssh/fly-ssh.pub fly-admin@ + $ scp -o IdentityAgent=$SECRETIVE -r ../fly-trader-dist/deploy ~/.ssh/fly-release.pub ~/fly-keys/fly-release-recovery.pub fly-admin@: + $ ssh -o IdentityAgent=$SECRETIVE fly-admin@ + ``` +5. The first pass. `TPM_UNLOCK=1` binds the disk encryption to the TPM, so security-update reboots come back up on + their own; it asks once for the disk passphrase. Without it, every boot waits for the passphrase at the console, + and automatic reboots are switched off. + ``` + server$ sudo TS_AUTHKEY= RECOVERY_PUBKEY="$(cat fly-release-recovery.pub)" \ + EXPECT_MANIFEST= TPM_UNLOCK=1 ADMIN_USER=fly-admin bash deploy/bootstrap-host.sh "$(cat fly-release.pub)" + ``` + It hardens the box (roughly CIS level 1): + - **The machine:** Wi-Fi and Bluetooth drivers can never load; it warns if the IOMMU is off or the disk is not + encrypted. + - **SSH:** only `fly-admin`, key only (ed25519, P-256, FIDO), root off. + - **Firewall:** in and out; containers reach only HTTPS, HTTP (apt) and DNS. + - **Kernel and Docker:** user namespaces, no new privileges. + - **Monitoring:** fail2ban; an audit log of every read of the wallet keys, with a Telegram alert when anything but + the signer reads them; a daily AIDE file-integrity check (05:30 UTC) that reports to Telegram when files changed; + automatic security updates (reboot 04:00 UTC when needed and the TPM unlocks the disk). + - **Keys:** it makes the trading and payout keys and prints them, **once**, as `bot_key:
` (the trading + key) and `payout_key:
`. Copy both lines now. Later, once the containers run: + `server$ sudo fly signer status` (as `"trading"` and `"payout"`). +6. **Keep this session open.** From a second terminal on the Mac, log in over Tailscale: `$ ssh fly-vault`. Only when + that works, run the second pass in the first session. It removes SSH from the local network, leaving Tailscale as + the only way in: + ``` + server$ sudo CLOSE_PUBLIC_SSH=1 bash deploy/bootstrap-host.sh "$(cat fly-release.pub)" + ``` + The recovery key already installed is kept. After this the machine has no open port except SSH over Tailscale. +7. Physical access is now the way around every other control: keep the machine somewhere only you reach. The disk is + useless without the TPM on this board (or the passphrase), and the firmware password and USB-boot-off stop anyone + booting another system on it. Unbinding the TPM (a new board, a firmware reset) means typing the passphrase at the + console once and re-running step 5 with `TPM_UNLOCK=1`. + +### 4.3 The treasury (your phone) +1. Open `https://fly-trader.app/owner.html` in Solflare's browser and connect Solflare. +2. Paste the trading key (`bot_key`) and the payout key (`payout_key`), leave the multisig field empty, and tap + "Read the treasury". +3. Tap "Create the treasury multisig": you are its only member, threshold 1, no time lock. +4. Tap "Create L1" (2 SOL a day, to the trading wallet only) and "Create L2" (1 SOL a week, claims). +5. Every check the page shows must pass. It prints the three lines for `custody.env`. **Save the multisig, L1 and L2 + addresses in your password manager.** The page remembers them only in that browser, and the Revoke button needs + them. + +### 4.4 Settings, start, check +1. Fill in the settings with `sudoedit`: `server$ sudoedit /srv/fly/custody.env`, and so on. The files are root-only + (mode 600). + - **`/srv/fly/custody.env`:** the three lines from the owner page (`VAULT_MULTISIG`, `VAULT_LIMIT_TRADING`, + `VAULT_LIMIT_PAYOUT`) and `VAULT_BACKUP_RECIPIENT` (your `age1…`). The float and caps have working defaults. + - **`/srv/fly/signer.env`:** `SIGNER_RPC_URL` (the second provider). Required: the signer does not start without it. + - **`/srv/fly/vault.env`:** + - `HELIUS_API_KEY`, `JUPITER_API_KEY`, `SOLANA_CHECK_RPC_URL` + - `CAPITAL_SOL` (your principal, e.g. 5) + - `FUNDING_ADDRESSES`: the addresses you will send principal from, comma-separated; at least your Solflare address + - `VAULT_ADDRESS`, `VAULT_TIMELOCK`, `VAULT_START_BLOCK` (`blockNumber`), `VAULT_IMPL_CODEHASHES`, all from step 2 + - `RELAY_SECRET` (the relay's secret from step 3) + - `TELEGRAM_BOT_TOKEN`, `TELEGRAM_CHAT_ID`, `TELEGRAM_ADMIN_USER_ID`, `HEALTHCHECK_URL` + - `VAULT_BACKUP_BUCKET`, `VAULT_BACKUP_HF_TOKEN` (the bucket-only token) + - **`/etc/fly/update.json`:** `telegram_bot_token` and `telegram_chat_id` (the host updater, the key-read watcher and + AIDE report through these). `expect_manifest_sha256` was set by the bootstrap. + + Everything must be filled in before the next step: a first release that does not come up healthy is refused for + good. If that happens: fix the cause, publish a new release, and put its manifest sha256 into + `expect_manifest_sha256` in `/etc/fly/update.json`. +2. Start the updater. It installs the pinned release, builds the brain and signer images and starts them (the first + build takes several minutes): + ``` + server$ sudo systemctl enable --now fly-update.timer + server$ sudo systemctl start fly-update.service # the first round now, instead of in 2 minutes + server$ sudo journalctl -u fly-update -f + ``` +3. Check everything: + - `server$ sudo /usr/local/lib/fly/check-host.sh`: every control must pass (keys only in the signer, Tailnet Lock on, + no public SSH…) + - `server$ sudo fly vault custody`: no problems + - `server$ sudo python3 /usr/local/lib/fly/fly_audit_watch.py test`: the Telegram test message arrives +4. The console: `$ ssh -L 8501:127.0.0.1:8501 fly-vault`, then http://localhost:8501/vault. ## 5. Go live -1. Send 5 SOL to the vault wallet from an address listed in `FUNDING_ADDRESSES`. -2. Check the deposit was counted: `fly-trader vault status` (inside the container) should show `deposits: 5000000000`. - If it came from another address, run `fly-trader vault reclassify deposit`. -3. The fly trades live after the handover: at least 20 paper trades. -4. Settlements run every Monday at 00:00 UTC. Claims open from the first one. +1. From a funding address, send **0.05 SOL to the trading wallet** (the `bot_key` address). It pays the fees of its own + refills and of claims. +2. Send your principal to the **treasury** (the vault 0 address on the owner page) from a funding address. The fly + refills its 2 SOL float from it through L1 on its own. +3. `server$ sudo fly vault status`: `"ledger": {"deposits": …}` must equal the SOL you sent from funding addresses. If + some came from another address, it counts as profit: `server$ sudo fly vault reclassify deposit`. +4. The fly trades live after the handover (at least 20 paper trades). Settlements run every Monday 00:00 UTC; claims + open from the first one. +5. When claims are larger than L2 allows in a week, Telegram says a claim is waiting. Raise L2 as in "Day to day". + Waiting claims never fail. + +## Emergency +- **Something is wrong (possible compromise):** + 1. On your phone, open the owner page and tap **Revoke L1 and L2 now**. From then on the server cannot move any SOL + out of the treasury; trading keeps only its float. + 2. Send `/panic` to the bot. The fly halts; the signer refuses buys, top-ups and claims; open positions are sold; and + the float goes back to the treasury, all but the 0.3 SOL gas reserve. + 3. If $FLY locking should stop too: MetaMask, Blockscout, FlyVault `pause()`. +- **A key-read, halt, AIDE or setup-change alert:** look before resuming. Resuming is SSH-only: + ``` + server$ sudo fly vault resume + server$ sudo fly reset-circuit --kill + server$ sudo fly resume-entries + server$ sudo fly signer clear-panic + ``` +- **Rebuild after losing the server:** + 1. On the owner page, revoke L1 and L2 of the old keys. + 2. Publish nothing; take the manifest sha256 of the current release: + `$ curl -sL https://huggingface.co/bryceweiner/fly-trader/resolve/main/release.json | shasum -a 256`. + 3. New server: sections 4.2 (with that pin) to 4.4. On the owner page, clear the old L1/L2 fields and create new + limits for the new keys; put the new addresses in `custody.env`. + 4. Restore the ledger. Download the newest `ledger/.dump.age` from the backup bucket, then: + ``` + $ age -d -i ~/fly-keys/fly-backup.key .dump.age > ledger.dump + $ scp ledger.dump fly-vault: && rm ledger.dump + server$ sudo docker stop fly-fly-1 # the brain must not write while the tables are replaced + server$ sudo docker exec -i fly-db-1 pg_restore -U fly -d fly_trader --clean < ledger.dump && rm ledger.dump + server$ sudo docker exec fly-db-1 psql -U fly -d fly_trader -c "DELETE FROM vault_kv WHERE key IN ('backup_key_done','custody','halt','panic')" + server$ sudo python3 /usr/local/lib/fly/fly_update.py restart + ``` + 5. The old trading wallet's float and positions: `$ age -d -i ~/fly-keys/fly-backup.key keys/trading-
.age` + gives its secret (base58, 64 bytes), which you can import into a wallet to move them to the treasury. ## Day to day -- **New models or code:** run `publish_release.py`, with `--models` to refresh the models. The server applies the - release within 5 minutes. A release that changes Docker or compose files waits until you run - `sudo python3 /usr/local/lib/fly/fly_update.py approve ` on the server. -- **Take principal out:** `docker compose -p fly -f /srv/fly/docker-compose.yml exec fly fly-trader vault withdraw --sol X --to `. -- **A halt alert** (unknown transaction or anomaly): - 1. Find the cause on the console's `$FLY vault` page. - 2. Run `fly-trader vault resume`. - 3. Run `fly-trader resume-entries`. -- **Restore from backups:** - - Key: `age -d -i ~/.ssh/fly_release keys/.age` - - Ledger: `age -d … ledger/.dump.age | pg_restore` +- **Changing a limit** (e.g. raising L2 after a big settlement): on the owner page, "Set L2 to this amount". A new + limit has a **new address**, so put it in `/srv/fly/custody.env` and restart: + `server$ sudo python3 /usr/local/lib/fly/fly_update.py restart`. Until then claims wait. The same applies to any L1 + change. +- **After editing any `/srv/fly/*.env`:** `server$ sudo python3 /usr/local/lib/fly/fly_update.py restart`. +- **New models or code:** publish as in 4.1 step 3 (`--models` to refresh the models). + - Model-only releases apply within 5 minutes; they cannot carry code (weights-only torch files, skops selectors). + - A release that changes **code** is held 24 h, and Telegram says so. To apply it now: + `server$ sudo python3 /usr/local/lib/fly/fly_update.py approve `. To stop it: + `server$ sudo python3 /usr/local/lib/fly/fly_update.py veto `. If you did not publish it, the release key is + compromised: remove it from `/etc/fly/allowed_signers` and rotate. + - A release that changes `deploy/`, `docker/`, the Dockerfile, `.dockerignore` or a compose file waits for + `approve`. It replaces only the compose file. For changed host scripts (bootstrap, `fly_update.py`, + `check-host.sh`, the watchers), copy them over and re-run the bootstrap **with `CLOSE_PUBLIC_SSH=1`** (without it + the bootstrap opens public SSH again); your recovery key is kept: + ``` + $ scp -r ../fly-trader-dist/deploy ~/.ssh/fly-release.pub fly-vault: + server$ sudo CLOSE_PUBLIC_SSH=1 bash deploy/bootstrap-host.sh "$(cat fly-release.pub)" + ``` + - To sign with the recovery key: decrypt it (`$ age -d ~/fly-keys/fly-release-recovery.age > /tmp/k && chmod 600 + /tmp/k`), pass `--key /tmp/k`, then `rm /tmp/k`. +- **Take principal out:** `server$ sudo fly vault withdraw` prints how much you may take and how. You send it from the + treasury in the Squads app with your owner wallet; the fly books and reports it. +- **Watch it:** `/status` to the bot at any time. A daily report arrives at 09:00 UTC. If it does not, or + healthchecks.io alerts, the server or the fly is down. `server$ sudo fly vault status` gives the full ledger. +- **After an AIDE alert** you expected (a release, apt upgrades): `server$ sudo aide --update` then + `server$ sudo cp /var/lib/aide/aide.db.new /var/lib/aide/aide.db`. +- **Backups:** hot keys once, ledger nightly, both age-encrypted in the bucket. Until `VAULT_BACKUP_*` and the recipient + are set, Telegram reminds you daily. To read them: + - Hot keys: `$ age -d -i ~/fly-keys/fly-backup.key keys/-
.age` + - Ledger: see "Rebuild", step 4. + +## Ledger (when you're home): no redeploys +- **Solana:** in the Squads app, add the Ledger as a member, then remove Solflare (or keep both at threshold 2; the + owner page then proposes and approves, and the other member executes in the Squads app). L1 and L2 stay as they are. +- **EVM:** move MetaMask's roles to the Ledger through the timelock (8 days): + 1. Print the calldata: + ``` + $ (cd contracts && VAULT= TIMELOCK= OLD_OWNER= NEW_OWNER= MODE=calldata \ + forge script script/HandOver.s.sol --rpc-url robinhood) + ``` + 2. Send the printed "schedule" data from MetaMask to the timelock (Blockscout "Write contract", `scheduleBatch`, or + any raw-data send). + 3. After 8 days, send the "execute" data from any wallet. +- **SSH and releases:** add the Ledger's SSH key (ledger-agent) to `~fly-admin/.ssh/authorized_keys`. For releases, + make the Ledger's key the release key: re-run the bootstrap (as in "Day to day") with the Ledger's public key as the + argument (the bootstrap rewrites `/etc/fly/allowed_signers` from its arguments; a line added by hand would be lost), + and replace the `bryce` line in the distribution's `release/allowed_signers` (publish that change while the old key + still signs). Then remove the Secretive keys. diff --git a/docs/vault/SPEC.md b/docs/vault/SPEC.md index 9711e276de65c1156139190536edb2ae2cbdf25e..5e4f19887e392d84e523eeda6ebb8f48121646e6 100644 --- a/docs/vault/SPEC.md +++ b/docs/vault/SPEC.md @@ -1,11 +1,12 @@ # $FLY vault — interface spec v1 -The contract between the four parts of the vault system. Change it here first, then in code. +The contract between the parts of the vault system. Change it here first, then in code. - **FlyVault** (`contracts/`, Robinhood Chain): holds locked $FLY. - **The vault fly** (`fly_trader/vault/`, the hosted server): trades, keeps the ledger, settles weekly and pays claims. +- **The signer** (`fly_trader/signer/`, its own container on the server): the only holder of the two hot keys (§6). - **The relay** (`web/relay/`, on the site's Gandi app): passes stats and claims between the fly and browsers. -- **The site** (`web/`): the Trading and Vault pages. +- **The site** (`web/`): the Trading, Vault and (unlisted) Owner pages. Units, everywhere: - SOL amounts are integer **lamports**. @@ -19,18 +20,18 @@ Solidity `0.8.24+` on OpenZeppelin Contracts Upgradeable v5, using UUPS and ERC- (`fly.storage.FlyVault`). The constructor calls `_disableInitializers()`. ```solidity -uint256 public immutable WITHDRAW_DELAY; // constructor arg of the implementation: 7 days on mainnet (Deploy.s.sol refuses - // anything else on chain 4663); shorter on the testnet rehearsal. Changing it = an upgrade. +uint256 public immutable WITHDRAW_DELAY; // constructor arg of the implementation: 7 days on mainnet, 300 s by default + // on the rehearsal (DeployTestnet.s.sol). Changing it = an upgrade. bytes32 public constant PAUSER_ROLE = keccak256("PAUSER_ROLE"); bytes32 public constant UPGRADER_ROLE = keccak256("UPGRADER_ROLE"); -function initialize(address fly, address pauser, address timelock) external initializer; +function initialize(address fly, address pauser, address timelock) external initializer; // reverts on a zero address // DEFAULT_ADMIN_ROLE + UPGRADER_ROLE -> timelock ; PAUSER_ROLE -> pauser -function lock(uint256 amount) external; // whenNotPaused; reverts unless received == amount +function lock(uint256 amount) external; // whenNotPaused, nonReentrant; reverts unless received == amount function requestWithdrawal(uint256 amount) external returns (uint256 id); // never paused function cancelRequest(uint256 id) external; // whenNotPaused (it re-locks) -function withdraw(uint256 id) external; // never paused; block.timestamp >= readyAt +function withdraw(uint256 id) external; // never paused, nonReentrant; block.timestamp >= readyAt function pause() external; function unpause() external; // PAUSER_ROLE function token() external view returns (address); @@ -47,16 +48,29 @@ event WithdrawRequested(address indexed user, uint256 indexed id, uint256 amount event RequestCancelled(address indexed user, uint256 indexed id, uint256 amount, uint256 lockedAfter); event Withdrawn(address indexed user, uint256 indexed id, uint256 amount); // plus OZ: Paused(address), Unpaused(address), Upgraded(address indexed implementation), RoleGranted/RoleRevoked + +error ZeroAddress(); error ZeroAmount(); error TransferAmountMismatch(uint256 expected, uint256 received); +error InsufficientLocked(uint256 requested, uint256 available); error UnknownRequest(uint256 id); +error NotRequestOwner(uint256 id); error RequestNotPending(uint256 id); error WithdrawalNotReady(uint256 id, uint64 readyAt); ``` - **Request ids** start at 1 and are global. - **`lockedAfter`** is the user's earning balance after the event, so the indexer can resync to an absolute value. -- **Timelock.** Upgrades go through an OZ `TimelockController`: - - minDelay = 8 days - - proposer = canceller = the operator's hardware account +- **Timelock.** Upgrades and role changes go through an OZ `TimelockController`: + - minDelay = 8 days on mainnet (600 s by default on the rehearsal) + - proposer = canceller = the owner (MetaMask until the Ledger takes over; `OWNER` in the deploy) - executor = anyone (`address(0)`) - - admin = `address(0)` + - admin = `address(0)`: the timelock administers itself, so role changes also wait the full delay +- **Mainnet deploy rules.** On chain 4663 the deploy (`FlyVaultStack.enforceChainRules`) refuses anything but the real + $FLY, an 8-day timelock and a 7-day withdraw delay. `PAUSER` defaults to `OWNER`. - **No rescue function for FLY.** The token is set only in `initialize`. +- **The fly indexes** the vault's events and the timelock's `CallScheduled`, `CallExecuted` and `Cancelled` for + operations aimed at the vault or the timelock (an alert and `vault.upgrade_scheduled`, §4.1). +- **Scripts.** + - `HandOver.s.sol`: one timelock batch that grants PROPOSER, CANCELLER and PAUSER to a new owner and revokes them + from the old one. `MODE=schedule|execute`, or `MODE=calldata` to print the calldata for a browser wallet. + - `ScheduleUpgrade.s.sol` rehearses an upgrade locally and schedules `upgradeToAndCall`; `ExecuteUpgrade.s.sol` runs + it after the delay. ## 2. Claim messages (v1) @@ -66,13 +80,15 @@ newline. Vectors: `tests/vectors/claim_v1.json`. Python (`fly_trader/vault/claim Fields: - `domain`: e.g. `fly-trader.app`, or `localhost:5173` on staging -- `uri`: e.g. `https://fly-trader.app/vault.html` +- `uri`: e.g. `https://fly-trader.app/vault.html`, or `http://localhost:5173/vault.html` on staging; https (http only + for localhost/127.0.0.1), no spaces or line breaks - `chain_id`: `4663`, or `46630` on testnet - `sol_chain`: `mainnet` or `devnet` - `evm`: EIP-55 checksummed - `sol`: base58 pubkey - `nonce`: 32 lowercase hex characters, issued by the relay -- `issued_at`, `expires_at`: RFC 3339 `YYYY-MM-DDTHH:MM:SSZ`; expiry = issue + 900 s +- `issued_at`, `expires_at`: RFC 3339 `YYYY-MM-DDTHH:MM:SSZ`; expiry = issue + 900 s. The fly accepts no other TTL, so + the relay's `claim_ttl_s` must stay 900. - `request_id`: always `fly-vault-claim-v1` EVM text, signed with `personal_sign` (EIP-191); the signature is `0x…` hex, 65 bytes for an EOA and up to 1024 bytes @@ -108,60 +124,95 @@ Request ID: fly-vault-claim-v1 ``` What the fly checks: -- **The texts.** It re-renders both texts from the fields and never uses text supplied by the relay. +- **The texts.** Both are re-rendered from the fields; relay-supplied text is never used. `domain`/`uri` must equal + `VAULT_SITE_DOMAIN`/`VAULT_SITE_URI`, `chain_id` must equal `RH_CHAIN_ID`, and `sol_chain` must match `VAULT_CLUSTER`. +- **Freshness.** A claim is rejected if the fly first stored it more than 3600 s after `expires_at`, or if `issued_at` + is more than 300 s in the future. The relay already refuses expired nonces at submission. - **The EVM signature.** - - Low-s, and v ∈ {27, 28} (0/1 are normalized). - - Recovered by ecrecover. If the address has code, it instead calls EIP-1271 `isValidSignature(hashMessage(text), sig) == 0x1626ba7e`. -- **The Solana signature.** ed25519 over the text. -- **The nonce.** Unique per EVM address and not expired. -- **The amount.** Owed ≥ `CLAIM_MIN_LAMPORTS` (2,000,000). - -A valid claim pays **all** SOL owed to the EVM address, sent to `sol`. + - ecrecover first: exactly 65 bytes, 0 < r < n, low-s, v ∈ {27, 28} (0/1 are normalized). + - Only if that does not recover the address and the address has code: EIP-1271 `isValidSignature(hashMessage(text), sig)` + (65..1024 bytes, 200,000 gas cap) must return exactly `0x1626ba7e`. +- **The Solana signature.** 64 bytes, ed25519 over the UTF-8 text; `sol` must be an on-curve pubkey. +- **The nonce.** Unique per EVM address. +- **One in flight.** A claim is rejected while another claim for the address is `verified | waiting_liquidity | + sending`. +- **The amount.** Owed ≥ `CLAIM_MIN_LAMPORTS` (2,000,000), read when the claim moves to `sending`; the amount is fixed + then. The relay's `min_lamports` and the site's `claimMinLamports` must hold the same value. +- **Halts.** Nothing is verified or paid while the vault is halted (claims are still pulled from the relay). + +A valid claim pays **all** SOL owed to the EVM address, sent to `sol` from the **treasury** (Squads vault 0) under +spending limit L2 (§6): the payout key signs `spendingLimitUse` and the trading key pays the fee. Nothing is ever paid +from the trading wallet's own SOL. Neither the treasury, the trading wallet nor any payout signature is published. + +**Statuses:** `received → verified → (waiting_liquidity) → sending → paid | rejected | failed`. +- `waiting_liquidity`: L2 has too little left this week, the treasury is short, or the signer is in panic. The claim + retries every loop and alerts the operator; it never fails for these reasons. +- `failed`: four unlanded attempts, a failure on chain, or a signer policy refusal. +- The payment's signature and last valid block height are stored before broadcast; a claim is re-signed only once its + blockhash has expired unlanded, and the signer keeps its own once-per-claim ledger (§6). ## 3. Relay HTTP API (v1) The base is `{origin}/api`. Every response is JSON with `Cache-Control: no-store`. Errors look like `{"error": ""}` -with status 400, 401, 404, 409, 413, 429 or 503. +with status 400, 401, 404, 405 (with `Allow`), 409, 413, 429 (with `Retry-After`), 500 or 503. 503 means either +`relay not configured` (relay.json missing or invalid) or `storage unavailable` (SQLite error, with `Retry-After: 5`). ### Public - **`GET /api/stats`** returns the latest snapshot (§4), or 503 before the first push. - **`GET /api/history?kind=nav|trades|flows|settlements&before=&limit=<1..500, default 100>`** returns - `{"kind", "items": [...], "next": }`, newest first. The cursor is opaque; send it back as `before`. -- **`GET /api/account?evm=0x…`** returns the account (§4.3). An unknown address gets all zeros. + `{"kind", "items": [...], "next": }`, newest first by (time, id). The time is nav `ts`, trades + `closed_at`, flows `ts`, settlements `period_end`. The cursor is opaque; send it back as `before`. An unknown `kind`, + a bad `limit` or a bad cursor gets 400. +- **`GET /api/account?evm=0x…`** returns the account (§4.3). `evm` is accepted in any case. An unknown address gets all + zeros; the fly pushes only addresses with at least one allocation. - **`GET /api/claim/challenge?evm=0x…&sol=`**: - Returns `{"nonce", "issued_at", "expires_at", "domain", "uri", "chain_id", "sol_chain", "request_id", "min_lamports"}`. - - The relay stores the challenge and checks the syntax of `evm` and `sol`. + - The relay stores the challenge; a bad `evm` or `sol` gets 400 (checked before the rate limit). - **`POST /api/claim`** with body `{"nonce", "evm", "sol", "evm_sig", "sol_sig"}` (≤ 4 KB): - Returns `202 {"id", "status": "received"}`. - Checks, all cheap and none cryptographic: - the nonce is known, unexpired, unused, and issued for this `evm` + `sol` - the signatures have valid syntax: `evm_sig` matches `^0x([0-9a-fA-F]{2}){65,1024}$`; `sol_sig` is base58 of 64 bytes - - the published account's owed ≥ min - - no claim for this EVM address is in flight - - 409 if the nonce was already used or a claim is in flight. -- **`GET /api/claim/`** returns `{"id", "status", "reason", "lamports", "tx", "created_at", "updated_at"}`. + - the published account's owed ≥ `min_lamports` + - no claim for this EVM address that the fly has verified is in flight (`verified | waiting_liquidity | + sending`). An unverified `received` claim does not block, since it may be a stranger's junk; the fly pays + each address at most once whatever the relay accepts. + - 400 for bad syntax, an unknown, expired or mismatched nonce, or nothing to claim; 409 if the nonce was already used + or a verified claim is in flight; 429 per the limits below. +- **`GET /api/claim/`** (the relay's id, 1-18 digits) returns `{"id", "status", "reason", "lamports", "tx", + "created_at", "updated_at"}`, or 404. - Status is one of `received | verified | waiting_liquidity | sending | paid | rejected | failed`. -- **Rate limits (token buckets):** + - `tx` is always null: the fly never reports the payout signature. +- **Rate limits (token buckets; a 429 carries `Retry-After`):** - challenge: 30 per hour per IP - - claim: 10 per hour per IP and 5 per hour per EVM address - - reads: 600 per hour per IP. + - claim: 10 per hour per IP, charged after the syntax checks; and 5 per hour per EVM address and IP, charged only + after every other check passes (so junk from one client cannot lock a holder out) + - reads (stats, history, account, claim/): 600 per hour per IP + - IPv6 clients are bucketed per /64. Behind a proxy, `client_ip_header` names the header to read, and the relay takes + its rightmost entry (the one the proxy appended). The fly's HMAC routes are not rate-limited. ### Fly only (HMAC) - **`POST /api/fly/push`**: upserts. - Body: `{"stats": {…§4…}?, "history": {"nav": [...], "trades": [...], "flows": [...], "settlements": [...]}?, "accounts": [ …§4.3… ]?}`. - - History items upsert by `id` (nav by `ts`); accounts upsert by `evm`. + - Unknown top-level keys get 400. History items upsert by `id` (nav by `ts`); each needs an integer time field (nav + `ts`; trades `closed_at` or `opened_at`; settlements `period_end` or `period_start`; flows `ts`) and, except nav, an + `id` (an integer ≥ 0 or a string ≤ 200 characters). Accounts upsert by `evm` and need an integer `owed`. `stats` is + stored as is; NaN and Infinity become null. - Returns `{"ok": true}`. Body ≤ 2 MB. -- **`GET /api/fly/claims?after=&limit=<1..50>`**: +- **`GET /api/fly/claims?after=&limit=<1..50, default 50>`**: - Returns `{"claims": [{"id", "nonce", "evm", "sol", "evm_sig", "sol_sig", "domain", "uri", "chain_id", "sol_chain", "issued_at", "expires_at", "created_at"}]}`. - - Only claims in status `received`, in id order. -- **`POST /api/fly/claims/result`**: body `{"results": [{"id", "status", "reason"?, "lamports"?, "tx"?}]}` returns `{"ok": true}`. -- **`GET /api/fly/probe`** returns `{"python", "db_path", "writable", "wal", "pid", "remote_addr", "headers": {...}}`. + - Only claims in status `received`, in id order. `evm` is lowercase; `created_at` is unix seconds. +- **`POST /api/fly/claims/result`**: body `{"results": [{"id", "status", "reason"?, "lamports"?, "tx"?}]}` returns + `{"ok": true}`. `status` must be one of the seven statuses; `reason` is cut to 500 characters; omitted fields keep + their value; unknown ids are ignored. +- **`GET /api/fly/probe`** returns `{"python", "db_path", "writable", "wal", "pid", "remote_addr", "headers": {...}, + "sqlite", "time", "client_ip", "multiprocess", "multithread"}` (headers: every `HTTP_*` except Cookie). HMAC headers: - `X-Fly-Key`: the key id -- `X-Fly-Ts`: unix seconds +- `X-Fly-Ts`: unix seconds (1-15 digits) - `X-Fly-Nonce`: 32 hex characters - `X-Fly-Sig`: hex HMAC-SHA256(key = the UTF-8 bytes of the configured secret string, canonical) @@ -169,12 +220,15 @@ The canonical string: ``` FLY-RELAY-1\n{METHOD}\n{PATH}\n{QUERY}\n{TS}\n{NONCE}\n{sha256hex(body)} ``` -- `PATH` excludes the query string. +- `PATH` starts at `/api/…` and excludes the query string. - `QUERY` is `&`.join(`quote(k,safe='')=quote(v,safe='')` for (k, v) in sorted(parse_qsl(qs, keep_blank_values=True))). - The body is the raw bytes; an empty body hashes as sha256 of `b""`. -- The timestamp must be within ±300 s. Nonces are remembered for 15 minutes, and a reused nonce gets 401. +- The timestamp must be within ±300 s. Nonces are remembered for 15 minutes (case-insensitively), and a reused nonce + gets 401. The body is verified before the nonce is recorded. -Relay config: `relay.json`, kept next to `wsgi.py` and never inside `site/`: +Relay config: `relay.json`, kept next to `wsgi.py` and never inside `site/`, or wherever `FLY_RELAY_CONFIG` points. It +is re-read when it changes; a relative `db_path` is resolved against its folder. Secrets must be at least 32 characters +(64 hex recommended). ```json {"keys": {"k1": "<64 hex>"}, "db_path": "", "domain": "fly-trader.app", "uri": "https://fly-trader.app/vault.html", "chain_id": 4663, "sol_chain": "mainnet", "claim_ttl_s": 900, "min_lamports": 2000000, "client_ip_header": null} @@ -182,33 +236,56 @@ Relay config: `relay.json`, kept next to `wsgi.py` and never inside `site/`: ## 4. Payloads -### 4.1 Stats snapshot (`stats`, pushed every 60 s) +### 4.1 Stats snapshot (`stats`, pushed every vault-worker loop, about every 10 s) ```json { - "v": 1, "ts": 0, "cluster": "mainnet", - "fly": {"state": "starting|paper|live|halted", "wallet": "", "handover": false, "kill_switch": false, + "v": 1, "ts": 0, "book": "live", "cluster": "mainnet", + "fly": {"state": "starting|paper|live|halted", "handover": false, "kill_switch": false, "entries_paused": false, "model": {"fly": 62, "selector": 59, "release": 3}}, - "wallet": {"native": 0, "token_accounts": 0, "open_cost": 0, "positions_value": 0, "exit_cost": 0, "nav": 0}, + "wallet": {"native": 0, "nav": 0}, "ledger": {"deposits": 0, "withdrawals": 0, "claims_paid": 0, "realized": 0, "booked_realized": 0, "allocated": 0, "reserved": 0, "pot": 0}, "prices": {"sol_usd": 0.0, "fly_usd": 0.0, "ts": 0}, "vault": {"address": "0x…", "chain_id": 4663, "total_locked": "0", "total_pending": "0", "earners": 0, "finalized_block": 0, "paused": false, "impl": "0x…", "upgrade_scheduled": null}, - "settlement": {"next_at": 0, "last": null}, - "positions": [{"mint": "", "symbol": "", "opened_at": 0, "cost": 0, "value": 0, "entry_price": 0.0, "mark_price": 0.0, "hold_min": 0}], - "index": {"value": 1.0, "peak": 1.0, "drawdown": 0.0} + "settlement": {"last": null} } ``` -- `wallet.nav` = native + token_accounts + positions_value − exit_cost. -- `ledger.realized` is R. `pot` = max(0, R − allocated). `reserved` = allocated − claims_paid. -- `settlement.last` is a settlements item (§4.2) or null. -- `vault.upgrade_scheduled` is `{"eta": ts, "id": "0x…"}` or null. +- **Nothing that helps trade ahead of the fly.** `ts` is rounded down to the hour; `wallet.native` and `wallet.nav` + come from the last 5-minute mark and are rounded to 0.1 SOL (NAV history points likewise), so one buy cannot be + matched to an on-chain trade; `ledger.realized` is R at that same mark. Open positions, the performance index and + the next settlement time are not published; a trade appears in `trades` history once it closes. No address of the + fly's (trading wallet, treasury, keys) is published, nor anything that leads to one: no transaction signatures and no + deposit senders. +- `book` is `VAULT_BOOK` (`live`, or a paper book such as `paper_fly` on a dry run). `cluster` is `mainnet` or + `VAULT_CLUSTER` as is. +- `fly.state`: `halted` if the vault is halted or the kill switch is on; `live` after the handover once the live wallet + reports; `paper` once the fly reports status; else `starting`. `fly.model` = `{fly, selector, release}`, each may be + null. +- `wallet.native` = trading wallet + treasury lamports; `wallet.nav` = native + open positions' value − exit cost. +- `ledger`: `realized` is R (§7); `pot` = max(0, R − allocated); `reserved` = max(0, allocated − claims_paid); + `booked_realized` is the book's summed closed-trade profit; `deposits`, `withdrawals` (fees included) and + `claims_paid` are running totals. On a paper book `realized` is closed-trade profit since the vault started and + `deposits` is `CAPITAL_SOL`. +- `prices`: `sol_usd` from Jupiter Price v3, `fly_usd` from the GeckoTerminal pool, cached 60 s (a failed fetch keeps + the last value); `ts` is the older of the two fetch times, or 0. +- `vault`: `total_locked`, `total_pending`, `earners` and `paused` come from events in finalized blocks (about 18 minutes + behind the head), up to `finalized_block`. `address` is null until `VAULT_ADDRESS` is set. +- `settlement.last` is the newest `allocated` settlement (§4.2), or null. +- `vault.upgrade_scheduled` is `{"eta": ts, "id": "0x…"}` or null: the soonest pending timelock operation whose target is the + vault (upgrade, role change) or the timelock itself (`updateDelay`, proposer changes). `eta` is the block time of + `CallScheduled` plus its `delay`. ### 4.2 History items -- `nav`: `{"ts", "nav", "index", "sol_usd"}` -- `trades`: `{"id", "mint", "symbol", "opened_at", "closed_at", "cost", "proceeds", "realized", "exit_kind"}` -- `flows`: `{"id", "ts", "signature", "direction": "in|out", "kind": "deposit|profit|withdrawal|claim", "counterparty", "lamports"}` -- `settlements`: `{"id", "period_start", "period_end", "realized", "pot", "allocated", "carried", "earners", "total_weight": "", "status"}` +- `nav`: `{"ts", "nav", "sol_usd"}`: one point per 5-minute boundary (`ts`), from the newest consistent mark at or before + it, nav rounded to 0.1 SOL; `sol_usd` is the price at push time. +- `trades`: `{"id", "mint", "symbol", "opened_at", "closed_at", "cost", "proceeds", "realized", "exit_kind"}`: closed + positions of `VAULT_BOOK` only; `proceeds` = cost + realized; `exit_kind` is the forced-exit kind or `hold`. +- `flows`: `{"id", "ts", "direction": "in|out", "kind": "deposit|profit|withdrawal|claim", "lamports"}`. Internal moves + (`sweep`, `topup`) and `unknown_outbound`/`anomaly` rows are never published. +- `settlements`: `{"id", "period_start", "period_end", "realized", "pot", "allocated", "carried", "earners", + "total_weight": "", "status"}`: `status` is always `allocated`; `carried` = pot − allocated; `total_weight` is + the sum of wei·seconds; `realized` is R at the snapshot. ### 4.3 Account ```json @@ -216,15 +293,114 @@ Relay config: `relay.json`, kept next to `wsgi.py` and never inside `site/`: "allocations": [{"period_end": 0, "lamports": 0, "weight": "", "share": 0.0}], "claims": [{"id": 0, "status": "", "lamports": 0, "sol": "", "tx": "", "created_at": 0, "updated_at": 0}]} ``` -`owed` = allocated − claimed − in_flight, where in_flight counts claims in `verified | waiting_liquidity | sending`. +- `owed` = allocated − claimed − in_flight. `in_flight` counts claims from `sending` on (a `verified` claim has no amount + yet). +- `allocations`: the 52 most recent; `share` = weight / that settlement's total weight. +- `claims`: the 20 most recent; `id` is the relay's claim id (as in `GET /api/claim/`); `tx` is always `""`. ## 5. Networks | | mainnet | rehearsal | |---|---|---| +| Site build | `VITE_NETWORK=mainnet` (default) | `VITE_NETWORK=testnet` | | EVM chain | Robinhood Chain 4663, `https://rpc.mainnet.chain.robinhood.com` | RH testnet 46630, `https://rpc.testnet.chain.robinhood.com` | | Explorer | `https://robinhoodchain.blockscout.com` | `https://explorer.testnet.chain.robinhood.com` | | $FLY | `0x2fC7f9E2911f20b2C4660d2AEf808aa91bDdb3D3` | `MockFLY` (deployed for the rehearsal) | +| USDG | `0x5fc5360D0400a0Fd4f2af552ADD042D716F1d168` (6 decimals) | — | +| Multicall3 | `0xcA11bde05977b3631167028862bE2a173976CA11` | same | | Solana | mainnet-beta | devnet | +| Squads v4 | `SQDS4ep65T869zMMBKyuUq6aD6EgTu8psMjkvj52pCf` | same | +| Settlement period | 7 days (`VAULT_PERIOD_S=604800`) | 900 s | | Kyber | `https://aggregator-api.kyberswap.com/robinhood/api/v1/`, router `0x6131B5fae19EA4f9D964eAc0408E4408b66337b5` | — | | Chart | GeckoTerminal network `robinhood`, pool `0xe6925f7bdedf22c2714c749d0ff208ce9e4bc1540938fa2b9ab2379486322edd` | fixtures | + +- The fly's defaults are mainnet. A rehearsal sets `RH_CHAIN_ID=46630`, `RH_RPC_URL`, `VAULT_CLUSTER=devnet`, + `VAULT_SOLANA_RPC_URL`, `VAULT_PERIOD_S=900` and `VAULT_SITE_DOMAIN`/`VAULT_SITE_URI`. +- The site's Solana RPC (the Owner page) is WalletConnect's unless `VITE_SOLANA_RPC` is set. + +## 6. Custody (Solana) + +The fly's SOL is one book in two accounts: +- **The treasury** is vault 0 of a Squads v4 multisig. Its members are the owner's wallets only (Solflare until the + Ledger takes over); threshold 1 and no time lock while there is one member. +- **The trading wallet** is a hot key that keeps a float of about `TREASURY_FLOAT_SOL` (default 2 SOL). + +| Spending limit | Member | Destinations | Period | Used for | +|---|---|---|---|---| +| L1 | trading key | the trading wallet only | Day | refilling the float (`topup` flows, memo `fly-topup`) | +| L2 | payout key | any | Week | claims (`spendingLimitUse` from the treasury, memo `fly-vault-claim::`) | + +- **Rebalancing** (every minute, `vault/payout.py`): first the treasury must hold everything owed plus its 890,880-lamport + rent minimum, sweeping from the trading wallet if it is short; then the trading wallet is brought to the float, by a + top-up under L1 when low or a sweep when high. Moves under `TOPUP_MIN_SOL` (0.1) are skipped, except a sweep the + treasury needs. A sweep never takes the trading wallet below `GAS_RESERVE_SOL` + fee. Under `/panic` the float target + is 0. +- **Flows.** A trading → treasury transfer is a `sweep`, a treasury → trading one a `topup`: both internal, excluded + from R, the index and published history. SOL or wSOL arriving at either account from `FUNDING_ADDRESSES` is a + `deposit`; from anyone else it is `profit` (a gift). A treasury outflow signed by a multisig member is a `withdrawal` + (the owner's principal; alerted, louder when the destination is not a funding address). A transaction one of our + keys signed that no table knows is `unknown_outbound`; lamports leaving without our or an owner's signature are an + `anomaly`. Both halt the vault (§7), unless the second RPC provider sees the transaction differently. +- **Accounting.** R, NAV and the performance index count both accounts together. Positions are sized on the bankroll + = trading free SOL + open positions at cost + (treasury − owed); buys spend only the trading wallet's own SOL, less + any owed SOL the treasury does not hold. +- **The signer** (`fly_trader/signer`) is the only holder of the two hot keys. It builds and signs: + - Jupiter swaps: SOL ↔ one token only; Metis routes only (`excludeRouters=jupiterz,dflow,okx`); static rules on every + instruction (programs, the trading wallet's roles, a priority-fee cap of `SIGNER_MAX_PRIORITY_SOL`, 0.005); a + simulation, through its own RPC, of the trading wallet's SOL and every token account it owns (a buy may spend at + most the ordered SOL plus fees and must deliver the token to the trading wallet; a sell may spend at most the + ordered tokens and must return SOL; no other holding may shrink); a value floor of `SIGNER_MIN_VALUE_RATIO` (0.6) + at the signer's own Jupiter prices. Buys are capped per trade (`SIGNER_MAX_BUY_SOL`, default `MAX_POSITION_SOL`) and + per day (`SIGNER_DAILY_BUY_SOL`, 10); all swaps per minute (`SIGNER_SWAPS_PER_MIN`, 6). + - top-ups under L1 (at most `TREASURY_FLOAT_SOL` each; L1's destinations must be exactly the trading wallet) and + sweeps (a plain transfer to the treasury) + - claims under L2, once per claim id, with the same destination and amount; re-signed only after the earlier + signature failed or its blockhash expired + - closes of empty token accounts of the trading wallet (batches of 10) + - the hot keys, age-encrypted to `VAULT_BACKUP_RECIPIENT`, for the off-box backup + - after `panic`: no buys, top-ups or claims; `clear-panic` is a CLI command inside its container only + + On the server the brain reaches it over a Unix socket (one JSON request per line, 256 KB max, one at a time) and never + sees a key. The signer keeps its own SQLite ledger. Without `SIGNER_SOCKET` (self-hosting, tests) the same signer runs + inside the brain's process. +- **Custody checks** (every minute, `vault/custody.py`), alerted: neither server key is a multisig member; no config + authority; L1 is a SOL limit with member = the trading key and sole destination = the trading wallet; L2 is a SOL + limit with member = the payout key. Any change to members, threshold, time lock or the limits is reported. +- **The owner** changes the limits from `owner.html` (`web/src/lib/squads.ts`). Every new limit has a fresh address: + the server's `VAULT_LIMIT_TRADING` / `VAULT_LIMIT_PAYOUT` must be updated to it (the page prints the lines). A + limit's amount is changed by removing it and adding a new one, in two config transactions: the program rejects both + in one. + +## 7. Settlement, halts, kill switch + +- **Schedule.** Period ends fall at `VAULT_EPOCH + k·VAULT_PERIOD_S` (345600 = Monday 1970-01-05 00:00 UTC; 7 days). + The first settlement covers the period before the first end after the vault started; missed periods are settled as + one window. Settlements advance one step per minute, in period order. +- **Two phases.** + 1. Snapshot at the period end T1: both accounts at a finalized slot that contains every transaction the fly sent, + read under the wallet lock. The second RPC provider (`SOLANA_CHECK_RPC_URL`) must report the same balance, or the + settlement pauses, alerts and retries. Flows are scanned up to that slot. + 2. Allocate once Robinhood Chain is finalized past T1, and only if the vault's implementation code hash is in + `VAULT_IMPL_CODEHASHES` (else the vault halts). + + Statuses: `snapshotted | allocated | failed`. +- **Formulas.** + ``` + R = N + K + C − D + Wd + P N: both accounts' SOL; K: wSOL and empty token-account rent; C: open positions at + cost; D: deposits; Wd: withdrawals incl. fees; P: claims paid + pot = max(0, R − A) A: everything ever allocated + out = min(pot, N − (A − P) − GAS_RESERVE_SOL) + weight_i = ∫ lockedAfter_i dt over [T0, T1) (wei·s, from FlyVault events) + alloc_i = floor(out · weight_i / Σ weight); the remainder, dust and empty weeks carry forward (A grows only by alloc) + ``` +- **Halts.** The vault halts on `unknown_outbound`, `anomaly`, an unknown implementation code hash, or Telegram + `/panic` (the operator's user only). A halt stops claim processing, settlement and new entries. `/panic` also trips + the kill switch with liquidation, sets the float target to 0 and puts the signer in panic. Resume is SSH-only: + `fly-trader vault resume` (clears halt and panic), `fly-trader reset-circuit --kill`, `fly-trader resume-entries`, + and the signer's `clear-panic`. +- **Kill switch.** A flow-adjusted index `I_t = I_{t−1}·(W_t − F_t)/W_{t−1}` over consistent NAV marks of both + accounts, where F is every deposit/profit in and withdrawal/claim out since the previous mark. Its peak uses marks at + least 600 s old, over 30 days. It trips at `KILL_SWITCH_DRAWDOWN` (default 0.30; the vault server sets 0.20) and, with + `KILL_SWITCH_LIQUIDATE`, sells every open position. +- **Principal withdrawal.** Only the owner, from the Squads app. The cap `fly-trader vault withdraw` prints is + min(D − Wd + min(0, R − A), treasury − owed − 890,880). diff --git a/fly_trader/__main__.py b/fly_trader/__main__.py index c7a963ac976bff8edbdc44b3533d55be7d7884a6..ab471d066e7188824e87f8b1cf74ee23032f62eb 100644 --- a/fly_trader/__main__.py +++ b/fly_trader/__main__.py @@ -144,7 +144,8 @@ def _cmd_fly_replay(args): from .logging_setup import setup from .train import fly_replay setup("train"); _use_device(args) - out = fly_replay.main(days=args.days, start_day=args.start_day if args.start_day is not None else fly_replay.DEFAULT_START) + out = fly_replay.main(days=args.days, start_day=args.start_day if args.start_day is not None else fly_replay.DEFAULT_START, + only_chains=tuple(args.chains.split(",")) if args.chains else None) print(json.dumps({k: out.get(k) for k in ("S", "passed", "reason", "alpha", "half_life_days", "evaluation", "random", "frozen", "plastic_beats_frozen")}, indent=1, default=str)) @@ -183,6 +184,123 @@ def _cmd_worker(args): procs.worker_cli(args.action, args.name) +def _cmd_kalshi_history(args): + from .kalshi import history + history.main() + + +def _cmd_kalshi_build(args): + from .kalshi import mature + mature.main() + + +def _cmd_kalshi_train_selector(args): + from .kalshi import selector + from .logging_setup import setup + setup("kalshi_train") + out = selector.main(days=args.days, holdout_days=args.holdout_days) + print(json.dumps({k: out.get(k) for k in ("snapshot_id", "deployable", "deploy_reason")}, indent=1, default=str)) + + +def _cmd_kalshi_train_fly(args): + from .kalshi import fly + from .logging_setup import setup + setup("kalshi_train"); _use_device(args) + out = fly.main(days=args.days, epochs=args.epochs) + print(json.dumps({k: out.get(k) for k in ("snapshot_id", "S", "lines", "calibration", "diagnostics", "gates_ok", "gate_failures", "teacher")}, indent=1, default=str)) + + +def _cmd_kalshi_fly_replay(args): + from .kalshi import fly_replay + from .logging_setup import setup + setup("kalshi_train"); _use_device(args) + out = fly_replay.main(days=args.days, start_day=args.start_day if args.start_day is not None else fly_replay.DEFAULT_START, + only_chains=tuple(args.chains.split(",")) if args.chains else None) + print(json.dumps({k: out.get(k) for k in ("S", "passed", "reason", "alpha", "half_life_days", "evaluation", "random", "frozen", "plastic_beats_frozen")}, indent=1, default=str)) + + +def _cmd_kalshi_train(args): + from .kalshi import pipeline + pipeline.main() + + +def _cmd_kalshi_stream(args): + from .kalshi import stream + stream.main() + + +def _cmd_kalshi_run(args): + from .kalshi import engine + engine.main() + + +def _cmd_kalshi_subaccount(args): + from .kalshi import client + if args.action == "create": + print(f"subaccount {client.subaccount_create()} created and written to .env") + else: + print(json.dumps(client.rest().subaccount_balances(), indent=1, default=str)) + + +def _cmd_kalshi_fund(args): + from .kalshi import client + print(json.dumps(client.fund(args.usd), indent=1, default=str)) + + +def _cmd_rh_backfill(args): + from .rh import stream + stream.main() + + +def _cmd_rh_status(args): + from .rh import stream + print(json.dumps(stream.status(), indent=1, default=str)) + + +def _cmd_rh_probe(args): + from .rh import probe + print(json.dumps(probe.run(sample=args.sample), indent=1, default=str)) + + +def _cmd_rh_wallet(args): + from .rh import smoke + if args.action == "new": + print(f"RH bot wallet {smoke.new_key()} written to .env (RH_BOT_PRIVATE_KEY, RH_BOT_ADDRESS). Fund it with ETH on Robinhood Chain.") + else: + from . import config + from .rh.rpc import RhRpc + addr = (config.RH_BOT_ADDRESS or "").lower() + print(json.dumps({"address": addr or None, "eth": RhRpc().get_balance(addr) / 1e18 if addr else None, + "live_prerequisites_missing": config.rh_live_prerequisites_missing()}, indent=1)) + + +def _cmd_rh_smoke(args): + if not args.yes: + raise SystemExit("rh-smoke sends real Robinhood Chain transactions: pass --yes") + from .rh import smoke + print(json.dumps(smoke.run(args.size, args.cap), indent=1, default=str)) + + +def _cmd_kalshi_status(args): + from .kalshi import client + print(json.dumps(client.status(), indent=1, default=str)) + + +def _cmd_kalshi_pause(args): + from .agent import rails + rails.set_entries_paused(True, rails.KALSHI_CIRCUIT) + + +def _cmd_kalshi_resume(args): + from .agent import rails + rails.set_entries_paused(False, rails.KALSHI_CIRCUIT) + + +def _cmd_kalshi_reset_circuit(args): + from .agent import rails + rails.reset_circuit(kill=args.kill, circuit_id=rails.KALSHI_CIRCUIT) + + def build_parser() -> argparse.ArgumentParser: p = argparse.ArgumentParser(prog="fly-trader") sub = p.add_subparsers(dest="command", required=True) @@ -211,6 +329,7 @@ def build_parser() -> argparse.ArgumentParser: fr.add_argument("--start-day", type=int, default=None, help="day index of the bootstrap (the selector teaches on the days before it); default: train/fly_replay.DEFAULT_START, " "the earliest day that leaves the teacher's walk-forward enough days in each half") + fr.add_argument("--chains", default=None, help="store the verdict of these chains only (comma list, e.g. rh); the others keep theirs") fr.set_defaults(fn=_cmd_fly_replay) bc = sub.add_parser("backtest-corpus"); bc.add_argument("--max-tokens", type=int, default=None); bc.add_argument("--days", type=int, default=None) bc.add_argument("--fees", default="0.003,0.0055"); bc.add_argument("--only", default=None); bc.add_argument("--universe", default="graduation", choices=["graduation", "mature"]); bc.set_defaults(fn=_cmd_backtest_corpus) @@ -229,9 +348,42 @@ def build_parser() -> argparse.ArgumentParser: ar.add_argument("dir"); ar.add_argument("--bootstrap", action="store_true"); ar.set_defaults(fn=_cmd_apply_release) sub.add_parser("health", help="exit 0 when the database answers and workers run (container HEALTHCHECK)").set_defaults(fn=_cmd_health) sub.add_parser("ready-for-restart", help="exit 0 when no order, claim payment or settlement is in flight").set_defaults(fn=_cmd_ready) + mc = sub.add_parser("models", help="convert: rewrite this machine's own .joblib selectors as .skops (pickles can run code)") + mc.add_argument("action", choices=["convert"]); mc.set_defaults(fn=_cmd_models) from .vault import cli as vault_cli vault_cli.add(sub) wk = sub.add_parser("worker"); wk.add_argument("action", choices=["start", "stop", "status"]); wk.add_argument("name", nargs="?"); wk.set_defaults(fn=_cmd_worker) + # ---- Kalshi prediction markets (fly_trader/kalshi) ---- + ks = sub.add_parser("kalshi-subaccount", help="create the fly's dedicated Kalshi subaccount (writes KALSHI_SUBACCOUNT to .env) or list balances") + ks.add_argument("action", choices=["create", "list"]); ks.set_defaults(fn=_cmd_kalshi_subaccount) + kf = sub.add_parser("kalshi-fund", help="move dollars from the primary Kalshi account into the fly's subaccount"); kf.add_argument("--usd", type=float, required=True); kf.set_defaults(fn=_cmd_kalshi_fund) + sub.add_parser("rh-backfill", help="the Robinhood Chain index from Pons V2's deployment to the head, then live (the rh_stream worker)").set_defaults(fn=_cmd_rh_backfill) + sub.add_parser("rh-status", help="Robinhood Chain index cursors, row counts and stream status").set_defaults(fn=_cmd_rh_status) + rp = sub.add_parser("rh-probe", help="read-only check of the Robinhood Chain path: contracts, Kyber / v4 routability and gas on recent Pons graduates") + rp.add_argument("--sample", type=int, default=12); rp.set_defaults(fn=_cmd_rh_probe) + rw = sub.add_parser("rh-wallet", help="new: create the RH bot key in .env (refuses to replace one); show: its address, ETH and what live trading still needs") + rw.add_argument("action", choices=["new", "show"]); rw.set_defaults(fn=_cmd_rh_wallet) + rs = sub.add_parser("rh-smoke", help="three tiny real RH round trips (Kyber, v4 fallback, non-ETH quote) booked and reconciled to the wei") + rs.add_argument("--size", type=float, default=0.002); rs.add_argument("--cap", type=float, default=0.01); rs.add_argument("--yes", action="store_true") + rs.set_defaults(fn=_cmd_rh_smoke) + sub.add_parser("kalshi-status", help="exchange status, subaccount balance, resting orders, positions, settlements").set_defaults(fn=_cmd_kalshi_status) + sub.add_parser("kalshi-history", help="the Kalshi corpus worker: dataset seed, settled markets, candles, trades").set_defaults(fn=_cmd_kalshi_history) + sub.add_parser("kalshi-build", help="build the Kalshi feature parts from the corpus").set_defaults(fn=_cmd_kalshi_build) + kts = sub.add_parser("kalshi-train-selector", help="fit the Kalshi strategy stack walk-forward and save it"); kts.add_argument("--days", type=int, default=None) + kts.add_argument("--holdout-days", type=int, default=None, help="last days withheld from every fit and scored once (default 21)") + kts.set_defaults(fn=_cmd_kalshi_train_selector) + ktf = sub.add_parser("kalshi-train-fly", help="bootstrap the Kalshi fly (optic lobes) from the deployable Kalshi selector") + ktf.add_argument("--days", type=int, default=None); ktf.add_argument("--epochs", type=int, default=None); ktf.add_argument("--device", default=None) + ktf.set_defaults(fn=_cmd_kalshi_train_fly) + kfr = sub.add_parser("kalshi-fly-replay", help="bootstrap the Kalshi fly once, let it learn from settlements over the corpus, judge it") + kfr.add_argument("--days", type=int, default=None); kfr.add_argument("--device", default=None); kfr.add_argument("--start-day", type=int, default=None) + kfr.set_defaults(fn=_cmd_kalshi_fly_replay) + sub.add_parser("kalshi-train", help="the Kalshi training pipeline worker (selector weekly, fly bootstrap when needed)").set_defaults(fn=_cmd_kalshi_train) + sub.add_parser("kalshi-stream", help="the Kalshi live feed worker (websocket → kalshi_minutes / kalshi_quotes)").set_defaults(fn=_cmd_kalshi_stream) + sub.add_parser("kalshi-run", help="the Kalshi trading engine worker (the visual fly's paper arms, live mirror behind KALSHI_LIVE_ENABLED)").set_defaults(fn=_cmd_kalshi_run) + sub.add_parser("kalshi-pause-entries").set_defaults(fn=_cmd_kalshi_pause) + sub.add_parser("kalshi-resume-entries").set_defaults(fn=_cmd_kalshi_resume) + kr = sub.add_parser("kalshi-reset-circuit"); kr.add_argument("--kill", action="store_true"); kr.set_defaults(fn=_cmd_kalshi_reset_circuit) return p @@ -248,6 +400,12 @@ def _cmd_health(args) -> None: raise SystemExit(0 if ok else 1) +def _cmd_models(args) -> None: + from .train import model_io + for r in model_io.convert_legacy(): + print(f"selector #{r['id']} -> {r['path']}") + + def _cmd_ready(args) -> None: from .ops import release ok, why = release.ready_for_restart(); print(why) diff --git a/fly_trader/agent/fly_live.py b/fly_trader/agent/fly_live.py index 2e776fbd23fa3058efd22c752101260ff824c155..8ad2f348ce9d0ccdf9adf36aa60c6de855dad497 100644 --- a/fly_trader/agent/fly_live.py +++ b/fly_trader/agent/fly_live.py @@ -79,15 +79,18 @@ class LiveMirror: opens = ledger.open_positions(conn, BOOK); held_mints = {p["mint"] for p in opens} | set(inflight) c = rails.load_circuit(conn) blocked = "kill switch" if c.kill_switch else "circuit tripped" if c.tripped else "paused" if c.entries_paused else None - reserved = 0.0 - if config.VAULT_ENABLED: # SOL owed to vault lockers stays in the wallet but is never traded + reserved = treasury = 0.0 + if config.VAULT_ENABLED: # SOL owed to vault lockers is never traded; the treasury's idle SOL is bankroll from ..vault import nav as vault_nav - reserved = vault_nav.reserved_lamports(conn) / config.LAMPORTS_PER_SOL - bankroll = sol_free + sum(float(p["cost_sol"]) for p in opens) - reserved; cash = sol_free - reserved; n_enter = 0; skipped = [] + reserved = vault_nav.reserved_in_trading(conn) / config.LAMPORTS_PER_SOL + treasury = vault_nav.treasury_free(conn) / config.LAMPORTS_PER_SOL + # sized on the whole book, spent only from the trading wallet's own float (cash) + bankroll = sol_free + sum(float(p["cost_sol"]) for p in opens) - reserved + treasury; cash = sol_free - reserved; n_enter = 0; skipped = [] for e in entries: if e["mint"] in held_mints: skipped.append((e["mint"], "held")); continue - size, why = sizing.size_position(e["score"], e.get("threshold", line), e.get("table") or table or [], bankroll, cash, e["info"]["resq"]) + size, why = sizing.size_position(e["score"], e.get("threshold", line), e.get("table") or table or [], bankroll, cash, e["info"]["resq"], + flat=config.FLY_SIZING == "flat") if blocked or size <= 0: skipped.append((e["mint"], blocked or why)); continue if self.worker.submit(ExecRequest(decision_id=e["decision_id"], mint=e["mint"], pool=e["info"]["pool"], side="buy", amount_in=int(size * config.LAMPORTS_PER_SOL), @@ -109,13 +112,18 @@ class LiveMirror: "ON CONFLICT (beat_id, book) DO NOTHING", (beat_id, BOOK, m1, sol_free, gross_v - exit_cost, exit_cost, wealth, peak, (1.0 - wealth / peak) if peak > 0 else 0.0, exposure, len(opens))) if config.VAULT_ENABLED: # deposits, withdrawals and payouts are not performance - vault_nav.mark(conn, m1, snap.lamports, int(round(gross_v * config.LAMPORTS_PER_SOL)), + from ..vault import payout as vault_payout # trading wallet + treasury are one book: a top-up or sweep is no loss + vault_nav.mark(conn, m1, snap.lamports + vault_payout.cached_balance(), int(round(gross_v * config.LAMPORTS_PER_SOL)), int(round(exit_cost * config.LAMPORTS_PER_SOL))) ix = vault_nav.status(conn, since=rebase) - killed = rails.check_drawdown(conn, ix["value"], ix["peak"]) + killed = rails.check_drawdown(conn, ix["value"], ix["peak"], unit="index") else: killed = rails.check_drawdown(conn, wealth, peak) - n_liq = self.liquidate(conn, ctx, run_id, beat_id) if killed and config.KILL_SWITCH_LIQUIDATE else 0 + panic = False + if config.VAULT_ENABLED: # Telegram /panic: sell everything whatever KILL_SWITCH_LIQUIDATE says + from ..vault import state as vault_state + panic = bool(vault_state.get("panic")) + n_liq = self.liquidate(conn, ctx, run_id, beat_id) if killed and (config.KILL_SWITCH_LIQUIDATE or panic) else 0 gap = self.gap(conn) if ctx.m1_epoch - self.last_sweep >= SWEEP_S: self.last_sweep = ctx.m1_epoch; self.sweep(conn, snap, inflight, m1) diff --git a/fly_trader/agent/fly_session.py b/fly_trader/agent/fly_session.py index dacbefc123a47b3b3bd5b1c01c912bef5597344e..f7c2e3984b4f930c9cccf3a384970d21e8f9d019 100644 --- a/fly_trader/agent/fly_session.py +++ b/fly_trader/agent/fly_session.py @@ -35,7 +35,7 @@ from pathlib import Path import numpy as np import torch -from .. import config +from .. import config, markets from ..brain import activity, device as brain_device, plastic from ..brain.connectome import current_connectome_path from ..db.apilog import record_event @@ -43,12 +43,12 @@ from ..db.connection import transaction from ..execution import ledger from ..execution.broker_paper import PaperBroker from ..ops.reset import activity_dir, fly_state_dir -from ..train import fly_calibrate, fly_governance as gov, fly_selector +from ..train import fly_calibrate, fly_governance as gov, fly_selector, selector, strategies from ..train.decisions import X_COLS from . import handover, paper_trading log = logging.getLogger(__name__) -BOOK, KIND = "paper_fly", "fly" +BOOK, KIND = "paper_fly", "fly" # the Solana book's names (another chain's book takes its own from fly_trader/markets) BAR_KEEP = 270 # minutes of bars kept per watched mint (the longest strategy hold, 240, + next-open window + margin) NEXT_OPEN_S = 120.0 # entry at the next traded minute's open when it comes within two minutes (decisions.build) JUMP = 50.0 @@ -90,37 +90,57 @@ def resolve_label(bars: list, t: float, horizon_s: float) -> float | None: return float(exit_px / entry * (1 - rows[0][3]) * (1 - rows[j][3]) - 1) -def try_start(live: bool = False) -> tuple["FlyBook | None", str]: - """A FlyBook when the fly may trade, else (None, why).""" +def chain_passed(v: dict, chain: str) -> tuple[bool, str]: + """Whether the replay lets this chain's book trade: its own per-chain verdict (a combined replay), else the one verdict + for Solana (a Solana-only replay says nothing about another chain).""" + if "chains" in v: + c = v["chains"].get(chain) + return (bool(c and c.get("passed")), (c or {}).get("reason") or f"the replay has no {chain} verdict") + return (bool(v.get("passed")) and chain == "sol", v.get("reason") if chain == "sol" else f"the replay judged no {chain} trades") + + +def try_start(live: bool = False, chain: str = "sol", brain: "FlyBook | None" = None) -> tuple["FlyBook | None", str]: + """A FlyBook for ``chain`` when the fly may trade there, else (None, why). ``brain``: the Solana book whose fly and + plastic memory another chain's book shares (one model learns from both chains).""" from ..train import fly_replay v = fly_replay.verdict() if not v: return None, "the fly's replay has not run on the current definitions (fly-trader fly-replay)" - if not v.get("passed"): - return None, f"the fly's replay did not pass: {v.get('reason')}" + ok, why = chain_passed(v, chain) + if not ok: + return None, f"the fly's replay did not pass on {markets.for_chain(chain).name}: {why}" with transaction() as conn: boot = fly_selector.latest_deployable(conn) if boot is None: return None, "no deployable fly bootstrap (the training pipeline bootstraps one)" try: - return FlyBook(boot, v, live=live), "ok" + return FlyBook(boot, v, live=live, chain=chain, brain=brain), "ok" except Exception as e: log.exception("fly book failed to start") return None, f"fly book failed to start: {type(e).__name__}: {e}" -class FlyBook: - name = "fly" +def no_teacher(n: int) -> dict: + """The decision when the teacher gate is on (FLY_TEACHER_GATE) but the fly's teacher could not be loaded: no row is + allowed, each recorded with why. A fresh install once lost the teacher's snapshot id (ops/release.remap_teacher).""" + return {"strategy": np.full(n, None, dtype=object), "score": np.full(n, -1.0), "hold_s": np.zeros(n), "threshold": np.full(n, np.inf), + "tables": [[] for _ in range(n)], "allow": np.zeros(n, bool), + "reason": np.full(n, "fail closed: the fly's teacher (the selector that taught it) is not loaded", dtype=object)} + - def __init__(self, boot: dict, verdict: dict, live: bool = False): +class FlyBook: + def __init__(self, boot: dict, verdict: dict, live: bool = False, chain: str = "sol", brain: "FlyBook | None" = None): self.live = live; self.done = False; self.mirror = None self.verdict = verdict - self.state_path = fly_state_dir() / "state.pt" + self.chain = chain; self.market = markets.for_chain(chain); self.brain = brain + self.name = "fly" if chain == "sol" else f"fly_{chain}" + self.BOOK = self.market.fly_book; self.KIND = self.name; self.status_key = self.market.fly_status_key + self.state_path = fly_state_dir() / ("state.pt" if chain == "sol" else f"state_{chain}.pt") self._load_bootstrap(boot) - self.run_id = str(uuid.uuid4()); self.beat_no = 0; self.broker = PaperBroker(BOOK) + self.run_id = str(uuid.uuid4()); self.beat_no = 0; self.broker = PaperBroker(self.BOOK) with transaction() as conn: conn.execute("INSERT INTO runs (run_id, kind, config, brain_snapshot_id, status) VALUES (%s,%s,%s,%s,'running')", - (self.run_id, KIND, json.dumps({"channels": self.cfg, "book": BOOK}, default=str), self.boot_id)) + (self.run_id, self.KIND, json.dumps({"channels": self.cfg, "book": self.BOOK, "chain": chain}, default=str), self.boot_id)) self._restore_tags(conn) record_event("info", "fly", "fly session started", {"run_id": self.run_id, "bootstrap": self.boot_id, "channels": self.cfg, "lines": self.lines["plastic"], "pending": len(self.pending), "learning_frozen": self.learning_frozen}) @@ -128,54 +148,126 @@ class FlyBook: self.lines["plastic"], len(self.pending)) # ---- model and state ---- - def _load_bootstrap(self, boot: dict) -> None: - self.boot_id = int(boot["id"]) - self.fly = fly_selector.load(boot["path"]) - try: - self.connectome_name = current_connectome_path().name # stamped on the activity files the console draws - except Exception: - self.connectome_name = "" - self.capture_ok = True - self.idx = np.asarray([X_COLS.index(c) for c in self.fly.cols], dtype=int) - self.names = list(self.fly.strategies); self.holds = {k: float(self.fly.rules[k]["hold_min"]) * 60.0 for k in self.names} - self.H = max(self.holds.values()) - per = self.verdict.get("per_strategy") or {self.names[0]: {"alpha": self.verdict.get("alpha", 0.0), "half_life_days": self.verdict.get("half_life_days")}} - self.cfg = {k: (float((per.get(k) or {}).get("alpha") or 0.0), (per.get(k) or {}).get("half_life_days")) for k in self.names} - self.alpha = self.cfg[self.names[0]][0]; self.half_life = float(self.cfg[self.names[0]][1]) if self.cfg[self.names[0]][1] is not None else math.inf - net = self.fly.net - self.bank = plastic.PlasticBank(net, [(self.alpha, self.half_life)], fly_selector.SCALE, learn=net.learn.cpu().numpy(), read=net.read.cpu().numpy()) - for j, k in enumerate(self.names): # each strategy's channel learns and forgets at its own replay-chosen rate - a, h = self.cfg[k]; cols = self.bank.learn[j] - self.bank.alpha[0, cols] = a; self.bank.half_life_s[0, cols] = float(h) * 86400.0 if h is not None else float("inf") + def _load_bootstrap(self, boot: dict | None) -> None: + b = self.brain + if b is not None: # another chain's book: the Solana book's fly and plastic memory, this chain's own lines and tags + self.boot_id = b.boot_id; self.fly = b.fly; self.connectome_name = b.connectome_name; self.capture_ok = False + self.teacher_models = b.teacher_models; self.idx = b.idx; self.names = b.names; self.holds = b.holds; self.H = b.H + self.teacher_live, self.teacher_groups = b.teacher_live, b.teacher_groups + self.cfg = b.cfg; self.alpha = b.alpha; self.half_life = b.half_life; self.bank = b.bank + net = self.fly.net + else: + self.boot_id = int(boot["id"]) + self.fly = fly_selector.load(boot["path"]) + try: + self.connectome_name = current_connectome_path().name # stamped on the activity files the console draws + except Exception: + self.connectome_name = "" + self.capture_ok = True + self.teacher_models = self._teacher_models(boot) + self.idx = np.asarray([X_COLS.index(c) for c in self.fly.cols], dtype=int) + self.names = list(self.fly.strategies); self.holds = {k: float(self.fly.rules[k]["hold_min"]) * 60.0 for k in self.names} + self.H = max(self.holds.values()) + per = self.verdict.get("per_strategy") or {self.names[0]: {"alpha": self.verdict.get("alpha", 0.0), "half_life_days": self.verdict.get("half_life_days")}} + self.cfg = {k: (float((per.get(k) or {}).get("alpha") or 0.0), (per.get(k) or {}).get("half_life_days")) for k in self.names} + self.alpha = self.cfg[self.names[0]][0]; self.half_life = float(self.cfg[self.names[0]][1]) if self.cfg[self.names[0]][1] is not None else math.inf + net = self.fly.net + self.bank = plastic.PlasticBank(net, [(self.alpha, self.half_life)], fly_selector.SCALE, learn=net.learn.cpu().numpy(), read=net.read.cpu().numpy()) + for j, k in enumerate(self.names): # each strategy's channel learns and forgets at its own replay-chosen rate + a, h = self.cfg[k]; cols = self.bank.learn[j] + self.bank.alpha[0, cols] = a; self.bank.half_life_s[0, cols] = float(h) * 86400.0 if h is not None else float("inf") self.pending = plastic.PendingTags(net.n_kc, net.k_active) self.bars: dict[str, deque] = {}; self.watch: dict[str, int] = {} - self.lines = {"plastic": dict(self.fly.lines), "frozen": dict(self.fly.lines)} - self.sizing = {"plastic": {k: list(v) for k, v in self.fly.sizings.items()}, "frozen": {k: list(v) for k, v in self.fly.sizings.items()}} + own = {k: self.fly.line_for(k, self.chain) for k in self.names}; own_s = {k: list(self.fly.sizing_for(k, self.chain)) for k in self.names} + if config.FLY_SCORE == "teacher" and self.teacher_live: # it scores on its teacher's scale: it starts on its teacher's lines and bands + ts_ = self.teacher_live.get("strategies") or {} + own = {k: float(ts_[k]["line"]) if k in ts_ else v for k, v in own.items()} + own_s = {k: list(ts_[k].get("sizing") or v) if k in ts_ else v for k, v in own_s.items()} + self.lines = {"plastic": dict(own), "frozen": dict(own)} + self.sizing = {"plastic": {k: list(v) for k, v in own_s.items()}, "frozen": {k: list(v) for k, v in own_s.items()}} self.applied_through = -math.inf; self.learning_frozen = False; self.checks_paused_until = {k: 0.0 for k in self.names} self.race_started_at: float | None = None; self.last_day: int | None = None; self.last_hour: int | None = None self.rollbacks: dict = {k: [] for k in self.names}; self.nu_set = False; self.hour_stats = self._empty_stats(); self.last_checks: dict = {} s = self._read_state() + if b is not None: + self.nu_set = True # the shared memory is the Solana book's to load and save + if s and s.get("bootstrap_id") == self.boot_id and s.get("cfg") == self.cfg and s.get("score", "own") != config.FLY_SCORE: + self._archive_state(s) # learned under another scoring: it starts from its bootstrap + s = None if s and s.get("bootstrap_id") == self.boot_id and s.get("cfg") == self.cfg: - self.bank.load_state(s["bank"]); self.nu_set = True + if b is None: + self.bank.load_state(s["bank"]) + self.nu_set = True self.lines, self.sizing = s["lines"], s["sizing"] self.applied_through, self.learning_frozen, self.checks_paused_until = s["applied_through"], s["learning_frozen"], s["checks_paused_until"] self.race_started_at, self.last_day, self.last_hour, self.rollbacks = s["race_started_at"], s["last_day"], s["last_hour"], s["rollbacks"] elif s: self.race_started_at = s.get("race_started_at") # a new bootstrap keeps racing the same book with transaction() as conn: - conn.execute("UPDATE fly_scored SET state = 'dropped', x = NULL WHERE state = 'pending' AND bootstrap_id IS DISTINCT FROM %s", (self.boot_id,)) + conn.execute("UPDATE fly_scored SET state = 'dropped', x = NULL WHERE state = 'pending' AND chain = %s AND bootstrap_id IS DISTINCT FROM %s", + (self.chain, self.boot_id)) + + teacher_live: dict | None = None + teacher_groups: set = set() + + def _teacher_models(self, boot: dict) -> dict | None: + """The stack that taught this fly (its snapshot's ``teacher_snapshot``: the holdout-blind copy when that is what + taught it), whose final decision on each scored row is stored as ``teacher_allow`` -- the match calibration + rule's budget. None (with a warning) when the fly was taught by a refit stack or the snapshot is gone.""" + meta = boot.get("meta") or {} + sid = meta.get("teacher_snapshot") + if sid is None: + if fly_calibrate.RULE == "match": + log.warning("fly #%s names no teacher snapshot: the match rule keeps its bootstrap line", self.boot_id) + return None + try: + m = selector.load_snapshot(int(sid)) + except Exception: + log.exception("fly #%s: teacher snapshot #%s could not be loaded", self.boot_id, sid) + return None + blind = getattr(m, "blind", None) or {} + models = blind["models"] if "blind to the calibration week" in str(meta.get("teacher") or "") and blind.get("models") else getattr(m, "stack", None) + live = getattr(m, "stack", None) # the gate: the teacher's full stack, as its selector book trades it + if live and live.get("strategies") and all(c in X_COLS for v in live["strategies"].values() for c in v["cols"]): + self.teacher_live = live + self.teacher_groups = set((getattr(m, "metrics", None) or {}).get("groups") or []) + if not models or not models.get("strategies") or any(c not in X_COLS for v in models["strategies"].values() for c in v["cols"]): + log.warning("fly #%s: teacher #%s has no usable strategy stack", self.boot_id, sid) + return None + return models + + def _teacher_allow(self, X: np.ndarray, t: float) -> dict[str, np.ndarray]: + """Per strategy: would the teacher trade each row (trigger, line and every filter)? Empty when unknown.""" + if self.teacher_models is None: + return {} + try: + per = strategies.decide_all(self.teacher_models, X, X_COLS, t) + return {k: np.asarray(v["allow"], bool) for k, v in per.items()} + except Exception: + log.exception("teacher decisions failed for this minute (the fly still trades)") + return {} def _read_state(self) -> dict | None: if not self.state_path.exists(): return None try: - return torch.load(self.state_path, map_location="cpu", weights_only=False) + return torch.load(self.state_path, map_location="cpu", weights_only=True) except Exception: log.exception("fly state unreadable; starting from the bootstrap") return None + def _archive_state(self, s: dict) -> None: + """Keep a state learned under another scoring mode beside the new one (never loaded again).""" + try: + dst = self.state_path.with_name(f"{self.state_path.stem}_{s.get('score', 'own')}_{time.strftime('%Y%m%dT%H%M%S')}.pt") + os.replace(self.state_path, dst) + record_event("info", "fly", f"fly state learned with FLY_SCORE={s.get('score', 'own')} set aside: the fly starts from its bootstrap", + {"path": str(dst), "drift": None}) + except OSError: + log.exception("could not set the old fly state aside") + def _state(self) -> dict: - return {"bootstrap_id": self.boot_id, "cfg": self.cfg, "bank": self.bank.state(), "lines": self.lines, "sizing": self.sizing, + return {"bootstrap_id": self.boot_id, "cfg": self.cfg, "score": config.FLY_SCORE, "bank": self.bank.state() if self.brain is None else None, "lines": self.lines, "sizing": self.sizing, "applied_through": self.applied_through, "learning_frozen": self.learning_frozen, "checks_paused_until": self.checks_paused_until, "race_started_at": self.race_started_at, "last_day": self.last_day, "last_hour": self.last_hour, "rollbacks": self.rollbacks, "saved_at": time.time()} @@ -186,7 +278,8 @@ class FlyBook: torch.save(self._state(), tmp); os.replace(tmp, self.state_path) def _restore_tags(self, conn) -> None: - rows = conn.execute("SELECT ts, mint, strategy, x, score, line FROM fly_scored WHERE state = 'pending' AND bootstrap_id = %s ORDER BY ts", (self.boot_id,)).fetchall() + rows = conn.execute("SELECT ts, mint, strategy, x, score, line FROM fly_scored WHERE state = 'pending' AND bootstrap_id = %s AND chain = %s ORDER BY ts", + (self.boot_id, self.chain)).fetchall() rows = [r for r in rows if r["strategy"] in self.holds] applied = [(r["ts"], r["mint"], r["strategy"]) for r in rows if r["ts"].timestamp() + self.holds[r["strategy"]] + LABEL_LAG_S <= self.applied_through] if applied: @@ -198,14 +291,34 @@ class FlyBook: groups.setdefault((r["ts"].timestamp(), r["strategy"]), []).append(r) for (t, name), rs in sorted(groups.items()): j = self.names.index(name) - Y, u0, k = self.fly.parts_all(np.asarray([r["x"] for r in rs], dtype=np.float32)) - w = plastic.row_weights(torch.tensor([[float(r["score"]) for r in rs]]), torch.tensor([float(rs[0]["line"])])) - self.pending.push([(t, r["mint"], name) for r in rs], np.full(len(rs), t), t + self.holds[name] + LABEL_LAG_S, Y[:, j], u0, k, w, s=j) + Xr = np.asarray([r["x"] for r in rs], dtype=np.float32) + Y, u0, k = self.fly.parts_all(Xr[:, self.idx]) + tv = self._teacher_scores(Xr, t); loc = np.arange(len(rs)) + if tv is not None: + loc = loc[np.isfinite(tv[:, j])] + if not len(loc): + continue + rs = [rs[i] for i in loc]; Xr = Xr[loc]; li = torch.as_tensor(loc, device=Y.device); Y, u0, k = Y[li], u0[li], k[li] + y = self._head(Y[:, j], tv, loc, j, u0) + if config.FLY_TEACHER_GATE and self.teacher_live is not None: # gated: it learns from its teacher's trades, as live + d = strategies.decide(self.teacher_live, Xr, X_COLS, t) + w = torch.tensor([[float(d["strategy"][i] == name and bool(d["allow"][i])) for i in range(len(rs))]]) + elif config.FLY_TEACHER_GATE: + w = torch.zeros((1, len(rs))) + else: + w = plastic.row_weights(torch.tensor([[float(r["score"]) for r in rs]]), torch.tensor([float(rs[0]["line"])])) + self.pending.push([(t, r["mint"], name) for r in rs], np.full(len(rs), t), t + self.holds[name] + LABEL_LAG_S, y, u0, k, w, s=j) for r in rs: self.watch[r["mint"]] = self.watch.get(r["mint"], 0) + 1 def maybe_reload(self) -> bool: - """A newer deployable bootstrap (a re-bootstrap) replaces this one at once: D = 0, pending tags dropped, the book carries on.""" + """A newer deployable bootstrap (a re-bootstrap) replaces this one at once: D = 0, pending tags dropped, the book carries on. + Another chain's book follows its brain (the Solana book reloads first).""" + if self.brain is not None: + if self.brain.boot_id == self.boot_id: + return False + self._write_state(); self._load_bootstrap(None) + return True from ..train import fly_replay with transaction() as conn: boot = fly_selector.latest_deployable(conn) @@ -231,7 +344,7 @@ class FlyBook: q.append(b) def open_mints(self, conn) -> set[str]: - return {p["mint"] for p in ledger.open_positions(conn, BOOK)} + return {p["mint"] for p in ledger.open_positions(conn, self.BOOK)} def on_minute(self, ctx) -> dict: conn, t_now = ctx.conn, ctx.t_start @@ -254,11 +367,12 @@ class FlyBook: self.race_started_at = ctx.m1_epoch self.beat_no += 1 d = scored["decision"] - st = paper_trading.trade_minute(ctx, book=BOOK, run_id=self.run_id, beat_no=self.beat_no, broker=self.broker, kind=KIND, mints=ctx.mints, infos=ctx.infos, + st = paper_trading.trade_minute(ctx, book=self.BOOK, run_id=self.run_id, beat_no=self.beat_no, broker=self.broker, kind=self.KIND, mints=ctx.mints, infos=ctx.infos, scores=d["score"], threshold=d["threshold"], table=None, horizon_s=self.holds[self.names[0]], holds=d["hold_s"], - strategies=d["strategy"], tables=d["tables"]) + strategies=d["strategy"], tables=d["tables"], flat=config.FLY_SIZING == "flat", + allow=d.get("allow"), reasons=d.get("reason")) out.update({k: v for k, v in st.items() if k != "entries"}); out["stage"] = "trading" - if self.live and handover.state(conn) is not None: + if self.live and handover.state(conn, self.chain) is not None: out["live"] = self._live(ctx, st) else: out["stage"] = "learning (not trading this minute)" if ctx.trade else "catching up" @@ -273,12 +387,15 @@ class FlyBook: cur.executemany("UPDATE fly_scored SET label = %s, state = %s, resolved_at = now(), x = NULL WHERE ts = %s AND mint = %s AND strategy = %s", learned.pop("keys")) learned.pop("keys", None) out["checks"] = self.last_checks; out["updated_at"] = datetime.now(timezone.utc).isoformat() - conn.execute("INSERT INTO ui_settings (key, value) VALUES ('fly_status', %s) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()", - (json.dumps(out, default=str),)) + conn.execute("INSERT INTO ui_settings (key, value) VALUES (%s, %s) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()", + (self.status_key, json.dumps({**out, "chain": self.chain}, default=str))) return out def _live(self, ctx, st: dict) -> dict: """The fly holds the seat: mirror this minute on the bot wallet (agent/fly_live.py) while signing is allowed.""" + if self.chain != "sol": + from ..rh import live as rh_live # Robinhood Chain: the ETH wallet (rh/live.py) + return rh_live.minute(self, ctx, st) from ..chain.cluster_guard import signing_allowed if not signing_allowed(): return {"stage": "live gated: signing is not allowed (LIVE_ENABLED=1 and the live prerequisites)"} @@ -343,21 +460,37 @@ class FlyBook: self._capture(H, trig, ctx.t_start); del H if not self.nu_set: self.bank.estimate_nu(Y[:, 0], u0, k); self.nu_set = True - t = ctx.t_start; ts = datetime.fromtimestamp(t, timezone.utc); rows = [] - V = np.full((n, len(self.names)), -np.inf) + t = ctx.t_start; ts = datetime.fromtimestamp(t, timezone.utc); rows = []; ta = self._teacher_allow(ctx.X, t) + V = np.full((n, len(self.names)), -np.inf); F = np.full((n, len(self.names)), np.nan) + gated = config.FLY_TEACHER_GATE + if config.FLY_SCORE == "teacher" and self.teacher_live is None: # it starts from its teacher's score: none, no score + return {"decision": no_teacher(n), "picks": 0} + td = self._teacher_decision(ctx) if gated and self.teacher_live is not None else None + tv = self._teacher_scores(ctx.X, t) for j, name in enumerate(self.names): loc = np.flatnonzero(trig[:, j]) + if tv is not None: + loc = loc[np.isfinite(tv[loc, j])] # a candidate its teacher scores if not len(loc): continue li = torch.as_tensor(loc, device=Y.device) + y = self._head(Y[li, j], tv, loc, j, u0[li]) with torch.no_grad(): - sc, _ = self.bank.predict(Y[li, j], u0[li], k[li], s=np.full(len(loc), j)); sc = sc[0] - fr = self.bank.frozen(Y[li, j], u0[li], s=np.full(len(loc), j)) - v, f = sc.float().cpu().numpy(), fr.float().cpu().numpy(); V[loc, j] = v + sc, _ = self.bank.predict(y, u0[li], k[li], s=np.full(len(loc), j)); sc = sc[0] + fr = self.bank.frozen(y, u0[li], s=np.full(len(loc), j)) + v, f = sc.float().cpu().numpy(), fr.float().cpu().numpy(); V[loc, j] = v; F[loc, j] = f line, fline = self.lines["plastic"][name], self.lines["frozen"][name] - rows += [(ts, ctx.mints[i], name, int(self.holds[name] // 60), ctx.X[i].tolist(), float(v[q]), float(f[q]), line, fline, self.boot_id) for q, i in enumerate(loc)] - w = plastic.row_weights(sc[None], torch.tensor([line], dtype=sc.dtype, device=sc.device)) - self.pending.push([(t, ctx.mints[i], name) for i in loc], np.full(len(loc), t), t + self.holds[name] + LABEL_LAG_S, Y[li, j], u0[li], k[li], w, s=j) + al = ta.get(name) + rows += [(ts, ctx.mints[i], name, int(self.holds[name] // 60), ctx.X[i].tolist(), float(v[q]), float(f[q]), line, fline, self.boot_id, + None if al is None else bool(al[i])) for q, i in enumerate(loc)] + if td is not None: # gated: its decisions are the teacher's trades of this strategy (before its own vetoes), so it learns from those + mine = np.array([td["strategy"][i] == name and bool(td["allow"][i]) for i in loc], dtype=np.float32) + w = torch.as_tensor(mine[None], dtype=sc.dtype, device=sc.device) + elif gated: # the gate is on but its teacher is not loaded: nothing to learn from until it is + w = torch.zeros((1, len(loc)), dtype=sc.dtype, device=sc.device) + else: + w = plastic.row_weights(sc[None], torch.tensor([line], dtype=sc.dtype, device=sc.device)) + self.pending.push([(t, ctx.mints[i], name) for i in loc], np.full(len(loc), t), t + self.holds[name] + LABEL_LAG_S, y, u0[li], k[li], w, s=j) for i in loc: m = ctx.mints[i]; self.watch[m] = self.watch.get(m, 0) + 1 q_ = self.bars.setdefault(m, deque(maxlen=BAR_KEEP)); b = ctx.bars.get(m) @@ -365,12 +498,64 @@ class FlyBook: q_.append(b) if rows: with conn.cursor() as cur: - cur.executemany("INSERT INTO fly_scored (ts, mint, strategy, hold_min, x, score, frozen_score, line, frozen_line, bootstrap_id) " - "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT (ts, mint, strategy) DO NOTHING", rows) - self.fly.lines = dict(self.lines["plastic"]); self.fly.sizings = {k: list(v) for k, v in self.sizing["plastic"].items()} + cur.executemany("INSERT INTO fly_scored (ts, mint, strategy, hold_min, x, score, frozen_score, line, frozen_line, bootstrap_id, teacher_allow, chain) " + "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT (ts, mint, strategy) DO NOTHING", [r + (self.chain,) for r in rows]) + if self.chain == "sol": + self.fly.lines = dict(self.lines["plastic"]); self.fly.sizings = {k: list(v) for k, v in self.sizing["plastic"].items()} + else: # the shared model decides this chain's rows on this chain's lines + self.fly.chain_lines[self.chain] = dict(self.lines["plastic"]); self.fly.chain_sizings[self.chain] = {k: list(v) for k, v in self.sizing["plastic"].items()} + if td is not None: + d = self._gated(ctx, V, F, td) + return {"decision": d, "picks": int(d["allow"].sum())} + if gated: # never the ungated fly: it bought what the selector rejected and lost (2026-10-08) + return {"decision": no_teacher(n), "picks": 0} d = fly_selector.fly_decide(self.fly, ctx.X, X_COLS, np.where(np.isfinite(V), V, -np.inf)) return {"decision": d, "picks": int(sum(1 for x in d["strategy"] if x is not None))} + def _teacher_scores(self, X: np.ndarray, t) -> np.ndarray | None: + """[B, S] the teacher's own score of each row per strategy (NaN off its trigger) when the fly scores as its teacher + plus its learned correction (config.FLY_SCORE = teacher), else None.""" + if config.FLY_SCORE != "teacher" or self.teacher_live is None: + return None + return fly_selector.teacher_scores(self.teacher_live, X, t, self.names, X_COLS) + + def _head(self, y_own: torch.Tensor, tv: np.ndarray | None, loc: np.ndarray, j: int, u0: torch.Tensor) -> torch.Tensor: + """The head output the bank scores with: the fly's own decoder output, or (FLY_SCORE = teacher) the one under which + its frozen score is the teacher's score exactly (PlasticBank.head_for), so only its learning moves it.""" + if tv is None: + return y_own + return self.bank.head_for(torch.as_tensor(tv[loc, j], dtype=torch.float32, device=self.bank.dev), u0, s=np.full(len(loc), j)) + + def _teacher_decision(self, ctx) -> dict: + """The deployed selector's decision on this minute's rows, with its live fail-closed rules.""" + from .selector_session import fail_closed + return fail_closed(self.teacher_groups, ctx.infos, strategies.decide(self.teacher_live, ctx.X, X_COLS, ctx.t_start)) + + def _gated(self, ctx, V: np.ndarray, F: np.ndarray, td: dict | None = None) -> dict: + """The fly's decision under its teacher: each row's strategy, trigger, line, win filter, gates, dump veto and + fail-closed rules are the deployed selector's (agent/selector_session.fail_closed), so a fresh bootstrap trades the + selector's set; the fly drops a teacher trade only where its learning has moved its score more than + ``FLY_LEARNED_VETO`` below its bootstrap score. A row the teacher blocks is recorded with the teacher's reason.""" + n = len(ctx.mints) + td = td if td is not None else self._teacher_decision(ctx) + strat = np.full(n, None, dtype=object); score = np.full(n, -1.0); hold = np.zeros(n); thr = np.full(n, np.inf) + tables = [[] for _ in range(n)]; allow = np.zeros(n, bool); reason = np.full(n, "", dtype=object) + for i in range(n): + s_ = td["strategy"][i] + if s_ is None or s_ not in self.names: + continue + j = self.names.index(s_); v, f = V[i, j], F[i, j] + # the selector's own score, line and sizing table for the row: a gated fly buy is sized exactly as the selector + # sizes it (agent/sizing.size_position; FLY_SIZING=kelly: the same Kelly band, fraction and caps; flat: flat) + strat[i] = s_; score[i] = float(td["score"][i]); thr[i] = float(td["threshold"][i]); hold[i] = self.holds[s_]; tables[i] = td["tables"][i] + if not td["allow"][i]: + reason[i] = f"selector: {td['reason'][i]}" + elif np.isfinite(v) and np.isfinite(f) and v < f - config.FLY_LEARNED_VETO: + reason[i] = f"learned veto: score {v:.3f} < bootstrap {f:.3f} - {config.FLY_LEARNED_VETO:g}" + else: + allow[i] = True; reason[i] = "trade" + return {"strategy": strat, "score": score, "hold_s": hold, "threshold": thr, "tables": tables, "allow": allow, "reason": reason} + def _capture(self, H: torch.Tensor, trig: np.ndarray, t: float) -> None: """The console's brain view: this minute's mean activity over the rows some strategy's trigger fired on (every row when none did), one file per minute (brain/activity.py). Never allowed to interrupt scoring.""" @@ -387,25 +572,39 @@ class FlyBook: # ---- daily and hourly ---- def _resolved(self, conn, since_resolved: float, cols: str, name: str) -> list[dict]: """A strategy's resolved rows whose labels became known after ``since_resolved`` (epoch s).""" - return conn.execute(f"SELECT {cols} FROM fly_scored WHERE state = 'resolved' AND label IS NOT NULL AND bootstrap_id = %s AND strategy = %s AND ts >= %s ORDER BY ts", - (self.boot_id, name, datetime.fromtimestamp(since_resolved - self.holds[name] - LABEL_LAG_S, timezone.utc))).fetchall() + return conn.execute(f"SELECT {cols} FROM fly_scored WHERE state = 'resolved' AND label IS NOT NULL AND bootstrap_id = %s AND strategy = %s AND chain = %s " + "AND ts >= %s ORDER BY ts", (self.boot_id, name, self.chain, + datetime.fromtimestamp(since_resolved - self.holds[name] - LABEL_LAG_S, timezone.utc))).fetchall() def _calibrate(self, conn, now: float) -> None: day = datetime.fromtimestamp(now, timezone.utc).date() + rule = fly_calibrate.RULE for name in self.names: - rows = self._resolved(conn, now - fly_calibrate.WINDOW_DAYS * 86400.0, "ts, mint, score, frozen_score, label", name) + rows = self._resolved(conn, now - fly_calibrate.WINDOW_DAYS * 86400.0, "ts, mint, score, frozen_score, label, teacher_allow", name) + target = None + if rule == "match": + # the teacher's trades over the same rows are the budget; rows scored before it was recorded cannot count + rows = [r for r in rows if r["teacher_allow"] is not None] + if not rows: + log.info("fly %s: no scored rows with the teacher's decision yet; the line stays %.4f", name, self.lines["plastic"][name]) + continue if not rows: continue ts = np.array([r["ts"].timestamp() for r in rows]); mint = np.array([r["mint"] for r in rows]); lab = np.array([r["label"] for r in rows], dtype=np.float64) + if rule == "match": + al = np.array([bool(r["teacher_allow"]) for r in rows]) & np.isfinite(lab) + target = len(fly_calibrate.taken_idx(ts, mint, self.holds[name], np.flatnonzero(al))) for arm, col in (("plastic", "score"), ("frozen", "frozen_score")): cal = fly_calibrate.calibrate(ts, mint, self.holds[name], np.array([r[col] for r in rows], dtype=np.float64), lab, - prev_line=self.lines[arm][name], prev_sizing=self.sizing[arm][name]) + prev_line=self.lines[arm][name], prev_sizing=self.sizing[arm][name], rule=rule, target_trades=target) if cal.changed: - record_event("info", "fly", f"{name} {arm} line {self.lines[arm][name] * 100:+.2f}% → {cal.line * 100:+.2f}%", {"trades": cal.trades, "mean": cal.mean, "total": cal.total}) + record_event("info", "fly", f"{name} {arm} line {self.lines[arm][name] * 100:+.2f}% → {cal.line * 100:+.2f}%", + {"trades": cal.trades, "mean": cal.mean, "total": cal.total, "rule": rule, "teacher_trades": target}) self.lines[arm][name], self.sizing[arm][name] = cal.line, cal.sizing conn.execute("INSERT INTO fly_calibrations (day, arm, line, sizing, trades, total, mean, window_days) VALUES (%s,%s,%s,%s,%s,%s,%s,%s) " "ON CONFLICT (day, arm) DO UPDATE SET line = EXCLUDED.line, sizing = EXCLUDED.sizing, trades = EXCLUDED.trades, total = EXCLUDED.total, mean = EXCLUDED.mean", - (day, f"{arm}:{name}", cal.line, json.dumps(cal.sizing), cal.trades, cal.total, cal.mean, fly_calibrate.WINDOW_DAYS)) + (day, f"{arm}:{name}" if self.chain == "sol" else f"{arm}:{self.chain}:{name}", cal.line, json.dumps(cal.sizing), cal.trades, + cal.total, cal.mean, fly_calibrate.WINDOW_DAYS)) def _drift(self, j: int) -> float: return float(self.bank.drift(j)[0]) if self.bank.learn.shape[0] > 1 else float(self.bank.drift()[0]) @@ -427,6 +626,10 @@ class FlyBook: return out def _hourly(self, conn, now: float) -> None: + if self.brain is not None: + # another chain's book: its own handover; the shared memory's checks, snapshots and rollbacks are the Solana book's + self._handover_check(conn, now) + return hour = datetime.fromtimestamp(math.floor(now / 3600) * 3600 - 3600, timezone.utc) checks = self._checks(conn, now); self.last_checks = checks h = self.hour_stats; first = checks.get(self.names[0]) or {} @@ -447,7 +650,7 @@ class FlyBook: d = fly_state_dir(); d.mkdir(parents=True, exist_ok=True) path = d / f"snap_{datetime.fromtimestamp(now, timezone.utc).strftime('%Y%m%dT%H%M')}.pt" torch.save({"bank": self.bank.state(), "bootstrap_id": self.boot_id, "lines": self.lines, "sizing": self.sizing}, path) - note = {"bootstrap_id": self.boot_id, "data": fly_selector.FLY_VERSION, "good": good, "checks": checks, + note = {"bootstrap_id": self.boot_id, "data": fly_selector.FLY_VERSION, "score": config.FLY_SCORE, "good": good, "checks": checks, "drift": max((c["drift"] for c in checks.values()), default=0.0), "ic24": {k: c.get("ic") for k, c in checks.items()}} return int(conn.execute("INSERT INTO brain_snapshots (run_id, path, kind, note) VALUES (%s,%s,'fly_plastic',%s) RETURNING id", (self.run_id, str(path), json.dumps(note, default=str))).fetchone()["id"]) @@ -476,11 +679,11 @@ class FlyBook: (datetime.fromtimestamp(now - ROLLBACK_MIN_AGE_S, timezone.utc),)).fetchall(): n = json.loads(r["note"] or "{}") ok = n.get("good") or not ((n.get("checks") or {}).get(name) or {}).get("triggers") - if ok and n.get("bootstrap_id") == self.boot_id and Path(r["path"]).exists(): + if ok and n.get("bootstrap_id") == self.boot_id and n.get("score", "own") == config.FLY_SCORE and Path(r["path"]).exists(): target = r; break if target is not None: snap = plastic.PlasticBank(self.fly.net, [(0.0, math.inf)], fly_selector.SCALE, learn=self.bank.learn.cpu().numpy(), read=self.bank.read.cpu().numpy()) - st = torch.load(target["path"], map_location="cpu", weights_only=False)["bank"] + st = torch.load(target["path"], map_location="cpu", weights_only=True)["bank"] snap.D = torch.zeros_like(snap.D); snap.D[:, snap.mask.bool()] = st["D"].to(snap.dev).float() self.bank.D[0][:, cols] = snap.D[0][:, cols] else: @@ -519,15 +722,17 @@ class FlyBook: whole race), closed ``config.HANDOVER_MIN_TRADES`` paper trades and, when ``config.HANDOVER_BEAT_SELECTOR``, realized at least the paper selector's P&L over that window.""" days = config.HANDOVER_DAYS - if handover.state(conn) is not None or self.race_started_at is None or now - self.race_started_at < days * 86400.0: + if handover.state(conn, self.chain) is not None or self.race_started_at is None or now - self.race_started_at < days * 86400.0: return since = datetime.fromtimestamp(self.race_started_at if days <= 0 else now - days * 86400.0, timezone.utc) + fb, sb = self.BOOK, self.market.selector_book # this chain's race: its fly book against its selector book pnl = {b: conn.execute("SELECT COALESCE(sum(realized_sol), 0) AS s, count(*) AS n FROM positions WHERE book = %s AND status = 'closed' AND closed_at >= %s", - (b, since)).fetchone() for b in (BOOK, "paper_selector")} - fly_pnl, sel_pnl, n = float(pnl[BOOK]["s"]), float(pnl["paper_selector"]["s"]), int(pnl[BOOK]["n"]) + (b, since)).fetchone() for b in (fb, sb)} + fly_pnl, sel_pnl, n = float(pnl[fb]["s"]), float(pnl[sb]["s"]), int(pnl[fb]["n"]) if n >= config.HANDOVER_MIN_TRADES and (fly_pnl >= sel_pnl or not config.HANDOVER_BEAT_SELECTOR): detail = {"at": datetime.fromtimestamp(now, timezone.utc).isoformat(), "fly_pnl_sol": fly_pnl, "selector_pnl_sol": sel_pnl, "fly_trades": n, - "selector_trades": int(pnl["paper_selector"]["n"]), "days": days, "beat_selector": config.HANDOVER_BEAT_SELECTOR, "bootstrap": self.boot_id} - handover.record(conn, detail) + "selector_trades": int(pnl[sb]["n"]), "days": days, "beat_selector": config.HANDOVER_BEAT_SELECTOR, "bootstrap": self.boot_id, + "chain": self.chain, "unit": self.market.unit} + handover.record(conn, detail, self.chain) record_event("info", "handover", f"the fly takes the selector's seat: {fly_pnl:+.3f} SOL vs {sel_pnl:+.3f} SOL over {n} trades" + (f" in {days} days" if days > 0 else ""), detail) diff --git a/fly_trader/agent/handover.py b/fly_trader/agent/handover.py index f7532d135ce25c444db76c80e39ce36e934ef55d..fda3ad49fd357c8757e6cc6a53ded3f22d78a9f2 100644 --- a/fly_trader/agent/handover.py +++ b/fly_trader/agent/handover.py @@ -11,14 +11,22 @@ import json KEY = "handover" -def state(conn) -> dict | None: - r = conn.execute("SELECT value FROM ui_settings WHERE key = %s", (KEY,)).fetchone() +def _key(chain: str | None) -> str: + """Each chain's fly takes its own chain's seat (fly_trader/markets: 'handover', 'handover_rh').""" + if chain in (None, "sol"): + return KEY + from ..markets import for_chain + return for_chain(chain).handover_key + + +def state(conn, chain: str | None = "sol") -> dict | None: + r = conn.execute("SELECT value FROM ui_settings WHERE key = %s", (_key(chain),)).fetchone() if not r or r["value"] is None: return None v = r["value"] if isinstance(r["value"], dict) else json.loads(r["value"]) return v or None -def record(conn, detail: dict) -> None: +def record(conn, detail: dict, chain: str | None = "sol") -> None: conn.execute("INSERT INTO ui_settings (key, value) VALUES (%s, %s) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()", - (KEY, json.dumps(detail, default=str))) + (_key(chain), json.dumps(detail, default=str))) diff --git a/fly_trader/agent/minute_engine.py b/fly_trader/agent/minute_engine.py index 6b39ce44d69915cb3a400a133a7d3589fb3acfe9..1be386f8111917c2e3a779b59b0006ddc8f70a69 100644 --- a/fly_trader/agent/minute_engine.py +++ b/fly_trader/agent/minute_engine.py @@ -30,6 +30,7 @@ from datetime import datetime, timezone import numpy as np +from .. import config, markets from ..db.apilog import record_event from ..db.connection import transaction from ..market.exit_cost import PUMP_SUPPLY @@ -78,6 +79,7 @@ class Minute: fresh: bool trade: bool engine: "MinuteEngine" + market: object = None # fly_trader/markets.MarketSpec of these rows (None: Solana) @property def t_start(self) -> float: @@ -91,7 +93,14 @@ class Minute: class MinuteEngine: - def __init__(self, books: list | None = None, live: bool = False): + """One market's minutes (``market``: Solana — pump_minutes, corpus_meta — or Robinhood Chain — rh_minutes, rh_meta, + amounts in ETH). The engines of both markets share one list of books; each minute goes only to its market's books.""" + + def __init__(self, books: list | None = None, live: bool = False, market=None): + self.spec = market or markets.SOL; self.rh = self.spec.chain != "sol" + # how old a minute may be and still be traded: Solana's feed flushes within seconds; the RH stream's minutes land + # ~2–3 min late (confirmations, the index pass, the price marks), so on RH "the current minute" never arrives + self.trade_lag_s = float(config.RH_TRADE_LAG_S) if self.rh else 0.0 self.books = list(books or []); self.live = live self.states: dict[str, MintState] = {} self.meta_cache: dict[str, dict] = {}; self.last_sweep = time.time(); self.last_minute: float | None = None @@ -100,21 +109,34 @@ class MinuteEngine: def has_book(self, name: str) -> bool: return any(b.name == name for b in self.books) + def tradeable(self, minute: float, m1: float) -> bool: + """Whether minute ``minute`` (its end, epoch s) is recent enough to trade at wall-clock minute ``m1``.""" + return int(m1) - self.trade_lag_s <= int(minute) <= int(m1) + + def my_books(self) -> list: + return [b for b in self.books if getattr(b, "chain", "sol") == self.spec.chain] + # ---- lookups ---- def _meta(self, conn, mint: str) -> dict | None: """``corpus_meta`` row (graduation time + creation/creator features), cached for 10 minutes: rows appear at graduation and the archive rebuild later replaces stream values, so the cache is refreshed.""" hit = self.meta_cache.get(mint) if hit is None or time.time() - hit["at"] > META_TTL_S: - r = conn.execute("SELECT graduated_at, supply, " + ", ".join(META_COLS) + " FROM corpus_meta WHERE mint = %s", (mint,)).fetchone() + if self.rh: # Robinhood Chain: rh_meta, the same columns (rh/meta.py), plus the quote class + r = conn.execute("SELECT graduated_at, supply, quote_class, " + ", ".join(META_COLS) + " FROM rh_meta WHERE mint = %s", (mint,)).fetchone() + else: + r = conn.execute("SELECT graduated_at, supply, " + ", ".join(META_COLS) + " FROM corpus_meta WHERE mint = %s", (mint,)).fetchone() hit = self.meta_cache[mint] = {"row": dict(r) if r else None, "at": time.time()} return hit["row"] def _state(self, conn, mint: str, pool: str | None, program_label: str | None) -> MintState: s = self.states.get(mint) if s is None: - t = conn.execute("SELECT decimals FROM tokens WHERE mint = %s", (mint,)).fetchone() - s = MintState(mint, int((t or {}).get("decimals") or 6), pool, program_label, None, None) + if self.rh: # every Pons launch has 18 decimals + s = MintState(mint, 18, pool, program_label, None, None); s.meta.ec_size = 0.1 * self.spec.k() + else: + t = conn.execute("SELECT decimals FROM tokens WHERE mint = %s", (mint,)).fetchone() + s = MintState(mint, int((t or {}).get("decimals") or 6), pool, program_label, None, None) self.states[mint] = s row = self._meta(conn, mint) s.meta_row = row @@ -128,8 +150,11 @@ class MinuteEngine: s = self.states.get(mint) if s is not None and s.st.last_res_quote_sol: return s.st.last_res_quote_sol - r = conn.execute("SELECT resq_sol FROM pump_minutes WHERE mint = %s AND resq_sol IS NOT NULL ORDER BY ts DESC LIMIT 1", (mint,)).fetchone() - return float(r["resq_sol"]) if r else None + if self.rh: + r = conn.execute("SELECT resq_eth AS q FROM rh_minutes WHERE mint = %s AND resq_eth IS NOT NULL ORDER BY ts DESC LIMIT 1", (mint,)).fetchone() + else: + r = conn.execute("SELECT resq_sol AS q FROM pump_minutes WHERE mint = %s AND resq_sol IS NOT NULL ORDER BY ts DESC LIMIT 1", (mint,)).fetchone() + return float(r["q"]) if r else None def _mcap(self, mint: str, price: float | None) -> float | None: """Market cap in SOL (price × supply), which sets the pool fee tier; None without a price.""" @@ -138,7 +163,16 @@ class MinuteEngine: # ---- one minute ---- def _aggregate(self, conn, m0: datetime, m1: datetime) -> dict[str, dict]: - """PumpAPI minutes: every SOL-quoted PumpSwap pump.fun token, the same fields as the archive.""" + """PumpAPI minutes: every SOL-quoted PumpSwap pump.fun token, the same fields as the archive. Robinhood Chain: the + rh_minutes of graduated Pons pools in ETH (rh/minutes.py), under the same keys.""" + if self.rh: + rows = conn.execute("SELECT mint, pool_id, open, high, low, close, buy_eth, sell_eth, n_buys, n_sells, n_traders, resq_eth, fee_rate, " + "n_buyers, wash_eth, wash_buy_eth, top_sell_eth, insider_sell_eth, skill_buy FROM rh_minutes WHERE ts = %s", (m0,)).fetchall() + return {r["mint"]: {"open": r["open"], "high": r["high"], "low": r["low"], "close": r["close"], "buy": r["buy_eth"] or 0.0, "sell": r["sell_eth"] or 0.0, + "nb": r["n_buys"] or 0, "ns": r["n_sells"] or 0, "n_traders": int(r["n_traders"] or 0), "resq": r["resq_eth"], + "pool": r["pool_id"], "program_label": self.spec.program_label, "fee_rate": r["fee_rate"], + "n_buyers": r["n_buyers"], "wash_sol": r["wash_eth"], "wash_buy_sol": r["wash_buy_eth"], "top_sell_sol": r["top_sell_eth"], + "insider_sell_sol": r["insider_sell_eth"], "skill_buy": r["skill_buy"]} for r in rows if r["close"]} rows = conn.execute("SELECT mint, pool_id, open, high, low, close, buy_sol, sell_sol, n_buys, n_sells, n_traders, resq_sol, fee_rate, " "n_buyers, wash_sol, wash_buy_sol, top_sell_sol, insider_sell_sol, skill_buy FROM pump_minutes WHERE ts = %s", (m0,)).fetchall() return {r["mint"]: {"open": r["open"], "high": r["high"], "low": r["low"], "close": r["close"], "buy": r["buy_sol"] or 0.0, "sell": r["sell_sol"] or 0.0, @@ -151,7 +185,9 @@ class MinuteEngine: """The full feature vector (``X_COLS``) of a mint for the minute ending ``t_end``, and what trading needs.""" s = self._state(conn, mint, a["pool"], a["program_label"]) st, price, resq = s.st, float(a["close"]), a["resq"] - if (s.prev_close and not (1 / 50 <= price / s.prev_close <= 50)) or (resq is not None and resq > 1e5): + if self.rh: + s.meta.pool_fee = a.get("fee_rate") # the Pons hook fee this minute charged (the cost model's fee) + if (s.prev_close and not (1 / 50 <= price / s.prev_close <= 50)) or (resq is not None and resq > 1e5 * self.spec.k()): s.broken = True # train/decisions.py: a scale break makes the mint ineligible from that minute on s.prev_close = price if a["buy"] > 0: @@ -175,21 +211,27 @@ class MinuteEngine: "meta_known": 1.0 if meta.get("ttg_min") is not None else 0.0, **s.flow.features(t_end), "mkt_vol_1h": self.market.value(t_end)} by_name = {**{n: float(f[i]) for i, n in enumerate(FEATURES)}, **extra, **{c: float(meta.get(c)) if meta.get(c) is not None else 0.0 for c in META_COLS}} + if self.rh: # the chain inputs (train/decisions.CHAIN_COLS) + qc = meta.get("quote_class") + by_name.update(chain_rh=1.0, qc_stable=float(qc == "stable"), qc_btc=float(qc == "btc"), qc_stock=float(qc == "stock")) x = np.nan_to_num(np.asarray([by_name.get(c, 0.0) for c in X_COLS], dtype=np.float32), nan=0.0, posinf=0.0, neginf=0.0) info = {"price": price, "open": float(a["open"]) if a.get("open") else price, "resq": resq, "logvol_15m": f[FIDX["logvol_15m"]], # "ec" is the training label's own cost model (market/exit_cost.cost_at_size), at the size the book trades - "ec": float(cost_at_size(f[FIDX["exit_cost_0p1"]], resq)), "age_h": (t_end - s.graduated_at) / 3600 if s.graduated_at else None, + "ec": float(cost_at_size(f[FIDX["exit_cost_0p1"]], resq, market=self.spec if self.rh else None)), + "age_h": (t_end - s.graduated_at) / 3600 if s.graduated_at else None, "mcap": price * s.meta.supply, "fee_rate": a.get("fee_rate"), "decimals": s.decimals, "pool": s.pool, "program_label": s.program_label, "broken": s.broken, - "skill_missing": a.get("skill_buy") is None, "curve_known": bool(meta.get("curve_known"))} + "skill_missing": a.get("skill_buy") is None, "curve_known": bool(meta.get("curve_known")), "chain": self.spec.chain, + "quote_class": meta.get("quote_class")} return x, info - @staticmethod - def eligible(info: dict) -> bool: + def eligible(self, info: dict) -> bool: + if self.rh: # the same gates in the chain's unit (x K), as training (train/decisions) + return not info["broken"] and info["resq"] is not None and info["resq"] >= self.spec.min_resq() and info["logvol_15m"] >= math.log1p(self.spec.min_vol_15m()) return not info["broken"] and info["resq"] is not None and info["resq"] >= MIN_RESQ_SOL and info["logvol_15m"] >= math.log1p(MIN_VOL_15M_SOL) def _stream_through(self) -> float | None: with transaction() as conn: - r = conn.execute("SELECT value->>'flushed_through' AS ft FROM ui_settings WHERE key = 'pumpstream_status'").fetchone() + r = conn.execute("SELECT value->>'flushed_through' AS ft FROM ui_settings WHERE key = %s", (self.spec.stream_status_key,)).fetchone() return datetime.fromisoformat(r["ft"]).timestamp() if r and r["ft"] else None def ready_through(self, m1_epoch: float) -> float: @@ -224,7 +266,7 @@ class MinuteEngine: for mint, a in agg.items(): _x, info = self._features(conn, mint, a, t1); n += 1 bars[mint] = (t1 - 60.0, info["open"], info["price"], info["ec"]) - for b in self.books: + for b in self.my_books(): b.on_bars(t1 - 60.0, bars) log.info("warm-up: %d token-minutes over the last %d minutes (%d tokens)", n, minutes, len(self.states)) return n @@ -247,9 +289,10 @@ class MinuteEngine: X, infos, mints, bars = self._minute_rows(conn, agg, m1_epoch) ctx = Minute(conn=conn, m0=m0, m1=m1, m1_epoch=m1_epoch, agg=agg, X=X, infos=infos, mints=mints, bars=bars, prices={m: float(a["close"]) for m, a in agg.items()}, resqs={m: a["resq"] for m, a in agg.items()}, - fees={m: a.get("fee_rate") for m, a in agg.items()}, fresh=trade and self.stream_fresh(m1_epoch - 60), trade=trade, engine=self) + fees={m: a.get("fee_rate") for m, a in agg.items()}, fresh=trade and self.stream_fresh(m1_epoch - 60), trade=trade, engine=self, + market=self.spec) out.update(mints_traded=len(agg), eligible=len(mints)); held: set[str] = set() - for b in list(self.books): + for b in list(self.my_books()): try: with conn.transaction(): # a savepoint per book b.on_bars(m1_epoch - 60.0, bars) @@ -259,7 +302,7 @@ class MinuteEngine: log.exception("%s: minute %s failed", b.name, m1.isoformat()) record_event("error", b.name, f"minute failed: {m1.isoformat()}") self._sweep(m1_epoch, held) - for b in [b for b in self.books if b.done]: + for b in [b for b in self.my_books() if b.done]: b.finish(); self.books.remove(b) return out @@ -288,35 +331,45 @@ def admit_books(engine: MinuteEngine, live: bool) -> list: from . import fly_session, handover from .selector_session import NoModel, SelectorBook new = [] - if not engine.has_book("fly"): - fb, why = fly_session.try_start(live) - if fb is not None: - new.append(fb) - else: - _status("fly_status", {"stage": "not trading", "detail": why}) - fly_on = engine.has_book("fly") or bool(new) - with transaction() as conn: - handed = handover.state(conn) is not None - if not engine.has_book("selector") and not (handed and fly_on): - try: - new.append(SelectorBook()) - except NoModel as e: - _status("selector_status", {"stage": "waiting for a model", "detail": str(e)}) + for m in markets.enabled(): # Solana first: another chain's fly shares the Solana fly's brain + fly_name = "fly" if m.chain == "sol" else f"fly_{m.chain}"; sel_name = "selector" if m.chain == "sol" else f"selector_{m.chain}" + if not engine.has_book(fly_name) and not any(b.name == fly_name for b in new): + brain = next((b for b in engine.books + new if b.name == "fly"), None) if m.chain != "sol" else None + fb, why = fly_session.try_start(live, chain=m.chain, brain=brain) + if fb is not None: + new.append(fb) + else: + _status(m.fly_status_key, {"stage": "not trading", "detail": why, "chain": m.chain}) + fly_on = engine.has_book(fly_name) or any(b.name == fly_name for b in new) + with transaction() as conn: + handed = handover.state(conn, m.chain) is not None + if not engine.has_book(sel_name) and not (handed and fly_on): + try: + new.append(SelectorBook(m.chain)) + except NoModel as e: + _status(m.selector_status_key, {"stage": "waiting for a model", "detail": str(e), "chain": m.chain}) return new def main(stop_event: threading.Event | None = None, live: bool = False) -> None: - engine = MinuteEngine(live=live) + """One engine per enabled chain (markets.enabled()), all sharing one list of books: each engine feeds its minutes, as + its own stream writes them, to its chain's books only — a lagging chain never holds another chain's minutes back.""" + engines = [MinuteEngine(live=live, market=m) for m in markets.enabled()] + lead = engines[0]; books: list = [] + for e in engines: + e.books = books # the same list object: admission and removal are seen by every engine while not (stop_event is not None and stop_event.is_set()): - engine.books = admit_books(engine, live) - if engine.books: + books += admit_books(lead, live) + if books: break log.info("trading engine waiting: no book qualifies yet") (stop_event or threading.Event()).wait(MODEL_CHECK_S) else: return m_start = math.floor(time.time() / 60) * 60 - engine.warm_up(m_start); engine.last_minute = m_start - 60; last_check = time.time(); n_min = 0 + for e in engines: + e.warm_up(m_start); e.last_minute = m_start - 60 + last_check = time.time(); n_min = 0 last_release, last_stamp_check = release_stamp(), 0.0 try: while not (stop_event is not None and stop_event.is_set()): @@ -327,31 +380,35 @@ def main(stop_event: threading.Event | None = None, live: bool = False) -> None: last_release, last_check = stamp, 0.0 if now - last_check >= MODEL_CHECK_S: last_check = now - for b in engine.books: + for b in books: try: b.maybe_reload() except Exception: log.exception("%s: model reload check failed", b.name) try: - engine.books += admit_books(engine, live) + books += admit_books(lead, live) except Exception: log.exception("book admission failed") - if engine.books and m1 > engine.last_minute and now - m1 >= 4.0 and (upto := engine.ready_through(m1)) > engine.last_minute: # only minutes the stream has written + for engine in engines: + if not engine.my_books() or m1 <= engine.last_minute or now - m1 < 4.0: + continue + if (upto := engine.ready_through(m1)) <= engine.last_minute: # only minutes this chain's stream has written + continue for minute in range(int(engine.last_minute) + 60, int(upto) + 1, 60): try: - st = engine.run_minute(float(minute), trade=(minute == int(m1))) # missed minutes update features (and the fly's learning) only + st = engine.run_minute(float(minute), trade=engine.tradeable(minute, m1)) # older minutes update features (and the fly's learning) only n_min += 1 - if minute == int(m1) and n_min % 10 == 0: - log.info("minute %s: traded %d eligible %d | %s", st["minute"], st.get("mints_traded", 0), st.get("eligible", 0), + if engine.tradeable(minute, m1) and n_min % 10 == 0: + log.info("%s minute %s: traded %d eligible %d | %s", engine.spec.chain, st["minute"], st.get("mints_traded", 0), st.get("eligible", 0), " | ".join(f"{b.name}: picks {(st.get(b.name) or {}).get('picks')} open {(st.get(b.name) or {}).get('open')} wealth {(st.get(b.name) or {}).get('wealth')}" - for b in engine.books)) + for b in engine.my_books())) except Exception: - log.exception("minute %s failed", minute); record_event("error", "engine", f"minute failed: {minute}") + log.exception("%s minute %s failed", engine.spec.chain, minute); record_event("error", "engine", f"{engine.spec.chain} minute failed: {minute}") engine.last_minute = upto time.sleep(0.5) finally: - for b in engine.books: + for b in books: try: b.finish() except Exception: - log.exception("%s: finish failed", b.name) + log.exception("%s: finish failed", b.name) \ No newline at end of file diff --git a/fly_trader/agent/paper_trading.py b/fly_trader/agent/paper_trading.py index 0f255f720af8117e32480a20e8f2023c3d4b3290..42b0fe5850c014fff6f2f01993b98e6cb68a4f8c 100644 --- a/fly_trader/agent/paper_trading.py +++ b/fly_trader/agent/paper_trading.py @@ -20,10 +20,11 @@ from ..market.exit_cost import exit_cost_fraction def trade_minute(ctx, *, book: str, run_id: str, beat_no: int, broker, kind: str, mints: list, infos: list, scores, threshold, table: list | None, horizon_s: float, block: str | None = None, holds=None, strategies=None, allow=None, reasons=None, - tables: list | None = None) -> dict: + tables: list | None = None, flat: bool = False) -> dict: """Optional per-row arguments (the selector's strategy stack, train/strategies.decide): ``threshold`` may be per row, ``holds`` (s), ``strategies``, ``tables`` (sizing) and ``allow`` / ``reasons`` (a blocked row is recorded with its reason). - Left out, the call behaves exactly as the single-strategy engine.""" + Left out, the call behaves exactly as the single-strategy engine. ``flat``: every buy the same share of the bankroll + (agent/sizing.size_position), the fly's rule (``config.FLY_SIZING``).""" conn, m1 = ctx.conn, ctx.m1 beat = conn.execute("INSERT INTO beats (run_id, ts, beat_no, n_slots_active, notes) VALUES (%s,%s,%s,%s,%s) RETURNING id", (run_id, m1, beat_no, len(mints), json.dumps({"minute": m1.isoformat(), "mints_traded": len(ctx.agg), "book": book}))).fetchone()["id"] @@ -41,7 +42,9 @@ def trade_minute(ctx, *, book: str, run_id: str, beat_no: int, broker, kind: str opens_now = ledger.open_positions(conn, book); open_mints = {p["mint"] for p in opens_now} cash = ledger.paper_cash(conn, book); n_enter = 0; picks = []; entries = [] bankroll = cash + sum(float(p["cost_sol"]) for p in opens_now) # wealth at cost: the sizing base (agent/sizing.py) - circuit = conn.execute("SELECT kill_switch, entries_paused FROM circuit_state WHERE id = 1").fetchone() + from ..markets import for_book + market = for_book(book) # the book's market: its circuit, money settings and sizes + circuit = conn.execute("SELECT kill_switch, entries_paused FROM circuit_state WHERE id = %s", (market.circuit_id,)).fetchone() blocked = (block or ("kill switch" if circuit and circuit["kill_switch"] else None) or ("paused" if circuit and circuit["entries_paused"] else None) or ("book halted" if rails.book_halted(conn, book) else None)) thr_v = np.broadcast_to(np.asarray(threshold, dtype=np.float64), (len(mints),)) if len(mints) else np.array([]) @@ -51,7 +54,7 @@ def trade_minute(ctx, *, book: str, run_id: str, beat_no: int, broker, kind: str continue picks.append((m, float(sc))) tab_k = tables[k] if tables is not None else table - size, why = sizing.size_position(float(sc), thr_k, tab_k or [], bankroll, cash, i["resq"]) + size, why = sizing.size_position(float(sc), thr_k, tab_k or [], bankroll, cash, i["resq"], flat=flat, market=market) denied = allow is not None and not bool(allow[k]) if m in open_mints or blocked or denied or size <= 0: rail = blocked or ("held" if m in open_mints else (str(reasons[k]) if denied and reasons is not None else "filtered") if denied else "sizing") @@ -84,6 +87,6 @@ def trade_minute(ctx, *, book: str, run_id: str, beat_no: int, broker, kind: str peak_row = conn.execute("SELECT max(wealth) AS pk FROM wealth_marks WHERE book = %s", (book,)).fetchone(); peak = max(float(peak_row["pk"] or 0.0), wealth) conn.execute("INSERT INTO wealth_marks (beat_id, book, ts, sol_free, positions_value, exit_cost, wealth, peak, drawdown, exposure, n_open) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", (beat, book, m1, cash, positions_net, exit_cost, wealth, peak, (1.0 - wealth / peak) if peak > 0 else 0.0, exposure, len(opens))) - rails.check_book_drawdown(conn, book, wealth, peak) # the book's own kill switch: blocks its next minutes' entries + rails.check_book_drawdown(conn, book, wealth, peak, drawdown=market.kill_drawdown(), unit=market.unit) # the book's own kill switch return {"beat_id": beat, "entries": entries, "entered": n_enter, "exited": n_exit, "open": len(opens), "cash": cash, "wealth": wealth, "picks": len(picks), "blocked": blocked, "top_scores": sorted(picks, key=lambda x: -x[1])[:5], "open_mints": sorted({p["mint"] for p in opens})} diff --git a/fly_trader/agent/rails.py b/fly_trader/agent/rails.py index 422cadf8c96b99af56ad0b5c9f24dff41f2afee3..de7c6ba78d5783729cefac863f2b5deb13ff6489 100644 --- a/fly_trader/agent/rails.py +++ b/fly_trader/agent/rails.py @@ -1,6 +1,9 @@ """Risk rails outside the model. KillSwitch (−KILL_SWITCH_DRAWDOWN from peak wealth → no entries), CircuitBreaker (CIRCUIT_THRESHOLD consecutive execution failures), the rolling 24 h notional ledger, and PauseEntries (console toggle). State lives in circuit_state / circuit_events / notional_ledger; the trading engine reads it before entries. +One circuit per trading type: ``circuit_id`` 1 is the memecoin live book (the default everywhere), 2 the Kalshi live +books (``KALSHI_CIRCUIT``, fly_trader/kalshi), 3 the Robinhood Chain memecoin live book (``RH_CIRCUIT``, +fly_trader/rh); ``check_drawdown`` takes the drawdown limit of the caller's type. """ from __future__ import annotations @@ -13,6 +16,7 @@ from ..db.apilog import record_event from ..db.connection import transaction log = logging.getLogger(__name__) +SOLANA_CIRCUIT, KALSHI_CIRCUIT, RH_CIRCUIT = 1, 2, 3 @dataclass @@ -25,63 +29,80 @@ class CircuitState: entries_paused: bool = False -def load_circuit(conn) -> CircuitState: - r = conn.execute("SELECT * FROM circuit_state WHERE id = 1").fetchone() +def load_circuit(conn, circuit_id: int = SOLANA_CIRCUIT) -> CircuitState: + r = conn.execute("SELECT * FROM circuit_state WHERE id = %s", (circuit_id,)).fetchone() if not r: return CircuitState() return CircuitState(r["fail_count"], r["tripped"], r["kill_switch"], r["kill_reason"], r["peak_wealth"], r["entries_paused"]) -def _event(conn, kind: str, detail: dict | None = None) -> None: - conn.execute("INSERT INTO circuit_events (kind, detail) VALUES (%s, %s)", (kind, json.dumps(detail or {}, default=str))) +def _event(conn, kind: str, detail: dict | None = None, circuit_id: int = SOLANA_CIRCUIT) -> None: + conn.execute("INSERT INTO circuit_events (kind, detail) VALUES (%s, %s)", (kind, json.dumps({**(detail or {}), "circuit": circuit_id}, default=str))) -def record_failure(conn, reason: str) -> CircuitState: +def record_failure(conn, reason: str, circuit_id: int = SOLANA_CIRCUIT) -> CircuitState: r = conn.execute( - "UPDATE circuit_state SET fail_count = fail_count + 1, last_failure_ts = now(), updated_at = now() WHERE id = 1 RETURNING fail_count, tripped" + "UPDATE circuit_state SET fail_count = fail_count + 1, last_failure_ts = now(), updated_at = now() WHERE id = %s RETURNING fail_count, tripped", (circuit_id,) ).fetchone() tripped = r["tripped"] if r["fail_count"] >= config.CIRCUIT_THRESHOLD and not tripped: - conn.execute("UPDATE circuit_state SET tripped = true, updated_at = now() WHERE id = 1") - _event(conn, "tripped", {"reason": reason, "fail_count": r["fail_count"]}) - record_event("error", "rails", "circuit breaker tripped", {"reason": reason}) - return load_circuit(conn) + conn.execute("UPDATE circuit_state SET tripped = true, updated_at = now() WHERE id = %s", (circuit_id,)) + _event(conn, "tripped", {"reason": reason, "fail_count": r["fail_count"]}, circuit_id) + record_event("error", "rails", "circuit breaker tripped", {"reason": reason, "circuit": circuit_id}) + return load_circuit(conn, circuit_id) -def record_success(conn) -> None: - conn.execute("UPDATE circuit_state SET fail_count = 0, updated_at = now() WHERE id = 1") +def record_success(conn, circuit_id: int = SOLANA_CIRCUIT) -> None: + conn.execute("UPDATE circuit_state SET fail_count = 0, updated_at = now() WHERE id = %s", (circuit_id,)) -def check_drawdown(conn, wealth: float, peak: float) -> bool: - """Trip the kill switch when ``wealth`` has fallen KILL_SWITCH_DRAWDOWN below ``peak`` (the book's own peak: the - trading engine passes the highest wealth mark of the current run). Returns whether the kill switch is on.""" - st = load_circuit(conn) +def check_drawdown(conn, wealth: float, peak: float, circuit_id: int = SOLANA_CIRCUIT, drawdown: float | None = None, unit: str = "SOL") -> bool: + """Trip the kill switch when ``wealth`` has fallen ``drawdown`` (default KILL_SWITCH_DRAWDOWN) below ``peak`` (the book's + own peak: the trading engine passes the highest wealth mark of the current run). Returns whether the kill switch is on.""" + st = load_circuit(conn, circuit_id) if st.kill_switch: return True - if peak > 0 and wealth <= peak * (1.0 - config.KILL_SWITCH_DRAWDOWN): - conn.execute("UPDATE circuit_state SET kill_switch = true, kill_reason = %s, peak_wealth = %s, updated_at = now() WHERE id = 1", - (f"wealth {wealth:.4f} SOL <= {1 - config.KILL_SWITCH_DRAWDOWN:.2f} x peak {peak:.4f} SOL", peak)) - _event(conn, "kill_switch", {"wealth": wealth, "peak": peak}) - record_event("error", "rails", "kill switch tripped: entries blocked", {"wealth": wealth, "peak": peak}) + dd = config.KILL_SWITCH_DRAWDOWN if drawdown is None else float(drawdown) + if peak > 0 and wealth <= peak * (1.0 - dd): + conn.execute("UPDATE circuit_state SET kill_switch = true, kill_reason = %s, peak_wealth = %s, updated_at = now() WHERE id = %s", + (f"wealth {wealth:.4f} {unit} <= {1 - dd:.2f} x peak {peak:.4f} {unit}", peak, circuit_id)) + _event(conn, "kill_switch", {"wealth": wealth, "peak": peak}, circuit_id) + record_event("error", "rails", "kill switch tripped: entries blocked", {"wealth": wealth, "peak": peak, "circuit": circuit_id}) + if config.VAULT_ENABLED: # the hosted fly: tell the operator at once + try: + from ..vault import alerts + alerts.send(f"KILL SWITCH tripped: {wealth:.4f} {unit} <= {1 - dd:.2f} x peak {peak:.4f} {unit}. Entries blocked" + + (", open positions are being liquidated" if config.KILL_SWITCH_LIQUIDATE else "") + + ". Clear it with `fly-trader reset-circuit --kill` once you know why.") + except Exception: + pass return True return False -def kill_rebase_at(conn, circuit_id: int = 1): - """When the kill switch was last cleared: peaks are measured from marks after it, so clearing really re-bases. - (Before this, the live book recomputed its peak from every wealth mark ever and re-tripped the next minute.)""" - r = conn.execute("SELECT max(ts) AS ts FROM circuit_events WHERE kind = 'reset' AND (detail->>'kill')::boolean " +def kill_rebase_at(conn, circuit_id: int = SOLANA_CIRCUIT): + """When the kill switch was last cleared, or money last withdrawn from the live wallet: peaks are measured from marks + after it, so clearing really re-bases and a withdrawal is never a drawdown. (Before this, the live book recomputed its + peak from every wealth mark ever and re-tripped the next minute.)""" + r = conn.execute("SELECT max(ts) AS ts FROM circuit_events WHERE ((kind = 'reset' AND (detail->>'kill')::boolean) OR kind = 'withdrawal') " "AND COALESCE((detail->>'circuit')::int, 1) = %s", (circuit_id,)).fetchone() return r["ts"] if r else None -def check_book_drawdown(conn, book: str, wealth: float, peak: float) -> bool: - """A paper race book's own kill switch: KILL_SWITCH_DRAWDOWN below the book's peak halts that book's entries only, - so one book's drawdown never stops the other during the race. Returns whether the book is halted.""" +def record_withdrawal(conn, detail: dict, circuit_id: int = SOLANA_CIRCUIT) -> None: + """The operator took money out of a live wallet (chain/withdraw.py, rh/withdraw.py): ``kill_rebase_at`` re-bases the + live book's peak at it.""" + _event(conn, "withdrawal", detail, circuit_id) + + +def check_book_drawdown(conn, book: str, wealth: float, peak: float, drawdown: float | None = None, unit: str = "SOL") -> bool: + """A paper race book's own kill switch: ``drawdown`` (default KILL_SWITCH_DRAWDOWN) below the book's peak halts that + book's entries only, so one book's drawdown never stops the other during the race. Returns whether the book is halted.""" if book_halted(conn, book): return True - if peak > 0 and wealth <= peak * (1.0 - config.KILL_SWITCH_DRAWDOWN): - reason = f"wealth {wealth:.4f} SOL <= {1 - config.KILL_SWITCH_DRAWDOWN:.2f} x peak {peak:.4f} SOL" + dd = config.KILL_SWITCH_DRAWDOWN if drawdown is None else float(drawdown) + if peak > 0 and wealth <= peak * (1.0 - dd): + reason = f"wealth {wealth:.4f} {unit} <= {1 - dd:.2f} x peak {peak:.4f} {unit}" conn.execute("INSERT INTO book_state (book, halted, reason, peak, updated_at) VALUES (%s, true, %s, %s, now()) " "ON CONFLICT (book) DO UPDATE SET halted = true, reason = EXCLUDED.reason, peak = EXCLUDED.peak, updated_at = now()", (book, reason, peak)) @@ -103,28 +124,29 @@ def clear_book_halt(book: str) -> None: record_event("info", "rails", f"{book}: drawdown halt cleared") -def notional_24h(conn) -> float: - r = conn.execute("SELECT COALESCE(sum(sol), 0) AS s FROM notional_ledger WHERE ts > now() - interval '24 hours'").fetchone() +def notional_24h(conn, market: str = "sol") -> float: + """The last 24 h of entries in ``market``'s unit (``sol`` holds ETH on the ``rh`` market's rows).""" + r = conn.execute("SELECT COALESCE(sum(sol), 0) AS s FROM notional_ledger WHERE ts > now() - interval '24 hours' AND market = %s", (market,)).fetchone() return float(r["s"]) -def record_notional(conn, sol: float, kind: str = "entry") -> None: - conn.execute("INSERT INTO notional_ledger (sol, kind) VALUES (%s, %s)", (sol, kind)) +def record_notional(conn, sol: float, kind: str = "entry", market: str = "sol") -> None: + conn.execute("INSERT INTO notional_ledger (sol, kind, market) VALUES (%s, %s, %s)", (sol, kind, market)) -def reset_circuit(kill: bool = False) -> None: +def reset_circuit(kill: bool = False, circuit_id: int = SOLANA_CIRCUIT) -> None: with transaction() as conn: - conn.execute("UPDATE circuit_state SET fail_count = 0, tripped = false, updated_at = now() WHERE id = 1") - _event(conn, "reset", {"kill": kill}) + conn.execute("UPDATE circuit_state SET fail_count = 0, tripped = false, updated_at = now() WHERE id = %s", (circuit_id,)) + _event(conn, "reset", {"kill": kill}, circuit_id) if kill: - conn.execute("UPDATE circuit_state SET kill_switch = false, kill_reason = NULL, peak_wealth = NULL, updated_at = now() WHERE id = 1") - record_event("info", "rails", "circuit reset" + (" incl. kill switch (peak re-based)" if kill else "")) + conn.execute("UPDATE circuit_state SET kill_switch = false, kill_reason = NULL, peak_wealth = NULL, updated_at = now() WHERE id = %s", (circuit_id,)) + record_event("info", "rails", "circuit reset" + (" incl. kill switch (peak re-based)" if kill else ""), {"circuit": circuit_id}) print("circuit reset" + (" + kill switch cleared" if kill else "")) -def set_entries_paused(paused: bool) -> None: +def set_entries_paused(paused: bool, circuit_id: int = SOLANA_CIRCUIT) -> None: with transaction() as conn: - conn.execute("UPDATE circuit_state SET entries_paused = %s, updated_at = now() WHERE id = 1", (paused,)) - _event(conn, "paused" if paused else "resumed", {}) - record_event("info", "rails", "entries paused" if paused else "entries resumed") + conn.execute("UPDATE circuit_state SET entries_paused = %s, updated_at = now() WHERE id = %s", (paused, circuit_id)) + _event(conn, "paused" if paused else "resumed", {}, circuit_id) + record_event("info", "rails", "entries paused" if paused else "entries resumed", {"circuit": circuit_id}) print("entries paused" if paused else "entries resumed") diff --git a/fly_trader/agent/selector_session.py b/fly_trader/agent/selector_session.py index a67d9ff547e6f472147e28d4d75e073a6e957d95..92a7e5d4e5678637b117356b74ae0ba275b52a6e 100644 --- a/fly_trader/agent/selector_session.py +++ b/fly_trader/agent/selector_session.py @@ -50,23 +50,53 @@ def pinned_snapshot() -> int | None: return int(v) if v is not None else None -class SelectorBook: - name = "selector" +def chain_deployable(model, chain: str) -> tuple[bool, str]: + """Whether the selector may trade ``chain``: Solana by its own deployability (as before); another chain only when the + combined stack's holdout on that chain's rows showed an edge (train/strategies.score_holdout_by_chain).""" + if chain == "sol": + return True, "ok" + c = ((getattr(model, "metrics", None) or {}).get("chains") or {}).get(chain) + if not c: + return False, f"the selector was not judged on {chain} rows" + return bool(c.get("deployable")), f"{chain} holdout: {c.get('n', 0)} trades, mean {(c.get('mean') or 0) * 100:+.2f}% (random {(c.get('random_mean') or 0) * 100:+.2f}%)" + + +def fail_closed(groups: set, infos: list, d: dict) -> dict: + """The selector's live fail-closed rules on a ``strategies.decide`` result (in place): rows whose inputs training had + but live lacks are blocked. Shared by the fly's teacher gate (agent/fly_session.py), so both books obey the same rules.""" + for k, i in enumerate(infos): + if not d["allow"][k]: + continue + if "skill" in groups and i.get("skill_missing"): + d["allow"][k] = False; d["reason"][k] = "fail closed: no wallet-skill table for today" + elif "rug" in groups and i.get("age_h") is not None and not i.get("curve_known"): + d["allow"][k] = False; d["reason"][k] = "fail closed: curve facts not known yet" + return d + - def __init__(self): +class SelectorBook: + def __init__(self, chain: str = "sol"): + from .. import markets from ..train import selector as sel + self.chain = chain; self.market = markets.for_chain(chain) + self.name = "selector" if chain == "sol" else f"selector_{chain}" + self.BOOK = self.market.selector_book; self.KIND = self.name; self.status_key = self.market.selector_status_key pin = pinned_snapshot() self.model = sel.load_snapshot(pin) if pin is not None else sel.load_latest() if self.model is None: raise NoModel("no model has qualified to trade yet (trained on the current data, with a backtest that made money after costs and beat random picks)") + ok, why = chain_deployable(self.model, chain) + if not ok: + raise NoModel(f"the selector may not trade {self.market.name}: {why}") self.idx = _columns(self.model) self.horizon_s = self.model.horizon_min * 60 - self.broker = PaperBroker(BOOK); self.done = False + self.broker = PaperBroker(self.BOOK); self.done = False self.run_id = str(uuid.uuid4()); self.beat_no = 0 with transaction() as conn: self.snapshot_id = pin if pin is not None else sel.latest_current(conn)["id"] # the snapshot actually loaded conn.execute("INSERT INTO runs (run_id, kind, config, brain_snapshot_id, status) VALUES (%s,%s,%s,%s,'running')", - (self.run_id, KIND, json.dumps({"threshold": self.model.threshold, "horizon_min": self.model.horizon_min, "book": BOOK}, default=str), self.snapshot_id)) + (self.run_id, self.KIND, json.dumps({"threshold": self.model.threshold, "horizon_min": self.model.horizon_min, "book": self.BOOK, "chain": chain}, + default=str), self.snapshot_id)) record_event("info", "selector", "selector session started", {"run_id": self.run_id, "snapshot": self.snapshot_id, "threshold": self.model.threshold, "horizon_min": self.model.horizon_min}) log.info("selector session: snapshot %d threshold %.4f horizon %d min", self.snapshot_id, self.model.threshold, self.model.horizon_min) @@ -74,8 +104,7 @@ class SelectorBook: """Switch to a newer deployable model without a restart (the paper book and open positions carry on). While a snapshot is pinned (``ui_settings['pinned_selector_snapshot']``) the book stays on it, so a forward test measures one model instead of a blend of every retrain that lands during it.""" - import joblib - from ..train import selector as sel + from ..train import model_io, selector as sel pin = pinned_snapshot() with transaction() as conn: if pin is not None: # a pin freezes the book, but a NEW pin (a release) switches to it @@ -86,7 +115,7 @@ class SelectorBook: r = sel.latest_current(conn) if not r or r["id"] == self.snapshot_id: return False - m = joblib.load(r["path"]) + m = model_io.load_selector(r["path"]) try: idx = _columns(m) except RuntimeError as e: @@ -95,7 +124,7 @@ class SelectorBook: old = self.snapshot_id; self.model, self.idx, self.snapshot_id, self.horizon_s = m, idx, r["id"], m.horizon_min * 60 with transaction() as conn: conn.execute("UPDATE runs SET brain_snapshot_id = %s, config = %s WHERE run_id = %s", - (r["id"], json.dumps({"threshold": m.threshold, "horizon_min": m.horizon_min, "book": BOOK}, default=str), self.run_id)) + (r["id"], json.dumps({"threshold": m.threshold, "horizon_min": m.horizon_min, "book": self.BOOK}, default=str), self.run_id)) record_event("info", "selector", f"switched to model #{r['id']}", {"from": old, "to": r["id"], "threshold": m.threshold}) log.info("switched from model #%s to #%d (threshold %.4f)", old, r["id"], m.threshold) return True @@ -104,7 +133,7 @@ class SelectorBook: pass def open_mints(self, conn) -> set[str]: - return {p["mint"] for p in ledger.open_positions(conn, BOOK)} + return {p["mint"] for p in ledger.open_positions(conn, self.BOOK)} def _decide(self, ctx) -> dict | None: """The strategy stack's per-row decision (train/strategies.decide), with the live fail-closed rules: rows whose @@ -113,15 +142,7 @@ class SelectorBook: if not (getattr(self.model, "stack", None) or {}).get("strategies") or not len(ctx.mints): return None d = self.model.decide(ctx.X, X_COLS, ctx.t_start) - groups = set((getattr(self.model, "metrics", None) or {}).get("groups") or []) - for k, i in enumerate(ctx.infos): - if not d["allow"][k]: - continue - if "skill" in groups and i.get("skill_missing"): - d["allow"][k] = False; d["reason"][k] = "fail closed: no wallet-skill table for today" - elif "rug" in groups and i.get("age_h") is not None and not i.get("curve_known"): - d["allow"][k] = False; d["reason"][k] = "fail closed: curve facts not known yet" - return d + return fail_closed(set((getattr(self.model, "metrics", None) or {}).get("groups") or []), ctx.infos, d) def on_minute(self, ctx) -> dict: conn = ctx.conn @@ -141,10 +162,10 @@ class SelectorBook: status = {**summary, "stage": "stream stale: holding (no entries or exits)", "updated_at": datetime.now(timezone.utc).isoformat()} self._write_status(conn, status) return status - handed = handover.state(conn) is not None and ctx.engine.has_book("fly") + handed = handover.state(conn, self.chain) is not None and ctx.engine.has_book("fly" if self.chain == "sol" else f"fly_{self.chain}") self.beat_no += 1 extra = {} if d is None else {"holds": d["hold_s"], "strategies": d["strategy"], "allow": d["allow"], "reasons": d["reason"], "tables": d["tables"]} - st = paper_trading.trade_minute(ctx, book=BOOK, run_id=self.run_id, beat_no=self.beat_no, broker=self.broker, kind=KIND, mints=ctx.mints, infos=ctx.infos, + st = paper_trading.trade_minute(ctx, book=self.BOOK, run_id=self.run_id, beat_no=self.beat_no, broker=self.broker, kind=self.KIND, mints=ctx.mints, infos=ctx.infos, scores=scores, threshold=thr, table=getattr(self.model, "sizing", None), horizon_s=self.horizon_s, block="handed over to the fly" if handed else None, **extra) if handed and st["open"] == 0: @@ -155,10 +176,9 @@ class SelectorBook: self._write_status(conn, status) return status - @staticmethod - def _write_status(conn, status: dict) -> None: - conn.execute("INSERT INTO ui_settings (key, value) VALUES ('selector_status', %s) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()", - (json.dumps(status, default=str),)) + def _write_status(self, conn, status: dict) -> None: + conn.execute("INSERT INTO ui_settings (key, value) VALUES (%s, %s) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()", + (self.status_key, json.dumps({**status, "chain": self.chain}, default=str))) def finish(self) -> None: with transaction() as conn: diff --git a/fly_trader/agent/sizing.py b/fly_trader/agent/sizing.py index e5dcdaa723b21f2772b74aa92462c894ec482e93..dfda72469719c6c574fcc303040256c28ee2fd0f 100644 --- a/fly_trader/agent/sizing.py +++ b/fly_trader/agent/sizing.py @@ -4,16 +4,20 @@ Certainty is measured, not assumed. During training the backtest's out-of-sample score cleared the buy line (``build_table``: equal-count bands of ``score − threshold``). For each band the growth-optimal (Kelly) fraction is found on the band's actual net returns — the bet f that maximises the average of log(1 + f·r), so the rare −100 % rugs count in full; a band with no edge gets 0. Live (``size_position``): the current score's band → -``KELLY_FRACTION`` of that fraction of the deployable bankroll (wealth at cost minus the gas reserve), capped at -``MAX_POSITION_FRACTION`` of it, ``MAX_POOL_SHARE`` of the pool's quote reserve (bigger buys pay more price impact than -the 0.1 SOL the backtest measured) and the free cash above the reserve; below ``MIN_POSITION_SOL`` the buy is skipped. -A model without a table (trained before sizing) trades the fixed ``MAX_POSITION_SOL``. +``KELLY_FRACTION`` of that fraction of the deployable bankroll (wealth at cost minus the gas reserve); ``flat`` (the +fly, ``config.FLY_SIZING``): ``MAX_POSITION_FRACTION`` of it on every buy that cleared the line, no bands. Either way +capped at ``MAX_POSITION_FRACTION`` of the deployable bankroll, the free cash above the reserve, and the label size +(``market/exit_cost.label_size``: ``LABEL_SIZE_SOL`` or ``MAX_POOL_SHARE`` of the pool) -- the size every edge was +measured at, so the book behaves the same with 5 SOL or 500: a bigger bankroll buys no bigger than the backtest did, +it only runs out of cash later. Below ``MIN_POSITION_SOL`` the buy is skipped. A model without a table (trained +before sizing) trades the fixed ``MAX_POSITION_SOL``, under the same caps. """ from __future__ import annotations import numpy as np from .. import config +from ..market.exit_cost import label_size N_BANDS = 4 MIN_BAND_TRADES = 50 @@ -57,26 +61,31 @@ def band_for(table: list[dict], margin: float) -> dict | None: def size_position(score: float, threshold: float, table: list[dict] | None, bankroll_sol: float, cash_sol: float, - res_quote_sol: float | None) -> tuple[float, str]: - """(size in SOL, why). Size 0 means skip the buy.""" - reserve = config.GAS_RESERVE_SOL; free = cash_sol - reserve - if not table: - size = config.MAX_POSITION_SOL - return (size, f"fixed {size:g} SOL (model has no sizing table)") if free >= size else (0.0, "not enough free cash") - b = band_for(table, score - threshold) - if b is None or b["kelly"] <= 0: - return 0.0, "no edge in this score band" + res_quote_sol: float | None, flat: bool = False, market=None) -> tuple[float, str]: + """(size in the book's unit, why). Size 0 means skip the buy. ``flat``: ``MAX_POSITION_FRACTION`` of the deployable + bankroll on every buy that cleared the line (the fly); otherwise the Kelly band of ``table``. ``market`` + (fly_trader/markets; default Solana): the gas reserve, minimum position and label size of the book's own market.""" + rh = market is not None and market.chain != "sol" + reserve = market.gas_reserve() if rh else config.GAS_RESERVE_SOL; free = cash_sol - reserve + min_pos = market.min_position() if rh else config.MIN_POSITION_SOL deployable = max(0.0, bankroll_sol - reserve) - size = config.KELLY_FRACTION * b["kelly"] * deployable - caps = {"bankroll cap": config.MAX_POSITION_FRACTION * deployable, "free cash": free} - if res_quote_sol: - caps["pool depth"] = config.MAX_POOL_SHARE * res_quote_sol + if flat: + size = config.MAX_POSITION_FRACTION * deployable; why = f"flat {config.MAX_POSITION_FRACTION:g} of {deployable:.2f} SOL" + elif not table: + size = config.MAX_POSITION_SOL * (market.k() if rh else 1.0); why = f"fixed {size:g} {'ETH' if rh else 'SOL'} (model has no sizing table)" + else: + b = band_for(table, score - threshold) + if b is None or b["kelly"] <= 0: + return 0.0, "no edge in this score band" + size = config.KELLY_FRACTION * b["kelly"] * deployable + why = f"band from +{b['lo']:.3f} over the line: Kelly {b['kelly']:.2f} × {config.KELLY_FRACTION:g} of {deployable:.2f} SOL" + # the label size is the largest buy any backtest measured: past it the edge is an extrapolation that impact eats + caps = {"bankroll cap": config.MAX_POSITION_FRACTION * deployable, "free cash": free, "label size": label_size(res_quote_sol, market if rh else None)} limit = min(caps, key=caps.get) - why = f"band from +{b['lo']:.3f} over the line: Kelly {b['kelly']:.2f} × {config.KELLY_FRACTION:g} of {deployable:.2f} SOL" if caps[limit] < size: size = caps[limit]; why += f", capped by {limit}" - if size < config.MIN_POSITION_SOL: - return 0.0, why + f" → {size:.3f} SOL is below the {config.MIN_POSITION_SOL:g} SOL minimum" + if size < min_pos: + return 0.0, why + f" → {max(size, 0.0):.6f} is below the {min_pos:g} minimum" return float(size), why + f" → {size:.3f} SOL" diff --git a/fly_trader/brain/connectome.py b/fly_trader/brain/connectome.py index d1f2e737c5ab648255419912b3a4ddf088eacbc3..466f91417b919888ffd49b6d56f6d7a1dd286377 100644 --- a/fly_trader/brain/connectome.py +++ b/fly_trader/brain/connectome.py @@ -115,6 +115,9 @@ class Connectome: self.pop_ranges, self.pathway_gains) self.W_KM0 = torch.from_numpy(np.ascontiguousarray(z["W_KM0"])).to(torch.float32) self.M_KM = torch.from_numpy(np.ascontiguousarray(z["M_KM"])).to(torch.bool) + # FlyBrain's finer annotation per neuron (VIS_R1R6, VIS_ME, LPTC, ...): the Kalshi fly finds its photoreceptors by it + self.flybrain_group = np.asarray(z["flybrain_group"]) if "flybrain_group" in z.files else None + self.flybrain_group_names = [str(x) for x in z["flybrain_group_names"]] if "flybrain_group_names" in z.files else [] self.spectral_radius_raw = float(z["spectral_radius_raw"]) self.content_sha256 = str(z["content_sha256"]) self.s = float(scale["s"]) if scale and "s" in scale else float(z["s0"]) @@ -159,3 +162,27 @@ class SubConnectome: self.pathway_gains = getattr(c, "pathway_gains", {}) self.excluded = exclude self.keep = keep + fg = getattr(c, "flybrain_group", None) + self.flybrain_group = fg[keep.numpy()] if fg is not None else None + self.flybrain_group_names = list(getattr(c, "flybrain_group_names", [])) + + +# ---------------------------------------------------------------- the Kalshi fly's graph: the optic lobes kept, the memecoin senses dropped +VISUAL_SUB = tuple(AFFERENT_POPS) # exclude these and VISUAL stays: the second fly's sub-connectome +PHOTORECEPTOR_GROUPS = ("VIS_R1R6", "VIS_R7R8") # FlyBrain groups of the retina's photoreceptors (R7R8 is absent in the v783 join) + + +def photoreceptor_rows(graph) -> torch.Tensor: + """Rows of ``graph`` (a Connectome or SubConnectome) that are photoreceptors: VISUAL neurons whose FlyBrain group is + R1–R6 or R7–R8. The Kalshi fly's features enter here and travel the optic lobes into the central brain.""" + if "VISUAL" not in graph.pop_ranges: + raise ValueError("this graph has no VISUAL population: the Kalshi fly needs the optic lobes") + fg = getattr(graph, "flybrain_group", None); names = list(getattr(graph, "flybrain_group_names", []) or []) + lo, hi = graph.pop_ranges["VISUAL"] + if fg is None or not names: + raise ValueError("the connectome artifact carries no flybrain_group annotation; rebuild it (brain/build.py)") + want = [i for i, n in enumerate(names) if n in PHOTORECEPTOR_GROUPS] + rows = np.flatnonzero(np.isin(fg[lo:hi], want)) + lo + if not len(rows): + raise ValueError(f"no photoreceptor rows: VISUAL groups present are {sorted(set(names[int(g)] for g in set(fg[lo:hi].tolist()) if int(g) < len(names)))}") + return torch.from_numpy(rows.astype(np.int64)) diff --git a/fly_trader/brain/geometry.py b/fly_trader/brain/geometry.py index 79a3a5a2ed664e1eb6776bf10958b08f0c0b0c83..abdd290a2f7a5ecd84c259f69e46d1f5f02f6a56 100644 --- a/fly_trader/brain/geometry.py +++ b/fly_trader/brain/geometry.py @@ -25,6 +25,8 @@ EXCLUDE = ("VISUAL",) POS_COLS = ["pos_x", "pos_y", "pos_z"] SOMA_COLS = ["soma_x", "soma_y", "soma_z"] BIN, META = "geometry.bin", "geometry.json" +FULL_BIN, FULL_META = "geometry_full.bin", "geometry_full.json" # every neuron: the scene both flies are drawn into (ui/brain3d) +FLY_EXCLUDES = {"memecoin": EXCLUDE, "kalshi": None} # None: brain/connectome.VISUAL_SUB (the memecoin afferents), resolved lazily def keep_mask(pop_ranges: dict, n: int, exclude: tuple[str, ...] = EXCLUDE) -> np.ndarray: @@ -60,10 +62,11 @@ def read_meta(out_dir: Path) -> dict | None: def build_geometry(out_dir: Path, connectome_path: str | Path | None = None, annotations: str | Path | None = None, - exclude: tuple[str, ...] = EXCLUDE) -> dict: + exclude: tuple[str, ...] = EXCLUDE, names: tuple[str, str] = (BIN, META)) -> dict: """Write (once per connectome) and return the geometry metadata. Soma centroid where known, else the neuron's - annotation point, else its population's centroid (a handful of neurons); scaled to micrometres and centred.""" - out_dir = Path(out_dir); jpath, bpath = out_dir / META, out_dir / BIN + annotation point, else its population's centroid (a handful of neurons); scaled to micrometres and centred. + ``exclude=()`` with ``names=(FULL_BIN, FULL_META)`` is the whole brain (``build_full_geometry``).""" + out_dir = Path(out_dir); bpath, jpath = out_dir / names[0], out_dir / names[1] path = Path(connectome_path) if connectome_path else current_connectome_path() with np.load(path, allow_pickle=False) as z: sha = str(z["content_sha256"]); n = int(z["N"]); pop_ranges = {k: tuple(v) for k, v in json.loads(str(z["pop_ranges"])).items()} @@ -97,3 +100,39 @@ def build_geometry(out_dir: Path, connectome_path: str | Path | None = None, ann out_dir.mkdir(parents=True, exist_ok=True) _atomic(bpath, np.ascontiguousarray(xyz).tobytes()); _atomic(jpath, json.dumps(meta).encode()) return meta + + +def fly_index(pop_ranges: dict, n: int, exclude: tuple[str, ...]) -> np.ndarray: + """int32 map from a fly's sub-graph index to the full-graph index (the rows ``SubConnectome`` keeps, in order).""" + return np.flatnonzero(keep_mask(pop_ranges, n, exclude)).astype(np.int32) + + +def build_full_geometry(out_dir: Path, connectome_path: str | Path | None = None, annotations: str | Path | None = None) -> dict: + """The whole brain's geometry plus, per fly, the index map from its sub-graph into it (``index_.bin``): the + scene the console draws both flies into. ``geometry_full.json`` carries ``flies`` = {name: {"exclude", "n", "index"}}.""" + from .connectome import VISUAL_SUB + out_dir = Path(out_dir); jpath = out_dir / FULL_META + meta = read_meta_named(out_dir, FULL_META) + path = Path(connectome_path) if connectome_path else current_connectome_path() + with np.load(path, allow_pickle=False) as z: + sha = str(z["content_sha256"]); n = int(z["N"]); pop_ranges = {k: tuple(v) for k, v in json.loads(str(z["pop_ranges"])).items()} + if meta and meta.get("connectome_sha256") == sha and meta.get("version") == VERSION and meta.get("flies") and (out_dir / FULL_BIN).exists() \ + and all((out_dir / f["index"]).exists() for f in meta["flies"].values()): + return meta + meta = build_geometry(out_dir, path, annotations, exclude=(), names=(FULL_BIN, FULL_META)) + flies = {} + for name, exc in FLY_EXCLUDES.items(): + exc = tuple(exc) if exc is not None else tuple(VISUAL_SUB) + idx = fly_index(pop_ranges, n, exc); fname = f"index_{name}.bin" + _atomic(out_dir / fname, np.ascontiguousarray(idx).tobytes()) + flies[name] = {"exclude": list(exc), "n": int(len(idx)), "index": fname} + meta = {**meta, "flies": flies} + _atomic(jpath, json.dumps(meta).encode()) + return meta + + +def read_meta_named(out_dir: Path, name: str) -> dict | None: + try: + return json.loads((Path(out_dir) / name).read_text()) + except (FileNotFoundError, ValueError): + return None diff --git a/fly_trader/brain/pathways.py b/fly_trader/brain/pathways.py index 52dbf5f0c8fc8f9a61a486d9aa432be7f4dc02ae..a8855c7d961ce2445988b62fe66f94cd5aa31684 100644 --- a/fly_trader/brain/pathways.py +++ b/fly_trader/brain/pathways.py @@ -13,7 +13,8 @@ from datetime import datetime from pathlib import Path import numpy as np -import torch + +from ..train import model_io TOP_K = 200 @@ -27,7 +28,7 @@ def km_pairs(connectome_path: str | Path) -> tuple[np.ndarray, np.ndarray, int, def bootstrap_weights(boot_path: str | Path) -> tuple[np.ndarray, np.ndarray, list[str]]: """(base weight per pair, learn [S, J], strategy names) from a fly bootstrap checkpoint (``fly_selector.save``).""" - d = torch.load(boot_path, map_location="cpu", weights_only=False); sd = d["state_dict"] + d = model_io.torch_load(boot_path); sd = d["state_dict"] w0 = np.logaddexp(0.0, sd["theta_km"].detach().float().numpy()).astype(np.float32) # softplus n_mbon = int(sd["c_sign"].numel()) if "c_sign" in sd else int(sd["mask"].shape[1]) learn = sd["learn"].numpy().astype(bool) if "learn" in sd else np.ones((1, n_mbon), dtype=bool) @@ -37,7 +38,7 @@ def bootstrap_weights(boot_path: str | Path) -> tuple[np.ndarray, np.ndarray, li def bank_D(path: str | Path) -> tuple[np.ndarray, int | None]: """The learned change per pair from ``state.pt`` or a ``snap_*.pt`` (both carry ``bank`` and ``bootstrap_id``).""" - s = torch.load(path, map_location="cpu", weights_only=False) + s = model_io.torch_load(path) D = s["bank"]["D"]; D = D[0] if D.ndim == 2 else D bid = s.get("bootstrap_id") return D.detach().float().numpy(), (int(bid) if bid is not None else None) diff --git a/fly_trader/brain/plastic.py b/fly_trader/brain/plastic.py index 6914912c482f82d2ba2401c83bd52b2ee304f315..a558f3c4750ff80ad676251eb4944f568674e598 100644 --- a/fly_trader/brain/plastic.py +++ b/fly_trader/brain/plastic.py @@ -140,7 +140,7 @@ def row_weights(scores: torch.Tensor, lines: torch.Tensor) -> torch.Tensor: even at 10x the losing population outweighed the tradeable rows by more than 100:1 and dragged every prediction toward the candidate mean. Learning then made the fly worse than the same network frozen (PF 1.012 vs 1.098). A row below the line is not a decision, so it carries no weight.""" - return torch.where(scores >= lines[:, None], 1.0, 0.0) + return torch.where(scores >= (lines if lines.dim() == 2 else lines[:, None]), 1.0, 0.0) # lines [C] or per row [C, B] (each row's chain) class PlasticBank: @@ -187,6 +187,12 @@ class PlasticBank: m = torch.tanh(u) return (y_dn[None] + (((m - self.mu) / self.sigma) * self._read(s, len(y_dn))) @ self.c) / self.scale, m + def head_for(self, target: torch.Tensor, u0: torch.Tensor, s=None) -> torch.Tensor: + """The head output under which the frozen score (D = 0) is exactly ``target`` (net-return units) [B]: with it, + ``predict`` is ``target`` plus the plastic change of the MBON readout, and ``update`` learns the error of that + sum -- the fly that scores as its teacher plus a learned correction (config.FLY_SCORE = teacher).""" + return target * self.scale - (((torch.tanh(u0) - self.mu) / self.sigma) * self._read(s, len(target))) @ self.c + def frozen(self, y_dn: torch.Tensor, u0: torch.Tensor, s=None) -> torch.Tensor: """The shadow: the bootstrap fly's score (D = 0) [B].""" return (y_dn + (((torch.tanh(u0) - self.mu) / self.sigma) * self._read(s, len(y_dn))) @ self.c) / self.scale diff --git a/fly_trader/chain/cluster_guard.py b/fly_trader/chain/cluster_guard.py index d7026944014169c398cc8df4e825a21dea526232..aec0002f29adcbae65ac4a6d7a1e2247ef9a19f7 100644 --- a/fly_trader/chain/cluster_guard.py +++ b/fly_trader/chain/cluster_guard.py @@ -6,6 +6,9 @@ quotes, signature statuses) never do. Signing is allowed only when ALL of these exists), ``LIVE_ENABLED=1``, and ``config.live_prerequisites_missing()`` is empty (capital, position cap, gas reserve, bot key and API keys all set). ``config`` attributes are read at call time so a test can monkeypatch them. + +The one exception is a withdrawal to the operator's own wallet (``assert_withdraw_allowed``, +chain/withdraw.py): it takes money out of the fly's reach and never trades, so it needs no LIVE_ENABLED. """ from __future__ import annotations @@ -35,6 +38,18 @@ def signing_allowed() -> bool: return False +def assert_withdraw_allowed(dest: str) -> None: + """A withdrawal of the bot's SOL (chain/withdraw.py): mainnet, and only to a wallet listed in FUNDING_ADDRESSES — the + list lives in .env and is never typed into the console, so whoever reaches the console can at most send the bot's SOL + to its owner. Never on the hosted vault fly: its SOL leaves through the treasury (``fly-trader vault withdraw``).""" + if config.VAULT_ENABLED: + raise SigningRefused("refusing: the vault fly's SOL leaves through the treasury (fly-trader vault withdraw), not from the trading wallet") + if config.SOLANA_CLUSTER != "mainnet-beta": + raise SigningRefused(f"refusing to withdraw on cluster {config.SOLANA_CLUSTER!r}: the bot wallet is on mainnet-beta") + if dest not in config.FUNDING_ADDRESSES: + raise SigningRefused(f"refusing: {dest} is not one of your FUNDING_ADDRESSES (.env)") + + def assert_vault_signing_allowed() -> None: """The vault's own signing (claim payouts, principal withdrawals, token-account closes). On mainnet it is the same gate as trading. The devnet rehearsal (``VAULT_CLUSTER=devnet``) may sign with a throwaway key that is not a diff --git a/fly_trader/chain/jupiter_swap.py b/fly_trader/chain/jupiter_swap.py index b030cbe7c33123fe50918fe3e1cc1f7be9c771c2..945f19c58cf5a46037cb4937eb46b49dc389210e 100644 --- a/fly_trader/chain/jupiter_swap.py +++ b/fly_trader/chain/jupiter_swap.py @@ -58,7 +58,7 @@ JUPITER_EXECUTE_RPS = config.JUPITER_RPS # separate bucket, same rate JUPITER_LABELS_URL = "https://api.jup.ag/swap/v2/program-id-to-label" JUPITER_LABELS_CACHE = config.DATA_DIR / "jupiter_program_labels.json" JUPITER_LABELS_MAX_AGE_S = 86400.0 -DEFAULT_EXCLUDE_ROUTERS = "jupiterz" +DEFAULT_EXCLUDE_ROUTERS = "jupiterz,dflow,okx" # Metis only: the signer refuses other routers' programs (signer/policy.py) class JupiterError(RuntimeError): diff --git a/fly_trader/chain/rpc.py b/fly_trader/chain/rpc.py index 3b4400d0cf4e057dba088af885b3a0e61a867f30..3f7ec59a943f55ac4a82e131473e2e1d17bc3d0e 100644 --- a/fly_trader/chain/rpc.py +++ b/fly_trader/chain/rpc.py @@ -224,3 +224,9 @@ class HttpSolanaRpc: sig = self.call("sendTransaction", [tx_b64, {"encoding": "base64", "skipPreflight": bool(skip_preflight), "preflightCommitment": COMMITMENT, "maxRetries": int(max_retries)}]) return str(sig) + + +def check_rpc() -> "HttpSolanaRpc | None": + """A client for the second, independent provider (``SOLANA_CHECK_RPC_URL``) that money-moving reads must agree with, + or None when none is configured.""" + return HttpSolanaRpc(config.SOLANA_CHECK_RPC_URL) if config.SOLANA_CHECK_RPC_URL else None diff --git a/fly_trader/chain/withdraw.py b/fly_trader/chain/withdraw.py new file mode 100644 index 0000000000000000000000000000000000000000..c622de5939e80d35d693f107a0dedf69942d38fd --- /dev/null +++ b/fly_trader/chain/withdraw.py @@ -0,0 +1,129 @@ +"""Withdraw SOL from the bot wallet to one of the operator's own wallets (System → Safety & wallet in the console). + +- Only to a wallet listed in FUNDING_ADDRESSES and only on mainnet; never on the hosted vault fly, whose SOL leaves + through the treasury (cluster_guard.assert_withdraw_allowed). No LIVE_ENABLED: a withdrawal takes money out of the + fly's reach and never trades. +- What is left is 0 or at least the rent-exempt minimum, and while live positions are open at least the gas reserve + (their exits pay fees from it). An empty destination must receive at least the rent-exempt minimum. +- It holds the live broker's in-flight lock, so a swap's balance delta never contains a withdrawal. +- The signed transaction is recorded (wallet_events 'withdrawal', status 'signed') before it is broadcast; its outcome is + added to that row. A confirmed withdrawal re-bases the live book's kill-switch peak (agent/rails.record_withdrawal): + money taken out is not a drawdown. +""" +from __future__ import annotations + +import base64 +import logging +from decimal import Decimal + +from psycopg.types.json import Jsonb +from solders.hash import Hash +from solders.message import MessageV0 +from solders.pubkey import Pubkey +from solders.system_program import TransferParams, transfer +from solders.transaction import VersionedTransaction + +from .. import config +from ..agent import rails +from ..db.apilog import record_event +from ..db.connection import transaction +from ..logging_setup import scrub +from ..markets import SOL +from .cluster_guard import assert_withdraw_allowed +from .rpc import RpcError, RpcTransportError + +log = logging.getLogger(__name__) +LAMPORTS = config.LAMPORTS_PER_SOL +FEE_LAMPORTS = 5_000 # one signature, no priority fee +RENT_EXEMPT_MIN = 890_880 # a system account without data: a balance below it (other than 0) is refused +LOCK_WAIT_S = 120.0 # a swap holds the broker's lock until it confirms or expires + + +class WithdrawRefused(RuntimeError): + """Nothing was signed; the message says why.""" + + +def destinations() -> list[str]: + """The wallets SOL may go to: the valid Solana addresses in FUNDING_ADDRESSES.""" + out = [] + for a in config.FUNDING_ADDRESSES: + try: + Pubkey.from_string(a) + out.append(a) + except ValueError: + log.warning("FUNDING_ADDRESSES: %r is not a Solana address", a) + return out + + +def limits(rpc, me: str, to: str | None = None) -> dict: + """Fresh reads: the wallet's lamports, what must stay (the gas reserve while live positions are open), the most that + can go out, and the destination's lamports.""" + balance = int(rpc.get_balance(me)) + with transaction() as conn: + n_open = int(conn.execute("SELECT count(*) AS n FROM positions WHERE book = %s AND status = 'open'", (SOL.live_book,)).fetchone()["n"]) + keep = max(int(config.GAS_RESERVE_SOL * LAMPORTS), RENT_EXEMPT_MIN) if n_open else 0 + return {"balance": balance, "fee": FEE_LAMPORTS, "keep": keep, "n_open": n_open, "max": max(0, balance - FEE_LAMPORTS - keep), + "to_balance": int(rpc.get_balance(to)) if to else None} + + +def plan(lim: dict, sol: float | None) -> tuple[int, str | None]: + """(lamports to send, why not or None) for ``sol`` SOL, or everything withdrawable when ``sol`` is None.""" + lam = lim["max"] if sol is None else int(Decimal(str(sol)) * LAMPORTS) + if lam <= 0: + return 0, "nothing to withdraw" if sol is None else "enter an amount" + if lam > lim["max"]: + return lam, (f"at most {lim['max'] / LAMPORTS:.6f} SOL can go out" + + (f": {lim['keep'] / LAMPORTS:g} SOL stays for the fees of {lim['n_open']} open live position(s)" if lim["keep"] else " after the fee")) + left = lim["balance"] - lam - lim["fee"] + if 0 < left < RENT_EXEMPT_MIN: + return lam, f"Solana refuses to leave {left / LAMPORTS:.6f} SOL behind: leave at least {RENT_EXEMPT_MIN / LAMPORTS:g} SOL or withdraw everything" + if lim.get("to_balance") == 0 and lam < RENT_EXEMPT_MIN: + return lam, f"an empty wallet must receive at least {RENT_EXEMPT_MIN / LAMPORTS:g} SOL" + return lam, None + + +def withdraw(sol: float | None, to: str, rpc=None, keypair=None) -> dict: + """Send ``sol`` SOL (None: everything withdrawable) from the bot wallet to ``to``. Returns {status, signature, lamports, + to, slot}; status 'confirmed', 'failed_on_chain' (only the fee moved) or 'expired' (nothing moved). Raises + WithdrawRefused, cluster_guard.SigningRefused or keys.WalletKeyError when nothing was signed.""" + from ..execution.broker_live import _INFLIGHT, await_confirmation + assert_withdraw_allowed(to) + if keypair is None: + from .keys import load_keypair + keypair = load_keypair() + if rpc is None: + from .rpc import HttpSolanaRpc + rpc = HttpSolanaRpc() + me = str(keypair.pubkey()) + if not _INFLIGHT.acquire(timeout=LOCK_WAIT_S): + raise WithdrawRefused("a swap is still in flight: try again in a minute") + try: + lam, why = plan(limits(rpc, me, to), sol) + if why: + raise WithdrawRefused(why) + bh = rpc.get_latest_blockhash() + ix = transfer(TransferParams(from_pubkey=keypair.pubkey(), to_pubkey=Pubkey.from_string(to), lamports=lam)) + tx = VersionedTransaction(MessageV0.try_compile(keypair.pubkey(), [ix], [], Hash.from_string(bh["blockhash"])), [keypair]) + sig = str(tx.signatures[0]) + detail = {"to": to, "lamports": lam, "sol": lam / LAMPORTS, "signature": sig, "last_valid_block_height": bh["last_valid_block_height"], "status": "signed"} + with transaction() as conn: # recorded before the network ever sees it + eid = conn.execute("INSERT INTO wallet_events (kind, pubkey, detail) VALUES ('withdrawal', %s, %s) RETURNING id", (me, Jsonb(detail))).fetchone()["id"] + try: + rpc.send_transaction(base64.b64encode(bytes(tx)).decode()) + except RpcError as e: # refused by the node (preflight): never broadcast + with transaction() as conn: + conn.execute("UPDATE wallet_events SET detail = detail || %s WHERE id = %s", (Jsonb({"status": "rejected", "error": scrub(str(e))[:300]}), eid)) + raise WithdrawRefused(f"the network refused it: {scrub(str(e))[:200]}") from None + except RpcTransportError as e: # it may have gone out: its status decides + log.warning("withdrawal %s: send failed in transport (%s); polling its status", sig, e) + status, st = await_confirmation(rpc, sig, bh["last_valid_block_height"]) + slot = int((st or {}).get("slot") or 0) + with transaction() as conn: + conn.execute("UPDATE wallet_events SET detail = detail || %s WHERE id = %s", (Jsonb({"status": status, "slot": slot}), eid)) + if status == "confirmed": + rails.record_withdrawal(conn, {"sol": lam / LAMPORTS, "to": to, "signature": sig}) + finally: + _INFLIGHT.release() + record_event("info" if status == "confirmed" else "warning", "wallet", f"withdrawal of {lam / LAMPORTS:.6f} SOL to {to}: {status}", + {"signature": sig, "slot": slot}) + return {"status": status, "signature": sig, "lamports": lam, "to": to, "slot": slot} diff --git a/fly_trader/config.py b/fly_trader/config.py index 2a306ff6b338132dcd604dc4b849f0efc7828eb9..eda16efff0a1a5e9216443656b0081cd364c7ec3 100644 --- a/fly_trader/config.py +++ b/fly_trader/config.py @@ -14,8 +14,10 @@ from dotenv import load_dotenv REPO_ROOT = Path(__file__).resolve().parent.parent load_dotenv(REPO_ROOT / ".env", override=False) -SECRET_ENV_NAMES = ("HELIUS_API_KEY", "JUPITER_API_KEY", "BOT_PRIVATE_KEY", - "RELAY_SECRET", "TELEGRAM_BOT_TOKEN", "VAULT_BACKUP_HF_TOKEN", "VAULT_SOLANA_RPC_URL") +SECRET_ENV_NAMES = ("HELIUS_API_KEY", "JUPITER_API_KEY", "BOT_PRIVATE_KEY", "KALSHI_API_KEY_ID", "KALSHI_PRIVATE_KEY_PATH", + "RELAY_SECRET", "TELEGRAM_BOT_TOKEN", "VAULT_BACKUP_HF_TOKEN", "VAULT_SOLANA_RPC_URL", "SIGNER_RPC_URL", + "SOLANA_CHECK_RPC_URL", "HEALTHCHECK_URL", "RH_BOT_PRIVATE_KEY", "RH_RPC_URL_LOGS", "ENVIO_API_TOKEN", "QUICKNODE_API_KEY", + "QUICKNODE_RH_API_KEY", "QUICKNODE_RH_API_URL", "RH_RPC_URL_TX") def utcnow() -> datetime: @@ -76,9 +78,12 @@ KELLY_FRACTION = env_float("KELLY_FRACTION", 0.25) # quarter Kel MAX_POSITION_FRACTION = env_float("MAX_POSITION_FRACTION", 0.10) # of the deployable bankroll (wealth minus the gas reserve) MAX_POOL_SHARE = env_float("MAX_POOL_SHARE", 0.02) # of the pool's quote reserve (bounds price impact) # The training label charges what an exit really costs at the size the book takes. market/features.py prices -# exit_cost_0p1 for a fixed 0.1 SOL, but positions are Kelly-sized up to MAX_POSITION_FRACTION of the bankroll, and -# impact = value/(value + reserves) grows with size. 0 = the sizing rule's own ceiling (train/decisions.py). -LABEL_SIZE_SOL = env_float("LABEL_SIZE_SOL", 0.0) +# exit_cost_0p1 for a fixed 0.1 SOL, but impact = value/(value + reserves) grows with size. Labels are priced at +# min(LABEL_SIZE_SOL, MAX_POOL_SHARE of the pool) -- the "label size" -- and no position is ever bigger (agent/sizing.py): +# every edge the models show was measured at that size. A fixed reference, never a share of the bankroll: derived from +# CAPITAL_SOL it made the labels, and the sizes, grow with the bankroll, and in the 2026-09-26 replay a 500 SOL book +# sized past it lost 426 SOL over 60 days where one capped at it made the same +13 SOL as a 5 or 50 SOL book. +LABEL_SIZE_SOL = env_float("LABEL_SIZE_SOL", 0.5) MIN_POSITION_SOL = env_float("MIN_POSITION_SOL", 0.02) # smaller sizes are skipped KILL_SWITCH_DRAWDOWN = env_float("KILL_SWITCH_DRAWDOWN", 0.30) KILL_SWITCH_LIQUIDATE = env_bool("KILL_SWITCH_LIQUIDATE", False) # a tripped kill switch also sells every open live position (agent/fly_live.py) @@ -103,6 +108,25 @@ LAMPORTS_PER_SOL = 1_000_000_000 STATS_REFRESH_S = env_float("STATS_REFRESH_S", 600.0) # ---- connectome (the fly) ---- +# How the fly picks its buy line (train/fly_calibrate.py): total | selector | match. match = its teacher's selectivity: +# in the 2026-09-26 replays it took 92 trades at +10.1 % (58 % winners, PF 1.92) where total took 1,607 at +2.0 %. +FLY_CALIB_RULE = env_str("FLY_CALIB_RULE", "match") +# How the fly sizes a buy (agent/sizing.py): kelly = the selector's own Kelly band for the row (its score, line and sizing +# table; with FLY_TEACHER_GATE every fly buy is a selector buy, sized exactly as the selector sizes it) | flat = +# MAX_POSITION_FRACTION of the deployable bankroll on every buy. (Flat was the default while the fly sized from its own +# calibrated bands, which flipped between Kelly 0.97, 0 and 0.15 on its few trades a week; the selector's bands are fitted +# on its walk-forward and do not.) +FLY_SIZING = env_str("FLY_SIZING", "kelly") +# The fly buys only what its teacher (the deployed selector stack) would buy: the same trigger, line, win filter, gates, +# dump veto and fail-closed rules. Its own learning (plastic score below its bootstrap score by FLY_LEARNED_VETO) can +# only drop a teacher trade, never add one (2026-10-08: ungated, the fly bought rows the selector rejected and lost). +FLY_TEACHER_GATE = env_bool("FLY_TEACHER_GATE", True) +FLY_LEARNED_VETO = env_float("FLY_LEARNED_VETO", 0.02) +# How the fly scores a row. teacher: its teacher's own score plus what its mushroom body has learned (the plastic change of +# its KC->MBON synapses, zero at bootstrap), so it starts exactly as the selector and its learning only corrects the +# selector. own: its distilled copy of the selector (until 2026-10-11): that copy ranked the selector's top 1 % of +# candidates only ~0.6-correlated, so its own judgement at the line was half noise (fly #157 and #251 alike). +FLY_SCORE = env_str("FLY_SCORE", "teacher") DEVICE = env_str("DEVICE", "auto") # auto (CUDA, else MPS, else CPU) | cpu | cuda | cuda:N | mps — brain/device.py NT_SIGN_MODE = env_str("NT_SIGN_MODE", "shiu") # shiu | flybrain CELL_TYPES_SOURCE = env_str("CELL_TYPES_SOURCE", "annotations") # annotations | flybrain @@ -158,8 +182,8 @@ def live_prerequisites_missing() -> list[str]: missing.append("MAX_POSITION_SOL") if GAS_RESERVE_SOL <= 0: missing.append("GAS_RESERVE_SOL") - if not env_str("BOT_PRIVATE_KEY") and not env_str("BOT_PRIVATE_KEY_FILE"): - missing.append("BOT_PRIVATE_KEY") + if not SIGNER_SOCKET and not env_str("BOT_PRIVATE_KEY") and not env_str("BOT_PRIVATE_KEY_FILE"): + missing.append("BOT_PRIVATE_KEY") # the vault server's brain has no key: its signer does if not JUPITER_API_KEY: missing.append("JUPITER_API_KEY") if not HELIUS_API_KEY: @@ -168,9 +192,11 @@ def live_prerequisites_missing() -> list[str]: # ---- the $FLY vault (fly_trader/vault, docs/vault/SPEC.md): only the hosted vault fly sets VAULT_ENABLED ---- VAULT_ENABLED = env_bool("VAULT_ENABLED", False) +PAYOUT_PRIVATE_KEY_FILE = env_str("PAYOUT_PRIVATE_KEY_FILE") # the key that pays claims from the treasury (spending limit L2); read by the signer only +VAULT_BOOK = env_str("VAULT_BOOK", "live") # the book the vault shares: live; a paper book (e.g. paper_fly) for a dry run on real decisions without real SOL VAULT_CLUSTER = env_str("VAULT_CLUSTER", "mainnet-beta") # devnet only for the rehearsal (trading off, claims on devnet) VAULT_SOLANA_RPC_URL = env_str("VAULT_SOLANA_RPC_URL") # overrides Helius for the vault's reads/claims (the devnet rehearsal) -FUNDING_ADDRESSES = env_list("FUNDING_ADDRESSES", []) # SOL from these is a deposit; from anyone else, profit +FUNDING_ADDRESSES = env_list("FUNDING_ADDRESSES", []) # your own Solana wallets: SOL from these is a deposit (vault); the console withdraws only to these RH_CHAIN_ID = env_int("RH_CHAIN_ID", 4663) RH_RPC_URL = env_str("RH_RPC_URL", "https://rpc.mainnet.chain.robinhood.com") VAULT_ADDRESS = env_str("VAULT_ADDRESS") # the FlyVault proxy @@ -180,6 +206,8 @@ VAULT_IMPL_CODEHASHES = env_list("VAULT_IMPL_CODEHASHES", []) # implementa VAULT_PERIOD_S = env_int("VAULT_PERIOD_S", 7 * 86400) # one settlement period; the rehearsal uses 900 VAULT_EPOCH = env_int("VAULT_EPOCH", 345_600) # boundaries at EPOCH + k*PERIOD; 345600 = Monday 1970-01-05 00:00 UTC CLAIM_MIN_LAMPORTS = env_int("CLAIM_MIN_LAMPORTS", 2_000_000) # 0.002 SOL: above the rent-exempt minimum of a new wallet +CLAIM_MIN_WEI = env_int("CLAIM_MIN_WEI", 100_000_000_000_000) # 0.0001 ETH: smaller ETH owed waits for the next claim +VAULT_RH_GAS_RESERVE_WEI = env_int("VAULT_RH_GAS_RESERVE_WEI", 200_000_000_000_000) # ETH the settlement never allocates (the RH wallet pays payout gas) RELAY_URL = env_str("RELAY_URL") # the site origin; the fly calls {RELAY_URL}/api/... RELAY_KEY_ID = env_str("RELAY_KEY_ID", "k1") RELAY_SECRET = env_str("RELAY_SECRET") @@ -193,11 +221,161 @@ TELEGRAM_CHAT_ID = env_str("TELEGRAM_CHAT_ID") VAULT_BACKUP_RECIPIENT = env_str("VAULT_BACKUP_RECIPIENT") # age / ssh-ed25519 public key the backups are encrypted to VAULT_BACKUP_BUCKET = env_str("VAULT_BACKUP_BUCKET") # private HF storage bucket, e.g. bryceweiner/fly-vault-backups VAULT_BACKUP_HF_TOKEN = env_str("VAULT_BACKUP_HF_TOKEN") # scoped to that bucket only +# custody (docs/vault/SPEC.md "Custody"): a Squads v4 treasury the hot keys draw on only through its spending limits +VAULT_MULTISIG = env_str("VAULT_MULTISIG") # the Squads multisig; the treasury is its vault 0 +VAULT_LIMIT_TRADING = env_str("VAULT_LIMIT_TRADING") # L1: trading key -> the trading wallet only, per day +VAULT_LIMIT_PAYOUT = env_str("VAULT_LIMIT_PAYOUT") # L2: payout key -> any holder, per week +TREASURY_FLOAT_SOL = env_float("TREASURY_FLOAT_SOL", 2.0) # the most the trading wallet keeps idle; the rest goes back +TOPUP_MIN_SOL = env_float("TOPUP_MIN_SOL", 0.1) # smaller refills and returns are skipped +SIGNER_SOCKET = env_str("SIGNER_SOCKET") # the signer container's socket; unset: an in-process signer +SIGNER_RPC_URL = env_str("SIGNER_RPC_URL") # the signer's own RPC (a second provider) +SOLANA_CHECK_RPC_URL = env_str("SOLANA_CHECK_RPC_URL") # a second provider money-moving reads must agree with +TELEGRAM_ADMIN_USER_ID = env_str("TELEGRAM_ADMIN_USER_ID") # the only Telegram user whose /panic and /status count +HEALTHCHECK_URL = env_str("HEALTHCHECK_URL") # dead-man ping (healthchecks.io) every 5 min def vault_solana_rpc_url() -> str: return VAULT_SOLANA_RPC_URL or helius_http_url() + +def signer_env() -> dict: + """What an in-process signer reads (fly_trader/signer; the signer container has its own environment).""" + return {k: str(v) for k, v in {"VAULT_MULTISIG": VAULT_MULTISIG, "VAULT_LIMIT_TRADING": VAULT_LIMIT_TRADING, + "VAULT_LIMIT_PAYOUT": VAULT_LIMIT_PAYOUT, + "TREASURY_FLOAT_SOL": TREASURY_FLOAT_SOL, "VAULT_BACKUP_RECIPIENT": VAULT_BACKUP_RECIPIENT}.items() if v} + +# ---- Robinhood Chain memecoins (fly_trader/rh; Pons launches graduated into Uniswap v4; branch rh-memecoins) ---- +# The same selector and fly as Solana, trained on both chains' rows (a chain input tells them apart); RH books keep their +# money in native ETH. Amounts on RH rows are ETH (non-ETH-quoted pools converted at the minute's base/ETH price). +RH_ENABLED = env_bool("RH_ENABLED", True) # index RH and paper-trade it +RH_LIVE_ENABLED = env_bool("RH_LIVE_ENABLED", False) # sign and send real RH transactions (live_rh) +RH_TESTNET = env_bool("RH_TESTNET", False) # chain 46630 (the rehearsal); the guard refuses a mismatch +RH_EXPECTED_CHAIN_ID = 46630 if RH_TESTNET else RH_CHAIN_ID +RH_RPC_URL_LOGS = env_str("RH_RPC_URL_LOGS") # optional better endpoint for eth_getLogs / tx lookups (else RH_RPC_URL) + + +def quicknode_rh_url() -> str | None: + """The QuickNode RH endpoint: QUICKNODE_RH_API_URL, with QUICKNODE_RH_API_KEY appended when the URL does not carry it.""" + url, key = env_str("QUICKNODE_RH_API_URL"), env_str("QUICKNODE_RH_API_KEY") + if not url: + return None + return url if not key or key in url else url.rstrip("/") + "/" + key + + +# sending and reading transactions (sends, receipts, nonces, balances, eth_call, gas): every RhRpc() without an explicit URL. +# Never logs: QuickNode's free Discover plan caps eth_getLogs at 5 blocks (measured 2026-10-09), so logs stay on HyperSync / +# RH_RPC_URL_LOGS / RH_RPC_URL; its 10M credits a month cover the trading path, not a log tail. QuickNode serves mainnet +# only while RH_RPC_URL is the default: a run that points RH_RPC_URL elsewhere (anvil, the testnet rehearsal) sends there. +RH_RPC_URL_TX = env_str("RH_RPC_URL_TX") or (quicknode_rh_url() if env_str("RH_RPC_URL") is None and not RH_TESTNET else None) +# block headers for timing the index (rh/blocktime.py): PublicNode serves header batches fast but refuses eth_getLogs (403), +# the official RPC serves logs but rate-limits at ~20 requests/min (measured 2026-09-28) — so each does what it does well +RH_RPC_URL_BLOCKS = env_str("RH_RPC_URL_BLOCKS", "https://robinhood-rpc.publicnode.com") +RH_HYPERSYNC_URL = env_str("RH_HYPERSYNC_URL", "https://robinhood.hypersync.xyz") # log history in bulk when ENVIO_API_TOKEN is set (rh/hypersync.py) +RH_HYPERSYNC_RPM = env_float("RH_HYPERSYNC_RPM", 40) # HyperSync queries per minute (free tier: fair use, refused ~6 % at 60; Starter plan: 100) +RH_BOT_ADDRESS = env_str("RH_BOT_ADDRESS") # the address RH_BOT_PRIVATE_KEY must derive to (a guard, not a secret) +RH_FUNDING_ADDRESSES = env_list("RH_FUNDING_ADDRESSES", []) # your own EVM wallets: the console withdraws the RH bot's ETH only to these +RH_CONFIRMATIONS = env_int("RH_CONFIRMATIONS", 3) # blocks before a live receipt or a live log is applied +RH_START_BLOCK = env_int("RH_START_BLOCK", 0) # the indexer's first block (0: the Pons V2 factory's deployment block) +RH_START_DAY = env_str("RH_START_DAY", "2026-08-04") # Pons V2 went live: the first RH corpus day +# Fixed at build time by tools/rh_constants.py (2026-09-28 13:12 UTC: SOL/USD 119.51, ETH/USD 2682.05): the Solana sizes in ETH. +# Never derived live: they set the size every RH label is priced at (see LABEL_SIZE_SOL). +RH_ETH_PER_SOL = env_float("RH_ETH_PER_SOL", 0.0445583) # K: converts fixed-unit thresholds (eligibility, triggers) for RH rows +RH_CAPITAL_ETH = env_float("RH_CAPITAL_ETH", 0.222791) # the RH paper books' bankroll (5 SOL at build time) +RH_LABEL_SIZE_ETH = env_float("RH_LABEL_SIZE_ETH", 0.0222791) # 0.5 SOL at build time: RH labels are priced at it; no RH buy is bigger +RH_MIN_POSITION_ETH = env_float("RH_MIN_POSITION_ETH", 0.000891166) # 0.02 SOL at build time +RH_GAS_RESERVE_ETH = env_float("RH_GAS_RESERVE_ETH", 0.0029) # 200 round trips at 2x the gas rh-probe measured (356k gas, 0.02 gwei, 2026-09-28) +RH_MAX_FEE_GWEI = env_float("RH_MAX_FEE_GWEI", 5.0) # a transaction never offers more per gas +RH_TX_FEE_ETH = env_float("RH_TX_FEE_ETH", 7.2e-6) # one swap's gas in ETH (rh-probe 2026-09-28: 356k gas × 0.02 gwei); fixed like the SOL fee +RH_HOOK_FEE = env_float("RH_HOOK_FEE", 0.0099) # Pons hook fee + creator tax per side when a minute has none: the median of 52k indexed swaps (1 % of the net = 0.99 % of gross; high-tax launches reach 5.7 %) +RH_KILL_SWITCH_DRAWDOWN = env_float("RH_KILL_SWITCH_DRAWDOWN", 0.30) +RH_MIN_SWEEP_ETH = env_float("RH_MIN_SWEEP_ETH", 0.0002) # base-asset or dead-bag balances worth less are dust +RH_STREAM_POLL_S = env_float("RH_STREAM_POLL_S", 2.0) +RH_TRADE_LAG_S = env_float("RH_TRADE_LAG_S", 240.0) # an RH minute this old (s) is still traded: its stream lands ~2–3 min late +RH_MINUTES_KEEP_DAYS = env_int("RH_MINUTES_KEEP_DAYS", 14) +RH_DIR_NAME = "rh" # data/corpus/rh/... (the RH corpus beside the Solana one) +# contracts on Robinhood Chain (mainnet; verified to carry code 2026-09-28) +PONS_FACTORY = env_str("PONS_FACTORY", "0x7ed598bcef8bd9edd8c97a195c6d13f40801ec7e") +PONS_ROUTER = env_str("PONS_ROUTER", "0xe33e9e479df8802cb0866d5d05258bec4cf62948") +PONS_HOOK = env_str("PONS_HOOK", "0xe5e702641ea86f4ae6cc3cdaed2b886f976be044") +PONS_LOCKER = env_str("PONS_LOCKER", "0x267444d099b10fb5ed7c3cc7b7c767adca574952") +PONS_GRAD_EXECUTOR = env_str("PONS_GRAD_EXECUTOR", "0xc7819b64a1daecd7ec19856d026cb14efbd89046") +V4_POOL_MANAGER = env_str("V4_POOL_MANAGER", "0x8366a39cc670b4001a1121b8f6a443a643e40951") +V4_QUOTER = env_str("V4_QUOTER", "0x8dc178efb8111bb0973dd9d722ebeff267c98f94") +V4_STATE_VIEW = env_str("V4_STATE_VIEW", "0xf3334192d15450cdd385c8b70e03f9a6bd9e673b") +UNIVERSAL_ROUTER = env_str("UNIVERSAL_ROUTER", "0x8876789976decbfcbbbe364623c63652db8c0904") +PERMIT2 = env_str("PERMIT2", "0x000000000022D473030F116dDEE9F6B43aC78BA3") +V3_FACTORY = env_str("V3_FACTORY", "0x1f7d7550b1b028f7571e69a784071f0205fd2efa") +V3_QUOTER_V2 = env_str("V3_QUOTER_V2", "0x33e885ed0ec9bf04ecfb19341582aadcb4c8a9e7") +RH_WETH = env_str("RH_WETH", "0x0bd7d308f8e1639fab988df18a8011f41eacad73") # SwapRouter02.WETH9() +RH_USDG = env_str("RH_USDG", "0x5fc5360D0400a0Fd4f2af552ADD042D716F1d168") +KYBER_API = env_str("KYBER_API", "https://aggregator-api.kyberswap.com/robinhood/api/v1") +KYBER_ROUTER = env_str("KYBER_ROUTER", "0x6131B5fae19EA4f9D964eAc0408E4408b66337b5") # MetaAggregationRouterV2: the only router a built Kyber tx may call +KYBER_CLIENT_ID = env_str("KYBER_CLIENT_ID", "fly-trader") + + +def rh_live_prerequisites_missing() -> list[str]: + missing = [] + if not RH_LIVE_ENABLED: + missing.append("RH_LIVE_ENABLED=1") + if not env_str("RH_BOT_PRIVATE_KEY"): + missing.append("RH_BOT_PRIVATE_KEY") + if not RH_BOT_ADDRESS: + missing.append("RH_BOT_ADDRESS") + for name in ("RH_CAPITAL_ETH", "RH_LABEL_SIZE_ETH", "RH_MIN_POSITION_ETH", "RH_ETH_PER_SOL", "RH_GAS_RESERVE_ETH"): + if globals()[name] <= 0: + missing.append(name) + return missing + + +# ---- Kalshi prediction markets (fly_trader/kalshi; the same key names as better_bot, whose client is vendored) ---- +KALSHI_API_KEY_ID = env_str("KALSHI_API_KEY_ID") +KALSHI_PRIVATE_KEY_PATH = env_str("KALSHI_PRIVATE_KEY_PATH") +KALSHI_API_BASE_URL = env_str("KALSHI_API_BASE_URL", "https://api.elections.kalshi.com/trade-api/v2") +KALSHI_WS_URL = env_str("KALSHI_WS_URL", "wss://api.elections.kalshi.com/trade-api/ws/v2") +KALSHI_SUBACCOUNT = env_int("KALSHI_SUBACCOUNT", 0) # 0 = the primary account: only a dedicated subaccount may trade live +KALSHI_FEE_FACTOR = env_float("KALSHI_FEE_FACTOR", 0.07) # taker fee per contract = factor x multiplier x P(1-P) +KALSHI_MAKER_FEE_FACTOR = env_float("KALSHI_MAKER_FEE_FACTOR", 0.0175) +KALSHI_CAPITAL_USD = env_float("KALSHI_CAPITAL_USD", 100.0) # the cap both Kalshi arms share (paper start; live never sizes above it) +KALSHI_CASH_FLOOR_USD = env_float("KALSHI_CASH_FLOOR_USD", 5.0) # the gas reserve's analogue: never spent +KALSHI_MIN_PAYOUT_CENTS = env_int("KALSHI_MIN_PAYOUT_CENTS", 25) # a fill whose winning profit is below this is skipped +KALSHI_LIVE_ENABLED = env_bool("KALSHI_LIVE_ENABLED", False) +KALSHI_TAKER_LIVE = env_bool("KALSHI_TAKER_LIVE", True) +KALSHI_MAKER_LIVE = env_bool("KALSHI_MAKER_LIVE", True) +KALSHI_KILL_SWITCH_DRAWDOWN = env_float("KALSHI_KILL_SWITCH_DRAWDOWN", 0.30) +KALSHI_MAX_SLATE_FRACTION = env_float("KALSHI_MAX_SLATE_FRACTION", 0.50) # of the cap deployed at once across every open position +KALSHI_MAX_DAYS_TO_CLOSE = env_float("KALSHI_MAX_DAYS_TO_CLOSE", 10.0) # the evidence base (Whelan 2026) covers the last 10 days +KALSHI_MIN_MINUTES_TO_CLOSE = env_float("KALSHI_MIN_MINUTES_TO_CLOSE", 5.0) +KALSHI_MAX_SPREAD_CENTS = env_int("KALSHI_MAX_SPREAD_CENTS", 10) +KALSHI_MIN_OPEN_INTEREST = env_float("KALSHI_MIN_OPEN_INTEREST", 100.0) +KALSHI_MIN_VOLUME_24H = env_float("KALSHI_MIN_VOLUME_24H", 50.0) +KALSHI_MAKER_QUIET_MIN = env_float("KALSHI_MAKER_QUIET_MIN", 30.0) # no resting orders this close to the market's close +KALSHI_MAKER_TTL_H = env_float("KALSHI_MAKER_TTL_H", 6.0) +KALSHI_MAKER_MAX_RESTING = env_int("KALSHI_MAKER_MAX_RESTING", 40) +KALSHI_HISTORY_START = env_str("KALSHI_HISTORY_START", "2025-01-01") +KALSHI_DATASET_DIR = env_str("KALSHI_DATASET_DIR") # jon-becker/prediction-market-analysis checkout (data/kalshi/{markets,trades}) +KALSHI_MINUTES_KEEP_DAYS = env_int("KALSHI_MINUTES_KEEP_DAYS", 7) +KALSHI_MARKETS_PER_DAY = env_int("KALSHI_MARKETS_PER_DAY", 150) # the event sample's target: about this many markets per category per day (kalshi/history.sample_rates); + # events are kept by md5(event ticker) < the category's rate, blind to outcome and volume +KALSHI_DIR = DATA_DIR / "kalshi" +KALSHI_TRAIN_MAX_ROWS = env_int("KALSHI_TRAIN_MAX_ROWS", 8_000_000) # training rows per walk-forward block / final fit (uniform random subset) +KALSHI_FILL_THREADS = env_int("KALSHI_FILL_THREADS", 8) # markets filled at once by the history worker, all under the one KALSHI_RPS bucket +KALSHI_RPS = env_float("KALSHI_RPS", 16.0) # per process; the account's Advanced grant reads 200 tokens/s at 10 per call = 20 calls/s, the feed uses ~3 + + +def kalshi_live_prerequisites_missing() -> list[str]: + missing = [] + if not KALSHI_LIVE_ENABLED: + missing.append("KALSHI_LIVE_ENABLED=1") + if not KALSHI_API_KEY_ID: + missing.append("KALSHI_API_KEY_ID") + if not KALSHI_PRIVATE_KEY_PATH or not Path(KALSHI_PRIVATE_KEY_PATH).expanduser().exists(): + missing.append("KALSHI_PRIVATE_KEY_PATH (readable PEM file)") + if KALSHI_SUBACCOUNT <= 0: + missing.append("KALSHI_SUBACCOUNT (a dedicated subaccount, never the primary)") + if KALSHI_CAPITAL_USD <= 0: + missing.append("KALSHI_CAPITAL_USD") + return missing + # ---- historical corpus (every pump.fun graduation, pulled from the migration wallet + pump.fun's swap-api) ---- CORPUS_DIR = DATA_DIR / "corpus" CORPUS_DAYS = env_int("CORPUS_DAYS", 240) # enumerate graduations this far back (must reach before CORPUS_PULL_BEFORE) diff --git a/fly_trader/db/schema.py b/fly_trader/db/schema.py index 370965e565fe2565cf54333c29db1d4dee502c70..827a17217362b1374473d48c7e5f19785f6bfd1c 100644 --- a/fly_trader/db/schema.py +++ b/fly_trader/db/schema.py @@ -6,7 +6,9 @@ appending a version-gated block AND bumping SCHEMA_VERSION, or it never runs. """ from __future__ import annotations +import hashlib import logging +import time from datetime import datetime, timedelta, timezone import psycopg @@ -17,8 +19,15 @@ from .connection import connect, database_name, database_url log = logging.getLogger(__name__) -SCHEMA_VERSION = 6 # 5: the plastic fly's tables and per-book halts (BASE_DDL); 6: the selector's strategy stack (MIGRATIONS[6]) +SCHEMA_VERSION = 12 # 5: the plastic fly's tables and per-book halts (BASE_DDL); 6: the selector's strategy stack (MIGRATIONS[6]); 7: Kalshi (MIGRATIONS[7]); 8: Kalshi minute extremes _LOCK_KEY = 0x666C795F6D6967 # "fly_mig" +# Applying the DDL takes ACCESS EXCLUSIVE on the tables it touches (Postgres takes it for ALTER TABLE ... ADD COLUMN IF NOT +# EXISTS even when the column exists) and holds it to the end of the transaction, while live workers write those tables. +# 2026-10-07: the RH stream's start-up apply deadlocked with the trading engine on positions/decisions and crashed. So: +# unchanged DDL is not re-run at all (its fingerprint is stored), and a run never waits more than LOCK_TIMEOUT for a +# lock: it rolls back, releases everything it holds, and retries. +LOCK_TIMEOUT = "5s" +APPLY_TRIES = 30 BASE_DDL: list[str] = [ """CREATE TABLE IF NOT EXISTS schema_version ( @@ -150,11 +159,11 @@ BASE_DDL: list[str] = [ running_mean real[], running_var real[], n bigint NOT NULL DEFAULT 0, updated_at timestamptz NOT NULL DEFAULT now())""", """CREATE TABLE IF NOT EXISTS circuit_state ( - id int PRIMARY KEY CHECK (id = 1), fail_count int NOT NULL DEFAULT 0, last_failure_ts timestamptz, + id int PRIMARY KEY, fail_count int NOT NULL DEFAULT 0, last_failure_ts timestamptz, tripped boolean NOT NULL DEFAULT false, kill_switch boolean NOT NULL DEFAULT false, kill_reason text, peak_wealth double precision, entries_paused boolean NOT NULL DEFAULT false, updated_at timestamptz NOT NULL DEFAULT now())""", - "INSERT INTO circuit_state (id) VALUES (1) ON CONFLICT DO NOTHING", + "INSERT INTO circuit_state (id) VALUES (1) ON CONFLICT DO NOTHING", # id 2 (the Kalshi rails) is inserted by MIGRATIONS[7], after the CHECK is dropped """CREATE TABLE IF NOT EXISTS circuit_events ( id bigserial PRIMARY KEY, ts timestamptz NOT NULL DEFAULT now(), kind text NOT NULL, detail jsonb)""", """CREATE TABLE IF NOT EXISTS notional_ledger ( @@ -223,6 +232,8 @@ BASE_DDL: list[str] = [ label real, state text NOT NULL DEFAULT 'pending', resolved_at timestamptz, bootstrap_id bigint, PRIMARY KEY (ts, mint))""", "CREATE INDEX IF NOT EXISTS fly_scored_state_ts_idx ON fly_scored (state, ts)", + # would the fly's teacher have traded this row (its final decision)? the match calibration rule's budget (train/fly_calibrate.py) + "ALTER TABLE fly_scored ADD COLUMN IF NOT EXISTS teacher_allow boolean", """CREATE TABLE IF NOT EXISTS fly_calibrations ( day date NOT NULL, arm text NOT NULL, line real, sizing jsonb, trades int, total real, mean real, window_days int, created_at timestamptz NOT NULL DEFAULT now(), PRIMARY KEY (day, arm))""", @@ -261,6 +272,105 @@ MIGRATIONS: dict[int, list[str]] = { "ALTER TABLE fly_scored ADD COLUMN IF NOT EXISTS hold_min int", "ALTER TABLE fly_scored DROP CONSTRAINT IF EXISTS fly_scored_pkey", "ALTER TABLE fly_scored ADD PRIMARY KEY (ts, mint, strategy)"], + # 7: Kalshi prediction markets (fly_trader/kalshi): the exchange catalogue, live minutes and quotes, the corpus registry, + # the four Kalshi books' positions/orders/fills, the visual fly's tables, and a second circuit (id 2) for the Kalshi rails + 7: ["ALTER TABLE circuit_state DROP CONSTRAINT IF EXISTS circuit_state_id_check", + "INSERT INTO circuit_state (id) VALUES (2) ON CONFLICT DO NOTHING", + """CREATE TABLE IF NOT EXISTS kalshi_series ( + ticker text PRIMARY KEY, title text, category text, categories text[], tags text[], frequency text, fee_type text, + fee_multiplier double precision, settlement_sources jsonb, updated_at timestamptz NOT NULL DEFAULT now())""", + """CREATE TABLE IF NOT EXISTS kalshi_events ( + event_ticker text PRIMARY KEY, series_ticker text, title text, sub_title text, category text, mutually_exclusive boolean, + strike_date timestamptz, strike_period text, collateral_return_type text, exchange_index int, raw jsonb, + updated_at timestamptz NOT NULL DEFAULT now())""", + "CREATE INDEX IF NOT EXISTS kalshi_events_series_idx ON kalshi_events (series_ticker)", + """CREATE TABLE IF NOT EXISTS kalshi_markets ( + ticker text PRIMARY KEY, event_ticker text, market_type text, title text, yes_sub_title text, no_sub_title text, status text, + open_time timestamptz, close_time timestamptz, expected_expiration_time timestamptz, latest_expiration_time timestamptz, + settlement_ts timestamptz, result text, settlement_value double precision, strike_type text, floor_strike double precision, + cap_strike double precision, exchange_index int, price_level_structure text, is_provisional boolean, can_close_early boolean, + created_time timestamptz, source text, raw jsonb, updated_at timestamptz NOT NULL DEFAULT now())""", + "CREATE INDEX IF NOT EXISTS kalshi_markets_status_close_idx ON kalshi_markets (status, close_time)", + "CREATE INDEX IF NOT EXISTS kalshi_markets_event_idx ON kalshi_markets (event_ticker)", + "CREATE INDEX IF NOT EXISTS kalshi_markets_settled_idx ON kalshi_markets (settlement_ts)", + # the pump_minutes analogue: one row per (market, UTC minute) from the ticker and trade channels + """CREATE TABLE IF NOT EXISTS kalshi_minutes ( + ticker text NOT NULL, ts timestamptz NOT NULL, yes_bid real, yes_ask real, last real, bid_size real, ask_size real, + volume_fp double precision, open_interest_fp double precision, dollar_volume double precision, + taker_buy_yes double precision, taker_buy_no double precision, n_trades int, max_trade double precision, block_contracts double precision, + PRIMARY KEY (ticker, ts))""", + "CREATE INDEX IF NOT EXISTS kalshi_minutes_ts_idx ON kalshi_minutes (ts)", + # the live top of book (every ticker message), read by the maker arm and the vendored venue's spread tiers + """CREATE TABLE IF NOT EXISTS kalshi_quotes ( + ticker text PRIMARY KEY, ts timestamptz, yes_bid real, yes_ask real, mid real, bid_size real, ask_size real, last real, + updated_at timestamptz NOT NULL DEFAULT now())""", + """CREATE OR REPLACE VIEW venue_market_cache AS + SELECT 'kalshi'::text AS venue, ticker AS market_id, yes_ask AS yes_ask_cents, mid AS market_prob FROM kalshi_quotes""", + # the corpus registry (kalshi/history.py) and the daily feature builds (kalshi/mature.py) + """CREATE TABLE IF NOT EXISTS kalshi_corpus ( + ticker text PRIMARY KEY, status text NOT NULL DEFAULT 'pending', settled_ts timestamptz, result text, open_time timestamptz, + close_time timestamptz, candles_1m int, trades int, candle_path text, trade_path text, seeded_from text, last_error text, + updated_at timestamptz NOT NULL DEFAULT now())""", + "CREATE INDEX IF NOT EXISTS kalshi_corpus_status_idx ON kalshi_corpus (status, settled_ts DESC)", + """CREATE TABLE IF NOT EXISTS kalshi_days ( + day date PRIMARY KEY, markets int, rows int, took_s real, built_at timestamptz NOT NULL DEFAULT now())""", + # the four Kalshi books (paper_kalshi_taker, paper_kalshi_maker, live_kalshi_taker, live_kalshi_maker); money in cents + """CREATE TABLE IF NOT EXISTS kalshi_positions ( + id bigserial PRIMARY KEY, book text NOT NULL, ticker text NOT NULL, side text NOT NULL, contracts double precision NOT NULL, + cost_cents double precision NOT NULL, fee_cents double precision NOT NULL DEFAULT 0, avg_price_cents double precision, + opened_at timestamptz NOT NULL DEFAULT now(), closed_at timestamptz, entry_decision_id bigint, exit_decision_id bigint, + order_id text, status text NOT NULL DEFAULT 'open', result text, payout_cents double precision, realized_cents double precision, + strategy text, arm text, score real, line real, last_mark_cents real, last_mark_ts timestamptz)""", + "CREATE INDEX IF NOT EXISTS kalshi_positions_book_status_idx ON kalshi_positions (book, status)", + "CREATE INDEX IF NOT EXISTS kalshi_positions_ticker_idx ON kalshi_positions (ticker)", + """CREATE TABLE IF NOT EXISTS kalshi_orders ( + id bigserial PRIMARY KEY, book text NOT NULL, order_id text, client_order_id text, decision_id bigint, ticker text NOT NULL, + side text NOT NULL, action text NOT NULL DEFAULT 'buy', price_cents int NOT NULL, count double precision NOT NULL, tif text NOT NULL, + post_only boolean NOT NULL DEFAULT false, expiration_ts timestamptz, status text NOT NULL, fill_count double precision NOT NULL DEFAULT 0, + remaining double precision, avg_fill_cents double precision, fee_cents double precision, exchange_index int, subaccount int, + fair_cents real, margin_cents real, strategy text, request jsonb, response jsonb, error text, + ts timestamptz NOT NULL DEFAULT now(), updated_at timestamptz NOT NULL DEFAULT now())""", + "CREATE INDEX IF NOT EXISTS kalshi_orders_book_status_idx ON kalshi_orders (book, status)", + "CREATE UNIQUE INDEX IF NOT EXISTS kalshi_orders_order_id_uniq ON kalshi_orders (order_id) WHERE order_id IS NOT NULL", + """CREATE TABLE IF NOT EXISTS kalshi_fills ( + trade_id text PRIMARY KEY, order_id text, book text, ticker text NOT NULL, side text NOT NULL, price_cents double precision NOT NULL, + count double precision NOT NULL, fee_cents double precision, is_taker boolean, action text, ts timestamptz NOT NULL, + post_position double precision, raw jsonb)""", + "CREATE INDEX IF NOT EXISTS kalshi_fills_order_idx ON kalshi_fills (order_id)", + # the visual fly (kalshi/fly_session.py): scored (market, minute, side) rows waiting for settlement, calibrations, learning, rollbacks + """CREATE TABLE IF NOT EXISTS kalshi_fly_scored ( + ts timestamptz NOT NULL, ticker text NOT NULL, side text NOT NULL, strategy text NOT NULL, x real[], score real, frozen_score real, + line real, frozen_line real, label real, state text NOT NULL DEFAULT 'pending', due_ts timestamptz, resolved_at timestamptz, + bootstrap_id bigint, PRIMARY KEY (ts, ticker, side, strategy))""", + "CREATE INDEX IF NOT EXISTS kalshi_fly_scored_state_idx ON kalshi_fly_scored (state, due_ts)", + """CREATE TABLE IF NOT EXISTS kalshi_fly_calibrations ( + day date NOT NULL, arm text NOT NULL, line real, sizing jsonb, trades int, total real, mean real, window_days int, + created_at timestamptz NOT NULL DEFAULT now(), PRIMARY KEY (day, arm))""", + """CREATE TABLE IF NOT EXISTS kalshi_fly_updates ( + hour timestamptz PRIMARY KEY, n int, mean_delta real, mean_abs_delta real, step real, capped int, drift real, ic real, detail jsonb)""", + """CREATE TABLE IF NOT EXISTS kalshi_fly_rollbacks ( + id bigserial PRIMARY KEY, ts timestamptz NOT NULL DEFAULT now(), reason text, checks jsonb, from_snapshot bigint, to_snapshot bigint)"""], + # the minute's lowest YES ask and highest YES bid: whether a resting order would have filled (kalshi/maker.py) — the + # candle archive's yes_ask_low / yes_bid_high, which the label rule reads (kalshi/mature.py) + 8: ["ALTER TABLE kalshi_minutes ADD COLUMN IF NOT EXISTS yes_ask_low real", + "ALTER TABLE kalshi_minutes ADD COLUMN IF NOT EXISTS yes_bid_high real"], + # 9: a market's lifetime volume on its corpus row, so the corpus can keep the most traded markets per day (kalshi/history.prune_pending) + 9: ["ALTER TABLE kalshi_corpus ADD COLUMN IF NOT EXISTS volume double precision"], + # 10: every settled single market Kalshi listed (the dataset's and the exchange walk's), from which the corpus samples whole + # events outcome-blind (kalshi/history.select_corpus) + 10: ["""CREATE TABLE IF NOT EXISTS kalshi_universe ( + ticker text PRIMARY KEY, event_ticker text, series_ticker text, title text, open_time timestamptz, close_time timestamptz, + settled_ts timestamptz, result text, volume double precision, source text, selected boolean NOT NULL DEFAULT false, + updated_at timestamptz NOT NULL DEFAULT now())""", + "CREATE INDEX IF NOT EXISTS kalshi_universe_close_idx ON kalshi_universe (close_time)", + "CREATE INDEX IF NOT EXISTS kalshi_universe_selected_idx ON kalshi_universe (selected) WHERE selected"], + # 11: a market's early-close condition: such a market may close before its listed close when its event happens, so its + # realized close carries the outcome; its time anchor is the scheduled expected expiration (kalshi/mature.load_meta) + 11: ["ALTER TABLE kalshi_markets ADD COLUMN IF NOT EXISTS early_close_condition text"], + # 12: the minute's taker sales at their extremes: the lowest YES price a taker sold YES at (bought NO) and the highest a + # taker bought YES at (sold NO at 100 − it): a resting bid fills when a taker sells into it (kalshi/decisions.maker_filled) + 12: ["ALTER TABLE kalshi_minutes ADD COLUMN IF NOT EXISTS yes_sold_low real", + "ALTER TABLE kalshi_minutes ADD COLUMN IF NOT EXISTS yes_bought_high real"], } @@ -278,41 +388,93 @@ def ensure_database(url: str | None = None) -> bool: return True +def ddl_fingerprint() -> str: + """sha256 of every statement apply_schema runs (the ladder, the idempotent vault/RH/ETH DDL) and SCHEMA_VERSION.""" + from ..rh.schema import RH_DDL + from ..vault.eth_schema import ETH_DDL + from ..vault.schema import VAULT_DDL + h = hashlib.sha256() + for stmt in [str(SCHEMA_VERSION), *BASE_DDL, *(x for v in sorted(MIGRATIONS) for x in MIGRATIONS[v]), *VAULT_DDL, *RH_DDL, *ETH_DDL]: + h.update(stmt.encode()); h.update(b"\0") + return h.hexdigest() + + +def _applied(conn, fp: str) -> int | None: + """The schema version when exactly this DDL is already applied (nothing to run), else None.""" + try: + row = conn.execute("SELECT version, ddl_sha256 FROM schema_version WHERE singleton").fetchone() + except (psycopg.errors.UndefinedTable, psycopg.errors.UndefinedColumn): + conn.rollback() + return None + conn.commit() + if row and row["ddl_sha256"] == fp and int(row["version"]) >= SCHEMA_VERSION: + return int(row["version"]) + return None + + +def _apply_all(conn, fp: str) -> int: + with conn.cursor() as cur: + cur.execute(f"SET LOCAL lock_timeout = '{LOCK_TIMEOUT}'") + for stmt in BASE_DDL: + cur.execute(stmt) + cur.execute("ALTER TABLE schema_version ADD COLUMN IF NOT EXISTS ddl_sha256 text") + cur.execute("SELECT version FROM schema_version WHERE singleton") + row = cur.fetchone() + current = int(row["version"]) if row else 0 + for v in sorted(MIGRATIONS): + if current < v: + for stmt in MIGRATIONS[v]: + cur.execute(stmt) + current = v + from ..vault.schema import VAULT_DDL # idempotent, outside the ladder: same DDL on master and distribution + for stmt in VAULT_DDL: + cur.execute(stmt) + from ..rh.schema import RH_DDL # idempotent, outside the ladder: the Robinhood Chain market (fly_trader/rh) + for stmt in RH_DDL: + cur.execute(stmt) + from ..vault.eth_schema import ETH_DDL # the vault's ETH pot (Robinhood Chain profits paid in ETH) + for stmt in ETH_DDL: + cur.execute(stmt) + target = max(current, SCHEMA_VERSION) + cur.execute( + "INSERT INTO schema_version (singleton, version, ddl_sha256) VALUES (true, %s, %s) " + "ON CONFLICT (singleton) DO UPDATE SET version = EXCLUDED.version, ddl_sha256 = EXCLUDED.ddl_sha256, updated_at = now()", + (target, fp), + ) + return target + + def apply_schema(url: str | None = None) -> int: - """Apply the ladder under an advisory lock. Returns the resulting version.""" + """Apply the ladder under an advisory lock. Returns the resulting version. A no-op (no table locks) when this exact + DDL was applied before; otherwise every lock wait is bounded by LOCK_TIMEOUT and the whole run is retried.""" + fp = ddl_fingerprint() with connect(url) as conn: - with conn.cursor() as cur: - cur.execute("SELECT pg_advisory_lock(%s)", (_LOCK_KEY,)) - try: - for stmt in BASE_DDL: - cur.execute(stmt) - cur.execute("SELECT version FROM schema_version WHERE singleton") - row = cur.fetchone() - current = int(row["version"]) if row else 0 - for v in sorted(MIGRATIONS): - if current < v: - for stmt in MIGRATIONS[v]: - cur.execute(stmt) - current = v - from ..vault.schema import VAULT_DDL # idempotent, outside the ladder: same DDL on master and distribution - for stmt in VAULT_DDL: - cur.execute(stmt) - target = max(current, SCHEMA_VERSION) - cur.execute( - "INSERT INTO schema_version (singleton, version) VALUES (true, %s) " - "ON CONFLICT (singleton) DO UPDATE SET version = EXCLUDED.version, updated_at = now()", - (target,), - ) - conn.commit() - ensure_partitions(conn) - conn.commit() - return target - except Exception: - conn.rollback() - raise - finally: - cur.execute("SELECT pg_advisory_unlock(%s)", (_LOCK_KEY,)) - conn.commit() + conn.execute("SELECT pg_advisory_lock(%s)", (_LOCK_KEY,)) + conn.commit() # the advisory lock is the session's; no transaction stays open + try: + target = _applied(conn, fp) + if target is None: + for attempt in range(APPLY_TRIES): + try: + target = _apply_all(conn, fp) + conn.commit() + break + except (psycopg.errors.LockNotAvailable, psycopg.errors.DeadlockDetected) as e: + conn.rollback() # releases every table lock this run took + if attempt == APPLY_TRIES - 1: + raise + wait = min(30.0, 1.0 + attempt) + log.warning("schema: %s while applying the DDL (live workers hold the tables); retrying in %.0f s", type(e).__name__, wait) + time.sleep(wait) + ensure_partitions(conn) + conn.commit() + return target + except Exception: + conn.rollback() + raise + finally: + conn.execute("SELECT pg_advisory_unlock(%s)", (_LOCK_KEY,)) + conn.commit() def _partition_name(base: str, start: datetime, granularity: str) -> str: diff --git a/fly_trader/execution/broker_live.py b/fly_trader/execution/broker_live.py index c2e3121ec2b4e658150b2df03081c9f54210b787..b83fbe41f6e25ff2da45d849e40e54b0e7e1353a 100644 --- a/fly_trader/execution/broker_live.py +++ b/fly_trader/execution/broker_live.py @@ -23,7 +23,6 @@ Every write is committed immediately so a crash mid-poll leaves an auditable row """ from __future__ import annotations -import base64 import logging import re import threading @@ -31,20 +30,16 @@ import time from dataclasses import dataclass from psycopg.types.json import Jsonb -from solders.hash import Hash -from solders.instruction import AccountMeta, Instruction -from solders.message import MessageV0 -from solders.pubkey import Pubkey -from solders.transaction import VersionedTransaction - from .. import config from ..chain.balances import tx_deltas, Snapshot, compute_delta, snapshot_balances from ..chain.cluster_guard import assert_signing_allowed from ..chain.jupiter_swap import JupiterError, JupiterSwap from ..db.connection import transaction -from ..chain.keys import load_keypair from ..chain.rpc import TOKEN_2022_PROGRAM_ID, TOKEN_PROGRAM_ID, HttpSolanaRpc -from ..chain.signing import NotASigner, n_lookup_programs, sign_transaction_b64, static_program_ids, transaction_id +from ..chain.signing import n_lookup_programs, static_program_ids +from ..signer import client as signer_client +from ..signer.client import SignerUnavailable +from ..signer.policy import EXCLUDE_ROUTERS, PolicyError from ..db.apilog import record_event from ..db.connection import connect, transaction from ..logging_setup import scrub @@ -110,10 +105,14 @@ def await_confirmation(rpc, signature: str | None, last_valid_block_height: int class LiveBroker: - def __init__(self, rpc=None, jup=None, keypair=None): + """Swaps through Jupiter; the trading key never enters this process: ``signer`` (fly_trader/signer, the signer + container's socket on the vault server) checks and signs each transaction.""" + + def __init__(self, rpc=None, jup=None, signer=None): self._rpc = rpc self._jup = jup - self._keypair = keypair + self._signer = signer + self._pubkey: str | None = None self._labels: dict[str, str] | None = None @property @@ -129,14 +128,16 @@ class LiveBroker: return self._jup @property - def keypair(self): - if self._keypair is None: - self._keypair = load_keypair() - return self._keypair + def signer(self): + if self._signer is None: + self._signer = signer_client.get() + return self._signer @property def pubkey(self) -> str: - return str(self.keypair.pubkey()) + if self._pubkey is None: + self._pubkey = self.signer.call("pubkeys")["trading"] + return self._pubkey @property def labels(self) -> dict[str, str]: @@ -185,7 +186,7 @@ class LiveBroker: taker = self.pubkey pre = snapshot_balances(self.rpc, taker) request = {"inputMint": input_mint, "outputMint": output_mint, "amount": str(amount_in), "taker": taker, - "slippageBps": slippage_bps, "excludeRouters": "jupiterz"} + "slippageBps": slippage_bps, "excludeRouters": EXCLUDE_ROUTERS} base = dict(decision_id=decision_id, book=book, attempt=attempt, side=side, input_mint=input_mint, output_mint=output_mint, amount_in=amount_in, slippage_bps=slippage_bps, request=request) @@ -200,7 +201,7 @@ class LiveBroker: # 1. order try: - order = self.jup.order(input_mint, output_mint, amount_in, taker=taker, slippage_bps=slippage_bps) + order = self.jup.order(input_mint, output_mint, amount_in, taker=taker, slippage_bps=slippage_bps, exclude_routers=EXCLUDE_ROUTERS) except JupiterError as e: return fail("order_error", scrub(str(e))) err_code = _int_or_none(order.get("errorCode")) @@ -223,14 +224,18 @@ class LiveBroker: conn, **base, order_response=order, request_id=order.get("requestId"), router=order.get("router"), fee_bps=_int_or_none(order.get("feeBps")), program_ids=program_ids, lvbh=lvbh, status="ordered") - # 2. sign (our slot only) + # 2. sign: the signer checks the transaction (programs, roles, a simulation of our balances) and signs our slot try: - signed_b64, _idx = sign_transaction_b64(tx_b64, self.keypair) - except (NotASigner, ValueError) as e: - return fail("sign_error", f"{type(e).__name__}: {scrub(str(e))}", None, order_id=order_id) - if _idx != 0 and not order.get("gasless"): - return fail("sign_error", f"our key is signer {_idx}, but a non-gasless order must have the taker as fee payer", None, order_id=order_id) - tx_sig = transaction_id(signed_b64) + signed = self.signer.call("sign_swap", tx=tx_b64, in_mint=input_mint, out_mint=output_mint, in_amount=amount_in, + min_out=max(1, int(_int_or_none(order.get("otherAmountThreshold")) or 1))) + except PolicyError as e: + r = fail("sign_refused", f"{e.code}: {scrub(str(e))}", None, order_id=order_id) + if e.code == "policy": + record_event("error", "signer", f"swap refused: {e}", {"order_id": order_id, "side": side, "mint": mint}) + return r[0], e.code == "in_flight" # a simulation that failed (slippage, stale route): retry with a new order + except SignerUnavailable as e: + return fail("sign_error", scrub(str(e)), None, order_id=order_id) + signed_b64, tx_sig = signed["tx"], signed["signature"] # 3. execute exec_req = {"requestId": order.get("requestId"), "lastValidBlockHeight": lvbh, "signedTransaction": signed_b64} @@ -419,14 +424,13 @@ def swap_smoke(sol: float) -> None: print(f"net lamports delta: {net} ({net / config.LAMPORTS_PER_SOL:.9f} SOL)") -def close_empty_atas(rpc=None, keypair=None, url: str | None = None, batch: int = CLOSE_ATA_BATCH, skip_mints: set | None = None) -> dict: - """Close every zero-balance token account owned by the bot (rent back to the wallet). - SPL Token / Token-2022 CloseAccount = instruction index 9, accounts [account, destination, owner]. - One v0 transaction per ``batch`` accounts; each is recorded as wallet_events(kind='ata_closed').""" +def close_empty_atas(rpc=None, signer=None, url: str | None = None, batch: int = CLOSE_ATA_BATCH, skip_mints: set | None = None) -> dict: + """Close every zero-balance token account owned by the bot (rent back to the wallet). The signer builds and signs + the CloseAccount transactions (it re-checks that each account is ours and empty); each is recorded as + wallet_events(kind='ata_closed').""" assert_signing_allowed() - kp = keypair or load_keypair() - owner = kp.pubkey() - owner_s = str(owner) + signer = signer or signer_client.get() + owner_s = signer.call("pubkeys")["trading"] rpc = rpc or HttpSolanaRpc() skip = set(skip_mints or ()) empties = [a for a in rpc.get_token_accounts_by_owner(owner_s) if int(a["amount"]) == 0 and a["mint"] not in skip and not a.get("is_native")] @@ -436,16 +440,17 @@ def close_empty_atas(rpc=None, keypair=None, url: str | None = None, batch: int for i in range(0, len(empties), batch): chunk = empties[i:i + batch] accounts = [a["address"] for a in chunk] - ixs = [Instruction(Pubkey.from_string(a["program"]), bytes([9]), - [AccountMeta(Pubkey.from_string(a["address"]), False, True), - AccountMeta(owner, False, True), - AccountMeta(owner, True, False)]) for a in chunk] - bh = rpc.get_latest_blockhash() - msg = MessageV0.try_compile(owner, ixs, [], Hash.from_string(bh["blockhash"])) - tx = VersionedTransaction(msg, [kp]) - tx_b64 = base64.b64encode(bytes(tx)).decode() try: - sig = rpc.send_transaction(tx_b64) + built = signer.call("close_atas", accounts=accounts) + except (PolicyError, SignerUnavailable) as e: + result["failed"].append({"accounts": accounts, "error": str(e)}) + record_event("error", "close_empty_atas", "the signer refused", {"accounts": accounts, "error": str(e)}) + continue + if not built: + continue + bh = {"last_valid_block_height": built[0]["last_valid_block_height"]} + try: + sig = rpc.send_transaction(built[0]["tx"]) except Exception as e: err = f"{type(e).__name__}: {scrub(str(e))}" result["failed"].append({"accounts": accounts, "error": err}) diff --git a/fly_trader/execution/broker_paper.py b/fly_trader/execution/broker_paper.py index a3c73969c9e4fe8f782e1fcd8c5a93a924733a6c..789341439ecbbeae58b962cb7582c063b75fdc8f 100644 --- a/fly_trader/execution/broker_paper.py +++ b/fly_trader/execution/broker_paper.py @@ -32,7 +32,9 @@ class PaperFill: class PaperBroker: def __init__(self, book: str): - assert book in ("paper_selector", "paper_fly") or book.startswith("replay_"), book # the race books; replay_* for tests + from ..markets import MARKETS + paper = {b for m in MARKETS.values() for b in m.paper_books} + assert book in paper or book.startswith("replay_"), book # the race books of every market; replay_* for tests self.book = book def buy(self, conn, *, decision_id: int | None, mint: str, pool: str | None, size_sol: float, price: float | None, @@ -45,7 +47,7 @@ class PaperBroker: return PaperFill(False, None, None, 0.0, 0.0, 0, 0.0, "no price") if res_quote_sol is None or res_quote_sol <= 0: return PaperFill(False, None, None, price, 0.0, 0, 0.0, "no liquidity estimate") - fee_frac = fee_fraction(size_sol, mcap_sol, pool_fee) + fee_frac = fee_fraction(size_sol, mcap_sol, pool_fee, program_label) impact = impact_fraction(size_sol, res_quote_sol, program_label) if impact > config.EXECUTABILITY_MAX_IMPACT: # a router would not fill this either; refuse like the live executability check @@ -74,7 +76,7 @@ class PaperBroker: gross = qty_raw / (10 ** decimals) * price c = exit_cost_fraction(gross, res_quote_sol, mcap_sol, program_label, pool_fee) proceeds = gross * (1.0 - c) - fee_sol = gross * fee_fraction(gross, mcap_sol, pool_fee) + fee_sol = gross * fee_fraction(gross, mcap_sol, pool_fee, program_label) if fraction < 0.999: # partial: shrink the position, realise the sold part realized = ledger.realize_partial(conn, position_id=int(position["id"]), qty_raw=qty_raw, cost_part=float(position["cost_sol"]) * fraction, proceeds_sol=proceeds, diff --git a/fly_trader/execution/ledger.py b/fly_trader/execution/ledger.py index 8d4a5b15f30491a977aca6ad0a7409bf0f33cb68..44858727fc5e6e9c58dc7219811f45316c742571 100644 --- a/fly_trader/execution/ledger.py +++ b/fly_trader/execution/ledger.py @@ -20,7 +20,8 @@ log = logging.getLogger(__name__) def open_positions(conn, book: str) -> list[dict]: rows = conn.execute( - """SELECT p.*, t.decimals, t.graduated_at, wp.program_label, + """SELECT p.*, COALESCE(t.decimals, CASE WHEN p.chain = 'rh' THEN 18 END) AS decimals, t.graduated_at, + COALESCE(wp.program_label, CASE WHEN p.chain = 'rh' THEN 'Pons v4' END) AS program_label, COALESCE(p.hold_s, (d.detail->>'hold_s')::float) AS hold_s, COALESCE(p.strategy, d.detail->>'strategy') AS strategy FROM positions p LEFT JOIN tokens t ON t.mint = p.mint LEFT JOIN watch_pools wp ON wp.pool = p.pool @@ -38,7 +39,8 @@ def paper_cash(conn, book: str) -> float: FROM positions WHERE book = %s""", (book,), ).fetchone() - return config.CAPITAL_SOL + float(r["realized"]) - float(r["open_cost"]) + from ..markets import for_book + return for_book(book).capital() + float(r["realized"]) - float(r["open_cost"]) # the book's own market's capital (SOL / ETH) def record_fill(conn, *, order_id: int | None, book: str, mint: str, side: str, token_delta: int, @@ -55,6 +57,11 @@ def record_fill(conn, *, order_id: int | None, book: str, mint: str, side: str, return int(row["id"]) +def _chain_of(book: str) -> str: + from ..markets import for_book + return for_book(book).chain + + def open_position(conn, *, book: str, mint: str, pool: str | None, qty_raw: int, cost_sol: float, entry_price: float, decision_id: int | None, fees_sol: float, ts: datetime | None = None, hold_s: float | None = None, strategy: str | None = None) -> int: @@ -70,9 +77,9 @@ def open_position(conn, *, book: str, mint: str, pool: str | None, qty_raw: int, return int(existing["id"]) row = conn.execute( """INSERT INTO positions (book, mint, pool, opened_at, entry_decision_id, qty, cost_sol, entry_price, peak_price, - last_mark_price, last_mark_ts, fees_sol, status, last_swap_ts, hold_s, strategy) - VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,'open',%s,%s,%s) RETURNING id""", - (book, mint, pool, ts, decision_id, qty_raw, cost_sol, entry_price, entry_price, entry_price, ts, fees_sol, ts, hold_s, strategy), + last_mark_price, last_mark_ts, fees_sol, status, last_swap_ts, hold_s, strategy, chain) + VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,'open',%s,%s,%s,%s) RETURNING id""", + (book, mint, pool, ts, decision_id, qty_raw, cost_sol, entry_price, entry_price, entry_price, ts, fees_sol, ts, hold_s, strategy, _chain_of(book)), ).fetchone() return int(row["id"]) @@ -127,11 +134,11 @@ def mark_positions(conn, book: str, prices: dict[str, float], last_swaps: dict[s def verify_fills() -> None: """Reconcile live positions against on-chain token balances; prints mismatches, writes wallet_events.""" - from ..chain import keys from ..chain.rpc import HttpSolanaRpc from ..db.connection import transaction + from ..signer import client rpc = HttpSolanaRpc() - pub = keys.bot_pubkey() + pub = client.get().call("pubkeys")["trading"] accounts = rpc.get_token_accounts_by_owner(pub) onchain = {a["mint"]: int(a["amount"]) for a in accounts} with transaction() as conn: diff --git a/fly_trader/kalshi/__init__.py b/fly_trader/kalshi/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2da0574a53da7b4b64a14b926f5e12060aec6041 --- /dev/null +++ b/fly_trader/kalshi/__init__.py @@ -0,0 +1 @@ +"""Kalshi prediction markets: the fly's second trading type, on the optic lobes (see the plan of 2026-09-22).""" diff --git a/fly_trader/kalshi/client.py b/fly_trader/kalshi/client.py new file mode 100644 index 0000000000000000000000000000000000000000..83a1710c64cd1f347dda886df54237f3413a05d0 --- /dev/null +++ b/fly_trader/kalshi/client.py @@ -0,0 +1,342 @@ +"""fly-trader's Kalshi REST client: one persistent, authenticated SDK client on a private event-loop thread, paced under +the account's rate tier, every call logged to ``api_calls`` (service 'kalshi'; the key never appears). + +The vendored better_bot client (``kalshi/vendor/kalshi_client.py``) opens and closes an SDK client per call, which is +right for a daily batch and wrong for a history pull of tens of thousands of candlestick requests or a trading loop. +This wrapper keeps the same authentication (``Configuration.api_key_id`` / ``private_key_pem``, RSA-PSS by the SDK) and +the same ``call_api`` transport, adds the endpoints better_bot never needed — subaccounts, resting (GTC, post-only) +orders, cancel/amend, the historical tier, series and events — and passes ``KALSHI_SUBACCOUNT`` on every portfolio and +order call. Prices cross this boundary in cents (ints) and contracts as floats; the exchange's fixed-point dollar +strings are converted here and nowhere else. Paths and body fields verified against kalshi-python-async 3.10 on +2026-09-22 (``docs.kalshi.com``): V2 orders are ``POST /portfolio/orders`` with ``yes_price``/``no_price`` in cents. +""" +from __future__ import annotations + +import asyncio +import json +import logging +import threading +import time +import uuid +from urllib.parse import urlencode + +from .. import config +from ..chain.jupiter_tokens import TokenBucket +from ..db.apilog import record_api_call +from ..logging_setup import scrub + +log = logging.getLogger(__name__) +RETRIES_429 = 5 + + +class KalshiApiError(RuntimeError): + def __init__(self, method: str, path: str, status: int | None, body): + self.method, self.path, self.status, self.body = method, path, status, body + err = (body or {}).get("error") if isinstance(body, dict) else None + msg = (err.get("message") if isinstance(err, dict) else None) or (body.get("message") if isinstance(body, dict) else None) or str(body)[:200] + super().__init__(f"kalshi {method} {path}: HTTP {status}: {scrub(str(msg))}") + + +def _f(x) -> float | None: + try: + return float(x) if x not in (None, "") else None + except (TypeError, ValueError): + return None + + +def cents(dollars) -> int | None: + """A fixed-point dollar string ('0.5600') as whole cents; None when absent.""" + v = _f(dollars) + return int(round(v * 100)) if v is not None else None + + +def _summary(params: dict | None, body: dict | None) -> dict | None: + out = {} + if params: + out["params"] = {k: v for k, v in params.items()} + if body: + out["body"] = {k: v for k, v in body.items() if k not in ("signedTransaction",)} + return out or None + + +class KalshiRest: + """Synchronous facade over the async SDK client (one per process is enough: MPS and the minute loop are single-threaded).""" + + def __init__(self, rps: float | None = None, subaccount: int | None = None, base_url: str | None = None): + self.base = (base_url or config.KALSHI_API_BASE_URL).rstrip("/") + self.subaccount = config.KALSHI_SUBACCOUNT if subaccount is None else int(subaccount) + self.bucket = TokenBucket(rps or config.KALSHI_RPS) + self._loop: asyncio.AbstractEventLoop | None = None + self._thread: threading.Thread | None = None + self._client = None + self._lock = threading.Lock() + + # ---- lifecycle ---- + def _ensure(self): + with self._lock: + if self._client is not None: + return self._client + if not config.KALSHI_API_KEY_ID or not config.KALSHI_PRIVATE_KEY_PATH: + raise RuntimeError("KALSHI_API_KEY_ID and KALSHI_PRIVATE_KEY_PATH are not set") + from kalshi_python_async import Configuration, KalshiClient + cfg = Configuration(); cfg.host = self.base + cfg.api_key_id = config.KALSHI_API_KEY_ID + with open(config.KALSHI_PRIVATE_KEY_PATH) as f: + cfg.private_key_pem = f.read() + try: + import certifi + cfg.ssl_ca_cert = certifi.where() + except ImportError: + pass + self._loop = asyncio.new_event_loop() + self._thread = threading.Thread(target=self._loop.run_forever, name=f"{threading.current_thread().name}-kalshi-io", daemon=True) + self._thread.start() + self._client = asyncio.run_coroutine_threadsafe(self._make(cfg, KalshiClient), self._loop).result(30) + return self._client + + @staticmethod + async def _make(cfg, KalshiClient): + return KalshiClient(configuration=cfg) + + def close(self) -> None: + with self._lock: + if self._client is not None and self._loop is not None: + try: + asyncio.run_coroutine_threadsafe(self._client.close(), self._loop).result(10) + except Exception: + pass + self._loop.call_soon_threadsafe(self._loop.stop) + self._client = None + + # ---- transport ---- + async def _call(self, method: str, url: str, body: dict | None): + raw = await self._client.call_api(method, url, body=body, header_params={"Content-Type": "application/json"} if body is not None else None) + data = await raw.read() + text = data.decode("utf-8", "replace") if isinstance(data, (bytes, bytearray)) else str(data) + start = text.find("{") + try: + parsed = json.loads(text[start:]) if start >= 0 else {} + except ValueError: + parsed = {"raw": text[:300]} + return int(raw.status), parsed + + def request(self, method: str, path: str, params: dict | None = None, body: dict | None = None, timeout: float = 60.0) -> dict: + """One paced, logged call. Raises ``KalshiApiError`` on HTTP >= 400 (429 is retried with backoff first).""" + client = self._ensure() + q = {k: v for k, v in (params or {}).items() if v is not None} + url = self.base + path + (("?" + urlencode(q)) if q else "") + t0 = time.monotonic(); status = None; err = None; parsed = None + try: + for attempt in range(RETRIES_429 + 1): + self.bucket.acquire() + status, parsed = asyncio.run_coroutine_threadsafe(self._call(method, url, body), self._loop).result(timeout) + if status == 429 and attempt < RETRIES_429: + time.sleep(1.0 + attempt) + continue + break + if status >= 400: + raise KalshiApiError(method, path, status, parsed) + return parsed + except KalshiApiError as e: + err = str(e); raise + except Exception as e: + err = f"{type(e).__name__}: {scrub(str(e))}"; raise + finally: + record_api_call("kalshi", path, method, status, int((time.monotonic() - t0) * 1000), err is None, err, + request=_summary(q, body), response_bytes=len(json.dumps(parsed)) if parsed is not None else None) + + def get(self, path: str, **params) -> dict: + return self.request("GET", path, params) + + def pages(self, path: str, key: str, params: dict | None = None, limit: int = 1000, max_pages: int = 10_000, cursor: str | None = None, with_cursor: bool = False): + """Cursor walk: yields each page's ``key`` list until the cursor ends or a page is empty. ``cursor`` resumes a walk + from a page cursor seen before; ``with_cursor`` yields ``(items, next_cursor)`` so the caller can persist its place.""" + for _ in range(max_pages): + data = self.request("GET", path, {**(params or {}), "limit": limit, "cursor": cursor}) + items = data.get(key) or []; nxt = data.get("cursor") or None + if items: + yield (items, nxt) if with_cursor else items + cursor = nxt + if not cursor or not items: + break + + # ---- exchange / catalogue ---- + def exchange_status(self) -> dict: + return self.get("/exchange/status") + + def markets(self, cursor: str | None = None, with_cursor: bool = False, **params): + """Pages of market rows (``status`` one of unopened|open|closed|settled|...; ``min_settled_ts`` etc.).""" + yield from self.pages("/markets", "markets", params, cursor=cursor, with_cursor=with_cursor) + + def historical_markets(self, cursor: str | None = None, with_cursor: bool = False, **params): + yield from self.pages("/historical/markets", "markets", params, cursor=cursor, with_cursor=with_cursor) + + def historical_cutoff(self) -> dict: + return self.get("/historical/cutoff") + + def market(self, ticker: str) -> dict: + return (self.get(f"/markets/{ticker}") or {}).get("market") or {} + + def historical_market(self, ticker: str) -> dict: + return (self.get(f"/historical/markets/{ticker}") or {}).get("market") or {} + + def events(self, **params): + """Pages of events (``with_nested_markets=true`` nests their markets); limit is capped at 200 by the exchange.""" + yield from self.pages("/events", "events", params, limit=200) + + def event(self, event_ticker: str, with_nested_markets: bool = True) -> dict: + return self.get(f"/events/{event_ticker}", with_nested_markets=str(with_nested_markets).lower()) + + def series_list(self, category: str | None = None, **params) -> list[dict]: + return (self.get("/series", category=category, **params) or {}).get("series") or [] + + def series(self, series_ticker: str) -> dict: + return (self.get(f"/series/{series_ticker}") or {}).get("series") or {} + + def orderbook(self, ticker: str, depth: int | None = None) -> dict: + return self.get(f"/markets/{ticker}/orderbook", depth=depth) + + def candlesticks(self, series_ticker: str, ticker: str, start_ts: int, end_ts: int, period_interval: int = 1) -> list[dict]: + """1/60/1440-minute candles; a market archived past the historical cutoff answers on the historical path.""" + params = {"start_ts": int(start_ts), "end_ts": int(end_ts), "period_interval": int(period_interval)} + try: + return (self.request("GET", f"/series/{series_ticker}/markets/{ticker}/candlesticks", params) or {}).get("candlesticks") or [] + except KalshiApiError as e: + if e.status != 404: + raise + cs = (self.request("GET", f"/historical/markets/{ticker}/candlesticks", params) or {}).get("candlesticks") or [] + return [{**c, "_units": "dollars"} for c in cs] # the archive sends prices as dollar strings under plain keys ("close": "0.0100") + + def trades(self, ticker: str, min_ts: int | None = None, max_ts: int | None = None, historical: bool = False): + yield from self.pages("/historical/trades" if historical else "/markets/trades", "trades", {"ticker": ticker, "min_ts": min_ts, "max_ts": max_ts}) + + # ---- portfolio (always on this process's subaccount unless told otherwise) ---- + def _sub(self, subaccount) -> int: + return self.subaccount if subaccount is None else int(subaccount) + + def balance(self, subaccount: int | None = None) -> dict: + """``{"balance_cents", "portfolio_value_cents"}`` of one subaccount (0 = primary).""" + d = self.get("/portfolio/balance", subaccount=self._sub(subaccount)) + return {"balance_cents": int(d.get("balance") or 0), "portfolio_value_cents": int(d.get("portfolio_value") or 0), "updated_ts": d.get("updated_ts")} + + def subaccount_balances(self) -> list[dict]: + return (self.get("/portfolio/subaccounts/balances") or {}).get("subaccount_balances") or [] + + def create_subaccount(self) -> int: + d = self.request("POST", "/portfolio/subaccounts", body={}) + return int(d["subaccount_number"]) + + def transfer_between_subaccounts(self, from_subaccount: int, to_subaccount: int, amount_cents: int) -> dict: + body = {"client_transfer_id": str(uuid.uuid4()), "from_subaccount": int(from_subaccount), "to_subaccount": int(to_subaccount), "amount_cents": int(amount_cents)} + return self.request("POST", "/portfolio/subaccounts/transfer", body=body) + + def positions(self, subaccount: int | None = None, **params) -> list[dict]: + out = [] + for page in self.pages("/portfolio/positions", "market_positions", {**params, "subaccount": self._sub(subaccount)}, limit=1000): + out.extend(page) + return out + + def orders(self, status: str | None = None, subaccount: int | None = None, **params) -> list[dict]: + out = [] + for page in self.pages("/portfolio/orders", "orders", {**params, "status": status, "subaccount": self._sub(subaccount)}, limit=1000): + out.extend(page) + return out + + def order(self, order_id: str) -> dict: + return (self.get(f"/portfolio/orders/{order_id}") or {}).get("order") or {} + + def fills(self, subaccount: int | None = None, **params) -> list[dict]: + out = [] + for page in self.pages("/portfolio/fills", "fills", {**params, "subaccount": self._sub(subaccount)}, limit=1000): + out.extend(page) + return out + + def settlements(self, subaccount: int | None = None, **params) -> list[dict]: + out = [] + for page in self.pages("/portfolio/settlements", "settlements", {**params, "subaccount": self._sub(subaccount)}, limit=1000): + out.extend(page) + return out + + # ---- orders ---- + def create_order(self, ticker: str, side: str, price_cents: int, count: float, *, tif: str = "immediate_or_cancel", post_only: bool = False, + expiration_ts: int | None = None, client_order_id: str | None = None, subaccount: int | None = None, action: str = "buy") -> dict: + """Buy (or sell) ``count`` contracts of ``side`` ('yes'|'no') at ``price_cents`` each. ``tif``: immediate_or_cancel | + good_till_canceled | fill_or_kill; a resting order is GTC (+ ``post_only`` never crosses, ``expiration_ts`` ends it). + Returns the exchange's order object (``order_id``, ``status``, ``fill_count``/``remaining_count`` ...).""" + if side not in ("yes", "no"): + raise ValueError(f"side must be yes|no, got {side!r}") + if not 1 <= int(price_cents) <= 99: + raise ValueError(f"price {price_cents} outside 1..99 cents") + body = {"ticker": ticker, "client_order_id": client_order_id or f"fly-{uuid.uuid4().hex[:16]}", "side": side, "action": action, + "count_fp": f"{float(count):.2f}", ("yes_price" if side == "yes" else "no_price"): int(price_cents), "time_in_force": tif, + "post_only": bool(post_only), "self_trade_prevention_type": "maker" if post_only else "taker_at_cross", + "subaccount": self._sub(subaccount)} + if expiration_ts is not None: + body["expiration_ts"] = int(expiration_ts) + d = self.request("POST", "/portfolio/orders", body=body) + return d.get("order") or d + + def cancel_order(self, order_id: str, subaccount: int | None = None) -> dict: + d = self.request("DELETE", f"/portfolio/orders/{order_id}", {"subaccount": self._sub(subaccount)}) + return d.get("order") or d + + def amend_order(self, order_id: str, ticker: str, side: str, price_cents: int, count: float, client_order_id: str, subaccount: int | None = None, + action: str = "buy") -> dict: + body = {"ticker": ticker, "side": side, "action": action, "client_order_id": client_order_id, "updated_client_order_id": f"fly-{uuid.uuid4().hex[:16]}", + ("yes_price" if side == "yes" else "no_price"): int(price_cents), "count_fp": f"{float(count):.2f}", "subaccount": self._sub(subaccount)} + d = self.request("POST", f"/portfolio/orders/{order_id}/amend", body=body) + return d.get("order") or d + + +_REST: KalshiRest | None = None +_REST_LOCK = threading.Lock() + + +def rest() -> KalshiRest: + """The process-wide client (built on first use).""" + global _REST + with _REST_LOCK: + if _REST is None: + _REST = KalshiRest() + return _REST + + +# ---- operator commands (fly-trader kalshi-subaccount / kalshi-fund / kalshi-status) ---- +def subaccount_create(write_env: bool = True) -> int: + """Create the dedicated subaccount and record it in .env (``KALSHI_SUBACCOUNT``); refuses when one is already set.""" + import re + from ..db.apilog import record_event + if config.KALSHI_SUBACCOUNT > 0: + raise RuntimeError(f"KALSHI_SUBACCOUNT={config.KALSHI_SUBACCOUNT} is already set; refusing to create another") + n = KalshiRest(subaccount=0).create_subaccount() + if write_env: + path = config.REPO_ROOT / ".env"; text = path.read_text() if path.exists() else "" + text = re.sub(r"^KALSHI_SUBACCOUNT=.*$", f"KALSHI_SUBACCOUNT={n}", text, flags=re.M) if re.search(r"^KALSHI_SUBACCOUNT=", text, re.M) else text.rstrip("\n") + f"\nKALSHI_SUBACCOUNT={n}\n" + path.write_text(text) + record_event("info", "kalshi", f"subaccount {n} created", {"subaccount": n}) + return n + + +def fund(usd: float, to_subaccount: int | None = None) -> dict: + """Move ``usd`` from the primary account into the fly's subaccount.""" + from ..db.apilog import record_event + to = config.KALSHI_SUBACCOUNT if to_subaccount is None else int(to_subaccount) + if to <= 0: + raise RuntimeError("no subaccount: run fly-trader kalshi-subaccount create first") + out = KalshiRest(subaccount=0).transfer_between_subaccounts(0, to, int(round(usd * 100))) + record_event("info", "kalshi", f"funded subaccount {to} with ${usd:.2f}", {"transfer": out}) + return out + + +def status() -> dict: + r = rest() + out = {"exchange": r.exchange_status(), "subaccount": r.subaccount, "balances": r.subaccount_balances(), + "prerequisites_missing": config.kalshi_live_prerequisites_missing()} + try: + out["balance"] = r.balance() + out["resting_orders"] = r.orders(status="resting") + out["positions"] = [p for p in r.positions() if _f(p.get("position_fp") or p.get("position")) not in (None, 0.0)] + out["settlements_24h"] = r.settlements(min_ts=int(time.time()) - 86400) + except KalshiApiError as e: + out["error"] = str(e) + return out diff --git a/fly_trader/kalshi/data.py b/fly_trader/kalshi/data.py new file mode 100644 index 0000000000000000000000000000000000000000..a3130871aac6aadbdf6fdc3dd6d61f8c31ca65fe --- /dev/null +++ b/fly_trader/kalshi/data.py @@ -0,0 +1,164 @@ +"""The Kalshi corpus on disk and the catalogue rows both the history puller and the live stream write. + +Layout (``config.KALSHI_DIR``): +- ``candles/.parquet`` — 1-minute candles (``CANDLE_SCHEMA``: period end, YES bid/ask OHLC in cents, trade price + OHLC when traded, contracts traded in the period, open interest at its end); +- ``trades/.parquet`` — public trades (``TRADE_SCHEMA``: time, YES price in cents, contracts, the taker's side); +- ``features//part.parquet`` — the daily feature rows (kalshi/mature.py). +Rows of ``kalshi_markets`` / ``kalshi_events`` / ``kalshi_series`` are upserted from the exchange's JSON by +``upsert_market`` and friends; prices are kept in cents, contracts as floats, times as UTC timestamps. +""" +from __future__ import annotations + +import json +from datetime import datetime, timezone +from pathlib import Path + +import pyarrow as pa + +from .. import config + +CANDLES_DIR = config.KALSHI_DIR / "candles" +TRADES_DIR = config.KALSHI_DIR / "trades" +FEATURES_DIR = config.KALSHI_DIR / "features" +CANDLE_SCHEMA = pa.schema([("end_ts", pa.int64()), ("yes_bid_open", pa.float32()), ("yes_bid_high", pa.float32()), ("yes_bid_low", pa.float32()), + ("yes_bid_close", pa.float32()), ("yes_ask_open", pa.float32()), ("yes_ask_high", pa.float32()), ("yes_ask_low", pa.float32()), + ("yes_ask_close", pa.float32()), ("price_open", pa.float32()), ("price_high", pa.float32()), ("price_low", pa.float32()), + ("price_close", pa.float32()), ("price_mean", pa.float32()), ("volume", pa.float64()), ("open_interest", pa.float64())]) +TRADE_SCHEMA = pa.schema([("ts", pa.timestamp("ms", tz="UTC")), ("yes_price", pa.int16()), ("count", pa.float64()), ("taker_side", pa.string()), ("is_block", pa.bool_())]) +# discovery categories the exchange uses, folded into the fixed set the features one-hot (kalshi/features.CATEGORIES) +CATEGORY_KEYS = (("politic", "politics"), ("election", "politics"), ("sport", "sports"), ("crypto", "crypto"), ("bitcoin", "crypto"), ("econom", "economics"), + ("financ", "economics"), ("weather", "weather"), ("climate", "weather"), ("entertain", "entertainment"), ("culture", "entertainment"), + ("science", "science"), ("tech", "science"), ("compan", "companies"), ("world", "world"), ("health", "health")) + + +def category_key(raw: str | None) -> str: + s = raw.lower() if isinstance(raw, str) else "" # None or NaN (a market whose series is not catalogued yet): 'other' + for needle, key in CATEGORY_KEYS: + if needle in s: + return key + return "other" + + +def ts(v) -> datetime | None: + """An ISO-8601 string (or epoch seconds) from the exchange as an aware UTC datetime.""" + if v in (None, ""): + return None + if isinstance(v, (int, float)): + return datetime.fromtimestamp(float(v), timezone.utc) + s = str(v).replace("Z", "+00:00") + try: + d = datetime.fromisoformat(s) + except ValueError: + return None + return d if d.tzinfo else d.replace(tzinfo=timezone.utc) + + +def _f(v) -> float | None: + try: + return float(v) if v not in (None, "") else None + except (TypeError, ValueError): + return None + + +def price_cents(row: dict, key: str) -> float | None: + """``_dollars`` (fixed point) or ```` (cents) of a market/ticker row, as cents.""" + d = _f(row.get(f"{key}_dollars")) + if d is not None: + return d * 100.0 + return _f(row.get(key)) + + +def series_ticker_of(ticker: str | None) -> str | None: + return str(ticker).split("-")[0] if ticker else None + + +MARKET_COLS = ("ticker", "event_ticker", "market_type", "title", "yes_sub_title", "no_sub_title", "status", "open_time", "close_time", + "expected_expiration_time", "latest_expiration_time", "settlement_ts", "result", "settlement_value", "strike_type", "floor_strike", + "cap_strike", "exchange_index", "price_level_structure", "is_provisional", "can_close_early", "early_close_condition", "created_time", "source", "raw") + + +def market_row(m: dict, source: str) -> dict: + return {"ticker": m.get("ticker"), "event_ticker": m.get("event_ticker"), "market_type": m.get("market_type"), "title": m.get("title"), + "yes_sub_title": m.get("yes_sub_title"), "no_sub_title": m.get("no_sub_title"), "status": m.get("status"), "open_time": ts(m.get("open_time")), + "close_time": ts(m.get("close_time")), "expected_expiration_time": ts(m.get("expected_expiration_time")), + "latest_expiration_time": ts(m.get("latest_expiration_time")), "settlement_ts": ts(m.get("settlement_ts") or m.get("settled_time")), + "result": m.get("result") or None, "settlement_value": _f(m.get("settlement_value_dollars")) if m.get("settlement_value_dollars") is not None else _f(m.get("settlement_value")), + "strike_type": m.get("strike_type"), "floor_strike": _f(m.get("floor_strike")), "cap_strike": _f(m.get("cap_strike")), + "exchange_index": m.get("exchange_index"), "price_level_structure": m.get("price_level_structure"), "is_provisional": m.get("is_provisional"), + "can_close_early": m.get("can_close_early"), "early_close_condition": m.get("early_close_condition") or None, "created_time": ts(m.get("created_time")), "source": source, + "raw": json.dumps({k: v for k, v in m.items() if k not in ("rules_primary", "rules_secondary")}, default=str)} + + +def upsert_markets(conn, markets: list[dict], source: str) -> int: + rows = [market_row(m, source) for m in markets if m.get("ticker")] + if not rows: + return 0 + cols = MARKET_COLS + sets = ", ".join(f"{c} = COALESCE(EXCLUDED.{c}, kalshi_markets.{c})" for c in cols if c not in ("ticker", "source", "raw")) + conn.cursor().executemany( + f"INSERT INTO kalshi_markets ({', '.join(cols)}) VALUES ({', '.join(['%s'] * len(cols))}) ON CONFLICT (ticker) DO UPDATE SET {sets}, " + "source = EXCLUDED.source, raw = EXCLUDED.raw, updated_at = now()", [tuple(r[c] for c in cols) for r in rows]) + return len(rows) + + +def upsert_event(conn, e: dict) -> None: + if not e.get("event_ticker"): + return + conn.execute("INSERT INTO kalshi_events (event_ticker, series_ticker, title, sub_title, category, mutually_exclusive, strike_date, strike_period, " + "collateral_return_type, exchange_index, raw) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT (event_ticker) DO UPDATE SET " + "series_ticker = COALESCE(EXCLUDED.series_ticker, kalshi_events.series_ticker), title = COALESCE(EXCLUDED.title, kalshi_events.title), " + "sub_title = COALESCE(EXCLUDED.sub_title, kalshi_events.sub_title), category = COALESCE(EXCLUDED.category, kalshi_events.category), " + "mutually_exclusive = COALESCE(EXCLUDED.mutually_exclusive, kalshi_events.mutually_exclusive), strike_date = COALESCE(EXCLUDED.strike_date, kalshi_events.strike_date), " + "strike_period = COALESCE(EXCLUDED.strike_period, kalshi_events.strike_period), collateral_return_type = COALESCE(EXCLUDED.collateral_return_type, kalshi_events.collateral_return_type), " + "exchange_index = COALESCE(EXCLUDED.exchange_index, kalshi_events.exchange_index), raw = EXCLUDED.raw, updated_at = now()", + (e["event_ticker"], e.get("series_ticker") or series_ticker_of(e["event_ticker"]), e.get("title"), e.get("sub_title"), e.get("category"), + e.get("mutually_exclusive"), ts(e.get("strike_date")), e.get("strike_period"), e.get("collateral_return_type"), e.get("exchange_index"), + json.dumps({k: v for k, v in e.items() if k != "markets"}, default=str))) + + +def upsert_series(conn, s: dict) -> None: + if not s.get("ticker"): + return + conn.execute("INSERT INTO kalshi_series (ticker, title, category, categories, tags, frequency, fee_type, fee_multiplier, settlement_sources) " + "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT (ticker) DO UPDATE SET title = EXCLUDED.title, category = EXCLUDED.category, " + "categories = EXCLUDED.categories, tags = EXCLUDED.tags, frequency = EXCLUDED.frequency, fee_type = EXCLUDED.fee_type, " + "fee_multiplier = EXCLUDED.fee_multiplier, settlement_sources = EXCLUDED.settlement_sources, updated_at = now()", + (s["ticker"], s.get("title"), s.get("category"), list(s.get("categories") or []), list(s.get("tags") or []), s.get("frequency"), + s.get("fee_type"), _f(s.get("fee_multiplier")), json.dumps(s.get("settlement_sources") or [], default=str))) + + +def candle_rows(candles: list[dict]) -> list[dict]: + """Exchange candlesticks → ``CANDLE_SCHEMA`` rows (cents; missing trade prices stay NaN).""" + out = [] + for c in candles: + yb, ya, pr = c.get("yes_bid") or {}, c.get("yes_ask") or {}, c.get("price") or {} + plain = 100.0 if c.get("_units") == "dollars" else 1.0 # live: _dollars or in cents; archive: in dollars + + def q(d, k, plain=plain): + v = _f(d.get(f"{k}_dollars")) + return v * 100.0 if v is not None else (_f(d.get(k)) * plain if d.get(k) is not None and _f(d.get(k)) is not None else float("nan")) + out.append({"end_ts": int(c.get("end_period_ts") or 0), + "yes_bid_open": q(yb, "open"), "yes_bid_high": q(yb, "high"), "yes_bid_low": q(yb, "low"), "yes_bid_close": q(yb, "close"), + "yes_ask_open": q(ya, "open"), "yes_ask_high": q(ya, "high"), "yes_ask_low": q(ya, "low"), "yes_ask_close": q(ya, "close"), + "price_open": q(pr, "open"), "price_high": q(pr, "high"), "price_low": q(pr, "low"), "price_close": q(pr, "close"), "price_mean": q(pr, "mean"), + "volume": _f(c.get("volume_fp") or c.get("volume")) or 0.0, "open_interest": _f(c.get("open_interest_fp") or c.get("open_interest")) or 0.0}) + return out + + +def trade_rows(trades: list[dict]) -> list[dict]: + out = [] + for t in trades: + p = price_cents(t, "yes_price") + if p is None: + continue + out.append({"ts": ts(t.get("created_time")), "yes_price": int(round(p)), "count": _f(t.get("count_fp") or t.get("count")) or 0.0, + "taker_side": t.get("taker_outcome_side") or t.get("taker_side") or "", "is_block": bool(t.get("is_block_trade"))}) + return [r for r in out if r["ts"] is not None] + + +def write_parquet(rows: list[dict], schema: pa.Schema, path: Path) -> None: + import pyarrow.parquet as pq + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_name(path.name + ".tmp") + pq.write_table(pa.Table.from_pylist(rows, schema=schema), tmp, compression="zstd"); tmp.replace(path) diff --git a/fly_trader/kalshi/decisions.py b/fly_trader/kalshi/decisions.py new file mode 100644 index 0000000000000000000000000000000000000000..496f411917ef2fb971d67de8f2a76a561f1cbbf3 --- /dev/null +++ b/fly_trader/kalshi/decisions.py @@ -0,0 +1,149 @@ +"""Decision points for the Kalshi selector and fly: one row per (market, minute, side) of the feature parts +(kalshi/mature.py), with the settlement labels both arms trade on. + +Reuses ``train.decisions.DecisionSet`` unchanged: ``mint`` = ``":"``, ``ts`` = minute start, ``day`` = +the minute's date, and — because every position is held to settlement — a per-row hold ``hold_s`` = settlement − ts +(``taken_idx`` / ``evaluate`` already take per-row holds). Labels, per dollar staked at the fee-inclusive price: +- ``fwd_pess`` (taker): ``(100·y − eff) / eff`` with ``eff = effective_price_cents(side ask)`` and ``y`` the settlement + of the side (1 when it paid); +- ``fwd`` (reference): the same at the side's mid; +- ``fwd_h["maker"]``: the maker arm's order (kalshi/maker.py): a bid resting one tick above the side's bid, never at or + through its ask (``maker_rest``), for the order's lifetime (``KALSHI_MAKER_TTL_H``, ending ``KALSHI_MAKER_QUIET_MIN`` + before close); filled (``maker_filled``) when the side's ask came down to it — every bid at that price, ours included, + was taken — or a taker sold the side below it, or at it when our bid set a new best price and so stood first in its + queue; the maker fee where the series charges one; NaN when unfilled. Until feature version 5 the order rested one tick + inside the ask, for the market's whole life, and filled only when the ask later fell strictly below it: it paid away + the spread makers earn and counted only the fills that came with an adverse move (2026-10-09). +Eligibility (the live engine applies the same, ``kalshi/engine.KalshiMinuteEngine.eligible``): both quotes present, +spread ≤ KALSHI_MAX_SPREAD_CENTS, side ask in 1..99, open interest ≥ KALSHI_MIN_OPEN_INTEREST, 24 h volume ≥ +KALSHI_MIN_VOLUME_24H, settlement yes/no, time to close inside the entry window. +""" +from __future__ import annotations + +import glob +import math +from datetime import date + +import numpy as np +import pandas as pd +import pyarrow.parquet as pq + +from .. import config +from ..train.corpus_features import _epoch_s +from ..train.decisions import DecisionSet +from . import data as D +from .features import K_COLS, KALSHI_FEATURE_VERSION, KIDX +from .mature import part_current +from .vendor import kalshi_client as kc + +X_COLS = list(K_COLS) + + +def maker_rest(side_bid, side_ask) -> np.ndarray: + """The maker arm's bid (cents): one tick above the side's best bid, at most one tick below its ask (``post_only`` + never crosses); with a one-tick spread it joins the bid's queue.""" + return np.clip(np.minimum(np.round(np.asarray(side_bid, dtype=float)) + 1.0, np.round(np.asarray(side_ask, dtype=float)) - 1.0), 1.0, 99.0) + + +def maker_filled(rest, side_bid, ask_low, sold_low) -> np.ndarray: + """Would a bid at ``rest``, posted when the side's bid was ``side_bid``, have filled, given the lowest ask the side + showed (``ask_low``) and the lowest price a taker sold the side at (``sold_low``) while it rested (NaN: none). The ask + reaching ``rest`` means every bid at ``rest`` was taken; a taker sale below ``rest`` means the same; a sale at + ``rest`` reaches us only when we were first there — our bid improved on the best bid, so nobody was ahead of us.""" + rest = np.asarray(rest, dtype=float); ask_low = np.asarray(ask_low, dtype=float); sold_low = np.asarray(sold_low, dtype=float) + first = rest > np.round(np.asarray(side_bid, dtype=float)) + with np.errstate(invalid="ignore"): + return (np.isfinite(ask_low) & (ask_low <= rest)) | (np.isfinite(sold_low) & ((sold_low < rest) | (first & (sold_low <= rest)))) + + +def maker_fee_cents(price_cents, fee_mult, charged) -> np.ndarray: + """Per-contract maker fee in cents where the series charges makers (KALSHI_MAKER_FEE_FACTOR·mult·p(1−p)·100), else 0.""" + p = np.asarray(price_cents, dtype=float) / 100.0 + return np.where(np.asarray(charged, dtype=float) > 0, config.KALSHI_MAKER_FEE_FACTOR * np.asarray(fee_mult, dtype=float) * p * (1 - p) * 100.0, 0.0) + + +def eligible_mask(X: np.ndarray, cols: list[str], to_close_s: np.ndarray) -> np.ndarray: + c = {n: i for i, n in enumerate(cols)} + return ((X[:, c["spread"]] <= config.KALSHI_MAX_SPREAD_CENTS) & (X[:, c["side_ask"]] >= 1.0) & (X[:, c["side_ask"]] <= 99.0) + & (X[:, c["side_bid"]] >= 1.0) & (np.expm1(X[:, c["log_oi"]]) >= config.KALSHI_MIN_OPEN_INTEREST) + & (np.expm1(X[:, c["log_vol_24h"]]) >= config.KALSHI_MIN_VOLUME_24H) + & (to_close_s >= config.KALSHI_MIN_MINUTES_TO_CLOSE * 60.0) & (to_close_s <= config.KALSHI_MAX_DAYS_TO_CLOSE * 86400.0)) + + +def build(days: int | None = None, feature_dir=None, label_thr: float = 0.0) -> DecisionSet: + """The decision rows of the most recent ``days`` feature days (all of them when None). Each part is reduced to its + eligible rows as it is read — float32 inputs, labels, integer market ids — so the whole corpus (tens of millions of + rows) fits: reading every part into one pandas frame with its string columns peaked near three times the final size.""" + root = feature_dir or D.FEATURES_DIR + dirs = sorted(glob.glob(str(root / "*"))) + if days: + dirs = dirs[-days:] + files = [f for d in dirs for f in sorted(glob.glob(str(root / d.split("/")[-1] / "part-*.parquet")))] + if not files: + raise RuntimeError("no Kalshi feature parts; run kalshi-build") + stale = [f for f in files if not part_current(f)] + if stale: + raise RuntimeError(f"{len(stale)} Kalshi feature part(s) from another version (e.g. {stale[0]}); run kalshi-build") + need = list(dict.fromkeys([*X_COLS, "ticker", "side", "ts", "category", "close_ts", "end_ts", "settled_ts", "result", "fill_ask_low", "fill_sold_low"])) + tick_id: dict[str, int] = {}; tick_names: list[str] = []; cat_id: dict[str, int] = {}; cat_names: list[str] = [] + acc: dict[str, list] = {k: [] for k in ("X", "ts", "settled", "y", "fwd", "fwd_pess", "maker", "tick", "side", "cat", "ord")} + for f in files: + t = pq.read_table(f, columns=need) + if not t.num_rows: + continue + res = np.asarray(t["result"].to_numpy(zero_copy_only=False), dtype=object); side = np.asarray(t["side"].to_numpy(zero_copy_only=False), dtype=object) + ok = (res == "yes") | (res == "no") + if not ok.any(): + continue + X = np.column_stack([t[c].to_numpy(zero_copy_only=False).astype(np.float32) for c in X_COLS]) + X[~np.isfinite(X)] = 0.0 + ts = t["ts"].cast("int64").to_numpy().astype(np.float64) / 1000.0 + close = t["close_ts"].to_numpy(zero_copy_only=False).astype(np.float64); st_ = t["settled_ts"].to_numpy(zero_copy_only=False).astype(np.float64) + end = t["end_ts"].to_numpy(zero_copy_only=False).astype(np.float64) + settled = np.where(np.isfinite(st_), st_, end); to_close = close - ts # window on the anchor; a label's timing on the realized end + y = (res == side).astype(np.float64) + eff = X[:, KIDX["eff_price"]].astype(np.float64); mid = np.clip(X[:, KIDX["side_mid"]].astype(np.float64), 1.0, 99.0) + fwd_pess = (100.0 * y - eff) / eff; fwd = (100.0 * y - mid) / mid + rest = maker_rest(X[:, KIDX["side_bid"]], X[:, KIDX["side_ask"]]) + filled = maker_filled(rest, X[:, KIDX["side_bid"]], t["fill_ask_low"].to_numpy(zero_copy_only=False).astype(np.float64), + t["fill_sold_low"].to_numpy(zero_copy_only=False).astype(np.float64)) + m_eff = rest + maker_fee_cents(rest, X[:, KIDX["fee_mult"]], X[:, KIDX["maker_fee"]]) + maker = np.where(filled, (100.0 * y - m_eff) / m_eff, np.nan) + elig = ok & eligible_mask(X, X_COLS, to_close) & np.isfinite(fwd_pess) & (settled > ts) + if not elig.any(): + continue + tk = t["ticker"].to_numpy(zero_copy_only=False)[elig]; cats = t["category"].to_numpy(zero_copy_only=False)[elig] + acc["X"].append(X[elig]); acc["ts"].append(ts[elig]); acc["settled"].append(settled[elig]); acc["y"].append(y[elig]) + acc["fwd"].append(fwd[elig].astype(np.float32)); acc["fwd_pess"].append(fwd_pess[elig].astype(np.float32)); acc["maker"].append(maker[elig].astype(np.float32)) + acc["tick"].append(np.fromiter((tick_id.setdefault(x, len(tick_id)) for x in tk), dtype=np.int64, count=len(tk))) + acc["side"].append((side[elig] == "yes").astype(np.int8)) + acc["cat"].append(np.fromiter((cat_id.setdefault(x, len(cat_id)) for x in cats), dtype=np.int16, count=len(cats))) + acc["ord"].append((np.floor(ts[elig] / 86400.0) + 719163).astype(np.int32)) # date.fromordinal(719163) is 1970-01-01 + del X, t + if not acc["X"]: + raise RuntimeError("no eligible Kalshi decision rows") + cat = lambda k: np.concatenate(acc.pop(k)) + X = cat("X"); ts = cat("ts"); settled = cat("settled"); y = cat("y"); fwd = cat("fwd"); fwd_pess = cat("fwd_pess"); maker = cat("maker") + tick = cat("tick"); side = cat("side"); catc = cat("cat"); ords = cat("ord") + tick_names = np.array([None] * len(tick_id), dtype=object) + for k_, v in tick_id.items(): + tick_names[v] = k_ + cat_names = np.array([None] * len(cat_id), dtype=object) + for k_, v in cat_id.items(): + cat_names[v] = k_ + day_objs = {o: date.fromordinal(int(o)) for o in np.unique(ords)} + day_arr = np.array([day_objs[o] for o in sorted(day_objs)], dtype=object)[np.searchsorted(np.array(sorted(day_objs)), ords)] + ds = DecisionSet(X=X, y=(fwd_pess > label_thr).astype(np.int8), fwd=fwd, fwd_pess=fwd_pess, day=day_arr, ts=ts, mint=tick * 2 + side.astype(np.int64), + cols=list(X_COLS), horizon_s=float(np.median(settled - ts)), fwd_h={"maker": maker}) + ds._day_ord = ords + ds.train_max_rows = int(config.KALSHI_TRAIN_MAX_ROWS) or None # training rows per fit: a random subset (train/strategies.cap_rows) + ds.hold_s = (settled - ts).astype(np.float64) # per-row: every position is held to its own settlement + ds.data_version = f"kalshi-features-{KALSHI_FEATURE_VERSION}" # in the walk-forward cache key: another feature version never reuses a fit + ds.label_ts = (settled + 60.0).astype(np.float64) # when the outcome is known: training for a period may use only rows known before it + ds.outcome = y.astype(np.float32) + ds.side = np.array(["no", "yes"], dtype=object)[side]; ds.ticker = tick_names[tick]; ds.category = cat_names[catc] + # the stack's bars (train/strategies.Score): a positive mean with a weekly sign test, not the memecoin PF ≥ 1.3 and 65 % + # winners — a contract bought at 95c that pays 5c when it wins needs a 1.3 % edge for PF 1.3, about all the edge the + # literature finds left on Kalshi, and its win rate is set by its price (operator decision, 2026-10-09) + ds.gate = "significance" + return ds diff --git a/fly_trader/kalshi/engine.py b/fly_trader/kalshi/engine.py new file mode 100644 index 0000000000000000000000000000000000000000..456fa4a81121a1e108aebb030b9cd730964bda0e --- /dev/null +++ b/fly_trader/kalshi/engine.py @@ -0,0 +1,300 @@ +"""The Kalshi trading engine — the ``kalshi_runner`` worker (agent/minute_engine.py for prediction markets). + +Every UTC minute, once the stream has written it (``kalshi_stream_status.flushed_through``), the minute's rows of +``kalshi_minutes`` become ``Bar``s fed to the same feature engine the corpus was built with (kalshi/features.py): one +``MarketState`` per market, one ``EventState`` per event (the siblings' quotes), metadata from ``kalshi_markets`` / +``kalshi_events`` / ``kalshi_series`` (kalshi/mature.load_meta). A market is scored on a minute it had a message on — +as a corpus row exists only for minutes with a candle — when it lies inside the entry window (``KALSHI_MIN_MINUTES_TO_CLOSE`` +… ``KALSHI_MAX_DAYS_TO_CLOSE`` before close) and passes the eligibility gate (kalshi/decisions.eligible_mask); both sides +go to every book as ``K_COLS`` vectors. Books implement ``name``, ``on_minute(ctx)``, ``tickers_watched()``, +``maybe_reload()``, ``done`` and ``finish()``; each trades in its own savepoint. Warm-up feeds the last ``WARMUP_MIN`` +minutes without trading; no entries while the stream is stale; missed minutes are replayed through the features only. +""" +from __future__ import annotations + +import json +import logging +import math +import threading +import time +from dataclasses import dataclass, field +from datetime import datetime, timezone + +import numpy as np + +from .. import config +from ..db.apilog import record_event +from ..db.connection import connect, transaction +from ..logging_setup import setup +from . import stream as KS +from .decisions import eligible_mask +from .features import K_COLS, Bar, EventState, MarketMeta, MarketState, warm_fees +from .mature import load_meta +from .strategies import in_universe + +log = logging.getLogger(__name__) +WARMUP_MIN = 1440 +STREAM_STALE_S = 180 +META_TTL_S = 3600 +MODEL_CHECK_S = 600 +IDLE_EVICT_S = 7 * 86400 +KALSHI_LOCK_KEY = 0x6B616C5F72756E # "kal_run": one Kalshi runner per database + + +@dataclass +class KMinute: + """What every Kalshi book sees of one minute.""" + conn: object + m0: datetime + m1: datetime + m1_epoch: float + X: np.ndarray # [n, len(K_COLS)] eligible rows + keys: list # [(ticker, side)] per row + hold_s: np.ndarray # per row: seconds from the minute start to the market's close (the position's hold) + metas: dict # ticker → MarketMeta (every market with a state) + quotes: dict # ticker → (yes_bid, yes_ask, bid_size, ask_size) latest known + extremes: dict # ticker → (yes_ask_low, yes_bid_high, yes_sold_low, yes_bought_high) of this minute (markets with a row) + n_rows: int + fresh: bool + trade: bool + engine: "KalshiMinuteEngine" + results: dict = field(default_factory=dict) # ticker → 'yes'|'no' for markets asked about (``settled``) + + @property + def t_start(self) -> float: + return self.m1_epoch - 60.0 + + def settled(self, tickers) -> dict: + """Results of the given markets that have resolved (from ``kalshi_markets``; the stream writes them).""" + tickers = [t for t in set(tickers) if t not in self.results] + if tickers: + for r in self.conn.execute("SELECT ticker, result FROM kalshi_markets WHERE ticker = ANY(%s) AND result IN ('yes','no')", (tickers,)).fetchall(): + self.results[r["ticker"]] = r["result"] + return self.results + + +class KalshiMinuteEngine: + def __init__(self, books: list | None = None): + self.books = list(books or []) + self.states: dict[str, MarketState] = {}; self.events: dict[str, EventState] = {} + self.metas: dict[str, MarketMeta] = {}; self.meta_at: dict[str, float] = {}; self.missing_meta: dict[str, float] = {} + self.quotes: dict[str, tuple] = {} + self.last_minute: float | None = None; self.last_sweep = time.time(); self.fees_warmed = False + + def has_book(self, name: str) -> bool: + return any(b.name == name for b in self.books) + + # ---- metadata ---- + def _metas(self, conn, tickers: list[str]) -> None: + now = time.time() + todo = [t for t in tickers if (t not in self.metas or now - self.meta_at.get(t, 0) > META_TTL_S) and now - self.missing_meta.get(t, 0) > META_TTL_S] + if not todo: + return + if not self.fees_warmed: + warm_fees(conn); self.fees_warmed = True + got = load_meta(conn, todo) + for t in todo: + if t in got: + self.metas[t] = got[t]; self.meta_at[t] = now; self.missing_meta.pop(t, None) + else: + self.missing_meta[t] = now # not in the catalogue yet (the stream fetches lifecycle announcements) + + # ---- one minute ---- + def _rows(self, conn, m0: datetime) -> list[dict]: + return [dict(r) for r in conn.execute("SELECT ticker, yes_bid, yes_ask, last, bid_size, ask_size, volume_fp, open_interest_fp, taker_buy_yes, taker_buy_no, n_trades, max_trade, " + "block_contracts, yes_ask_low, yes_bid_high, yes_sold_low, yes_bought_high FROM kalshi_minutes WHERE ts = %s", (m0,)).fetchall()] + + @staticmethod + def _bar(r: dict, t_end: float) -> Bar: + f = lambda v: float(v) if v is not None and 0 < float(v) < 100 else math.nan + return Bar(t_end, f(r["yes_bid"]), f(r["yes_ask"]), float(r["last"]) if r["last"] is not None else math.nan, float(r["bid_size"] or 0.0), float(r["ask_size"] or 0.0), + float(r["volume_fp"] or 0.0), float(r["open_interest_fp"] or 0.0), float(r["taker_buy_yes"] or 0.0), float(r["taker_buy_no"] or 0.0), + float(r["n_trades"] or 0.0), float(r["max_trade"] or 0.0), float(r["block_contracts"] or 0.0)) + + def feed(self, conn, rows: list[dict], m1_epoch: float) -> list[str]: + """Append this minute's bars to the states (and the events' sibling quotes); returns the tickers with a bar.""" + self._metas(conn, [r["ticker"] for r in rows]) + fed = [] + for r in rows: + tk = r["ticker"]; bar = self._bar(r, m1_epoch) + st = self.states.get(tk) + if st is None: + st = self.states[tk] = MarketState(tk) + st.append(bar); fed.append(tk) + self.quotes[tk] = (bar.yes_bid if math.isfinite(bar.yes_bid) else None, bar.yes_ask if math.isfinite(bar.yes_ask) else None, bar.bid_size, bar.ask_size) + meta = self.metas.get(tk) + if meta is not None and meta.event_ticker: + ev = self.events.get(meta.event_ticker) + if ev is None: + ev = self.events[meta.event_ticker] = EventState() + ev.update(tk, bar, meta.strike) + return fed + + def rows_for(self, fed: list[str], m1_epoch: float) -> tuple[np.ndarray, list, np.ndarray, int]: + """K_COLS vectors of both sides of every fed market inside the entry window that passes the gate.""" + xs, keys, holds = [], [], []; n_rows = 0 + min_s = config.KALSHI_MIN_MINUTES_TO_CLOSE * 60.0; max_s = config.KALSHI_MAX_DAYS_TO_CLOSE * 86400.0 + for tk in fed: + meta = self.metas.get(tk); st = self.states.get(tk) + if meta is None or st is None or meta.close_ts is None: + continue + to_close = meta.close_ts - (m1_epoch - 60.0) + if not (min_s <= to_close <= max_s): + continue + ev = self.events.get(meta.event_ticker) if meta.event_ticker else None + for side in ("yes", "no"): + xs.append(st.features(m1_epoch, meta, side, ev)); keys.append((tk, side)); holds.append(to_close); n_rows += 1 + if not xs: + return np.zeros((0, len(K_COLS)), np.float32), [], np.zeros(0), 0 + X = np.nan_to_num(np.asarray(xs, dtype=np.float32), nan=0.0, posinf=0.0, neginf=0.0); holds = np.asarray(holds, dtype=np.float64) + ok = eligible_mask(X, K_COLS, holds) & in_universe(X, K_COLS) + return X[ok], [k for k, o in zip(keys, ok) if o], holds[ok], n_rows + + def _stream_through(self) -> float | None: + with transaction() as conn: + r = conn.execute("SELECT value->>'flushed_through' AS ft FROM ui_settings WHERE key = %s", (KS.STATUS_KEY,)).fetchone() + return datetime.fromisoformat(r["ft"]).timestamp() if r and r["ft"] else None + + def ready_through(self, m1_epoch: float) -> float: + ft = self._stream_through() + return min(m1_epoch, ft + 60.0) if ft is not None else float("-inf") + + def stream_fresh(self, m0_epoch: float) -> bool: + ft = self._stream_through() + return ft is not None and m0_epoch - ft <= STREAM_STALE_S + + def warm_up(self, m1_epoch: float, minutes: int = WARMUP_MIN) -> int: + n = 0 + with transaction() as conn: + for k in range(minutes, 0, -1): + t1 = m1_epoch - 60 * k + rows = self._rows(conn, datetime.fromtimestamp(t1 - 60, timezone.utc)) + if rows: + self.feed(conn, rows, t1); n += len(rows) + log.info("kalshi warm-up: %d market-minutes over the last %d minutes (%d markets, %d with metadata)", n, minutes, len(self.states), len(self.metas)) + return n + + def _sweep(self, now_s: float, held: set[str]) -> None: + if now_s - self.last_sweep < 3600: + return + self.last_sweep = now_s + idle = [t for t, s in self.states.items() if t not in held and (not s.ts or s.ts[-1] < now_s - IDLE_EVICT_S)] + for t in idle: + self.states.pop(t, None); self.metas.pop(t, None); self.meta_at.pop(t, None); self.quotes.pop(t, None) + if idle: + log.info("evicted %d market states idle for more than %d days", len(idle), IDLE_EVICT_S // 86400) + + def run_minute(self, m1_epoch: float, trade: bool = True) -> dict: + m1 = datetime.fromtimestamp(m1_epoch, timezone.utc); m0 = datetime.fromtimestamp(m1_epoch - 60, timezone.utc) + out: dict = {"minute": m1.isoformat()} + with transaction() as conn: + rows = self._rows(conn, m0) + fed = self.feed(conn, rows, m1_epoch) + X, keys, holds, n_rows = self.rows_for(fed, m1_epoch) + extremes = {r["ticker"]: (r["yes_ask_low"], r["yes_bid_high"], r["yes_sold_low"], r["yes_bought_high"]) for r in rows} + ctx = KMinute(conn=conn, m0=m0, m1=m1, m1_epoch=m1_epoch, X=X, keys=keys, hold_s=holds, metas=self.metas, quotes=self.quotes, extremes=extremes, + n_rows=n_rows, fresh=trade and self.stream_fresh(m1_epoch - 60), trade=trade, engine=self) + out.update(markets_active=len(rows), in_window=n_rows // 2, eligible=len(keys)); held: set[str] = set() + for b in list(self.books): + try: + with conn.transaction(): + out[b.name] = b.on_minute(ctx) + held |= set(b.tickers_watched()) + except Exception: + log.exception("%s: minute %s failed", b.name, m1.isoformat()) + record_event("error", b.name, f"minute failed: {m1.isoformat()}") + self._sweep(m1_epoch, held) + for b in [b for b in self.books if b.done]: + b.finish(); self.books.remove(b) + return out + + +def _status(key: str, value: dict) -> None: + try: + with transaction() as conn: + conn.execute("INSERT INTO ui_settings (key, value) VALUES (%s, %s) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()", + (key, json.dumps({**value, "updated_at": datetime.now(timezone.utc).isoformat()}, default=str))) + except Exception: + log.debug("status write failed", exc_info=True) + + +def admit_books(engine: KalshiMinuteEngine) -> list: + from . import fly_session + new = [] + if not engine.has_book(fly_session.NAME): + fb, why = fly_session.try_start() + if fb is not None: + new.append(fb) + else: + _status(fly_session.STATUS_KEY, {"stage": "not trading", "detail": why}) + return new + + +def acquire_lock(): + conn = connect(autocommit=True) + try: + ok = conn.execute("SELECT pg_try_advisory_lock(%s) AS ok", (KALSHI_LOCK_KEY,)).fetchone()["ok"] + except Exception: + conn.close(); raise + if not ok: + conn.close() + raise RuntimeError("another Kalshi runner holds the lock; refusing to start a second one") + return conn + + +def main(stop_event: threading.Event | None = None) -> None: + setup("kalshi_runner") + try: + lock_conn = acquire_lock() + except RuntimeError as e: + log.error("%s", e); record_event("error", "kalshi_runner", str(e)); raise + try: + record_event("info", "kalshi_runner", "kalshi runner started", {"live": bool(config.KALSHI_LIVE_ENABLED)}) + _main(stop_event) + finally: + lock_conn.close() + + +def _main(stop_event: threading.Event | None) -> None: + engine = KalshiMinuteEngine() + while not (stop_event is not None and stop_event.is_set()): + engine.books = admit_books(engine) + if engine.books: + break + log.info("kalshi engine waiting: no book qualifies yet") + (stop_event or threading.Event()).wait(MODEL_CHECK_S) + else: + return + m_start = math.floor(time.time() / 60) * 60 + engine.warm_up(m_start); engine.last_minute = m_start - 60; last_check = time.time(); n_min = 0 + try: + while not (stop_event is not None and stop_event.is_set()): + now = time.time(); m1 = math.floor(now / 60) * 60 + if now - last_check >= MODEL_CHECK_S: + last_check = now + for b in engine.books: + try: + b.maybe_reload() + except Exception: + log.exception("%s: model reload check failed", b.name) + try: + engine.books += admit_books(engine) + except Exception: + log.exception("book admission failed") + if engine.books and m1 > engine.last_minute and now - m1 >= 4.0 and (upto := engine.ready_through(m1)) > engine.last_minute: + for minute in range(int(engine.last_minute) + 60, int(upto) + 1, 60): + try: + st = engine.run_minute(float(minute), trade=(minute == int(m1))); n_min += 1 + if minute == int(m1) and n_min % 10 == 0: + log.info("kalshi minute %s: active %d in window %d eligible %d | %s", st["minute"], st.get("markets_active", 0), st.get("in_window", 0), st.get("eligible", 0), + " | ".join(f"{b.name}: {(st.get(b.name) or {}).get('stage')} picks {(st.get(b.name) or {}).get('picks')}" for b in engine.books)) + except Exception: + log.exception("kalshi minute %s failed", minute); record_event("error", "kalshi_runner", f"minute failed: {minute}") + engine.last_minute = upto + time.sleep(0.5) + finally: + for b in engine.books: + try: + b.finish() + except Exception: + log.exception("%s: finish failed", b.name) diff --git a/fly_trader/kalshi/features.py b/fly_trader/kalshi/features.py new file mode 100644 index 0000000000000000000000000000000000000000..7511ca3bf4211dd6c65558454fe4ab54349628f6 --- /dev/null +++ b/fly_trader/kalshi/features.py @@ -0,0 +1,255 @@ +"""One feature engine for the Kalshi corpus and the live minute engine: per (market, minute, side) the inputs the +literature says carry information about a binary contract's settlement (plan of 2026-09-22, "Literature basis"). + +A market's state is a rolling window of minute bars (``Bar``: the YES top of book, sizes, contracts traded, open interest +and the taker flow by side) covering ``WINDOW_S`` (7 days). ``features(t_end, meta, side, event)`` returns the ``K_COLS`` +vector for one side: every price is the side's own (a NO contract's ask is 100 minus the YES bid), so both sides share +one definition and one model. ``EventState`` holds the latest YES quote of every sibling market of an event, for the +sibling-implied probability, the count of near-certain legs and the strike-ladder residual. Category, recurrence, time +to close and the fee multiplier come from ``MarketMeta``. Fees are the vendored better_bot model +(``effective_price_cents``): warm its per-series cache from ``kalshi_series`` with ``warm_fees`` before building. +""" +from __future__ import annotations + +import math +from bisect import bisect_left, bisect_right +from collections import deque +from dataclasses import dataclass +from typing import NamedTuple + +from .vendor import kalshi_client as kc + +KALSHI_FEATURE_VERSION = 5 # 2: archive candles rescaled; 3: outcome-blind event sample; 4: time features anchored on what was known at decision time; + # 5: the maker label is the arms' order (one tick above the bid, the order's lifetime, fills from taker-sell prints) + # (an early-closing market's realized close is when its event happened: a leak, 2026-10-08) +WINDOW_S = 7 * 86400.0 +WINDOWS = {"5m": 300.0, "15m": 900.0, "1h": 3600.0, "6h": 21600.0, "24h": 86400.0, "7d": WINDOW_S} +CATEGORIES = ("politics", "sports", "crypto", "economics", "weather", "entertainment", "science", "companies", "world", "health", "other") +SIDES = ("yes", "no") + +K_COLS: list[str] = [ + # the side's quotes (cents) and the fee-inclusive price it would pay + "side_yes", "yes_bid", "yes_ask", "mid", "spread", "last", "side_ask", "side_bid", "side_mid", "eff_price", "dist_mid", "fee_mult", "maker_fee", + # the side's price history (cents) from the minute bars + "ret_5m", "ret_15m", "ret_1h", "ret_6h", "ret_24h", "ret_7d", "rvol_1h", "rvol_24h", "dd_24h", "range_24h", "log_min_since_trade", "log_history_min", + # taker flow on the side (contracts) and its imbalance + "log_taker_side_5m", "log_taker_side_1h", "log_taker_side_24h", "taker_imb_5m", "taker_imb_1h", "taker_imb_24h", "log_trades_1h", "log_trades_24h", + "log_max_trade_24h", "log_mean_trade_24h", "block_share_24h", + # size and depth (top-of-book sizes are not in the archive's candles, so they are not inputs) + "log_oi", "log_vol_24h", "log_vol_7d", "oi_change_24h", + # time + "log_h_to_close", "log_h_since_open", "life_frac", "last_day", "last_hour", "hod_s", "hod_c", "dow", + # structure + *[f"cat_{c}" for c in CATEGORIES], "is_recurring", "log_n_siblings", "mutually_exclusive", "sib_sum_bid", "sib_sum_ask", "implied_gap", + "ladder_rank", "ladder_resid", "n_sib_ge90", "has_strike", +] +KIDX = {n: i for i, n in enumerate(K_COLS)} + + +class Bar(NamedTuple): + t_end: float # minute end (epoch s) + yes_bid: float # cents; NaN when no bid + yes_ask: float + last: float # last trade price (cents) or NaN + bid_size: float # contracts at the YES bid (0 when unknown) + ask_size: float + volume: float # contracts traded in the minute + oi: float # open interest at the minute's end + taker_yes: float # contracts bought by takers on the YES side in the minute + taker_no: float + n_trades: float + max_trade: float + block: float # contracts in block trades + + +@dataclass +class MarketMeta: + ticker: str + event_ticker: str | None = None + series_ticker: str | None = None + category: str = "other" + is_recurring: float = 0.0 + open_ts: float | None = None + close_ts: float | None = None # the time ANCHOR known at decision time (listed close, or the scheduled expected expiration + # for markets that may close early when their event happens): every feature and window uses it + end_ts: float | None = None # the realized close: labels only (a maker order's fill window), never a feature + mutually_exclusive: float = 0.0 + fee_multiplier: float = 1.0 + maker_fee: float = 0.0 + strike: float | None = None + + +def warm_fees(conn) -> int: + """The vendored fee model's per-series cache from ``kalshi_series`` (no network).""" + n = 0 + for r in conn.execute("SELECT ticker, fee_type, fee_multiplier FROM kalshi_series").fetchall(): + kc._series_fee_cache[r["ticker"]] = {"fee_type": r["fee_type"], "fee_multiplier": float(r["fee_multiplier"] if r["fee_multiplier"] is not None else kc.DEFAULT_FEE_MULTIPLIER)} + n += 1 + return n + + +def side_prices(bar: Bar, side: str) -> tuple[float, float, float]: + """(ask, bid, mid) of ``side`` in cents; NaN where the book is one-sided.""" + if side == "yes": + ask, bid = bar.yes_ask, bar.yes_bid + else: + ask, bid = 100.0 - bar.yes_bid, 100.0 - bar.yes_ask + mid = (ask + bid) / 2.0 if math.isfinite(ask) and math.isfinite(bid) else (ask if math.isfinite(ask) else bid) + return ask, bid, mid + + +def _mid(bar: Bar) -> float: + if math.isfinite(bar.yes_bid) and math.isfinite(bar.yes_ask): + return (bar.yes_bid + bar.yes_ask) / 2.0 + return bar.yes_ask if math.isfinite(bar.yes_ask) else bar.yes_bid + + +class EventState: + """The latest YES quote and strike of every market of one event (the siblings).""" + __slots__ = ("quotes", "strikes") + + def __init__(self): + self.quotes: dict[str, tuple[float, float]] = {} + self.strikes: dict[str, float] = {} + + def update(self, ticker: str, bar: Bar, strike: float | None = None) -> None: + self.quotes[ticker] = (bar.yes_bid, bar.yes_ask) + if strike is not None: + self.strikes[ticker] = float(strike) + + def features(self, ticker: str) -> dict: + others = [(b, a) for t, (b, a) in self.quotes.items() if t != ticker] + bids = [b for b, _ in others if math.isfinite(b)]; asks = [a for _, a in others if math.isfinite(a)] + out = {"n_siblings": float(len(others)), "sib_sum_bid": float(sum(bids)), "sib_sum_ask": float(sum(asks)), + "n_sib_ge90": float(sum(1 for b, a in others if math.isfinite(b) and b >= 90.0)), "ladder_rank": 0.0, "ladder_resid": 0.0} + st = self.strikes.get(ticker) + if st is not None and len(self.strikes) > 1: + ranked = sorted(self.strikes.items(), key=lambda kv: kv[1]); ks = [t for t, _ in ranked] + i = ks.index(ticker); out["ladder_rank"] = i / max(len(ks) - 1, 1) + # a higher strike must not price above a lower one ("above X" ladders): the amount by which this leg breaks that + own = self.quotes.get(ticker); own_mid = (own[0] + own[1]) / 2.0 if own and math.isfinite(own[0]) and math.isfinite(own[1]) else float("nan") + resid = 0.0 + if math.isfinite(own_mid): + if i > 0: + lo = self.quotes.get(ks[i - 1]) + if lo and math.isfinite(lo[0]) and math.isfinite(lo[1]): + resid = max(resid, own_mid - (lo[0] + lo[1]) / 2.0) + if i < len(ks) - 1: + hi = self.quotes.get(ks[i + 1]) + if hi and math.isfinite(hi[0]) and math.isfinite(hi[1]): + resid = max(resid, (hi[0] + hi[1]) / 2.0 - own_mid) + out["ladder_resid"] = float(resid) + return out + + +class MarketState: + """A market's rolling minute bars; O(log n) window statistics by bisect over the bar ends (as market/features.py).""" + __slots__ = ("ticker", "bars", "ts", "mids", "last_trade_ts", "first_ts") + + def __init__(self, ticker: str): + self.ticker = ticker + self.bars: deque = deque(); self.ts: deque = deque(); self.mids: deque = deque() + self.last_trade_ts: float | None = None; self.first_ts: float | None = None + + def append(self, bar: Bar) -> None: + if self.ts and bar.t_end <= self.ts[-1]: + return # never rewinds: a late bar for a minute already seen is dropped + self.bars.append(bar); self.ts.append(bar.t_end); self.mids.append(_mid(bar)) + if bar.n_trades > 0 or bar.volume > 0: + self.last_trade_ts = bar.t_end + if self.first_ts is None: + self.first_ts = bar.t_end + while self.ts and self.ts[0] <= bar.t_end - WINDOW_S - 60.0: + self.bars.popleft(); self.ts.popleft(); self.mids.popleft() + + @property + def last(self) -> Bar | None: + return self.bars[-1] if self.bars else None + + def _idx(self, t0: float) -> int: + return bisect_left(self.ts, t0) + + def _mid_at_or_before(self, t: float) -> float: + i = bisect_right(self.ts, t) + if i == 0: + return float("nan") + return self.mids[i - 1] + + def features(self, t_end: float, meta: MarketMeta, side: str, event: EventState | None = None) -> list[float]: + f = [0.0] * len(K_COLS) + bar = self.last + if bar is None: + return f + yes = side == "yes"; sgn = 1.0 if yes else -1.0 + ask, bid, mid_s = side_prices(bar, side) + mid = _mid(bar) + f[KIDX["side_yes"]] = 1.0 if yes else 0.0 + f[KIDX["yes_bid"]] = bar.yes_bid if math.isfinite(bar.yes_bid) else 0.0 + f[KIDX["yes_ask"]] = bar.yes_ask if math.isfinite(bar.yes_ask) else 100.0 + f[KIDX["mid"]] = mid if math.isfinite(mid) else 50.0 + f[KIDX["spread"]] = (bar.yes_ask - bar.yes_bid) if math.isfinite(bar.yes_ask) and math.isfinite(bar.yes_bid) else 99.0 + f[KIDX["last"]] = (bar.last if yes else 100.0 - bar.last) if math.isfinite(bar.last) else f[KIDX["mid"]] + f[KIDX["side_ask"]] = ask if math.isfinite(ask) else 100.0 + f[KIDX["side_bid"]] = bid if math.isfinite(bid) else 0.0 + f[KIDX["side_mid"]] = mid_s if math.isfinite(mid_s) else 50.0 + f[KIDX["eff_price"]] = kc.effective_price_cents(min(max(f[KIDX["side_ask"]], 1.0), 99.0), meta.ticker) if math.isfinite(ask) else 100.0 + f[KIDX["dist_mid"]] = abs(f[KIDX["mid"]] - 50.0) + f[KIDX["fee_mult"]] = meta.fee_multiplier; f[KIDX["maker_fee"]] = meta.maker_fee + # history of the side's mid (cents); returns are differences, not ratios: a contract's scale is fixed + m_now = mid if math.isfinite(mid) else 50.0 + for k in ("5m", "15m", "1h", "6h", "24h", "7d"): + m_then = self._mid_at_or_before(t_end - WINDOWS[k]) + if not math.isfinite(m_then): + m_then = self.mids[0] if math.isfinite(self.mids[0]) else m_now + f[KIDX[f"ret_{k}"]] = sgn * (m_now - m_then) + for k, w in (("1h", 3600.0), ("24h", 86400.0)): + i = self._idx(t_end - w); seq = [m for m in list(self.mids)[max(i - 1, 0):] if math.isfinite(m)] + if len(seq) > 1: + d = [b - a for a, b in zip(seq[:-1], seq[1:])]; mu = sum(d) / len(d) + f[KIDX[f"rvol_{k}"]] = math.sqrt(sum((x - mu) ** 2 for x in d) / len(d)) + i24 = self._idx(t_end - 86400.0); win = [m for m in list(self.mids)[i24:] if math.isfinite(m)] + if win: + side_win = [sgn * m + (0.0 if yes else 100.0) for m in win]; cur = sgn * m_now + (0.0 if yes else 100.0) + f[KIDX["dd_24h"]] = cur - max(side_win); f[KIDX["range_24h"]] = max(side_win) - min(side_win) + f[KIDX["log_min_since_trade"]] = math.log1p(max(0.0, (t_end - self.last_trade_ts) / 60.0)) if self.last_trade_ts else math.log1p(WINDOW_S / 60.0) + f[KIDX["log_history_min"]] = math.log1p(max(0.0, (t_end - self.first_ts) / 60.0)) if self.first_ts else 0.0 + # flow + bars = list(self.bars) + for k in ("5m", "1h", "24h"): + i = self._idx(t_end - WINDOWS[k]); w = bars[i:] + ty = sum(b.taker_yes for b in w); tn = sum(b.taker_no for b in w); own = ty if yes else tn + f[KIDX[f"log_taker_side_{k}"]] = math.log1p(own) + f[KIDX[f"taker_imb_{k}"]] = sgn * (ty - tn) / (ty + tn) if ty + tn > 0 else 0.0 + if k in ("1h", "24h"): + f[KIDX[f"log_trades_{k}"]] = math.log1p(sum(b.n_trades for b in w)) + if k == "24h": + vol = sum(b.volume for b in w); nt = sum(b.n_trades for b in w) + f[KIDX["log_max_trade_24h"]] = math.log1p(max((b.max_trade for b in w), default=0.0)) + f[KIDX["log_mean_trade_24h"]] = math.log1p(vol / nt) if nt > 0 else 0.0 + f[KIDX["block_share_24h"]] = sum(b.block for b in w) / vol if vol > 0 else 0.0 + f[KIDX["log_vol_24h"]] = math.log1p(vol) + oi_then = w[0].oi if w else bar.oi + f[KIDX["oi_change_24h"]] = math.log1p(bar.oi) - math.log1p(oi_then) + f[KIDX["log_vol_7d"]] = math.log1p(sum(b.volume for b in bars)) + f[KIDX["log_oi"]] = math.log1p(bar.oi) + # time + h_close = max(0.0, (meta.close_ts - t_end) / 3600.0) if meta.close_ts else 24.0 * 365 + h_open = max(0.0, (t_end - meta.open_ts) / 3600.0) if meta.open_ts else 0.0 + f[KIDX["log_h_to_close"]] = math.log1p(h_close); f[KIDX["log_h_since_open"]] = math.log1p(h_open) + f[KIDX["life_frac"]] = h_open / (h_open + h_close) if h_open + h_close > 0 else 0.0 + f[KIDX["last_day"]] = 1.0 if h_close <= 24.0 else 0.0; f[KIDX["last_hour"]] = 1.0 if h_close <= 1.0 else 0.0 + hod = ((t_end - 60.0) % 86400) / 3600.0 + f[KIDX["hod_s"]] = math.sin(2 * math.pi * hod / 24); f[KIDX["hod_c"]] = math.cos(2 * math.pi * hod / 24) + f[KIDX["dow"]] = float(int((t_end - 60.0) // 86400 + 3) % 7) # 0 = Monday (epoch day 0 was a Thursday) + # structure + f[KIDX[f"cat_{meta.category if meta.category in CATEGORIES else 'other'}"]] = 1.0 + f[KIDX["is_recurring"]] = meta.is_recurring; f[KIDX["mutually_exclusive"]] = meta.mutually_exclusive + f[KIDX["has_strike"]] = 1.0 if meta.strike is not None else 0.0 + if event is not None: + e = event.features(meta.ticker) + f[KIDX["log_n_siblings"]] = math.log1p(e["n_siblings"]); f[KIDX["sib_sum_bid"]] = e["sib_sum_bid"]; f[KIDX["sib_sum_ask"]] = e["sib_sum_ask"] + f[KIDX["n_sib_ge90"]] = e["n_sib_ge90"]; f[KIDX["ladder_rank"]] = e["ladder_rank"]; f[KIDX["ladder_resid"]] = e["ladder_resid"] + if meta.mutually_exclusive and e["n_siblings"] > 0 and math.isfinite(ask): + implied_yes = 100.0 - e["sib_sum_bid"] # what the siblings' bids leave for this leg + f[KIDX["implied_gap"]] = sgn * (f[KIDX["yes_ask"]] - implied_yes) if yes else -(f[KIDX["yes_bid"]] - implied_yes) + return f diff --git a/fly_trader/kalshi/fly.py b/fly_trader/kalshi/fly.py new file mode 100644 index 0000000000000000000000000000000000000000..6a25b776d80d1b59fa33bc20e2e0b961cb9dba65 --- /dev/null +++ b/fly_trader/kalshi/fly.py @@ -0,0 +1,270 @@ +"""The Kalshi fly: a second FlyNet on the optic lobes (brain/connectome.VISUAL_SUB — the memecoin senses dropped, VISUAL +kept), its features entering the photoreceptors (``photoreceptor_rows``), taught by the Kalshi selector's classifier +stack (kalshi/selector.py) and calibrated, snapshotted and governed by the memecoin fly's machinery (train/fly_selector.py) +under its own kind ('kalshi_fly_selector'), directory (``data/brain/kalshi_policies``) and version. + +The head predicts **p̂(side pays)** per strategy (``scale`` 1: the readout is linear in probability, clipped to [0, 1] at +use) so the mushroom body's three-factor rule (brain/plastic.py: δ = r − ŷ, ``RET_CLIP`` 1) learns from the settlement +outcome y ∈ {0, 1} unchanged. The trading quantity is the **edge** p̂ − the arm's fee-inclusive price (kalshi/strategies +.effective): the fly's lines are minimum edges, its sizing bands are bands of edge over the line, and the calibration +labels are the arm's realised returns (taker at the ask, maker one tick inside it when filled). +""" +from __future__ import annotations + +import logging +import threading +import time +from datetime import date, datetime, timedelta, timezone + +import numpy as np +import torch + +from .. import config +from ..agent import sizing +from ..brain.connectome import Connectome, SubConnectome, VISUAL_SUB, photoreceptor_rows +from ..db.apilog import record_event +from ..db.connection import transaction +from ..train import fly_calibrate +from ..train import fly_selector as F +from ..train import progress as prog +from ..train import strategies as S +from ..train.decisions import DecisionSet, taken_idx +from ..train.scaling import RobustScaler +from . import decisions as KD +from . import selector as kselector +from .strategies import KALSHI, effective, in_universe, label, probability + +log = logging.getLogger(__name__) + +SCALE = 1.0 +KIND = "kalshi_fly_selector" +SUBDIR = "kalshi_policies" +PREFIX = "kalshi_fly" +CALIB_DAYS, PURGE_DAYS = F.CALIB_DAYS, F.PURGE_DAYS +KALSHI_FLY_VERSION = {**kselector.DATA_VERSION, "fly": "visual-1", "plastic": "mb4-ch", "teacher": "deployed-1"} +TEACHER_NOTE = "a Kalshi stack refit on the fly's training days (no deployable Kalshi selector to copy)" +HOLD_CEILING_S = float(config.KALSHI_MAX_DAYS_TO_CLOSE) * 86400.0 # a position runs to its market's settlement; this is the longest +_GRAPH = None + + +def graph(): + """The Kalshi fly's sub-connectome (cached): the whole brain minus the memecoin afferents.""" + global _GRAPH + if _GRAPH is None: + _GRAPH = SubConnectome(Connectome.load(), exclude=VISUAL_SUB) + return _GRAPH + + +def net_kwargs(g=None) -> dict: + return {"aff_rows": photoreceptor_rows(g if g is not None else graph()), "scale": SCALE} + + +class KalshiFlyModel(F.FlyModel): + """A trained Kalshi fly: per strategy (each on its own dopamine channel) p̂ of the side paying; ``rules[name]["hold_min"]`` + is the strategy's **arm** ('taker' | 'maker'), not a hold — every position runs to settlement.""" + + def arm(self, s: int) -> str: + return str(self.rules[self.strategies[s]]["hold_min"]) + + def hold_s(self, s: int) -> float: + return HOLD_CEILING_S + + @torch.no_grad() + def score_all(self, X: np.ndarray, batch: int | None = None) -> np.ndarray: + """[B, S] p̂(side pays) per strategy, clipped to [0, 1].""" + return np.clip(super().score_all(X, batch), 0.0, 1.0) + + def edge_all(self, X: np.ndarray, P: np.ndarray | None = None) -> np.ndarray: + """[B, S] each strategy's edge: p̂ − its arm's fee-inclusive price (per dollar of payout).""" + P = self.score_all(X) if P is None else P + X = np.atleast_2d(X); out = np.empty_like(P) + for j in range(len(self.strategies)): + out[:, j] = P[:, j] - effective(X, self.cols, self.arm(j)) / 100.0 + return out + + def triggers(self, X: np.ndarray, cols: list[str]) -> np.ndarray: + X = np.atleast_2d(X); out = np.zeros((len(X), len(self.strategies)), bool) + for j, name in enumerate(self.strategies): + r = self.rules[name] + out[:, j] = KALSHI.base_mask(name, X, cols, r.get("high")) & S.trigger_mask(name, r.get("thr") or {}, X, cols, KALSHI) + return out + + def universe(self, X: np.ndarray) -> np.ndarray: + return in_universe(X, self.cols) + + +class KalshiTeacher(F.StackTeacher): + """The Kalshi selector's stack as the fly's teacher: per strategy, on its candidates, the classifier's p̂ where the + stack would trade and, where a filter blocks it, a p̂ one score-spread below the line (so the fly learns the decision).""" + + def __init__(self, models: dict, scaler: RobustScaler, cols: list[str]): + self.models, self.scaler, self.cols = models, scaler, list(cols) + self.strategies = list(models.get("strategies") or {}) + self.rules = {k: {"thr": dict(v["thr"]), "high": v["high"], "hold_min": str(v["hold_min"])} for k, v in models["strategies"].items()} + self.lines = {k: float(v["line"]) for k, v in models["strategies"].items()} + self.threshold = self.lines[self.strategies[0]] if self.strategies else 0.0 + self.combine = models.get("combine", "score"); self.eps: dict = {} + + def targets(self, X: np.ndarray, ts: np.ndarray) -> tuple[np.ndarray, np.ndarray]: + per = S.decide_all(self.models, X, self.cols, ts, spec=KALSHI); X = np.atleast_2d(X) + T = np.full((len(X), len(self.strategies)), np.nan); A = np.zeros_like(T, dtype=bool) + for j, k in enumerate(self.strategies): + d = per[k]; m = self.models["strategies"][k]; trig = d["trig"]; sc = d["score"] + if k not in self.eps: + al = sc[d["allow"] & np.isfinite(sc)] + self.eps[k] = float(np.median(np.abs(al - np.median(al)))) if len(al) else 0.0 + if trig.any(): + ti = np.flatnonzero(trig); ci = [self.cols.index(c) for c in m["cols"]] + p = probability(m, X[np.ix_(ti, ci)]); eff = effective(X[ti], self.cols, str(m["hold_min"])) / 100.0 + T[ti, j] = np.where(d["allow"][ti], p, np.minimum(p, eff + self.lines[k] - self.eps[k])) + A[:, j] = d["allow"] + return T, A + + +def _stack_teacher(ds: DecisionSet, train: np.ndarray, stop=None): + sub = ds.subset(train) + for a in ("hold_s", "outcome", "side", "ticker", "category", "label_ts"): + setattr(sub, a, getattr(ds, a)[train]) + stack = S.fit_stack(sub, stop, holdout_days_n=0, spec=KALSHI) + if not stack.fits: + return None, stack + return KalshiTeacher(S.final_models(sub, stack, spec=KALSHI), RobustScaler.fit(sub.X, seed=7), ds.cols), stack + + +def deployed_teacher(ds: DecisionSet, train: np.ndarray) -> tuple[KalshiTeacher | None, str, int | None]: + """The newest deployable Kalshi selector as the teacher (its models are the ones the fly is judged against).""" + with transaction() as conn: + r = kselector.latest_current(conn) + if r is None: + return None, TEACHER_NOTE, None + m = kselector.load_snapshot(int(r["id"])) + if m is None or not (m.stack or {}).get("strategies"): + return None, f"kalshi selector #{r['id']} could not be loaded; {TEACHER_NOTE}", None + missing = sorted({c for v in m.stack["strategies"].values() for c in v["cols"]} - set(ds.cols)) + if missing: + return None, f"kalshi selector #{r['id']} wants columns the corpus lacks ({', '.join(missing[:4])}); {TEACHER_NOTE}", None + names = ", ".join(f"{k} (edge ≥ {v['line']:.3f}, {v['hold_min']})" for k, v in m.stack["strategies"].items()) + return KalshiTeacher(m.stack, RobustScaler.fit(ds.X[np.flatnonzero(train)], seed=7), list(ds.cols)), f"deployed kalshi selector #{r['id']}: {names}", int(r["id"]) + + +def kalshi_decide(fly: KalshiFlyModel, X: np.ndarray, cols: list[str], edges: np.ndarray | None = None, hold_s: np.ndarray | None = None, + only: str | None = None, trig: np.ndarray | None = None) -> dict: + """The fly's per-row decision: among the strategies whose trigger fires and whose edge clears the fly's line, the one + with the larger sizing-band Kelly fraction or margin. ``hold_s``: each row's time to settlement (the position's hold); + ``only``: an arm ('taker' | 'maker') restricts the choice to that arm's strategies (each arm trades its own book).""" + X = np.atleast_2d(X); E = fly.edge_all(X) if edges is None else edges; trig = fly.triggers(X, cols) if trig is None else trig; n = len(X) + strat = np.full(n, None, dtype=object); score = np.full(n, -1.0); thr = np.full(n, np.inf); tables = [[] for _ in range(n)]; arm = np.full(n, "", dtype=object) + key = np.full(n, -np.inf) + for j, name in enumerate(fly.strategies): + if only is not None and fly.arm(j) != only: + continue + v = E[:, j]; line = fly.lines[name]; ok = trig[:, j] & (v >= line) + kv = (np.array([((sizing.band_for(fly.sizings[name], m) or {}).get("kelly") or 0.0) for m in v - line]) + 1e-9 * v + if fly.combine == "kelly" and fly.sizings[name] else v - line) + take = ok & (kv > key); key = np.where(take, kv, key) + for i in np.flatnonzero(take): + strat[i] = name; score[i] = v[i]; thr[i] = line; tables[i] = fly.sizings[name]; arm[i] = fly.arm(j) + hold = np.asarray(hold_s, dtype=np.float64) if hold_s is not None else np.full(n, HOLD_CEILING_S) + return {"strategy": strat, "score": score, "hold_s": np.where(strat != None, hold, 0.0), "threshold": thr, "tables": tables, "arm": arm} # noqa: E711 + + +def bootstrap(ds: DecisionSet, Sday: date, epochs: int = F.EPOCHS, stop: threading.Event | None = None, seed: int = 0, g=None, device=None, + teacher=None, teacher_note: str | None = None) -> tuple[KalshiFlyModel | None, dict]: + """A Kalshi fly ready to trade from day ``Sday`` on: distilled from its teacher on the days before ``Sday − 8``, its + edge lines and sizing calibrated on the week before ``Sday`` from the positions that had settled by then.""" + train_end = Sday - timedelta(days=CALIB_DAYS + PURGE_DAYS) + train = (ds.day < train_end) & S.known_before(ds, datetime(train_end.year, train_end.month, train_end.day, tzinfo=timezone.utc).timestamp()) # settled before the purge + uni = in_universe(ds.X, ds.cols) + if not train.any(): + raise RuntimeError(f"no training days before {train_end}") + stack_info = None + if teacher is None: + prog.update("kalshi fly: fitting its teacher (the Kalshi stack, training days only)", 0, 1, force=True) + teacher, stack = _stack_teacher(ds, train, stop) + stack_info = {"components": stack.components, "deployable": stack.deployable, "reason": stack.reason} + if teacher is None: + return None, {"S": str(Sday), "gates_ok": False, "gate_failures": ["the teacher stack has no strategy on the training days"], "teacher_stack": stack_info} + g = g if g is not None else graph() + fly, fit_info = F.train_fly(ds, train & uni, teacher, epochs=epochs, stop=stop, seed=seed, graph=g, device=device, net_kwargs=net_kwargs(g), + model_cls=KalshiFlyModel, target_clip=(0.0, 1.0), horizon_min=0) + if fly is None: + return None, {"stopped": True, **fit_info} + s_epoch = datetime(Sday.year, Sday.month, Sday.day, tzinfo=timezone.utc).timestamp() + week = (ds.day >= Sday - timedelta(days=CALIB_DAYS)) & (ds.day < Sday) & uni + prog.update("kalshi fly: calibrating its own edge lines", 0, 1, force=True) + wi = np.flatnonzero(week) + P = fly.score_all(ds.X[wi]) if len(wi) else np.zeros((0, len(fly.strategies))) + E = fly.edge_all(ds.X[wi], P) if len(wi) else P; trig = fly.triggers(ds.X[wi], ds.cols) if len(wi) else np.zeros((0, len(fly.strategies)), bool) + settled = ds.ts[wi] + ds.hold_s[wi] + 60.0 <= s_epoch # labels known at S + per = {} + for j, name in enumerate(fly.strategies): + arm = fly.arm(j); y = label(ds, arm) + known = trig[:, j] & settled; ci = wi[known] + cal = fly_calibrate.calibrate(ds.ts[ci], ds.mint[ci], ds.hold_s[ci], E[known, j], y[ci]) + fly.lines[name], fly.sizings[name] = cal.line, cal.sizing + per[name] = {"line": cal.line, "trades": cal.trades, "mean": cal.mean, "total": cal.total, "arm": arm} + d = kalshi_decide(fly, ds.X[wi], ds.cols, E, ds.hold_s[wi]) if len(wi) else {"strategy": np.array([], dtype=object)} + picked = np.array([x is not None for x in d["strategy"]], bool) + ret = np.array([label(ds, fly.arm(fly.strategies.index(x)))[i] if x is not None else np.nan for i, x in zip(wi, d["strategy"])], dtype=np.float64) + known = picked & settled & np.isfinite(ret) + tr = taken_idx(ds.ts[wi], ds.mint[wi], ds.hold_s[wi], np.flatnonzero(known)); r = ret[tr] + comb = {"trades": int(len(r)), "mean": float(np.mean(r)) if len(r) else None, "total": float(np.sum(r)) if len(r) else 0.0, "per_strategy": per} + ci = wi[settled] + di = ci if len(ci) <= F.DIAG_ROWS else np.sort(np.random.default_rng(seed).choice(ci, F.DIAG_ROWS, replace=False)) + t_sc = teacher.targets(ds.X[di], ds.ts[di])[0][:, 0] if len(di) else None + if t_sc is not None: + ok_t = np.isfinite(t_sc); di, t_sc = di[ok_t], t_sc[ok_t] + diag = F.diagnose(fly, ds.X[di], t_sc) if len(di) else {"mbon_share": 0.0, "mbon_saturated": 1.0, "kc_active": 0.0, "rows": 0} + ok, why = F.gate_check(diag) + info = {"S": str(Sday), "train_through": str(train_end - timedelta(days=1)), "calibration_days": [str(Sday - timedelta(days=CALIB_DAYS)), str(Sday - timedelta(days=1))], + "line": fly.threshold, "sizing": fly.sizing, "lines": fly.lines, "calibration": comb, "strategies": fly.strategies, "rules": fly.rules, + "diagnostics": diag, "gates_ok": ok, "gate_failures": why, "fit": fit_info, "teacher": teacher_note or TEACHER_NOTE, "teacher_line": getattr(teacher, "threshold", None), + "teacher_stack": stack_info, "photoreceptors": int(len(fly.net.aff_rows)), "neurons": int(fly.net.N)} + log.info("kalshi fly bootstrap for %s: strategies %s, edge lines %s | calibration %d trades, mean %s | MBON share %.1f%% saturated %.0f%% | gates %s", + Sday, fly.strategies, {k: round(v, 4) for k, v in fly.lines.items()}, comb["trades"], f"{comb['mean'] * 100:+.2f}%" if comb["mean"] is not None else "-", + diag["mbon_share"] * 100, diag["mbon_saturated"] * 100, "ok" if ok else "; ".join(why)) + return fly, info + + +# ---------------------------------------------------------------- snapshots (the memecoin machinery under this fly's kind and version) +def is_current(meta: dict | None) -> bool: + return F.is_current(meta, KALSHI_FLY_VERSION) + + +def deployable(meta: dict | None) -> tuple[bool, str]: + return F.deployable(meta, KALSHI_FLY_VERSION) + + +def save(fly: KalshiFlyModel, metrics: dict, run_id: str | None = None): + return F.save(fly, metrics, run_id, kind=KIND, subdir=SUBDIR, version=KALSHI_FLY_VERSION, prefix=PREFIX) + + +def load(path, g=None, device=None) -> KalshiFlyModel: + return F.load(path, g if g is not None else graph(), device, model_cls=KalshiFlyModel) + + +def latest_current(conn) -> dict | None: + return F.latest_current(conn, KIND, KALSHI_FLY_VERSION) + + +def latest_deployable(conn) -> dict | None: + return F.latest_deployable(conn, KIND, KALSHI_FLY_VERSION) + + +def main(days: int | None = None, epochs: int | None = None, stop_event: threading.Event | None = None) -> dict: + """Bootstrap a Kalshi fly to trade from tomorrow on, taught by the deployable Kalshi selector (or a refit stack).""" + from ..ops.reset import reset_training_stats + reset_training_stats(KIND, reason="kalshi fly bootstrap") + prog.set_stop_event(stop_event); prog.clear() + prog.update("kalshi fly: building decision points", 0, 1, force=True) + ds = KD.build(days=days); Sday = ds.days[-1] + timedelta(days=1) + teacher, note, taught_by = deployed_teacher(ds, ds.day < Sday - timedelta(days=CALIB_DAYS + PURGE_DAYS)) + log.info("kalshi fly bootstrap teacher: %s", note) + fly, info = bootstrap(ds, Sday, epochs=F.EPOCHS if epochs is None else epochs, stop=stop_event, teacher=teacher, teacher_note=note) + if fly is None: + return {"stopped": True, **info} + info["teacher_snapshot"] = taught_by + record_event("info", KIND, "kalshi fly bootstrap", {k: info.get(k) for k in ("S", "lines", "calibration", "diagnostics", "gates_ok", "gate_failures", "teacher")}) + path, sid = save(fly, info) + prog.update("kalshi fly: done", 1, 1, force=True, snapshot_id=sid, line=info["line"], gates_ok=info["gates_ok"], diagnostics=info["diagnostics"]) + return {**info, "snapshot_id": sid, "path": str(path)} diff --git a/fly_trader/kalshi/fly_replay.py b/fly_trader/kalshi/fly_replay.py new file mode 100644 index 0000000000000000000000000000000000000000..57f029ece342bf46d9c150fee0484472753c8216 --- /dev/null +++ b/fly_trader/kalshi/fly_replay.py @@ -0,0 +1,271 @@ +"""The Kalshi fly's replay: the bootstrap (kalshi/fly.py) traded through the corpus under mushroom-body plasticity +(brain/plastic.py) in every configuration of (α, τ½), learning from each position's **settlement** (a tag comes due when +its market settles, not after a fixed hold) — the memecoin replay (train/fly_replay.py) with per-row holds, edges instead +of raw scores, and outcomes y ∈ {0, 1} as the reward. The verdict (``ui_settings['kalshi_fly_replay']``) chooses the +configuration and decides whether the Kalshi fly may trade a paper book. +""" +from __future__ import annotations + +import json +import logging +import math +import threading +import time +from datetime import date, datetime, timezone + +import numpy as np +import torch + +from ..brain import device as brain_device +from ..brain import plastic +from ..db.apilog import record_event +from ..db.connection import transaction +from ..train import fly_calibrate +from ..train import fly_governance as gov +from ..train import fly_replay as R +from ..train import progress as prog +from ..train import selector +from ..train.decisions import DecisionSet, evaluate, random_trades, summarize, taken_idx +from . import decisions as KD +from . import fly as KF +from .strategies import effective, in_universe, label + +log = logging.getLogger(__name__) +KEY = "kalshi_fly_replay" +DEFAULT_START = R.DEFAULT_START +CONFIGS = R.CONFIGS +LABEL_LAG_S = R.LABEL_LAG_S +HOUR_S, DAY_S = R.HOUR_S, R.DAY_S +CHUNK = R.CHUNK + + +def _resolved_window(ts_o: np.ndarray, res_o: np.ndarray, upto: int, now: float, span_s: float) -> np.ndarray: + """Positions (< ``upto``) of the rows whose settlement label became known in (now − span, now].""" + hi = int(np.searchsorted(ts_o[:upto], now, side="right")) + r = res_o[:hi] + return np.flatnonzero((r > now - span_s) & (r <= now)) + + +def _recalibrate(ds, order, ts_o, res_o, upto, scores, lines, sizings, now, arms, line_log) -> None: + """One edge line and sizing per strategy from the frozen fly's edges on the positions settled in the last window.""" + for s_, arm in enumerate(arms): + pos = _resolved_window(ts_o, res_o, upto, now, fly_calibrate.WINDOW_DAYS * DAY_S) + sc = scores[0, s_, pos]; has = np.isfinite(sc); pos = pos[has]; rows = order[pos] + y = label(ds, arm)[rows] + cal = fly_calibrate.calibrate(ds.ts[rows], ds.mint[rows], ds.hold_s[rows], sc[has], y, prev_line=float(lines[0, s_]), prev_sizing=sizings[0][s_]) + for c in range(lines.shape[0]): + key = (c, s_) + if key not in line_log: + line_log[key] = [(-math.inf, float(lines[c, s_]))] + lines[c, s_] = cal.line; sizings[c][s_] = cal.sizing + line_log[key].append((now, float(cal.line))) + + +def _governance(ds, order, ts_o, res_o, upto, scores, lines, line_log, bank, now, arms, counts) -> None: + for s_, arm in enumerate(arms): + p3 = _resolved_window(ts_o, res_o, upto, now, gov.SHADOW_DAYS * DAY_S); p1 = _resolved_window(ts_o, res_o, upto, now, gov.IC_HOURS * HOUR_S) + r3 = order[p3]; r1 = order[p1]; y = label(ds, arm) + lf = R._line_at(line_log, (0, s_), ts_o[p3], float(lines[0, s_])) + rf = gov.picks_returns(ds.ts[r3], ds.mint[r3], ds.hold_s[r3], np.nan_to_num(scores[0, s_, p3], nan=-np.inf), lf, y[r3]) + drift = bank.drift(s_).tolist() if bank.learn.shape[0] > 1 else bank.drift().tolist() + for c in range(1, lines.shape[0]): + lc = R._line_at(line_log, (c, s_), ts_o[p3], float(lines[c, s_])) + rp = gov.picks_returns(ds.ts[r3], ds.mint[r3], ds.hold_s[r3], np.nan_to_num(scores[c, s_, p3], nan=-np.inf), lc, y[r3]) + s1 = scores[c, s_, p1]; has = np.isfinite(s1) + out = gov.check((rp[np.isfinite(rp)], rf[np.isfinite(rf)]), (s1[has], ds.outcome[r1][has]), drift[c]) + counts[(c, s_)]["checks"] += 1 + for t in out["triggers"]: + counts[(c, s_)][t] += 1 + counts[(c, s_)]["any"] += bool(out["triggers"]) + + +def _book(ds, order, ts_o, scores, line_log, choice, boot_lines, arms, mask_pos): + """The combined book of the per-strategy choices: per row, among the strategies at/above their line in force, the + one with the largest margin; (pick, per-row hold s = time to settlement, per-row return of that strategy's arm).""" + N = len(ds.y); key = np.full(N, -np.inf); pick = np.zeros(N, bool); ret = np.full(N, np.nan, np.float32) + for s_, c in enumerate(choice): + Sf, Lf = R._arm(ds, order, ts_o, scores[c, s_], line_log, (c, s_), boot_lines[s_], mask_pos) + lab = label(ds, arms[s_]) + ok = (Sf >= Lf) & np.isfinite(lab) + m = np.where(ok, Sf - Lf, -np.inf); take = ok & (m > key) + key = np.where(take, m, key); pick |= ok + ret = np.where(take, lab, ret) + return pick, np.where(pick, ds.hold_s, 0.0), ret + + +def run(days: int | None = None, start_day: int = DEFAULT_START, configs: list | None = None, stop: threading.Event | None = None, device=None, + ds: DecisionSet | None = None, fly=None, boot: dict | None = None, g=None, save_verdict: bool = True) -> dict: + configs = [(float(a), float(h)) for a, h in (configs or CONFIGS)] + if configs[0][0] != 0.0: + configs = [(0.0, math.inf)] + configs + t0 = time.time() + if ds is None: + prog.update("kalshi fly replay: building decision points", 0, 1, force=True) + ds = KD.build(days=days) + dl = ds.days + if len(dl) <= start_day + 2: + raise RuntimeError(f"the Kalshi corpus has {len(dl)} days; the replay needs more than {start_day + 2}") + Sday = dl[start_day] + if fly is None: + fly, boot = KF.bootstrap(ds, Sday, stop=stop, g=g, device=device) + if fly is None: + return {"stopped": True, **(boot or {})} + boot = boot or {} + names = fly.strategies; NS = len(names); arms = [fly.arm(s_) for s_ in range(NS)] + S_epoch = datetime(Sday.year, Sday.month, Sday.day, tzinfo=timezone.utc).timestamp() + uni = in_universe(ds.X, ds.cols) + hist_from = S_epoch - fly_calibrate.WINDOW_DAYS * DAY_S + rows = np.flatnonzero(uni & (ds.ts >= hist_from)) + order = rows[np.argsort(ds.ts[rows], kind="stable")] + ts_o = ds.ts[order]; n = len(order); res_o = ts_o + ds.hold_s[order] + LABEL_LAG_S # when each row's settlement label is known + eff_o = np.stack([effective(ds.X[order], ds.cols, a) / 100.0 for a in arms]).astype(np.float32) # [NS, n] each arm's fee-inclusive price + live_from = int(np.searchsorted(ts_o, S_epoch, side="left")) + bank = plastic.PlasticBank(fly.net, configs, KF.SCALE, learn=fly.net.learn.cpu().numpy(), read=fly.net.read.cpu().numpy()) + brain_device.empty_cache(bank.dev); chunk = fly.net.batch_rows(ceiling=CHUNK) + log.info("kalshi fly replay on %s: %d rows per batch, %d rows, %d strategies", brain_device.describe(bank.dev), chunk, n, NS) + C = bank.C + scores = np.full((C, NS, n), np.nan, np.float32) # edges (p̂ − the arm's price) per configuration + lines = np.array([[fly.lines[k] for k in names]] * C, dtype=np.float64); sizings = [[list(fly.sizings[k]) for k in names] for _ in range(C)] + line_log: dict = {} + pending = plastic.PendingTags(fly.net.n_kc, fly.net.k_active) + outcome = torch.tensor(ds.outcome, dtype=torch.float32) + starts = np.flatnonzero(np.r_[True, ts_o[1:] != ts_o[:-1]]); ends = np.r_[starts[1:], n] + day_done = R._day_start(ts_o[live_from]) if live_from < n else None + next_gov = (ts_o[live_from] + HOUR_S) if live_from < n else None + gov_counts = {(c, s): {"shadow": 0, "ic": 0, "drift": 0, "any": 0, "checks": 0} for c in range(C) for s in range(NS)} + learn: dict = {} + nu_set = False; g_ = 0; n_updates = 0 + fly.net.eval() + while g_ < len(starts): + if stop is not None and stop.is_set(): + return {"stopped": True} + g1 = g_; rows_in = 0 + while g1 < len(starts) and rows_in < chunk: + rows_in += ends[g1] - starts[g1]; g1 += 1 + a, b = starts[g_], ends[g1 - 1] + Y, u0, k = fly.parts_all(ds.X[order[a:b]]); trig = fly.triggers(ds.X[order[a:b]], ds.cols) + if not nu_set: + bank.estimate_nu(Y[:, 0], u0, k); nu_set = True + for gi in range(g_, g1): + s0, s1 = starts[gi], ends[gi]; now = float(ts_o[s0]); lo, hi = s0 - a, s1 - a + if s0 >= live_from: + tags = pending.pop_due(now, device=bank.dev) + if tags is not None: + r = torch.tensor([float(outcome[row]) for row, _ in tags.keys], dtype=torch.float32) + R._learn_add(learn, now, bank.update(tags, r, now)); n_updates += len(tags) + if day_done is not None and R._day_start(now) > day_done: + day_done = R._day_start(now) + _recalibrate(ds, order, ts_o, res_o, s0, scores, lines, sizings, now, arms, line_log) + if next_gov is not None and now >= next_gov: + next_gov = now + HOUR_S + _governance(ds, order, ts_o, res_o, s0, scores, lines, line_log, bank, now, arms, gov_counts) + for s_ in range(NS): + loc = np.flatnonzero(trig[lo:hi, s_]) + if not len(loc): + continue + li = torch.as_tensor(lo + loc, device=Y.device) + with torch.no_grad(): + p, _ = bank.predict(Y[li, s_], u0[li], k[li], s=np.full(len(loc), s_)) + edge = p.clamp(0.0, 1.0) - torch.as_tensor(eff_o[s_, s0 + loc], device=p.device)[None] + scores[:, s_, s0 + loc] = edge.float().cpu().numpy() + if s0 >= live_from: + w = plastic.row_weights(edge, torch.tensor(lines[:, s_], dtype=edge.dtype, device=edge.device)) + keep = np.flatnonzero((w > 0).any(0).cpu().numpy()) + if len(keep): + kl = loc[keep]; due = res_o[s0 + kl] + for d_ in np.unique(due): # one push per settlement time + sub = np.flatnonzero(due == d_); kk = kl[sub]; ki = torch.as_tensor(lo + kk, device=Y.device) + pending.push([(int(order[s0 + j]), s_) for j in kk], ts_o[s0 + kk], float(d_), + Y[ki, s_], u0[ki], k[ki], w[:, torch.as_tensor(keep[sub], device=w.device)], s=s_) + if g_ == 0 or (g1 // 50) != (g_ // 50): + prog.update("kalshi fly replay: learning from settlements", int(b), n, day=str(datetime.fromtimestamp(float(ts_o[b - 1]), timezone.utc).date()), + updates=n_updates, pending=len(pending), drift=[round(float(x), 4) for x in bank.drift()], eta_s=(n - b) * (time.time() - t0) / max(b, 1)) + g_ = g1 + return _verdict(ds, order, ts_o, live_from, scores, line_log, configs, bank, gov_counts, boot, Sday, save_verdict, time.time() - t0, fly, names, arms, learn) + + +def _verdict(ds, order, ts_o, live_from, scores, line_log, configs, bank, gov_counts, boot, Sday, save_verdict, secs, fly, names, arms, learn) -> dict: + days_r = sorted(set(ds.day[order[live_from:]].tolist())) + sel_days, ev_days = days_r[: len(days_r) // 2], days_r[len(days_r) // 2:] + day_o = ds.day[order] + live_pos = np.arange(len(order)) >= live_from + sel_pos = live_pos & np.isin(day_o, sel_days); ev_pos = live_pos & np.isin(day_o, ev_days) + boot_lines = [fly.lines[k] for k in names] + per_strategy = {}; choice = [] + for s_, name in enumerate(names): + y = label(ds, arms[s_]); table = [] + for c, (alpha, hl) in enumerate(configs): + Sf, Lf = R._arm(ds, order, ts_o, scores[c, s_], line_log, (c, s_), boot_lines[s_], sel_pos) + tr = taken_idx(ds.ts, ds.mint, ds.hold_s, np.flatnonzero((Sf >= Lf) & np.isfinite(y))); r = y[tr] + table.append({"config": c, "alpha": alpha, "half_life_days": hl if math.isfinite(hl) else None, "trades": int(len(r)), "total": float(np.nansum(r)), + "mean": float(np.nanmean(r)) if len(r) else None, "governance": gov_counts.get((c, s_))}) + ok = [t for t in table if t["alpha"] > 0 and t["trades"] >= selector.MIN_LINE_TRADES] + best = max(ok, key=lambda t: t["total"])["config"] if ok else 0 + choice.append(best) + per_strategy[name] = {"configs": table, "chosen": table[best] if ok else None, "alpha": table[best]["alpha"], "half_life_days": table[best]["half_life_days"], + "arm": arms[s_], "hold_min": arms[s_], "plastic": bool(ok)} + + def judge(ch): + pick, hold, ret = _book(ds, order, ts_o, scores, line_log, ch, boot_lines, arms, ev_pos) + test = np.zeros(len(ds.y), bool); test[order[ev_pos]] = True + ev = evaluate(ds, np.where(pick, 1.0, np.nan), test & pick, 0.5, "kalshi fly", hold_s=hold, returns=ret) + rnd = summarize(random_trades(ds, test, max(1, int(ev["pooled"]["n"] or 1)), hold_s=ds.hold_s, returns=ds.fwd_pess)) + return ev, rnd + + out = {"S": str(Sday), "selection_days": [str(sel_days[0]), str(sel_days[-1])] if sel_days else None, "evaluation_days": [str(ev_days[0]), str(ev_days[-1])] if ev_days else None, + "strategies": names, "arms": arms, "per_strategy": per_strategy, "configs": per_strategy[names[0]]["configs"], + "bootstrap": {k: boot.get(k) for k in ("lines", "calibration", "diagnostics", "gates_ok", "gate_failures")}, "data": KF.KALSHI_FLY_VERSION, + "secs": secs, "finished_at": datetime.now(timezone.utc).isoformat(), "learning": R._learning_series(learn or {})} + ev0, rnd0 = judge([0] * len(names)) + out["frozen"] = {**ev0["pooled"], "random": rnd0} + if not any(v["plastic"] for v in per_strategy.values()): + out.update(passed=False, reason="no strategy's plastic configuration made 100 trades on the selection half") + else: + ev, rnd = judge(choice) + passed, why = selector.deploy_decision(ev["pooled"], rnd) + out.update(passed=bool(passed), reason=why, chosen=per_strategy[names[0]]["chosen"], evaluation=ev["pooled"], random=rnd, per_day=ev["per_day"], + alpha=per_strategy[names[0]]["alpha"], half_life_days=per_strategy[names[0]]["half_life_days"], + plastic_beats_frozen=bool((ev["pooled"]["mean"] or -1) > (ev0["pooled"]["mean"] or -1))) + log.info("kalshi fly replay from %s: %s — %s | frozen %s", Sday, "PASSED" if out.get("passed") else "FAILED", out.get("reason"), out["frozen"]) + if save_verdict: + record_event("info" if out.get("passed") else "warning", "kalshi_fly_replay", "kalshi fly replay " + ("passed" if out.get("passed") else "failed"), + {k: out.get(k) for k in ("S", "passed", "reason", "evaluation", "random", "frozen", "plastic_beats_frozen")}) + try: + with transaction() as conn: + conn.execute("INSERT INTO ui_settings (key, value) VALUES (%s, %s) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()", + (KEY, json.dumps(prog._finite(out), default=str))) + except Exception: + log.exception("could not store the Kalshi replay verdict") + try: + c0 = choice[0] if choice else 0 + with transaction() as conn: + for rec in out["learning"]: + d = date.fromisoformat(rec["day"]); pick = lambda k: (rec[k][c0] if len(rec.get(k) or []) > c0 else None) + conn.execute("INSERT INTO kalshi_fly_updates (hour, n, mean_delta, mean_abs_delta, step, capped, drift, ic, detail) " + "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT (hour) DO NOTHING", + (datetime(d.year, d.month, d.day, tzinfo=timezone.utc), rec["n"], pick("mean_delta"), pick("mean_abs_delta"), + pick("step"), pick("capped"), pick("drift"), None, json.dumps({"source": "replay", "config": c0, "per_config_drift": rec["drift"]}, default=str))) + except Exception: + log.exception("could not store the Kalshi replay's learning statistics (the verdict is stored)") + prog.update("kalshi fly replay: done", 1, 1, force=True, passed=out.get("passed"), reason=out.get("reason")) + return out + + +def verdict() -> dict | None: + with transaction() as conn: + r = conn.execute("SELECT value FROM ui_settings WHERE key = %s", (KEY,)).fetchone() + v = (r["value"] if isinstance(r["value"], dict) else json.loads(r["value"] or "{}")) if r else None + return v if v and v.get("data") == KF.KALSHI_FLY_VERSION else None + + +def main(days: int | None = None, start_day: int = DEFAULT_START, stop_event: threading.Event | None = None) -> dict: + """The replay; when it passes and no Kalshi fly may trade yet, the live fly is bootstrapped at once.""" + prog.set_stop_event(stop_event); prog.clear() + out = run(days=days, start_day=start_day, stop=stop_event) + if out.get("passed") and not (stop_event is not None and stop_event.is_set()): + with transaction() as conn: + have = KF.latest_deployable(conn) + if have is None: + log.info("kalshi replay passed: bootstrapping the live Kalshi fly") + out["live_bootstrap"] = {k: v for k, v in KF.main(days=days, stop_event=stop_event).items() if k in ("snapshot_id", "lines", "gates_ok", "gate_failures", "calibration")} + return out diff --git a/fly_trader/kalshi/fly_session.py b/fly_trader/kalshi/fly_session.py new file mode 100644 index 0000000000000000000000000000000000000000..2b0965ab5d810db105f9724d00e114d7cb7cfef3 --- /dev/null +++ b/fly_trader/kalshi/fly_session.py @@ -0,0 +1,542 @@ +"""The plastic Kalshi fly: trading the two paper arms (``paper_kalshi_taker``, ``paper_kalshi_maker``) on the Kalshi minute +engine and learning from every settlement (agent/fly_session.py for prediction markets). + +It starts once its design passed the replay (kalshi/fly_replay.verdict) and a deployable bootstrap exists +(kalshi/fly.latest_deployable); the replay's chosen (α, half-life) per strategy is its mushroom body's learning rate +and forgetting (brain/plastic.py). Each minute (kalshi/engine.py): + +1. labels: tags whose market has now settled (the stream writes ``kalshi_markets.result``) teach the mushroom body the + outcome y ∈ {0, 1} of their side (δ = y − p̂); a tag due before its market resolved is re-armed hourly, and dropped + as unknown a week after close; +2. calibration: at the first minute of each UTC day, every strategy's edge line and sizing, plastic and frozen, from the + rows resolved in the last seven days (``kalshi_fly_scored``; the taker label is the return at the effective ask, the + maker label the return of a bid one tick inside the ask when a later minute's ask reached it); +3. scoring: eligible (market, minute, side) rows are scored by the plastic fly and its frozen shadow — p̂ per strategy, + the edge p̂ − the arm's fee-inclusive price — stored and tagged; only rows at or above the line teach it; +4. trading (fresh stream, trade minutes): the taker arm (kalshi/paper.taker_entries) and the maker arm (kalshi/maker.plan + + adjudicate) from the same decision, then settlements and marks per book; the live mirror (kalshi/live.py) when + ``KALSHI_LIVE_ENABLED`` and the prerequisites hold. + +Hourly: learning statistics (``kalshi_fly_updates``), a snapshot of the plastic state (kind 'kalshi_fly_plastic'), the +rollback checks (train/fly_governance.py; a third rollback in 7 days freezes learning and asks kalshi/pipeline.py for a +re-bootstrap). State: ``data/brain/plastic_kalshi/state.pt``; commands ``ui_settings['kalshi_fly_command']``; status +``ui_settings['kalshi_fly_status']``; activity for the 3D view in ``data/brain/activity_kalshi/``. +""" +from __future__ import annotations + +import json +import logging +import math +import os +import time +import uuid +from datetime import datetime, timezone +from pathlib import Path + +import numpy as np +import torch + +from .. import config +from ..brain import activity, device as brain_device, plastic +from ..brain.connectome import current_connectome_path +from ..db.apilog import record_event +from ..db.connection import transaction +from ..ops.reset import kalshi_activity_dir, kalshi_fly_state_dir +from ..train import fly_calibrate, fly_governance as gov +from . import fly as KF, maker, paper as P +from .decisions import maker_filled, maker_rest +from .features import K_COLS, KIDX +from .strategies import effective + +log = logging.getLogger(__name__) +NAME, KIND = "kalshi_fly", "kalshi_fly" +STATUS_KEY, COMMAND_KEY = "kalshi_fly_status", "kalshi_fly_command" +SNAP_KIND = "kalshi_fly_plastic" +LABEL_LAG_S = 60.0 +REARM_S, UNKNOWN_AFTER_S = 3600.0, 7 * 86400.0 +ROLLBACK_PAUSE_S, ROLLBACK_LIMIT, ROLLBACK_WINDOW_S, ROLLBACK_MIN_AGE_S = 86400.0, 3, 7 * 86400.0, 86400.0 +SNAP_HOURLY_DAYS, SCORED_KEEP_DAYS = 7, 35 + + +def send_command(cmd: str) -> None: + assert cmd in ("pause", "resume", "rollback"), cmd + with transaction() as conn: + conn.execute("INSERT INTO ui_settings (key, value) VALUES (%s, %s) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()", + (COMMAND_KEY, json.dumps({"cmd": cmd, "at": datetime.now(timezone.utc).isoformat()}))) + + +def try_start() -> tuple["KalshiFlyBook | None", str]: + from . import fly_replay + v = fly_replay.verdict() + if not v: + return None, "the Kalshi fly's replay has not run on the current definitions (fly-trader kalshi-fly-replay)" + if not v.get("passed"): + return None, f"the Kalshi fly's replay did not pass: {v.get('reason')}" + with transaction() as conn: + boot = KF.latest_deployable(conn) + if boot is None: + return None, "no deployable Kalshi fly bootstrap (the Kalshi training pipeline bootstraps one)" + try: + return KalshiFlyBook(boot, v), "ok" + except Exception as e: + log.exception("kalshi fly book failed to start") + return None, f"kalshi fly book failed to start: {type(e).__name__}: {e}" + + +def arm_label(x: np.ndarray, side_won: float, arm: str, ticker: str, filled: bool | None) -> float | None: + """The realised net return per dollar of the arm on a row (the training labels): taker at the effective ask; maker + at its bid (kalshi/decisions.maker_rest) plus the maker fee, when the resting bid filled (None when it did not, or is + not known).""" + eff = float(effective(x[None], K_COLS, arm)[0]) + if arm == "maker" and not filled: + return None + if eff <= 0: + return None + return float((100.0 * side_won - eff) / eff) + + +class KalshiFlyBook: + name = NAME + + def __init__(self, boot: dict, verdict: dict): + self.done = False; self.mirror = None; self.verdict = verdict + self.state_path = kalshi_fly_state_dir() / "state.pt" + self._load_bootstrap(boot) + self.run_id = str(uuid.uuid4()); self.beat_no = 0 + with transaction() as conn: + conn.execute("INSERT INTO runs (run_id, kind, config, brain_snapshot_id, status) VALUES (%s,%s,%s,%s,'running')", + (self.run_id, KIND, json.dumps({"channels": self.cfg, "books": list(P.BOOKS.values())}, default=str), self.boot_id)) + self._restore_tags(conn) + record_event("info", NAME, "kalshi fly session started", {"run_id": self.run_id, "bootstrap": self.boot_id, "channels": self.cfg, "lines": self.lines["plastic"], + "pending": len(self.pending), "learning_frozen": self.learning_frozen}) + log.info("kalshi fly session: bootstrap %d on %s, channels %s, lines %s, %d pending tags", self.boot_id, brain_device.describe(self.fly.net.dev), self.cfg, + self.lines["plastic"], len(self.pending)) + + # ---- model and state ---- + def _load_bootstrap(self, boot: dict) -> None: + self.boot_id = int(boot["id"]) + self.fly = KF.load(boot["path"]) + try: + self.connectome_name = current_connectome_path().name + except Exception: + self.connectome_name = "" + self.capture_ok = True + self.idx = np.asarray([K_COLS.index(c) for c in self.fly.cols], dtype=int) + self.names = list(self.fly.strategies); self.arms = {k: self.fly.arm(j) for j, k in enumerate(self.names)} + per = self.verdict.get("per_strategy") or {self.names[0]: {"alpha": self.verdict.get("alpha", 0.0), "half_life_days": self.verdict.get("half_life_days")}} + self.cfg = {k: (float((per.get(k) or {}).get("alpha") or 0.0), (per.get(k) or {}).get("half_life_days")) for k in self.names} + a0, h0 = self.cfg[self.names[0]] + net = self.fly.net + self.bank = plastic.PlasticBank(net, [(a0, float(h0) if h0 is not None else math.inf)], KF.SCALE, learn=net.learn.cpu().numpy(), read=net.read.cpu().numpy()) + for j, k in enumerate(self.names): + a, h = self.cfg[k]; cols = self.bank.learn[j] + self.bank.alpha[0, cols] = a; self.bank.half_life_s[0, cols] = float(h) * 86400.0 if h is not None else float("inf") + self.pending = plastic.PendingTags(net.n_kc, net.k_active) + self.watch: dict[str, int] = {} # ticker → pending tags on it + self.lines = {"plastic": dict(self.fly.lines), "frozen": dict(self.fly.lines)} + self.sizing = {"plastic": {k: list(v) for k, v in self.fly.sizings.items()}, "frozen": {k: list(v) for k, v in self.fly.sizings.items()}} + self.applied_through = -math.inf; self.learning_frozen = False; self.checks_paused_until = {k: 0.0 for k in self.names} + self.last_day: int | None = None; self.last_hour: int | None = None + self.rollbacks: dict = {k: [] for k in self.names}; self.nu_set = False; self.hour_stats = self._empty_stats(); self.last_checks: dict = {} + s = self._read_state() + if s and s.get("bootstrap_id") == self.boot_id and s.get("cfg") == self.cfg: + self.bank.load_state(s["bank"]); self.nu_set = True + self.lines, self.sizing = s["lines"], s["sizing"] + self.applied_through, self.learning_frozen, self.checks_paused_until = s["applied_through"], s["learning_frozen"], s["checks_paused_until"] + self.last_day, self.last_hour, self.rollbacks = s["last_day"], s["last_hour"], s["rollbacks"] + elif s: + with transaction() as conn: + conn.execute("UPDATE kalshi_fly_scored SET state = 'dropped', x = NULL WHERE state = 'pending' AND bootstrap_id IS DISTINCT FROM %s", (self.boot_id,)) + + def _read_state(self) -> dict | None: + if not self.state_path.exists(): + return None + try: + return torch.load(self.state_path, map_location="cpu", weights_only=True) + except Exception: + log.exception("kalshi fly state unreadable; starting from the bootstrap") + return None + + def _state(self) -> dict: + return {"bootstrap_id": self.boot_id, "cfg": self.cfg, "bank": self.bank.state(), "lines": self.lines, "sizing": self.sizing, "applied_through": self.applied_through, + "learning_frozen": self.learning_frozen, "checks_paused_until": self.checks_paused_until, "last_day": self.last_day, "last_hour": self.last_hour, + "rollbacks": self.rollbacks, "saved_at": time.time()} + + def _write_state(self) -> None: + self.state_path.parent.mkdir(parents=True, exist_ok=True) + tmp = self.state_path.with_suffix(".tmp"); torch.save(self._state(), tmp); os.replace(tmp, self.state_path) + + def _restore_tags(self, conn) -> None: + rows = conn.execute("SELECT ts, ticker, side, strategy, x, score, line, due_ts FROM kalshi_fly_scored WHERE state = 'pending' AND bootstrap_id = %s AND x IS NOT NULL ORDER BY ts", + (self.boot_id,)).fetchall() + groups: dict[tuple, list] = {} + for r in rows: + if r["strategy"] in self.arms: + groups.setdefault((r["ts"].timestamp(), r["strategy"]), []).append(r) + for (t, name), rs in sorted(groups.items()): + j = self.names.index(name) + Y, u0, k = self.fly.parts_all(np.asarray([r["x"] for r in rs], dtype=np.float32)) + w = plastic.row_weights(torch.tensor([[float(r["score"]) for r in rs]]), torch.tensor([float(rs[0]["line"])])) + due = min(r["due_ts"].timestamp() if r["due_ts"] else t for r in rs) + self.pending.push([(t, r["ticker"], r["side"], name) for r in rs], np.full(len(rs), t), max(due, time.time()), Y[:, j], u0, k, w, s=j) + for r in rs: + self.watch[r["ticker"]] = self.watch.get(r["ticker"], 0) + 1 + + def maybe_reload(self) -> bool: + from . import fly_replay + with transaction() as conn: + boot = KF.latest_deployable(conn) + v = fly_replay.verdict() + if boot is None or v is None or not v.get("passed") or int(boot["id"]) == self.boot_id: + return False + old = self.boot_id; self.verdict = v + self._write_state(); self._load_bootstrap(boot) + with transaction() as conn: + conn.execute("UPDATE runs SET brain_snapshot_id = %s WHERE run_id = %s", (self.boot_id, self.run_id)) + self._write_state() + record_event("info", NAME, f"switched to kalshi fly bootstrap #{self.boot_id}", {"from": old, "to": self.boot_id, "lines": self.lines["plastic"]}) + return True + + # ---- engine interface ---- + def tickers_watched(self) -> set[str]: + return set(self.watch) + + def on_minute(self, ctx) -> dict: + conn, t_now = ctx.conn, ctx.t_start + self._commands(conn, ctx.m1_epoch) + learned = self._learn(conn, ctx) + day = int(t_now // 86400) + if self.last_day is None: + self.last_day = day + elif day > self.last_day: + self.last_day = day; self._calibrate(conn, ctx.m1_epoch) + scored = self._score(conn, ctx) + drift = {k: float(self.bank.drift(j)[0]) if self.bank.learn.shape[0] > 1 else float(self.bank.drift()[0]) for j, k in enumerate(self.names)} + out = {"minute": ctx.m1.isoformat(), "eligible": len(ctx.keys), "markets_active": ctx.n_rows // 2, "picks": scored["picks"], "lines": self.lines["plastic"], + "frozen_lines": self.lines["frozen"], "arms": self.arms, "drift": max(drift.values()) if drift else 0.0, "drift_by_strategy": drift, "pending": len(self.pending), + "learned": learned, "learning_frozen": self.learning_frozen, "channels": self.cfg, "bootstrap": self.boot_id, "device": str(self.fly.net.dev)} + self.beat_no += 1 + beat = conn.execute("INSERT INTO beats (run_id, ts, beat_no, n_slots_active, notes) VALUES (%s,%s,%s,%s,%s) RETURNING id", + (self.run_id, ctx.m1, self.beat_no, len(ctx.keys), json.dumps({"minute": ctx.m1.isoformat(), "markets_active": ctx.n_rows // 2}))).fetchone()["id"] + books = {} + for arm, book in P.BOOKS.items(): + results = ctx.settled([p["ticker"] for p in P.open_positions(conn, book)]) + settled = P.settle(conn, book=book, results=results, ts=ctx.m1, run_id=self.run_id, beat_id=beat) + books[arm] = {"settled": len(settled), "settled_pnl": sum(s["realized_cents"] for s in settled) / 100.0} + adj = maker.adjudicate(conn, book=P.BOOKS["maker"], run_id=self.run_id, beat_id=beat, ts=ctx.m1, now=ctx.m1_epoch, extremes=ctx.extremes, metas=ctx.metas) + books["maker"].update(filled=len(adj["filled"]), expired=adj["expired"]) + live_plan = {"taker_entries": [], "maker": {"posted": [], "canceled": 0, "replaced": 0}} + if ctx.trade and ctx.fresh: + tk = P.taker_entries(conn, book=P.BOOKS["taker"], run_id=self.run_id, beat_id=beat, ts=ctx.m1, picks=scored["taker"], quotes=ctx.quotes) + mk = maker.plan(conn, book=P.BOOKS["maker"], run_id=self.run_id, beat_id=beat, ts=ctx.m1, now=ctx.m1_epoch, picks=scored["maker"], quotes=ctx.quotes, metas=ctx.metas) + books["taker"].update(entered=tk["entered"], blocked=tk["blocked"]); books["maker"].update(posted=len(mk["posted"]), canceled=mk["canceled"], replaced=mk["replaced"], resting=mk["resting"]) + live_plan = {"taker_entries": tk["entries"], "maker": mk} + out["stage"] = "trading" + else: + out["stage"] = "learning (not trading this minute)" if ctx.trade else "catching up" + for arm, book in P.BOOKS.items(): + books[arm].update(P.mark(conn, book=book, quotes=ctx.quotes, ts=ctx.m1, beat_id=beat)) + out["books"] = books + if config.KALSHI_LIVE_ENABLED: + out["live"] = self._live(ctx, beat, live_plan) + hour = int(ctx.m1_epoch // 3600) + if self.last_hour is None: + self.last_hour = hour + elif hour > self.last_hour: + self.last_hour = hour; self._hourly(conn, ctx.m1_epoch) + self._write_state() + if learned.get("keys"): + with conn.cursor() as cur: + cur.executemany("UPDATE kalshi_fly_scored SET label = %s, state = %s, resolved_at = now(), x = NULL WHERE ts = %s AND ticker = %s AND side = %s AND strategy = %s", learned.pop("keys")) + if learned.get("rearmed"): + with conn.cursor() as cur: + cur.executemany("UPDATE kalshi_fly_scored SET due_ts = %s WHERE ts = %s AND ticker = %s AND side = %s AND strategy = %s", learned.pop("rearmed")) + learned.pop("keys", None); learned.pop("rearmed", None) + out["checks"] = self.last_checks; out["updated_at"] = datetime.now(timezone.utc).isoformat() + conn.execute("INSERT INTO ui_settings (key, value) VALUES (%s, %s) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()", (STATUS_KEY, json.dumps(out, default=str))) + return out + + def _live(self, ctx, beat_id: int, plan: dict) -> dict: + missing = config.kalshi_live_prerequisites_missing() + if missing: + return {"stage": "live gated: " + ", ".join(missing)} + if self.mirror is None: + from .live import KalshiLiveMirror + self.mirror = KalshiLiveMirror() + record_event("info", "kalshi_live", "the Kalshi fly trades the subaccount", {"run_id": self.run_id, "subaccount": config.KALSHI_SUBACCOUNT}) + try: + return self.mirror.minute(ctx, run_id=self.run_id, beat_id=beat_id, taker_entries=plan["taker_entries"], maker_plan=plan["maker"]) + except Exception as e: + log.exception("kalshi live mirror failed this minute") + return {"stage": "live error", "error": f"{type(e).__name__}: {e}"[:300]} + + def finish(self) -> None: + try: + self._write_state() + except Exception: + log.exception("kalshi fly state write failed at finish") + with transaction() as conn: + conn.execute("UPDATE runs SET ended_at = now(), status = 'finished' WHERE run_id = %s", (self.run_id,)) + record_event("info", NAME, "kalshi fly session stopped", {"run_id": self.run_id, "minutes": self.beat_no}) + + # ---- learning ---- + @staticmethod + def _empty_stats() -> dict: + return {"n": 0, "sum_delta": 0.0, "sum_abs": 0.0, "step": 0.0, "capped": 0, "unknown": 0} + + def _maker_filled(self, conn, ticker: str, side: str, t: float, x: np.ndarray, close_ts: float | None) -> bool: + """Did the maker arm's bid for the row scored in minute ``t`` (inputs ``x``) fill over the order's life — the + minutes after the decision, until ``KALSHI_MAKER_TTL_H`` later or ``KALSHI_MAKER_QUIET_MIN`` before close: the label's + rule (kalshi/decisions.maker_filled) on the stream's minute extremes.""" + bid = float(x[KIDX["side_bid"]]); rest = float(maker_rest(bid, x[KIDX["side_ask"]])) + end = t + 60.0 + config.KALSHI_MAKER_TTL_H * 3600.0 + if close_ts is not None: + end = min(end, close_ts - config.KALSHI_MAKER_QUIET_MIN * 60.0) + cols = "min(yes_ask_low) AS a, min(yes_sold_low) AS s" if side == "yes" else "100 - max(yes_bid_high) AS a, 100 - max(yes_bought_high) AS s" + r = conn.execute(f"SELECT {cols} FROM kalshi_minutes WHERE ticker = %s AND ts > %s AND ts + interval '1 minute' <= %s", + (ticker, datetime.fromtimestamp(t, timezone.utc), datetime.fromtimestamp(end, timezone.utc))).fetchone() + f = lambda v: float(v) if v is not None else math.nan + return r is not None and bool(maker_filled(rest, bid, f(r["a"]), f(r["s"]))) + + def _learn(self, conn, ctx) -> dict: + now = ctx.m1_epoch + tags = self.pending.pop_due(now, device=self.bank.dev) + if tags is None: + return {"n": 0} + results = ctx.settled({tk for _, tk, _, _ in tags.keys}) + labels = np.full(len(tags.keys), np.nan); keep = np.zeros(len(tags.keys), bool); rearm = [] + rows = {} + need = [(datetime.fromtimestamp(t, timezone.utc), tk, side, name) for t, tk, side, name in tags.keys] + if need: + for r in conn.execute("SELECT ts, ticker, side, strategy, x FROM kalshi_fly_scored WHERE (ts, ticker, side, strategy) IN (SELECT * FROM unnest(%s::timestamptz[], %s::text[], %s::text[], %s::text[]))", + ([n[0] for n in need], [n[1] for n in need], [n[2] for n in need], [n[3] for n in need])).fetchall(): + rows[(r["ts"].timestamp(), r["ticker"], r["side"], r["strategy"])] = r["x"] + out_keys = [] + for i, (t, tk, side, name) in enumerate(tags.keys): + res = results.get(tk) + if res in ("yes", "no"): + y = 1.0 if res == side else 0.0; labels[i] = y; keep[i] = True + x = rows.get((t, tk, side, name)); arm = self.arms[name]; lab = None + if x is not None: + xa = np.asarray(x, dtype=np.float32) + meta = ctx.metas.get(tk) + filled = self._maker_filled(conn, tk, side, t, xa, meta.close_ts if meta is not None else None) if arm == "maker" else None + lab = arm_label(xa, y, arm, tk, filled) + out_keys.append((lab, "resolved" if lab is not None else "unfilled", datetime.fromtimestamp(t, timezone.utc), tk, side, name)) + elif now - t > UNKNOWN_AFTER_S: + out_keys.append((None, "unknown", datetime.fromtimestamp(t, timezone.utc), tk, side, name)) + else: + rearm.append(i) + for i, (t, tk, side, name) in enumerate(tags.keys): + if i not in rearm: + c = self.watch.get(tk, 0) - 1 + if c > 0: + self.watch[tk] = c + else: + self.watch.pop(tk, None) + rearmed = [] + if rearm: + sub = tags.subset(np.isin(np.arange(len(tags.keys)), rearm)) + for j in sorted({int(s) for s in sub.s}): + m = sub.s == j; ss = sub.subset(m) + self.pending.push(ss.keys, ss.ts, now + REARM_S, ss.y_dn, ss.u0, ss.k, ss.w, s=j) + rearmed = [(datetime.fromtimestamp(now + REARM_S, timezone.utc), datetime.fromtimestamp(t, timezone.utc), tk, side, name) for t, tk, side, name in sub.keys] + stats = {"n": int(keep.sum()), "unknown": int(sum(1 for k in out_keys if k[1] == "unknown")), "rearmed": rearmed, "keys": out_keys, "waiting": len(rearm)} + if keep.any(): + tv = tags.subset(keep) + if self.learning_frozen: + self.bank.decay_to(now) + else: + st = self.bank.update(tv, torch.tensor(labels[keep], dtype=torch.float32), now) + stats.update(mean_delta=st["mean_delta"][0], mean_abs_delta=st["mean_abs_delta"][0], step=st["step"][0], capped=bool(st["capped"][0])) + h = self.hour_stats; h["n"] += stats["n"]; h["sum_delta"] += st["mean_delta"][0] * stats["n"]; h["sum_abs"] += st["mean_abs_delta"][0] * stats["n"] + h["step"] += st["step"][0]; h["capped"] += int(st["capped"][0]) + self.hour_stats["unknown"] += stats["unknown"] + self.applied_through = now + return stats + + def _score(self, conn, ctx) -> dict: + n = len(ctx.keys); empty = {"taker": [], "maker": [], "picks": 0} + if not n: + return empty + X = ctx.X[:, self.idx] + Y, u0, k, H = self.fly.parts_all_h(X); trig = self.fly.triggers(ctx.X, K_COLS) + self._capture(H, trig, ctx.t_start); del H + if not self.nu_set: + self.bank.estimate_nu(Y[:, 0], u0, k); self.nu_set = True + t = ctx.t_start; ts = datetime.fromtimestamp(t, timezone.utc); rows = [] + E = np.full((n, len(self.names)), -np.inf); Pm = np.full((n, len(self.names)), np.nan) + for j, name in enumerate(self.names): + loc = np.flatnonzero(trig[:, j]) + if not len(loc): + continue + li = torch.as_tensor(loc, device=Y.device); arm = self.arms[name] + eff = torch.as_tensor(effective(ctx.X[loc], K_COLS, arm) / 100.0, device=Y.device, dtype=Y.dtype) + with torch.no_grad(): + p, _ = self.bank.predict(Y[li, j], u0[li], k[li], s=np.full(len(loc), j)); p = p[0].clamp(0.0, 1.0) + fp = self.bank.frozen(Y[li, j], u0[li], s=np.full(len(loc), j)).clamp(0.0, 1.0) + sc, fr = p - eff, fp - eff + v, f = sc.float().cpu().numpy(), fr.float().cpu().numpy(); E[loc, j] = v; Pm[loc, j] = p.float().cpu().numpy() + line, fline = self.lines["plastic"][name], self.lines["frozen"][name] + due = [ctx.metas[ctx.keys[i][0]].close_ts + LABEL_LAG_S if ctx.metas.get(ctx.keys[i][0]) and ctx.metas[ctx.keys[i][0]].close_ts else t + ctx.hold_s[i] + LABEL_LAG_S for i in loc] + rows += [(ts, ctx.keys[i][0], ctx.keys[i][1], name, ctx.X[i].tolist(), float(v[q]), float(f[q]), line, fline, datetime.fromtimestamp(due[q], timezone.utc), self.boot_id) + for q, i in enumerate(loc)] + w = plastic.row_weights(sc[None], torch.tensor([line], dtype=sc.dtype, device=sc.device)) + keep = np.flatnonzero((w[0] > 0).cpu().numpy()) + if len(keep): + for d_ in sorted({due[q] for q in keep}): + qq = [q for q in keep if due[q] == d_]; ki = torch.as_tensor(loc[qq], device=Y.device) + self.pending.push([(t, ctx.keys[loc[q]][0], ctx.keys[loc[q]][1], name) for q in qq], np.full(len(qq), t), float(d_), Y[ki, j], u0[ki], k[ki], + w[:, torch.as_tensor(qq, device=w.device)], s=j) + for q in qq: + tk = ctx.keys[loc[q]][0]; self.watch[tk] = self.watch.get(tk, 0) + 1 + if rows: + with conn.cursor() as cur: + cur.executemany("INSERT INTO kalshi_fly_scored (ts, ticker, side, strategy, x, score, frozen_score, line, frozen_line, due_ts, bootstrap_id) " + "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT (ts, ticker, side, strategy) DO NOTHING", rows) + self.fly.lines = dict(self.lines["plastic"]); self.fly.sizings = {k_: list(v) for k_, v in self.sizing["plastic"].items()} + picks = {"taker": [], "maker": []} + for arm in ("taker", "maker"): + d = KF.kalshi_decide(self.fly, ctx.X, K_COLS, E, ctx.hold_s, only=arm, trig=trig) + for i, sname in enumerate(d["strategy"]): + if sname is None: + continue + j = self.names.index(sname) + picks[arm].append({"ticker": ctx.keys[i][0], "side": ctx.keys[i][1], "p": float(Pm[i, j]), "edge": float(d["score"][i]), "line": float(d["threshold"][i]), + "table": d["tables"][i], "strategy": sname, "hold_s": float(ctx.hold_s[i])}) + return {**picks, "picks": len(picks["taker"]) + len(picks["maker"])} + + def _capture(self, H: torch.Tensor, trig: np.ndarray, t: float) -> None: + if not self.capture_ok: + return + try: + sel = np.flatnonzero(trig.any(1)); n_c = int(len(sel)) + m = (H.index_select(0, torch.as_tensor(sel, device=H.device)) if n_c else H).mean(0) + activity.write(kalshi_activity_dir(), t, activity.quantise(m), n_rows=int(len(trig)), n_cand=n_c, connectome=self.connectome_name) + except Exception: + self.capture_ok = False + log.exception("kalshi brain activity capture disabled for this session") + + # ---- daily and hourly ---- + def _resolved(self, conn, since: float, cols: str, name: str) -> list[dict]: + return conn.execute(f"SELECT {cols} FROM kalshi_fly_scored WHERE state = 'resolved' AND label IS NOT NULL AND bootstrap_id = %s AND strategy = %s AND resolved_at >= %s ORDER BY ts", + (self.boot_id, name, datetime.fromtimestamp(since, timezone.utc))).fetchall() + + def _calibrate(self, conn, now: float) -> None: + day = datetime.fromtimestamp(now, timezone.utc).date() + for name in self.names: + rows = self._resolved(conn, now - fly_calibrate.WINDOW_DAYS * 86400.0, "ts, ticker, side, score, frozen_score, label, due_ts", name) + if not rows: + continue + ts = np.array([r["ts"].timestamp() for r in rows]); mint = np.array([f"{r['ticker']}:{r['side']}" for r in rows]); lab = np.array([r["label"] for r in rows], dtype=np.float64) + hold = np.array([max((r["due_ts"].timestamp() - r["ts"].timestamp()) if r["due_ts"] else 60.0, 60.0) for r in rows]) + for arm, col in (("plastic", "score"), ("frozen", "frozen_score")): + cal = fly_calibrate.calibrate(ts, mint, hold, np.array([r[col] for r in rows], dtype=np.float64), lab, prev_line=self.lines[arm][name], prev_sizing=self.sizing[arm][name]) + if cal.changed: + record_event("info", NAME, f"{name} {arm} edge line {self.lines[arm][name]:.3f} → {cal.line:.3f}", {"trades": cal.trades, "mean": cal.mean, "total": cal.total}) + self.lines[arm][name], self.sizing[arm][name] = cal.line, cal.sizing + conn.execute("INSERT INTO kalshi_fly_calibrations (day, arm, line, sizing, trades, total, mean, window_days) VALUES (%s,%s,%s,%s,%s,%s,%s,%s) " + "ON CONFLICT (day, arm) DO UPDATE SET line = EXCLUDED.line, sizing = EXCLUDED.sizing, trades = EXCLUDED.trades, total = EXCLUDED.total, mean = EXCLUDED.mean", + (day, f"{arm}:{name}", cal.line, json.dumps(cal.sizing), cal.trades, cal.total, cal.mean, fly_calibrate.WINDOW_DAYS)) + + def _drift(self, j: int) -> float: + return float(self.bank.drift(j)[0]) if self.bank.learn.shape[0] > 1 else float(self.bank.drift()[0]) + + def _checks(self, conn, now: float) -> dict: + out = {} + for j, name in enumerate(self.names): + rows = self._resolved(conn, now - gov.SHADOW_DAYS * 86400.0, "ts, ticker, side, score, line, frozen_score, frozen_line, label, due_ts, resolved_at", name) + if rows: + ts = np.array([r["ts"].timestamp() for r in rows]); mint = np.array([f"{r['ticker']}:{r['side']}" for r in rows]); lab = np.array([r["label"] for r in rows], dtype=np.float64) + hold = np.array([max((r["due_ts"].timestamp() - r["ts"].timestamp()) if r["due_ts"] else 60.0, 60.0) for r in rows]) + g = lambda c: np.array([r[c] for r in rows], dtype=np.float64) + rp = gov.picks_returns(ts, mint, hold, g("score"), g("line"), lab); rf = gov.picks_returns(ts, mint, hold, g("frozen_score"), g("frozen_line"), lab) + recent = np.array([r["resolved_at"].timestamp() for r in rows]) > now - gov.IC_HOURS * 3600.0 + out[name] = gov.check((rp, rf), (g("score")[recent], lab[recent]), self._drift(j)) + else: + out[name] = gov.check(None, None, self._drift(j)) + return out + + def _hourly(self, conn, now: float) -> None: + hour = datetime.fromtimestamp(math.floor(now / 3600) * 3600 - 3600, timezone.utc) + checks = self._checks(conn, now); self.last_checks = checks + h = self.hour_stats; first = checks.get(self.names[0]) or {} + conn.execute("INSERT INTO kalshi_fly_updates (hour, n, mean_delta, mean_abs_delta, step, capped, drift, ic, detail) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT (hour) DO NOTHING", + (hour, h["n"], h["sum_delta"] / h["n"] if h["n"] else None, h["sum_abs"] / h["n"] if h["n"] else None, h["step"], h["capped"], + max((c["drift"] for c in checks.values()), default=0.0), first.get("ic"), json.dumps({"unknown": h["unknown"], "checks": checks}, default=str))) + self.hour_stats = self._empty_stats() + good = not any(c["triggers"] for c in checks.values()) + sid = self._snapshot(conn, now, good, checks) + for name, c in checks.items(): + if c["triggers"] and now >= self.checks_paused_until.get(name, 0.0) and not self.learning_frozen: + self._rollback(conn, now, c, sid, name=name) + self._prune(conn, now) + activity.prune(kalshi_activity_dir(), now - activity.KEEP_S) + + def _snapshot(self, conn, now: float, good: bool, checks: dict) -> int: + d = kalshi_fly_state_dir(); d.mkdir(parents=True, exist_ok=True) + path = d / f"snap_{datetime.fromtimestamp(now, timezone.utc).strftime('%Y%m%dT%H%M')}.pt" + torch.save({"bank": self.bank.state(), "bootstrap_id": self.boot_id, "lines": self.lines, "sizing": self.sizing}, path) + note = {"bootstrap_id": self.boot_id, "data": KF.KALSHI_FLY_VERSION, "good": good, "checks": checks, "drift": max((c["drift"] for c in checks.values()), default=0.0), + "ic24": {k: c.get("ic") for k, c in checks.items()}} + return int(conn.execute("INSERT INTO brain_snapshots (run_id, path, kind, note) VALUES (%s,%s,%s,%s) RETURNING id", (self.run_id, str(path), SNAP_KIND, json.dumps(note, default=str))).fetchone()["id"]) + + def _commands(self, conn, now: float) -> None: + r = conn.execute("SELECT value FROM ui_settings WHERE key = %s", (COMMAND_KEY,)).fetchone() + if not r or not r["value"]: + return + v = r["value"] if isinstance(r["value"], dict) else json.loads(r["value"]) + conn.execute("DELETE FROM ui_settings WHERE key = %s", (COMMAND_KEY,)) + cmd = v.get("cmd") + if cmd == "pause": + self.learning_frozen = True; record_event("info", NAME, "learning paused by the operator") + elif cmd == "resume": + self.learning_frozen = False; self.rollbacks = {}; record_event("info", NAME, "learning resumed by the operator") + elif cmd == "rollback": + for j, name in enumerate(self.names): + self._rollback(conn, now, {"triggers": ["operator"], "drift": self._drift(j)}, None, count=False, name=name) + + def _rollback(self, conn, now: float, checks: dict, from_sid: int | None, count: bool = True, name: str | None = None) -> None: + name = name or self.names[0]; j = self.names.index(name); cols = self.bank.learn[j] + target = None + for r in conn.execute("SELECT id, path, note, ts FROM brain_snapshots WHERE kind = %s AND ts <= %s ORDER BY id DESC", + (SNAP_KIND, datetime.fromtimestamp(now - ROLLBACK_MIN_AGE_S, timezone.utc))).fetchall(): + n = json.loads(r["note"] or "{}") + ok = n.get("good") or not ((n.get("checks") or {}).get(name) or {}).get("triggers") + if ok and n.get("bootstrap_id") == self.boot_id and Path(r["path"]).exists(): + target = r; break + if target is not None: + snap = plastic.PlasticBank(self.fly.net, [(0.0, math.inf)], KF.SCALE, learn=self.bank.learn.cpu().numpy(), read=self.bank.read.cpu().numpy()) + st = torch.load(target["path"], map_location="cpu", weights_only=True)["bank"] + snap.D = torch.zeros_like(snap.D); snap.D[:, snap.mask.bool()] = st["D"].to(snap.dev).float() + self.bank.D[0][:, cols] = snap.D[0][:, cols] + else: + self.bank.D[0][:, cols] = 0.0 + self.checks_paused_until[name] = now + ROLLBACK_PAUSE_S + self.rollbacks[name] = [t for t in self.rollbacks.get(name, []) if t > now - ROLLBACK_WINDOW_S] + ([now] if count else []) + conn.execute("INSERT INTO kalshi_fly_rollbacks (reason, checks, from_snapshot, to_snapshot) VALUES (%s,%s,%s,%s)", + (f"{name}: " + ", ".join(checks["triggers"]), json.dumps(checks, default=str), from_sid, target["id"] if target is not None else None)) + record_event("warning", NAME, f"{name} channel rolled back ({', '.join(checks['triggers'])})", {"to_snapshot": target["id"] if target is not None else "bootstrap (D = 0)", "checks": checks}) + if count and len(self.rollbacks[name]) >= ROLLBACK_LIMIT: + self.learning_frozen = True + from . import pipeline + pipeline.request_run(fly=True, reason=f"{name}: {len(self.rollbacks[name])} rollbacks in 7 days") + record_event("error", NAME, f"learning frozen after {len(self.rollbacks[name])} {name} rollbacks in 7 days; re-bootstrap requested", {"rollbacks": self.rollbacks}) + + def _prune(self, conn, now: float) -> None: + rows = conn.execute("SELECT id, path, ts FROM brain_snapshots WHERE kind = %s AND ts < %s ORDER BY ts DESC", + (SNAP_KIND, datetime.fromtimestamp(now - SNAP_HOURLY_DAYS * 86400.0, timezone.utc))).fetchall() + kept: set = set(); drop = [] + for r in rows: + d = r["ts"].date() + if d in kept: + drop.append(r) + else: + kept.add(d) + for r in drop: + conn.execute("DELETE FROM brain_snapshots WHERE id = %s", (r["id"],)) + try: + Path(r["path"]).unlink(missing_ok=True) + except OSError: + pass + conn.execute("DELETE FROM kalshi_fly_scored WHERE state <> 'pending' AND ts < %s", (datetime.fromtimestamp(now - SCORED_KEEP_DAYS * 86400.0, timezone.utc),)) diff --git a/fly_trader/kalshi/history.py b/fly_trader/kalshi/history.py new file mode 100644 index 0000000000000000000000000000000000000000..4e044d6e8d323ae34db275ff328eb1539aa990b5 --- /dev/null +++ b/fly_trader/kalshi/history.py @@ -0,0 +1,616 @@ +"""The Kalshi corpus (worker ``kalshi_history``, CLI ``kalshi-history``): every settled market since ``KALSHI_HISTORY_START`` +with its 1-minute candles and public trades. + +Which markets: an **event sample** that never looks at a market's outcome or its later trading. Every settled single +market Kalshi listed goes into ``kalshi_universe`` (the dataset's, then the exchange walk's); per (Kalshi category of the +series, close day) whole events are taken in md5(event ticker) order until the day holds ``KALSHI_MARKETS_PER_DAY`` +markets (``select_corpus``) — a fixed rate per category could not follow Kalshi's listings, which grew several-fold in 2026. +Until 2026-10-02 the corpus kept the markets with the most lifetime volume, which selects on the future: a cheap strike is +traded heavily mostly when the price runs to it, so kept 1–5c crypto-ladder sides returned +94.5 % per dollar against +−78 % for the rest, and the selector learned the selection (snapshots 151–153). Point-in-time liquidity is the decision +rows' eligibility gate (spread, 24 h volume, open interest at that minute), never the corpus's membership. + +1. Seed (idempotent per rule): the open dataset at ``KALSHI_DATASET_DIR`` (jon-becker/prediction-market-analysis: + ``data/kalshi/markets/*.parquet`` and ``data/kalshi/trades/*.parquet``) → ``kalshi_markets`` rows (source 'dataset') + and ``trades/.parquet`` for every settled yes/no market of a sampled event that closed on or after the start. +2. Refresh from the exchange, resumable and paced (``config.KALSHI_RPS``): settled markets from the live tier + (``GET /markets?status=settled``) and the archive (``GET /historical/markets``), events (category, mutual exclusivity) + and series (category, frequency, fee multiplier) on first sight. +3. Candles: for every settled market without a candle file, 1-minute candles over the last ``CANDLE_DAYS`` days before + close (the strategies only enter inside 10 days; the features look back 7) in ≤ ``CHUNK_MIN``-minute requests, on the + series path with the historical path as the 404 fallback; trades for markets the dataset did not cover. +Registry: ``kalshi_corpus`` (status pending → done | empty | error); progress in ``ui_settings['kalshi_history_status']``. +""" +from __future__ import annotations + +import glob +import json +import logging +import threading +import time +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timedelta, timezone +from pathlib import Path + +from .. import config +from ..db.apilog import record_event +from ..db.connection import transaction +from ..logging_setup import setup +from . import data as D +from .client import KalshiApiError, KalshiRest + +log = logging.getLogger(__name__) +CANDLE_DAYS = 17.0 +CHUNK_MIN = 5000 +SEED_KEY = "kalshi_dataset_seed" +WALK_KEY = "kalshi_refresh_walk" # where the exchange walk is (tier, page cursor) and through when it last completed +RECOVER_DAYS = 3 # a completed walk is re-covered this far back next round: a market settles after it closes +STATUS_KEY = "kalshi_history_status" +IDLE_S = 300.0 + + +def _start() -> datetime: + return datetime.fromisoformat(config.KALSHI_HISTORY_START).replace(tzinfo=timezone.utc) + + +def _status(**kv) -> None: + try: + with transaction() as conn: + r = conn.execute("SELECT value FROM ui_settings WHERE key = %s", (STATUS_KEY,)).fetchone() + cur = (r["value"] if isinstance(r["value"], dict) else json.loads(r["value"] or "{}")) if r else {} + conn.execute("INSERT INTO ui_settings (key, value) VALUES (%s, %s) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()", + (STATUS_KEY, json.dumps({**cur, **kv, "updated_at": datetime.now(timezone.utc).isoformat()}, default=str))) + except Exception: + log.debug("history status write failed", exc_info=True) + + +def _sleep(s: float, stop: threading.Event | None) -> None: + end = time.time() + s + while time.time() < end and not (stop is not None and stop.is_set()): + time.sleep(min(1.0, end - time.time())) + + +# ---------------------------------------------------------------- 1. dataset seed +SAMPLE_RULE = "event-md5-per-category-day-v1" +EXCLUDED_CATEGORIES = ("exotics",) # Kalshi's multivariate combos: not single markets, not our universe +NOT_SAMPLED = "not in the event sample" + + +def event_u(event_ticker: str | None) -> float: + """A uniform number in [0, 1) fixed by the event ticker alone: independent of the market's outcome and trading.""" + import hashlib + return int(hashlib.md5(str(event_ticker or "").encode()).hexdigest()[:12], 16) / float(16 ** 12) + + +def series_categories(conn) -> dict[str, str]: + return {r["ticker"]: (r["category"] or "unknown").strip().lower() for r in conn.execute("SELECT ticker, category FROM kalshi_series").fetchall()} + + +def refresh_series(rest: KalshiRest) -> int: + """Every series with its category (one call): the sample's categories come from here.""" + try: + ss = rest.series_list() + except KalshiApiError as e: + log.warning("series list: %s", e); return 0 + with transaction() as conn: + for x in ss: + D.upsert_series(conn, x) + return len(ss) + + +def seed_from_dataset(stop: threading.Event | None = None) -> dict: + """The dataset's settled single markets since the start, all of them, into ``kalshi_universe`` (source 'dataset'). + Idempotent: skips when ``ui_settings[kalshi_dataset_seed]`` records the same files and rule.""" + root = Path(config.KALSHI_DATASET_DIR).expanduser() if config.KALSHI_DATASET_DIR else None + if root is None or not (root / "data" / "kalshi" / "markets").exists(): + return {"skipped": "no dataset at KALSHI_DATASET_DIR"} + import duckdb + mfiles = sorted(glob.glob(str(root / "data" / "kalshi" / "markets" / "*.parquet"))) + stamp = {"markets": [Path(f).name for f in mfiles], "start": config.KALSHI_HISTORY_START, "rule": SAMPLE_RULE} + with transaction() as conn: + r = conn.execute("SELECT value FROM ui_settings WHERE key = %s", (SEED_KEY,)).fetchone() + if r and (r["value"] if isinstance(r["value"], dict) else json.loads(r["value"] or "{}")).get("stamp") == stamp: + return {"skipped": "already seeded"} + t0 = time.time(); con = duckdb.connect() + cols = [c[0] for c in con.execute("SELECT * FROM read_parquet(?, union_by_name = true) LIMIT 0", [mfiles]).description] + title = "title" if "title" in cols else "NULL" + rows = con.execute(f"""SELECT ticker, event_ticker, split_part(event_ticker, '-', 1) AS series_ticker, {title} AS title, open_time, close_time, result, + CAST(volume AS DOUBLE) AS volume + FROM read_parquet(?, union_by_name = true) WHERE result IN ('yes', 'no') AND close_time >= ? AND event_ticker IS NOT NULL""", + [mfiles, _start().replace(tzinfo=None)]).fetchall() + con.close() + n = 0 + with transaction() as conn: + conn.execute("CREATE TEMP TABLE u_in (LIKE kalshi_universe INCLUDING DEFAULTS) ON COMMIT DROP") + with conn.cursor().copy("COPY u_in (ticker, event_ticker, series_ticker, title, open_time, close_time, settled_ts, result, volume, source) FROM STDIN") as cp: + for tk, et, st, ti, ot, ct, res, vol in rows: + ot = ot.replace(tzinfo=timezone.utc) if ot is not None and ot.tzinfo is None else ot; ct = ct.replace(tzinfo=timezone.utc) if ct.tzinfo is None else ct + cp.write_row((tk, et, st, ti, ot, ct, ct, res, vol, "dataset")); n += 1 + conn.execute("""INSERT INTO kalshi_universe (ticker, event_ticker, series_ticker, title, open_time, close_time, settled_ts, result, volume, source) + SELECT ticker, event_ticker, series_ticker, title, open_time, close_time, settled_ts, result, volume, source FROM u_in + ON CONFLICT (ticker) DO NOTHING""") + conn.execute("INSERT INTO ui_settings (key, value) VALUES (%s, %s) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()", + (SEED_KEY, json.dumps({"stamp": stamp, "markets": n, "at": datetime.now(timezone.utc).isoformat()}))) + log.info("dataset universe: %d settled single markets since %s in %.0fs", n, config.KALSHI_HISTORY_START, time.time() - t0) + return {"universe": n} + + +def seed_dataset_trades(stop: threading.Event | None = None) -> int: + """Trade files from the dataset for selected corpus markets that have none yet (the dataset covers its own months).""" + root = Path(config.KALSHI_DATASET_DIR).expanduser() if config.KALSHI_DATASET_DIR else None + tfiles = sorted(glob.glob(str(root / "data" / "kalshi" / "trades" / "*.parquet"))) if root else [] + if not tfiles: + return 0 + with transaction() as conn: + todo = [r["ticker"] for r in conn.execute("""SELECT k.ticker FROM kalshi_corpus k JOIN kalshi_universe u USING (ticker) + WHERE u.source = 'dataset' AND k.trade_path IS NULL AND k.status IN ('pending', 'done')""").fetchall()] + if not todo: + return 0 + import duckdb + import pandas as pd + con = duckdb.connect(); con.register("want", pd.DataFrame({"ticker": todo})) + tcols = [c[0] for c in con.execute("SELECT * FROM read_parquet(?, union_by_name = true) LIMIT 0", [tfiles]).description] + side = "taker_side" if "taker_side" in tcols else "NULL" + con.execute(f"""CREATE TEMP TABLE tr AS SELECT t.ticker, t.created_time AS ts, CAST(t.yes_price AS INTEGER) AS yes_price, CAST(t.count AS DOUBLE) AS count, + {side} AS taker_side FROM read_parquet(?, union_by_name = true) t JOIN want USING (ticker)""", [tfiles]) + have = [r[0] for r in con.execute("SELECT DISTINCT ticker FROM tr").fetchall()]; done: list = []; n = 0 + for i, tk in enumerate(have): + if stop is not None and stop.is_set(): + break + tab = con.execute("SELECT ts, yes_price, count, taker_side FROM tr WHERE ticker = ? ORDER BY ts", [tk]).fetch_arrow_table() + rows_t = [{"ts": r["ts"].replace(tzinfo=timezone.utc) if r["ts"].tzinfo is None else r["ts"], "yes_price": int(r["yes_price"]), "count": float(r["count"]), + "taker_side": r["taker_side"] or "", "is_block": False} for r in tab.to_pylist()] + path = D.TRADES_DIR / f"{tk}.parquet" + if not path.exists(): + D.write_parquet(rows_t, D.TRADE_SCHEMA, path) + done.append((str(path), len(rows_t), tk)); n += 1 + if len(done) >= 2000 or i == len(have) - 1: + with transaction() as conn: + conn.cursor().executemany("UPDATE kalshi_corpus SET trade_path = %s, trades = %s, updated_at = now() WHERE ticker = %s", done) + done = []; _status(stage="seeding dataset trades", seed_trades=n, seed_trades_total=len(have)) + if not (stop is not None and stop.is_set()): # markets the dataset has no trades for: an empty file, not an API call + missing = sorted(set(todo) - set(have)); empty: list = [] + for tk in missing: + path = D.TRADES_DIR / f"{tk}.parquet" + if not path.exists(): + D.write_parquet([], D.TRADE_SCHEMA, path) + empty.append((str(path), 0, tk)) + with transaction() as conn: + for i in range(0, len(empty), 5000): + conn.cursor().executemany("UPDATE kalshi_corpus SET trade_path = %s, trades = %s, updated_at = now() WHERE ticker = %s", empty[i:i + 5000]) + con.close() + return n + +# ---------------------------------------------------------------- 2. exchange refresh +def _known_events(conn) -> set[str]: + return {r["event_ticker"] for r in conn.execute("SELECT event_ticker FROM kalshi_events").fetchall()} + + +def _known_series(conn) -> set[str]: + return {r["ticker"] for r in conn.execute("SELECT ticker FROM kalshi_series").fetchall()} + + +def ensure_catalogue(rest: KalshiRest, conn, event_tickers: set[str], known_events: set[str], known_series: set[str]) -> None: + """Events and series first seen: one GET each (category, mutual exclusivity, frequency, fee multiplier).""" + for et in sorted(event_tickers - known_events): + try: + e = (rest.event(et, with_nested_markets=False) or {}).get("event") or {} + except KalshiApiError as ex: + log.warning("event %s: %s", et, ex); e = {"event_ticker": et} + D.upsert_event(conn, e); known_events.add(et) + st = e.get("series_ticker") or D.series_ticker_of(et) + if st and st not in known_series: + try: + D.upsert_series(conn, rest.series(st)) + except KalshiApiError as ex: + log.warning("series %s: %s", st, ex); D.upsert_series(conn, {"ticker": st}) + known_series.add(st) + + +def market_volume(m: dict) -> float | None: + v = m.get("volume_fp") if m.get("volume_fp") is not None else m.get("volume") + try: + return float(v) if v is not None else None + except (TypeError, ValueError): + return None + + +def walk_state() -> dict: + with transaction() as conn: + r = conn.execute("SELECT value FROM ui_settings WHERE key = %s", (WALK_KEY,)).fetchone() + return (r["value"] if isinstance(r["value"], dict) else json.loads(r["value"] or "{}")) if r else {} + + +def _save_walk(conn, st: dict) -> None: + conn.execute("INSERT INTO ui_settings (key, value) VALUES (%s, %s) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()", + (WALK_KEY, json.dumps(st, default=str))) + + +def refresh_markets(rest: KalshiRest, stop: threading.Event | None = None, max_pages: int = 10_000) -> int: + """Every settled single market closing after the dataset's newest day (or the start) into ``kalshi_universe``, from the + live tier then the archive (both page newest first); which of them the corpus takes is ``select_corpus``'s decision. The walk is resumable: after every page its tier and page + cursor are saved with that page's rows (``ui_settings[WALK_KEY]``), so a timeout, a crash or a restart continues from the + next page instead of walking the newest months again. A completed walk records when it began; later rounds walk only the + live tier for markets closing since then (less ``RECOVER_DAYS``), and the archive is never walked twice.""" + start = _start(); n = 0 + with transaction() as conn: + r = conn.execute("SELECT max(close_time) AS t FROM kalshi_universe WHERE source = 'dataset'").fetchone() + if r and r["t"] and r["t"] > start: + start = r["t"] # the dataset covers the days before; the API fills from there on + st = walk_state(); now = datetime.now(timezone.utc) + if st.get("complete_through"): + lower = max(start, datetime.fromisoformat(st["complete_through"]) - timedelta(days=RECOVER_DAYS)); tiers = ["live"] + else: + lower = start; tiers = ["live", "historical"] + tier = st.get("tier") if st.get("tier") in tiers else None; cursor = st.get("cursor") if tier else None + if tier is None: # a fresh walk: from the newest settlements down to ``lower`` + tier = tiers[0]; cursor = None; st = {**st, "walk_started": now.isoformat(), "tier": tier, "cursor": None, "lower": lower.isoformat()} + with transaction() as conn: + _save_walk(conn, st) + if st.get("lower"): + lower = max(lower, datetime.fromisoformat(st["lower"])) + for tier in tiers[tiers.index(tier):]: + gen = (rest.markets(status="settled", mve_filter="exclude", min_close_ts=int(lower.timestamp()), cursor=cursor, with_cursor=True) if tier == "live" + else rest.historical_markets(mve_filter="exclude", cursor=cursor, with_cursor=True)) + for k, (page, nxt) in enumerate(gen): + if stop is not None and stop.is_set() or k >= max_pages: + return n + settled = [m for m in page if (m.get("result") in ("yes", "no")) and (D.ts(m.get("close_time")) or lower) >= lower and m.get("event_ticker")] + closes = [D.ts(m.get("close_time")) for m in page if D.ts(m.get("close_time"))] + done_tier = not nxt or (closes and max(closes) < lower) + with transaction() as conn: + conn.cursor().executemany( + "INSERT INTO kalshi_universe (ticker, event_ticker, series_ticker, title, open_time, close_time, settled_ts, result, volume, source) " + "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT (ticker) DO UPDATE SET result = COALESCE(EXCLUDED.result, kalshi_universe.result), " + "settled_ts = COALESCE(EXCLUDED.settled_ts, kalshi_universe.settled_ts), volume = COALESCE(EXCLUDED.volume, kalshi_universe.volume), updated_at = now()", + [(m["ticker"], m["event_ticker"], D.series_ticker_of(m["event_ticker"]), m.get("title"), D.ts(m.get("open_time")), D.ts(m.get("close_time")), + D.ts(m.get("settlement_ts") or m.get("settled_time")), m.get("result"), market_volume(m), tier) for m in settled]) + nxt_tier = tiers[tiers.index(tier) + 1] if done_tier and tiers.index(tier) + 1 < len(tiers) else (None if done_tier else tier) + st = {**st, "tier": nxt_tier, "cursor": None if done_tier else nxt, "oldest_close": min(closes).isoformat() if closes else st.get("oldest_close")} + if done_tier and nxt_tier is None: # every tier walked: settlements since this walk began are the next round's work + st = {**st, "complete_through": st.get("walk_started") or now.isoformat(), "walk_started": None, "lower": None} + _save_walk(conn, st) # the page's rows and the place after them commit together + n += len(settled) + _status(stage=f"refreshing {tier} markets", refreshed=n, tier_page=k, oldest_close=st.get("oldest_close")) + if done_tier: + break + cursor = None + return n + + +def fetch_market_details(rest: KalshiRest, stop: threading.Event | None = None, threads: int | None = None) -> int: + """Full market objects for corpus events whose markets lack their scheduled ``expected_expiration_time`` (dataset and + universe rows carry only the realized close): one call per event, the live tier first, then the archive. They give each + market's point-in-time time anchor (kalshi/mature.time_anchor); a market without one builds no rows. Returns events done.""" + with transaction() as conn: + todo = [r["event_ticker"] for r in conn.execute("""SELECT DISTINCT m.event_ticker FROM kalshi_corpus k JOIN kalshi_markets m USING (ticker) + WHERE k.status IN ('pending', 'done', 'built') AND m.expected_expiration_time IS NULL + AND m.event_ticker IS NOT NULL""").fetchall()] + if not todo: + return 0 + lock = threading.Lock(); done = {"n": 0} + + def one(et: str) -> None: + if stop is not None and stop.is_set(): + return + ms: list = [] + for path in ("/markets", "/historical/markets"): + try: + for page in rest.pages(path, "markets", {"event_ticker": et}, limit=1000): + ms.extend(page) + except KalshiApiError as ex: + if ex.status is not None and (ex.status == 429 or ex.status >= 500): + return # retried next round + except (OSError, TimeoutError, ConnectionError): + return + if ms: + break + if ms: + with transaction() as conn: + D.upsert_markets(conn, ms, "details") + with lock: + done["n"] += 1 + if done["n"] % 1000 == 0: + _status(stage="fetching market details", details=done["n"], details_total=len(todo)) + + with ThreadPoolExecutor(max_workers=max(1, int(threads or config.KALSHI_FILL_THREADS)), thread_name_prefix="kalshi-details") as pool: + list(pool.map(one, todo)) + log.info("market details for %d of %d events", done["n"], len(todo)) + return done["n"] + + +def catalogue_missing(rest: KalshiRest, stop: threading.Event | None = None, threads: int | None = None) -> int: + """Events (and their series) of corpus markets that have none in the catalogue — the dataset seed never fetched them, so + those markets built with category 'other', fee multiplier 1, not mutually exclusive and no siblings. Fetched + ``threads`` at a time under the client's one rate bucket; returns events catalogued.""" + with transaction() as conn: + todo = [r["event_ticker"] for r in conn.execute("""SELECT DISTINCT m.event_ticker FROM kalshi_corpus k JOIN kalshi_markets m USING (ticker) + LEFT JOIN kalshi_events e ON e.event_ticker = m.event_ticker + WHERE k.status IN ('pending', 'done', 'built') AND m.event_ticker IS NOT NULL AND e.event_ticker IS NULL""").fetchall()] + known_s = _known_series(conn) + if not todo: + return 0 + lock = threading.Lock(); done = {"n": 0} + + def one(et: str) -> None: + if stop is not None and stop.is_set(): + return + try: + e = (rest.event(et, with_nested_markets=False) or {}).get("event") or {"event_ticker": et} + except KalshiApiError as ex: + if ex.status is not None and (ex.status == 429 or ex.status >= 500): + return # retried next round + e = {"event_ticker": et} + st = e.get("series_ticker") or D.series_ticker_of(et); series = None + with lock: + need = bool(st) and st not in known_s + if need: + known_s.add(st) + if need: + try: + series = rest.series(st) + except KalshiApiError: + series = {"ticker": st} + with transaction() as conn: + D.upsert_event(conn, e) + if series is not None: + D.upsert_series(conn, series) + with lock: + done["n"] += 1 + if done["n"] % 500 == 0: + _status(stage="cataloguing seeded markets", catalogued=done["n"], catalogue_total=len(todo)) + + with ThreadPoolExecutor(max_workers=max(1, int(threads or config.KALSHI_FILL_THREADS)), thread_name_prefix="kalshi-cat") as pool: + list(pool.map(one, todo)) + log.info("catalogued %d of %d missing events", done["n"], len(todo)) + return done["n"] + + +def ensure_series(rest: KalshiRest, stop: threading.Event | None = None, threads: int | None = None) -> int: + """Series of corpus markets missing from the catalogue (the dataset's markets carry no category; their series does).""" + with transaction() as conn: + todo = [r["st"] for r in conn.execute("""SELECT DISTINCT COALESCE(e.series_ticker, split_part(COALESCE(m.event_ticker, m.ticker), '-', 1)) AS st + FROM kalshi_corpus k JOIN kalshi_markets m USING (ticker) LEFT JOIN kalshi_events e USING (event_ticker) + WHERE k.status <> 'empty'""").fetchall() if r["st"]] + known = _known_series(conn) + todo = [t for t in todo if t not in known] + + def one(st: str) -> None: + if stop is not None and stop.is_set(): + return + try: + s_ = rest.series(st) + except KalshiApiError as ex: + if ex.status is not None and (ex.status == 429 or ex.status >= 500): + return + s_ = {"ticker": st} + with transaction() as conn: + D.upsert_series(conn, s_ if s_.get("ticker") else {**s_, "ticker": st}) + if todo: + with ThreadPoolExecutor(max_workers=max(1, int(threads or config.KALSHI_FILL_THREADS)), thread_name_prefix="kalshi-series") as pool: + list(pool.map(one, todo)) + log.info("catalogued %d series of corpus markets", len(todo)) + return len(todo) + + +COMPLETE_AFTER_DAYS = 3 # a close day is selected once every market of it has settled (settlement follows close) + + +def select_events(df, target: int) -> set: + """Whole events per (category, close day) in md5(event ticker) order until the day holds ``target`` markets (the event + that crosses the budget is kept whole). ``df``: ticker, event_ticker, cat, day. Blind to volume and outcome.""" + import pandas as pd + df = df[~df["cat"].isin(EXCLUDED_CATEGORIES)] + if not len(df): + return set() + ev = df.groupby("event_ticker").agg(n=("ticker", "size"), day=("day", "min"), cat=("cat", "first")).reset_index() + ev["u"] = [event_u(e) for e in ev["event_ticker"]] + ev = ev.sort_values(["cat", "day", "u"]); before = ev.groupby(["cat", "day"])["n"].cumsum() - ev["n"] + keep = set(ev.loc[before < target, "event_ticker"]) + return set(df.loc[df["event_ticker"].isin(keep), "ticker"]) + + +def select_corpus() -> dict: + """The corpus from the universe: ``select_events`` over every complete close day, marked in ``kalshi_universe.selected``; + newly selected markets enter ``kalshi_markets`` and ``kalshi_corpus`` (pending), and the corpus follows the selection + (``apply_sample``). Deterministic: the same universe always gives the same corpus.""" + import pandas as pd + t0 = time.time(); cutoff = datetime.now(timezone.utc) - timedelta(days=COMPLETE_AFTER_DAYS) + with transaction() as conn: + cats = series_categories(conn) + df = pd.DataFrame([dict(r) for r in conn.execute("SELECT ticker, event_ticker, series_ticker, close_time FROM kalshi_universe WHERE close_time < %s", (cutoff,)).fetchall()]) + if not len(df): + return {"selected": 0} + df["cat"] = [cats.get(x, "unknown") for x in df["series_ticker"]] + df["day"] = pd.to_datetime(df["close_time"], utc=True).dt.floor("D") + sel = select_events(df[["ticker", "event_ticker", "cat", "day"]], int(config.KALSHI_MARKETS_PER_DAY)) + sel_l = sorted(sel) + with transaction() as conn: + conn.execute("CREATE TEMP TABLE sel (ticker text PRIMARY KEY) ON COMMIT DROP") + with conn.cursor().copy("COPY sel (ticker) FROM STDIN") as cp: + for tk in sel_l: + cp.write_row((tk,)) + conn.execute("UPDATE kalshi_universe u SET selected = true, updated_at = now() FROM sel WHERE sel.ticker = u.ticker AND NOT u.selected") + conn.execute("UPDATE kalshi_universe u SET selected = false, updated_at = now() WHERE u.selected AND u.close_time < %s " + "AND NOT EXISTS (SELECT 1 FROM sel WHERE sel.ticker = u.ticker)", (cutoff,)) + new = conn.execute("""SELECT u.* FROM kalshi_universe u JOIN sel USING (ticker) LEFT JOIN kalshi_corpus k USING (ticker) WHERE k.ticker IS NULL""").fetchall() + mk = [{"ticker": r["ticker"], "event_ticker": r["event_ticker"], "title": r["title"], "status": "settled", "open_time": r["open_time"], "close_time": r["close_time"], + "settlement_ts": r["settled_ts"], "result": r["result"]} for r in new] + for i in range(0, len(mk), 5000): + D.upsert_markets(conn, mk[i:i + 5000], "universe") + conn.cursor().executemany("INSERT INTO kalshi_corpus (ticker, status, settled_ts, result, open_time, close_time, seeded_from, volume) VALUES (%s,'pending',%s,%s,%s,%s,%s,%s) " + "ON CONFLICT (ticker) DO NOTHING", + [(r["ticker"], r["settled_ts"], r["result"], r["open_time"], r["close_time"], r["source"], r["volume"]) for r in new]) + changed = apply_sample() + log.info("corpus selection: %d markets of %d in complete days (%d new to the corpus, %d corpus rows changed) in %.0fs", + len(sel), len(df), len(new), changed, time.time() - t0) + return {"selected": len(sel), "universe": int(len(df)), "new": len(new), "changed": changed} + + +def apply_sample() -> int: + """Corpus rows follow the selection: a market not selected is 'skipped' (its files stay on disk, it is never built or + trained on); a selected market an older rule skipped comes back ('done' when its candles are already on disk, else + 'pending'). Markets of days not yet selectable (the last COMPLETE_AFTER_DAYS) are left as they are. Returns rows changed.""" + cutoff = datetime.now(timezone.utc) - timedelta(days=COMPLETE_AFTER_DAYS) + with transaction() as conn: + a = conn.execute("""UPDATE kalshi_corpus k SET status = 'skipped', last_error = %s, updated_at = now() + WHERE k.status IN ('pending', 'done', 'built') AND k.close_time < %s + AND NOT EXISTS (SELECT 1 FROM kalshi_universe u WHERE u.ticker = k.ticker AND u.selected)""", (NOT_SAMPLED, cutoff)).rowcount + b = conn.execute("""UPDATE kalshi_corpus k SET status = CASE WHEN k.candle_path IS NOT NULL THEN 'done' ELSE 'pending' END, last_error = NULL, updated_at = now() + FROM kalshi_universe u WHERE u.ticker = k.ticker AND u.selected AND k.status = 'skipped'""").rowcount + log.info("event sample: %d corpus rows left, %d came back", a, b) + return int(a or 0) + int(b or 0) + +# ---------------------------------------------------------------- 3. candles and trades +def _series_for(conn, ticker: str) -> str: + r = conn.execute("SELECT e.series_ticker FROM kalshi_markets m LEFT JOIN kalshi_events e USING (event_ticker) WHERE m.ticker = %s", (ticker,)).fetchone() + return (r["series_ticker"] if r and r["series_ticker"] else None) or D.series_ticker_of(ticker) + + +def pull_candles(rest: KalshiRest, ticker: str, series: str, open_time: datetime | None, close_time: datetime, settled: datetime | None) -> list[dict]: + end = settled or close_time + lo = max(open_time or (close_time - timedelta(days=CANDLE_DAYS)), close_time - timedelta(days=CANDLE_DAYS)) + t = int(lo.timestamp()) // 60 * 60; t_end = int(end.timestamp()) // 60 * 60 + 60 + out: list[dict] = [] + while t < t_end: + hi = min(t + CHUNK_MIN * 60, t_end) + out.extend(D.candle_rows(rest.candlesticks(series, ticker, t, hi, 1))) + t = hi + seen = set(); uniq = [] + for c in out: + if c["end_ts"] not in seen: + seen.add(c["end_ts"]); uniq.append(c) + return sorted(uniq, key=lambda c: c["end_ts"]) + + +_CUTOFF: dict = {"at": 0.0, "value": None} +_CUTOFF_LOCK = threading.Lock() +CUTOFF_TTL_S = 3600.0 + + +def trades_cutoff(rest: KalshiRest) -> datetime | None: + """The exchange's historical cutoff for trades, fetched at most hourly (it moves daily; every market used to ask for it).""" + with _CUTOFF_LOCK: + if time.time() - _CUTOFF["at"] < CUTOFF_TTL_S: + return _CUTOFF["value"] + try: + _CUTOFF["value"] = D.ts((rest.historical_cutoff() or {}).get("trades_created_ts")); _CUTOFF["at"] = time.time() + except KalshiApiError: + pass + return _CUTOFF["value"] + + +def pull_trades(rest: KalshiRest, ticker: str, close_time: datetime, settled: datetime | None) -> list[dict]: + lo = int((close_time - timedelta(days=CANDLE_DAYS)).timestamp()); hi = int((settled or close_time).timestamp()) + 60 + cutoff = trades_cutoff(rest) + historical = bool(cutoff and close_time < cutoff) + rows: list[dict] = [] + for page in rest.trades(ticker, min_ts=lo, max_ts=hi, historical=historical): + rows.extend(D.trade_rows(page)) + return sorted(rows, key=lambda r: r["ts"]) + + +TRANSIENT_WAIT_S = 60.0 + + +def fill(rest: KalshiRest, stop: threading.Event | None = None, limit: int | None = None, threads: int | None = None) -> int: + """Candles (and trades where missing) for pending corpus rows, newest first, ``threads`` markets at a time; every call + still passes the client's one rate bucket (KALSHI_RPS), so the threads only overlap the calls' latency. A market whose + fetch failed on the network stays pending (retried next batch); a batch that failed entirely waits ``TRANSIENT_WAIT_S``. + Returns the markets finished (done, empty or error).""" + threads = max(1, int(threads or config.KALSHI_FILL_THREADS)); limit = limit or max(200, threads * 25) + with transaction() as conn: + todo = conn.execute("SELECT c.ticker, c.open_time, c.close_time, c.settled_ts, c.candle_path, c.trade_path FROM kalshi_corpus c " + "WHERE c.status = 'pending' AND c.close_time IS NOT NULL ORDER BY c.close_time DESC LIMIT %s", (limit,)).fetchall() + series = {r["ticker"]: _series_for(conn, r["ticker"]) for r in todo} + if not todo: + return 0 + with ThreadPoolExecutor(max_workers=threads, thread_name_prefix="kalshi-fill") as pool: + results = list(pool.map(lambda r: _fill_one(rest, r, series[r["ticker"]], stop), todo)) + n = sum(1 for x in results if x == "finished") + if n == 0 and any(x == "transient" for x in results): + log.warning("kalshi fill: every market of the batch failed on the network; waiting %.0f s", TRANSIENT_WAIT_S) + _sleep(TRANSIENT_WAIT_S, stop) + return n if n else (0 if not any(x == "transient" for x in results) else -1) + + +def _fill_one(rest: KalshiRest, r: dict, series: str, stop: threading.Event | None) -> str: + """One market's candles and trades; 'finished' (row updated), 'transient' (left pending) or 'stopped'.""" + if stop is not None and stop.is_set(): + return "stopped" + tk = r["ticker"]; upd = {"status": "done", "last_error": None} + try: + if not r["candle_path"]: + cs = pull_candles(rest, tk, series, r["open_time"], r["close_time"], r["settled_ts"]) + if cs: + p = D.CANDLES_DIR / f"{tk}.parquet"; D.write_parquet(cs, D.CANDLE_SCHEMA, p); upd.update(candle_path=str(p), candles_1m=len(cs)) + else: + upd["status"] = "empty" + if not r["trade_path"]: + tr = pull_trades(rest, tk, r["close_time"], r["settled_ts"]) + p = D.TRADES_DIR / f"{tk}.parquet"; D.write_parquet(tr, D.TRADE_SCHEMA, p); upd.update(trade_path=str(p), trades=len(tr)) + except KalshiApiError as e: + if e.status is not None and (e.status == 429 or e.status >= 500): # the exchange is busy or down: retry later + _mark_transient(tk, str(e)); return "transient" + upd = {"status": "error" if e.status not in (404,) else "empty", "last_error": str(e)[:300]} + except (OSError, TimeoutError, ConnectionError) as e: # the network: the market stays pending + _mark_transient(tk, f"{type(e).__name__}: {e}"); return "transient" + except Exception as e: + if type(e).__module__.startswith(("aiohttp", "asyncio", "concurrent")) or "Timeout" in type(e).__name__: + _mark_transient(tk, f"{type(e).__name__}: {e}"); return "transient" + log.exception("candles %s failed", tk); upd = {"status": "error", "last_error": f"{type(e).__name__}: {e}"[:300]} + with transaction() as conn: + conn.execute("UPDATE kalshi_corpus SET " + ", ".join(f"{k} = %s" for k in upd) + ", updated_at = now() WHERE ticker = %s", (*upd.values(), tk)) + return "finished" + + +def _mark_transient(tk: str, err: str) -> None: + try: + with transaction() as conn: + conn.execute("UPDATE kalshi_corpus SET last_error = %s, updated_at = now() WHERE ticker = %s", (f"retrying: {err}"[:300], tk)) + except Exception: + log.warning("could not note the transient failure of %s", tk) + + +def counts() -> dict: + with transaction() as conn: + rows = conn.execute("SELECT status, count(*) AS n FROM kalshi_corpus GROUP BY status").fetchall() + first = conn.execute("SELECT min(close_time) AS a, max(close_time) AS b FROM kalshi_corpus WHERE status = 'done'").fetchone() + return {**{r["status"]: int(r["n"]) for r in rows}, "first_done": first["a"], "last_done": first["b"]} + + +def main(stop_event: threading.Event | None = None) -> None: + setup("kalshi_history"); stop = stop_event + record_event("info", "kalshi_history", "kalshi history started", {"start": config.KALSHI_HISTORY_START, "dataset": config.KALSHI_DATASET_DIR}) + rest = KalshiRest(subaccount=0) + try: + while not (stop is not None and stop.is_set()): + try: + _status(stage="seeding the universe from the dataset"); refresh_series(rest); seeded = seed_from_dataset(stop) + _status(stage="refreshing markets", seed=seeded); n_new = refresh_markets(rest, stop) + refresh_series(rest); sel = select_corpus(); skipped = sel.get("changed") # the outcome-blind event sample, + seed_dataset_trades(stop); ensure_series(rest, stop); catalogue_missing(rest, stop) # then only the selected markets' files and events + fetch_market_details(rest, stop) # and their scheduled expirations (time anchors) + c = counts(); _status(stage="filling candles", refreshed=n_new, skipped_now=skipped, **{k: v for k, v in c.items()}) + t0 = time.time(); done = 0 + while not (stop is not None and stop.is_set()): + k = fill(rest, stop) + if k == 0: + break + if k < 0: # the whole batch failed on the network: fill() already waited + continue + done += k; c = counts() + rate = done / max(time.time() - t0, 1e-9) + _status(stage="filling candles", filled_this_run=done, markets_per_h=rate * 3600, eta_h=(c.get("pending", 0) / rate / 3600) if rate else None, **c) + if not (stop is not None and stop.is_set()): + from . import mature # filled markets become feature rows (the training corpus) + _status(stage="building features", **counts()); built = mature.loop_once() + ok, why = mature.build_complete(); _status(stage="idle", built_this_round=built, build_complete=ok, build_note=why) + _status(stage="idle", **counts()) + except Exception as e: + log.exception("kalshi history round failed"); record_event("error", "kalshi_history", f"round failed: {type(e).__name__}: {e}") + _status(stage="error", last_error=str(e)[:200]) + _sleep(IDLE_S, stop) + finally: + rest.close(); _status(stage="stopped") + record_event("info", "kalshi_history", "kalshi history stopped", counts()) diff --git a/fly_trader/kalshi/live.py b/fly_trader/kalshi/live.py new file mode 100644 index 0000000000000000000000000000000000000000..377d44866024222378b23ec8fcb0270858f6d3c3 --- /dev/null +++ b/fly_trader/kalshi/live.py @@ -0,0 +1,317 @@ +"""Live execution for the Kalshi fly (books ``live_kalshi_taker`` / ``live_kalshi_maker``): the paper arms' decisions of +each minute mirrored on the dedicated subaccount, only when ``KALSHI_LIVE_ENABLED=1`` and every prerequisite holds +(config.kalshi_live_prerequisites_missing). Per minute: + +- taker: every paper taker entry becomes an IOC order at the walk's worst level for as many contracts as the paper fill, + sized down to the subaccount's cash (the cap is never exceeded); the exchange's fill count, cost and fees open the live + position (``client_order_id`` 'fly-', ``subaccount`` on every order); +- maker: every paper post becomes a GTC ``post_only`` order with the paper expiry; every paper cancel or replace cancels + the live order carrying the same decision; fills arrive on the stream's ``fill`` channel (``kalshi_fills``, keyed by + order id) and open live positions; resting orders are reconciled with ``GET /portfolio/orders`` every ``RECONCILE_S``; +- settlements: ``GET /portfolio/settlements`` closes live positions with the exchange's payout and fee; a market whose + result the stream wrote settles at 100c per winning contract when the exchange row is late; +- wealth: subaccount cash + positions at the bid + resting collateral → ``wealth_marks`` per live book (USD); the kill + switch (circuit 2, ``KALSHI_KILL_SWITCH_DRAWDOWN`` on the combined live wealth) blocks entries and, with + ``KILL_SWITCH_LIQUIDATE``, cancels every resting order and sells every position IOC at the bid; +- gap: over the last ``GAP_TRADES`` settled live positions per arm, a mean return trailing the paper mirror's same + decisions by more than ``GAP_MAX`` pauses entries (circuit 2) and raises an alert. Learning is unaffected. +Order failures feed the circuit breaker (``rails.record_failure``, circuit 2); a tripped circuit blocks entries. +""" +from __future__ import annotations + +import json +import logging +import math +import time +from datetime import datetime, timezone + +import numpy as np + +from .. import config +from ..agent import rails +from ..db.apilog import record_event +from . import paper as P +from .client import KalshiApiError +from .vendor.kalshi_client import kalshi_fee_rate_cents + +log = logging.getLogger(__name__) +BOOKS = P.LIVE_BOOKS +MIRROR = P.BOOKS +GAP_TRADES, GAP_MAX = 50, 0.02 +RECONCILE_S, SETTLE_LOOKBACK_S = 300.0, 3 * 86400.0 + + +def _c(v) -> float: + try: + return float(v) * 100.0 if v is not None else math.nan + except (TypeError, ValueError): + return math.nan + + +def _n(v, default=0.0) -> float: + try: + return float(v) if v is not None else default + except (TypeError, ValueError): + return default + + +def order_facts(o: dict) -> dict: + """What an exchange order object says about its fills: count, cost and fee in cents (the V2 ``*_dollars`` / ``*_fp`` fields).""" + filled = _n(o.get("fill_count_fp") if "fill_count_fp" in o else o.get("fill_count")) + remaining = _n(o.get("remaining_count_fp") if "remaining_count_fp" in o else o.get("remaining_count")) + cost = _c(o.get("taker_fill_cost_dollars")) if o.get("taker_fill_cost_dollars") is not None else _n(o.get("taker_fill_cost"), math.nan) + if not math.isfinite(cost): + cost = _c(o.get("maker_fill_cost_dollars")) if o.get("maker_fill_cost_dollars") is not None else _n(o.get("maker_fill_cost"), math.nan) + fee = _c(o.get("taker_fees_dollars")) if o.get("taker_fees_dollars") is not None else _n(o.get("taker_fees"), math.nan) + if not math.isfinite(fee): + fee = _c(o.get("maker_fees_dollars")) if o.get("maker_fees_dollars") is not None else _n(o.get("maker_fees"), 0.0) + price = _c(o.get("yes_price_dollars")) if o.get("yes_price_dollars") is not None else _n(o.get("yes_price"), math.nan) + if o.get("side") == "no": + np_ = _c(o.get("no_price_dollars")) if o.get("no_price_dollars") is not None else _n(o.get("no_price"), math.nan) + price = np_ if math.isfinite(np_) else (100.0 - price if math.isfinite(price) else math.nan) + return {"order_id": o.get("order_id"), "status": o.get("status"), "filled": filled, "remaining": remaining, "cost_cents": cost, "fee_cents": fee if math.isfinite(fee) else 0.0, + "price_cents": price} + + +class KalshiLiveMirror: + def __init__(self, rest=None): + if rest is None: + from .client import rest as _rest + rest = _rest() + self.rest = rest; self.sub = config.KALSHI_SUBACCOUNT + self.last_reconcile = 0.0; self.last_settle_ts = int(time.time() - SETTLE_LOOKBACK_S); self.cash_cents: float | None = None + + # ---- exchange state ---- + def balance_cents(self) -> float: + b = self.rest.balance(self.sub) + v = b.get("balance_dollars") + return _c(v) if v is not None else _n(b.get("balance"), 0.0) + + def _record_order(self, conn, *, book: str, decision_id: int | None, ticker: str, side: str, price: int, count: float, tif: str, post_only: bool, exp_ts: float | None, + req: dict, resp: dict | None, error: str | None, strategy: str | None, action: str = "buy") -> int: + f = order_facts(resp or {}) + return int(conn.execute( + "INSERT INTO kalshi_orders (book, order_id, client_order_id, decision_id, ticker, side, action, price_cents, count, tif, post_only, expiration_ts, status, fill_count, remaining, " + "avg_fill_cents, fee_cents, subaccount, strategy, request, response, error) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) RETURNING id", + (book, f["order_id"], req.get("client_order_id"), decision_id, ticker, side, action, price, count, tif, post_only, + datetime.fromtimestamp(exp_ts, timezone.utc) if exp_ts else None, "error" if error else (f["status"] or "submitted"), f["filled"], f["remaining"], + (f["cost_cents"] / f["filled"]) if f["filled"] and math.isfinite(f["cost_cents"]) else None, f["fee_cents"], self.sub, strategy, + json.dumps(req, default=str), json.dumps(resp, default=str) if resp is not None else None, error)).fetchone()["id"]) + + def _submit(self, conn, *, book: str, decision_id: int | None, ticker: str, side: str, price: int, count: float, tif: str, post_only: bool, exp_ts: float | None, + strategy: str | None, action: str = "buy") -> tuple[dict | None, int]: + coid = f"fly-{decision_id}" if decision_id is not None else None + req = {"ticker": ticker, "side": side, "price_cents": price, "count": count, "tif": tif, "post_only": post_only, "expiration_ts": int(exp_ts) if exp_ts else None, + "client_order_id": coid, "subaccount": self.sub, "action": action} + try: + resp = self.rest.create_order(ticker, side, price, count, tif=tif, post_only=post_only, expiration_ts=int(exp_ts) if exp_ts else None, client_order_id=coid, + subaccount=self.sub, action=action) + rails.record_success(conn, rails.KALSHI_CIRCUIT) + return resp, self._record_order(conn, book=book, decision_id=decision_id, ticker=ticker, side=side, price=price, count=count, tif=tif, post_only=post_only, exp_ts=exp_ts, + req=req, resp=resp, error=None, strategy=strategy, action=action) + except (KalshiApiError, RuntimeError, OSError) as e: + rails.record_failure(conn, f"kalshi order: {e}"[:300], rails.KALSHI_CIRCUIT) + log.warning("kalshi order failed (%s %s %s x%s @%sc): %s", action, side, ticker, count, price, e) + return None, self._record_order(conn, book=book, decision_id=decision_id, ticker=ticker, side=side, price=price, count=count, tif=tif, post_only=post_only, exp_ts=exp_ts, + req=req, resp=None, error=str(e)[:300], strategy=strategy, action=action) + + # ---- the minute ---- + def minute(self, ctx, *, run_id: str, beat_id: int, taker_entries: list, maker_plan: dict) -> dict: + conn, m1, now = ctx.conn, ctx.m1, ctx.m1_epoch + try: + self.cash_cents = self.balance_cents() + except Exception as e: + log.warning("kalshi balance: %s", e) + out = {"stage": "trading live", "cash": (self.cash_cents or 0.0) / 100.0} + out["settled"] = self.settlements(conn, ctx, run_id, beat_id) + out["maker_fills"] = self.maker_fills(conn, ctx, run_id, beat_id) + c = rails.load_circuit(conn, rails.KALSHI_CIRCUIT) + blocked = "kill switch" if c.kill_switch else "circuit tripped" if c.tripped else "paused" if c.entries_paused else None + out["blocked"] = blocked + out["taker"] = self.taker(conn, ctx, run_id, beat_id, taker_entries, blocked) if config.KALSHI_TAKER_LIVE else {"skipped": "KALSHI_TAKER_LIVE=0"} + out["maker"] = self.maker(conn, ctx, run_id, beat_id, maker_plan, blocked) if config.KALSHI_MAKER_LIVE else {"skipped": "KALSHI_MAKER_LIVE=0"} + if now - self.last_reconcile >= RECONCILE_S: + self.last_reconcile = now; out["reconciled"] = self.reconcile(conn) + wealth = 0.0 + for arm, book in BOOKS.items(): + cash_part = (self.cash_cents or 0.0) if arm == "taker" else 0.0 # the subaccount's cash counts once, on the taker book + m = P.mark(conn, book=book, quotes=ctx.quotes, ts=m1, beat_id=beat_id, cash=cash_part); out[f"wealth_{arm}"] = m["wealth"]; wealth += m["wealth"] + pk = conn.execute("SELECT max(wealth) AS pk FROM wealth_marks WHERE book = 'live_kalshi'").fetchone(); peak = max(float(pk["pk"] or 0.0), wealth) + conn.execute("INSERT INTO wealth_marks (beat_id, book, ts, sol_free, positions_value, exit_cost, wealth, peak, drawdown, exposure, n_open) VALUES (%s,'live_kalshi',%s,%s,%s,0,%s,%s,%s,%s,%s) " + "ON CONFLICT (beat_id, book) DO NOTHING", (beat_id, m1, (self.cash_cents or 0.0) / 100.0, wealth - (self.cash_cents or 0.0) / 100.0, wealth, peak, + (1.0 - wealth / peak) if peak > 0 else 0.0, wealth - (self.cash_cents or 0.0) / 100.0, + len(P.open_positions(conn, BOOKS["taker"])) + len(P.open_positions(conn, BOOKS["maker"])))) + out["wealth"] = wealth + killed = rails.check_drawdown(conn, wealth, peak, rails.KALSHI_CIRCUIT, drawdown=config.KALSHI_KILL_SWITCH_DRAWDOWN, unit="USD") + out["liquidated"] = self.liquidate(conn, ctx, run_id, beat_id) if killed and config.KILL_SWITCH_LIQUIDATE else 0 + out["gap"] = {arm: self.gap(conn, arm) for arm in BOOKS} + return out + + def taker(self, conn, ctx, run_id, beat_id, entries: list, blocked: str | None) -> dict: + book = BOOKS["taker"]; opens = P.open_positions(conn, book); held = {p["ticker"] for p in opens} + cash = self.cash_cents if self.cash_cents is not None else 0.0 + floor = config.KALSHI_CASH_FLOOR_USD * 100.0; n = 0; skipped = [] + for e in entries: + tk, side = e["ticker"], e["side"] + if tk in held: + skipped.append((tk, "held")); continue + if blocked: + skipped.append((tk, blocked)); continue + price = int(e["limit_price_cents"] or round(e["vwap_cents"])) + eff = price + kalshi_fee_rate_cents(price, tk) + count = min(float(e["contracts"]), math.floor(max(cash - floor, 0.0) / eff)) + if count < 1: + skipped.append((tk, "cash")); continue + resp, row = self._submit(conn, book=book, decision_id=e["decision_id"], ticker=tk, side=side, price=price, count=count, tif="immediate_or_cancel", post_only=False, + exp_ts=None, strategy=e.get("strategy")) + if resp is None: + continue + f = order_facts(resp) + if f["filled"] > 0: + cost = f["cost_cents"] if math.isfinite(f["cost_cents"]) and f["cost_cents"] > 0 else f["filled"] * price + P.open_position(conn, book=book, ticker=tk, side=side, contracts=f["filled"], cost_cents=cost, fee_cents=f["fee_cents"], avg_price=cost / f["filled"], + decision_id=e["decision_id"], order_id=f["order_id"], strategy=e.get("strategy"), arm="taker", score=e.get("edge"), line=e.get("line"), ts=ctx.m1) + conn.execute("UPDATE decisions SET book_targets = array_append(COALESCE(book_targets, '{}'), %s) WHERE id = %s", (book, e["decision_id"])) + cash -= cost + f["fee_cents"]; held.add(tk); n += 1 + else: + skipped.append((tk, f"no fill ({f['status']})")) + self.cash_cents = cash + return {"entered": n, "skipped": skipped[:10]} + + def maker(self, conn, ctx, run_id, beat_id, plan: dict, blocked: str | None) -> dict: + book = BOOKS["maker"]; n_post = n_cancel = 0 + # cancels and replaces first: the paper order rows that left 'resting' this minute carry the decision ids of the live orders to cancel + gone = conn.execute("SELECT o.decision_id, o.ticker FROM kalshi_orders o WHERE o.book = %s AND o.status IN ('canceled','replaced') AND o.updated_at >= %s", + (MIRROR["maker"], ctx.m0)).fetchall() + for g in gone: + for lo in conn.execute("SELECT id, order_id FROM kalshi_orders WHERE book = %s AND decision_id = %s AND status IN ('resting','submitted') AND order_id IS NOT NULL", + (book, g["decision_id"])).fetchall(): + try: + self.rest.cancel_order(lo["order_id"], self.sub); rails.record_success(conn, rails.KALSHI_CIRCUIT) + conn.execute("UPDATE kalshi_orders SET status = 'canceled', updated_at = now() WHERE id = %s", (lo["id"],)); n_cancel += 1 + except (KalshiApiError, RuntimeError, OSError) as e: + rails.record_failure(conn, f"kalshi cancel: {e}"[:300], rails.KALSHI_CIRCUIT); log.warning("cancel %s failed: %s", lo["order_id"], e) + skipped = [] + for o in plan.get("posted") or []: + if blocked: + skipped.append((o["ticker"], blocked)); continue + resp, _row = self._submit(conn, book=book, decision_id=o["decision_id"], ticker=o["ticker"], side=o["side"], price=int(o["price_cents"]), count=float(o["count"]), + tif="good_till_canceled", post_only=True, exp_ts=o.get("expiration_ts"), strategy=o.get("strategy")) + if resp is not None: + n_post += 1 + conn.execute("UPDATE decisions SET book_targets = array_append(COALESCE(book_targets, '{}'), %s) WHERE id = %s", (book, o["decision_id"])) + return {"posted": n_post, "canceled": n_cancel, "skipped": skipped[:10]} + + def maker_fills(self, conn, ctx, run_id, beat_id) -> int: + """Fills of our live maker orders (the stream writes ``kalshi_fills``; the fill row joins its order's book) open positions.""" + book = BOOKS["maker"]; n = 0 + rows = conn.execute("""SELECT f.trade_id, f.order_id, f.ticker, f.side, f.price_cents, f.count, f.fee_cents, f.ts, o.decision_id, o.strategy, o.id AS oid + FROM kalshi_fills f JOIN kalshi_orders o ON o.order_id = f.order_id WHERE o.book = %s AND f.action = 'buy' + AND NOT EXISTS (SELECT 1 FROM kalshi_positions p WHERE p.order_id = f.order_id AND p.book = %s)""", (book, book)).fetchall() + by_order: dict[str, list] = {} + for r in rows: + by_order.setdefault(r["order_id"], []).append(r) + for oid, fs in by_order.items(): + count = sum(float(f["count"]) for f in fs); px = float(fs[0]["price_cents"]) if fs[0]["side"] == "yes" else 100.0 - float(fs[0]["price_cents"]) + if fs[0]["side"] == "yes": + px = float(fs[0]["price_cents"]) + cost = sum(float(f["count"]) * (float(f["price_cents"]) if f["side"] == "yes" else 100.0 - float(f["price_cents"])) for f in fs) + fee = sum(float(f["fee_cents"] or 0.0) for f in fs) + P.open_position(conn, book=book, ticker=fs[0]["ticker"], side=fs[0]["side"], contracts=count, cost_cents=cost, fee_cents=fee, avg_price=cost / count, + decision_id=fs[0]["decision_id"], order_id=oid, strategy=fs[0]["strategy"], arm="maker", score=None, line=None, ts=ctx.m1) + conn.execute("UPDATE kalshi_orders SET fill_count = %s, status = CASE WHEN remaining IS NOT NULL AND remaining - %s <= 0 THEN 'filled' ELSE status END, updated_at = now() WHERE id = %s", + (count, count, fs[0]["oid"])) + n += 1 + return n + + def reconcile(self, conn) -> dict: + """Resting live maker orders against the exchange: gone or filled there → closed here.""" + book = BOOKS["maker"]; ours = conn.execute("SELECT id, order_id FROM kalshi_orders WHERE book = %s AND status IN ('resting','submitted') AND order_id IS NOT NULL", (book,)).fetchall() + if not ours: + return {"resting": 0} + try: + live = {o.get("order_id"): o for o in self.rest.orders(status="resting", subaccount=self.sub)} + except Exception as e: + log.warning("kalshi orders: %s", e); return {"error": str(e)[:200]} + closed = 0 + for r in ours: + o = live.get(r["order_id"]) + if o is None: + try: + o = self.rest.order(r["order_id"]) + except Exception: + o = {} + f = order_facts(o.get("order") or o) + conn.execute("UPDATE kalshi_orders SET status = %s, fill_count = %s, remaining = %s, updated_at = now() WHERE id = %s", + (f["status"] or "gone", f["filled"], f["remaining"], r["id"])); closed += 1 + return {"resting": len(ours) - closed, "closed": closed} + + def settlements(self, conn, ctx, run_id, beat_id) -> int: + """Exchange settlement rows (payout and fee per market) close live positions; a market the stream marked resolved + without a row yet settles at face value.""" + payouts: dict[str, tuple] = {}; results: dict[str, str] = {} + try: + for s in self.rest.settlements(self.sub, min_ts=self.last_settle_ts): + tk = s.get("ticker"); res = s.get("market_result") + if tk and res in ("yes", "no"): + rev = _c(s.get("revenue_dollars")) if s.get("revenue_dollars") is not None else _n(s.get("revenue"), math.nan) + fee = _c(s.get("fee_dollars")) if s.get("fee_dollars") is not None else _n(s.get("fee_cost") or s.get("fee"), 0.0) + if math.isfinite(rev): + payouts[tk] = (rev, fee) + results[tk] = res + self.last_settle_ts = int(ctx.m1_epoch) - 3600 + except Exception as e: + log.warning("kalshi settlements: %s", e) + n = 0 + for book in BOOKS.values(): + opens = P.open_positions(conn, book) + res = {**ctx.settled([p["ticker"] for p in opens]), **results} + n += len(P.settle(conn, book=book, results=res, ts=ctx.m1, run_id=run_id, beat_id=beat_id, payouts=payouts)) + return n + + def liquidate(self, conn, ctx, run_id, beat_id) -> int: + n = 0 + for lo in conn.execute("SELECT id, order_id FROM kalshi_orders WHERE book = %s AND status IN ('resting','submitted') AND order_id IS NOT NULL", (BOOKS["maker"],)).fetchall(): + try: + self.rest.cancel_order(lo["order_id"], self.sub); conn.execute("UPDATE kalshi_orders SET status = 'canceled', error = 'kill switch', updated_at = now() WHERE id = %s", (lo["id"],)) + except Exception as e: + log.warning("kill switch cancel %s: %s", lo["order_id"], e) + for book in BOOKS.values(): + for p in P.open_positions(conn, book): + bid, _ = P.side_quote(ctx.quotes, p["ticker"], p["side"]) + if not math.isfinite(bid) or not (1 <= bid <= 99): + continue + did = P.decision(conn, beat_id=beat_id, run_id=run_id, ts=ctx.m1, ticker=p["ticker"], side=p["side"], kind="kalshi_exit", edge=None, usd=float(p["cost_cents"]) / 100.0, + reason="kill switch: liquidating", detail={"position": int(p["id"])}, book=book) + resp, _ = self._submit(conn, book=book, decision_id=did, ticker=p["ticker"], side=p["side"], price=int(bid), count=float(p["contracts"]), tif="immediate_or_cancel", + post_only=False, exp_ts=None, strategy=p.get("strategy"), action="sell") + if resp is not None: + f = order_facts(resp) + if f["filled"] > 0: + proceeds = f["cost_cents"] if math.isfinite(f["cost_cents"]) and f["cost_cents"] > 0 else f["filled"] * bid + realized = proceeds - float(p["cost_cents"]) * (f["filled"] / float(p["contracts"])) - float(p["fee_cents"]) - f["fee_cents"] + if f["filled"] >= float(p["contracts"]) - 1e-9: + conn.execute("UPDATE kalshi_positions SET status = 'closed', closed_at = %s, payout_cents = %s, realized_cents = %s, exit_decision_id = %s WHERE id = %s", + (ctx.m1, proceeds, realized, did, int(p["id"]))) + else: + conn.execute("UPDATE kalshi_positions SET contracts = contracts - %s, cost_cents = cost_cents * (1 - %s / contracts) WHERE id = %s", (f["filled"], f["filled"], int(p["id"]))) + n += 1 + if n: + record_event("error", "kalshi_live", f"kill switch: liquidated {n} live position(s)", {"positions": n}) + return n + + def gap(self, conn, arm: str) -> dict: + rows = conn.execute("""SELECT l.realized_cents / NULLIF(l.cost_cents + l.fee_cents, 0) AS rl, p.realized_cents / NULLIF(p.cost_cents + p.fee_cents, 0) AS rp + FROM kalshi_positions l JOIN kalshi_positions p ON p.entry_decision_id = l.entry_decision_id AND p.book = %s AND p.status <> 'open' + WHERE l.book = %s AND l.status <> 'open' AND l.entry_decision_id IS NOT NULL ORDER BY l.closed_at DESC LIMIT %s""", (MIRROR[arm], BOOKS[arm], GAP_TRADES)).fetchall() + rl = np.array([r["rl"] for r in rows if r["rl"] is not None and r["rp"] is not None], dtype=np.float64) + rp = np.array([r["rp"] for r in rows if r["rl"] is not None and r["rp"] is not None], dtype=np.float64) + out = {"trades": int(len(rl)), "live_mean": float(rl.mean()) if len(rl) else None, "mirror_mean": float(rp.mean()) if len(rp) else None} + if len(rl) >= GAP_TRADES and rl.mean() - rp.mean() < -GAP_MAX: + c = rails.load_circuit(conn, rails.KALSHI_CIRCUIT) + if not c.entries_paused: + conn.execute("UPDATE circuit_state SET entries_paused = true, updated_at = now() WHERE id = %s", (rails.KALSHI_CIRCUIT,)) + conn.execute("INSERT INTO circuit_events (kind, detail) VALUES ('gap_pause', %s)", (json.dumps({**out, "arm": arm, "circuit": rails.KALSHI_CIRCUIT}),)) + record_event("error", "kalshi_live", f"{arm}: entries paused: live trails its paper mirror by {(rp.mean() - rl.mean()) * 100:.1f} points per trade over {len(rl)} trades", out) + out["paused"] = True + return out diff --git a/fly_trader/kalshi/maker.py b/fly_trader/kalshi/maker.py new file mode 100644 index 0000000000000000000000000000000000000000..9336c0dacefedd862871409a309f4438253667fb --- /dev/null +++ b/fly_trader/kalshi/maker.py @@ -0,0 +1,137 @@ +"""The maker arm (paper book ``paper_kalshi_maker``; mirrored live by kalshi/live.py): for every pick whose maker edge +clears the fly's line, a ``post_only`` bid one tick above the side's best bid and below its ask — the order the training +label assumes (kalshi/decisions.maker_rest / maker_filled). Each minute the plan is re-made: + +- a new pick posts (``kalshi_orders`` status 'resting', expiring at min(close − ``KALSHI_MAKER_QUIET_MIN``, now + + ``KALSHI_MAKER_TTL_H``)); nothing posts inside the quiet margin or beyond ``KALSHI_MAKER_MAX_RESTING`` open orders; +- a resting order stays at its price until it fills or expires, as the label's order does: re-pricing it when the book + moves would be another order than the one trained on (and live, our own bid would be the best bid it chased); +- a resting order whose (market, side) the fly no longer picks is cancelled; +- paper adjudication against the minute's extremes (``kalshi_minutes``): a bid at ``r`` fills when the side's lowest ask + reached ``r``, or a taker sold the side below ``r`` — or at ``r`` when the bid was first at its price (it improved on + the best bid when posted); a fill opens a position with the maker fee where the series charges one; an order past its + expiry expires. +""" +from __future__ import annotations + +import json +import logging +import math +from datetime import datetime, timezone + +from .. import config +from . import paper as P +from .decisions import maker_filled, maker_rest + +log = logging.getLogger(__name__) +BOOK = P.BOOKS["maker"] + + +def rest_price(side_bid: float, side_ask: float) -> int: + return int(maker_rest(side_bid, side_ask)) + + +def resting(conn, book: str = BOOK) -> list[dict]: + return [dict(r) for r in conn.execute("SELECT * FROM kalshi_orders WHERE book = %s AND status = 'resting' ORDER BY id", (book,)).fetchall()] + + +def _set_status(conn, oid: int, status: str, reason: str | None = None) -> None: + conn.execute("UPDATE kalshi_orders SET status = %s, error = COALESCE(%s, error), updated_at = now() WHERE id = %s", (status, reason, oid)) + + +def plan(conn, *, book: str, run_id: str, beat_id: int, ts: datetime, now: float, picks: list[dict], quotes: dict, metas: dict, maker_fee: dict | None = None) -> dict: + """One minute of the maker arm. ``picks``: [{ticker, side, p, edge, line, table, strategy}] with edge ≥ line (the + maker strategies'); ``quotes``: ticker → (yes_bid, yes_ask, ...); ``metas``: ticker → MarketMeta (close_ts, maker_fee).""" + have = resting(conn, book); by_key = {(o["ticker"], o["side"]): o for o in have} + opens = P.open_positions(conn, book); held = {(p["ticker"], p["side"]) for p in opens}; held_markets = {p["ticker"] for p in opens} + cash = P.cash_cents(conn, book); deployed = P.open_cost_cents(conn, book) + P.resting_cents(conn, book); bankroll = cash + deployed + blocked = P.blocked_reason(conn, book) + wanted: dict[tuple, dict] = {} + for pk in picks: + key = (pk["ticker"], pk["side"]); meta = metas.get(pk["ticker"]) + if meta is None or meta.close_ts is None or key in held or pk["ticker"] in held_markets: + continue + if meta.close_ts - now <= config.KALSHI_MAKER_QUIET_MIN * 60.0: + continue + bid, ask = P.side_quote(quotes, pk["ticker"], pk["side"]) + if not (math.isfinite(bid) and math.isfinite(ask)) or not (1 <= bid < ask <= 99): + continue + wanted[key] = {**pk, "rest": rest_price(bid, ask), "bid": bid, "ask": ask, "close_ts": meta.close_ts, "maker_fee": bool(meta.maker_fee)} + posted, canceled, replaced = [], 0, 0 + # cancellations: no longer picked (a picked order keeps its price: see the module docstring) + for key, o in by_key.items(): + if wanted.get(key) is None: + _set_status(conn, int(o["id"]), "canceled", "no longer picked"); canceled += 1 + P.decision(conn, beat_id=beat_id, run_id=run_id, ts=ts, ticker=o["ticker"], side=o["side"], kind="kalshi_maker_cancel", edge=None, usd=float(o["price_cents"]) * float(o["count"]) / 100.0, + reason="no longer picked", detail={"order": int(o["id"]), "price_cents": o["price_cents"]}, book=book) + n_rest = sum(1 for o in by_key.values() if o is not None) + for key, w in wanted.items(): + if by_key.get(key) is not None: + continue # still resting at the right price + tk, side = key; edge = float(w["edge"]) + if blocked: + P.decision(conn, beat_id=beat_id, run_id=run_id, ts=ts, ticker=tk, side=side, kind="blocked", edge=edge, usd=0.0, reason="maker pick not posted", rail=blocked, + detail={"edge": edge, "line": w["line"], "strategy": w.get("strategy")}, book=book) + continue + if n_rest >= config.KALSHI_MAKER_MAX_RESTING: + P.decision(conn, beat_id=beat_id, run_id=run_id, ts=ts, ticker=tk, side=side, kind="blocked", edge=edge, usd=0.0, reason="maker pick not posted", rail="max resting", + detail={"edge": edge, "line": w["line"], "resting": n_rest}, book=book) + continue + size, why = P.budget_cents(edge, w["line"], w.get("table"), bankroll, cash, deployed) + rest = w["rest"]; fee1 = P.maker_fee_cents(rest, tk, w["maker_fee"]); eff = rest + fee1 + count = math.floor(size / eff) if size > 0 else 0 + if count < 1 or count * (100.0 - eff) < config.KALSHI_MIN_PAYOUT_CENTS: + P.decision(conn, beat_id=beat_id, run_id=run_id, ts=ts, ticker=tk, side=side, kind="blocked", edge=edge, usd=size / 100.0, + reason=why if size <= 0 else f"{count} contracts at {rest}c pay too little", rail="sizing", detail={"edge": edge, "line": w["line"], "budget_cents": size}, book=book) + continue + exp_ts = min(w["close_ts"] - config.KALSHI_MAKER_QUIET_MIN * 60.0, now + config.KALSHI_MAKER_TTL_H * 3600.0) + did = P.decision(conn, beat_id=beat_id, run_id=run_id, ts=ts, ticker=tk, side=side, kind="kalshi_maker_post", edge=edge, usd=count * rest / 100.0, + reason=f"{w.get('strategy') or 'ev'}: maker edge {edge:.3f} ≥ {w['line']:.3f}; {why}; {count} × {rest}c resting (bid {w['bid']:.0f}c, ask {w['ask']:.0f}c)", + detail={"edge": edge, "line": w["line"], "strategy": w.get("strategy"), "p": w.get("p"), "count": count, "rest_cents": rest, "fee_cents_each": fee1, + "expiration_ts": exp_ts}, book=book) + oid = int(conn.execute("INSERT INTO kalshi_orders (book, decision_id, ticker, side, action, price_cents, count, tif, post_only, expiration_ts, status, fill_count, remaining, " + "fair_cents, margin_cents, strategy, request) VALUES (%s,%s,%s,%s,'buy',%s,%s,'good_till_canceled',true,%s,'resting',0,%s,%s,%s,%s,%s) RETURNING id", + (book, did, tk, side, rest, float(count), datetime.fromtimestamp(exp_ts, timezone.utc), float(count), float(w.get("p") or 0.0) * 100.0, + float(w["ask"]) - rest, w.get("strategy"), json.dumps({"p": w.get("p"), "edge": edge, "line": w["line"], "bid": w["bid"], "ask": w["ask"]}, + default=str))).fetchone()["id"]) + n_rest += 1; deployed += count * rest; cash -= count * rest + posted.append({"order_row": oid, "decision_id": did, "ticker": tk, "side": side, "price_cents": rest, "count": count, "expiration_ts": exp_ts, "p": w.get("p"), "edge": edge, + "line": w["line"], "strategy": w.get("strategy"), "maker_fee": w["maker_fee"]}) + return {"posted": posted, "canceled": canceled, "replaced": replaced, "resting": n_rest, "blocked": blocked} + + +def side_extremes(ex: tuple, side: str) -> tuple[float, float]: + """(lowest ask, lowest taker sale) of one side from a minute's ``(yes_ask_low, yes_bid_high, yes_sold_low, + yes_bought_high)``: the NO ask is 100 − the YES bid, and a taker sold NO when it bought YES.""" + f = lambda v: float(v) if v is not None and math.isfinite(float(v)) else math.nan + ask_low, bid_high, sold_low, bought_high = (tuple(ex) + (None, None))[:4] + if side == "yes": + return f(ask_low), f(sold_low) + return 100.0 - f(bid_high), 100.0 - f(bought_high) + + +def adjudicate(conn, *, book: str, run_id: str, beat_id: int, ts: datetime, now: float, extremes: dict, metas: dict) -> dict: + """Paper fills and expiries for the resting orders. ``extremes``: ticker → (yes_ask_low, yes_bid_high, yes_sold_low, + yes_bought_high) of the minute (absent when the market had no message this minute: nothing traded).""" + filled, expired = [], 0 + for o in resting(conn, book): + exp = o["expiration_ts"].timestamp() if o.get("expiration_ts") else math.inf + if now >= exp: + _set_status(conn, int(o["id"]), "expired"); expired += 1 + continue + ex = extremes.get(o["ticker"]) + if not ex: + continue + r = float(o["price_cents"]); ask_low, sold_low = side_extremes(ex, o["side"]) + bid_then = float((o.get("request") or {}).get("bid") or r) # an order without its posting bid is taken as joining the queue + if not bool(maker_filled(r, bid_then, ask_low, sold_low)): + continue + meta = metas.get(o["ticker"]); charged = bool(meta.maker_fee) if meta is not None else False + count = float(o["count"]); fee = P.maker_fee_cents(r, o["ticker"], charged) * count; cost = r * count + conn.execute("UPDATE kalshi_orders SET status = 'filled', fill_count = %s, remaining = 0, avg_fill_cents = %s, fee_cents = %s, updated_at = now() WHERE id = %s", (count, r, fee, int(o["id"]))) + did = P.decision(conn, beat_id=beat_id, run_id=run_id, ts=ts, ticker=o["ticker"], side=o["side"], kind="kalshi_maker_fill", edge=None, usd=cost / 100.0, + reason=f"resting {int(count)} × {int(r)}c filled (side ask low {ask_low:.0f}c, taker sale low {sold_low:.0f}c)", detail={"order": int(o["id"]), "fee_cents": fee}, book=book) + pid = P.open_position(conn, book=book, ticker=o["ticker"], side=o["side"], contracts=count, cost_cents=cost, fee_cents=fee, avg_price=r, decision_id=o.get("decision_id"), + order_id=o.get("order_id"), strategy=o.get("strategy"), arm="maker", score=None, line=None, ts=ts) + filled.append({"order_row": int(o["id"]), "position_id": pid, "decision_id": did, "ticker": o["ticker"], "side": o["side"], "count": count, "price_cents": r}) + return {"filled": filled, "expired": expired} diff --git a/fly_trader/kalshi/mature.py b/fly_trader/kalshi/mature.py new file mode 100644 index 0000000000000000000000000000000000000000..9d145943a77d4498b0c12e7d362a5070440e0f0c --- /dev/null +++ b/fly_trader/kalshi/mature.py @@ -0,0 +1,315 @@ +"""The Kalshi corpus as feature rows (``kalshi-build``; also run by the history worker after each fill round). + +Per event, every settled market's candles and trades are merged into minute bars and stepped through together (the +siblings' quotes feed ``EventState``), and every ``STRIDE_MIN`` minutes inside the entry window (the last +``KALSHI_MAX_DAYS_TO_CLOSE`` days before close, at least ``KALSHI_MIN_MINUTES_TO_CLOSE`` before it) a row per side is +written with the ``K_COLS`` vector, the settlement (``result``), the times the labels need, and what decides whether the +maker arm's bid would have filled (kalshi/decisions.maker_filled) over the order's life — the minutes after the decision +up to ``KALSHI_MAKER_TTL_H`` later, ending ``KALSHI_MAKER_QUIET_MIN`` before the close known at decision time (the arm's +expiry): ``fill_ask_low``, the lowest ask the side showed, and ``fill_sold_low``, the lowest price a taker sold the side +at (from the trades). The order's window is stamped on every part (``MAKER_WINDOW``): another window is another corpus. +Output: ``data/kalshi/features//part-.parquet`` (day = the row's UTC date), +version-stamped like train/mature.py; markets go ``done → built`` in ``kalshi_corpus``; ``kalshi_days`` counts rows. +""" +from __future__ import annotations + +import logging +import math +import time +from collections import defaultdict +from datetime import datetime, timezone +from pathlib import Path + +import numpy as np +import pandas as pd +import pyarrow as pa +import pyarrow.parquet as pq + +from .. import config +from ..db.connection import transaction +from ..train.corpus_features import _epoch_s +from . import data as D +from .features import CATEGORIES, K_COLS, KALSHI_FEATURE_VERSION, Bar, EventState, MarketMeta, MarketState, warm_fees + +log = logging.getLogger(__name__) +STRIDE_MIN = 5 +META_COLS = ["ticker", "side", "ts", "event_ticker", "category", "close_ts", "end_ts", "settled_ts", "result", "fill_ask_low", "fill_sold_low"] +SCHEMA = pa.schema([("ticker", pa.string()), ("side", pa.string()), ("ts", pa.timestamp("ms", tz="UTC")), ("event_ticker", pa.string()), ("category", pa.string()), + ("close_ts", pa.float64()), ("end_ts", pa.float64()), ("settled_ts", pa.float64()), ("result", pa.string()), ("fill_ask_low", pa.float32()), ("fill_sold_low", pa.float32())] + + [(c, pa.float32()) for c in K_COLS]) +MAKER_WINDOW = f"{config.KALSHI_MAKER_TTL_H:g}h-quiet{config.KALSHI_MAKER_QUIET_MIN:g}m" # the maker order's life the labels assume +RECURRING = ("hourly", "daily", "15min", "minute", "4h", "weekly") + + +def part_version(path) -> int: + md = pq.read_schema(path).metadata or {} + try: + return int(md.get(b"fly_version", b"0")) + except ValueError: + return 0 + + +def part_current(path) -> bool: + p = Path(path) + return p.exists() and part_version(p) == KALSHI_FEATURE_VERSION and (pq.read_schema(p).metadata or {}).get(b"maker_window") == MAKER_WINDOW.encode() + + +def build_complete() -> tuple[bool, str]: + with transaction() as conn: + c = {r["status"]: int(r["n"]) for r in conn.execute("SELECT status, count(*) AS n FROM kalshi_corpus GROUP BY status").fetchall()} + if not c: + return False, "no Kalshi corpus yet (kalshi-history)" + if c.get("pending"): + return False, f"{c['pending']} market(s) still to fetch" + if c.get("done"): + return False, f"{c['done']} market(s) still to build" + stale = [f for f in D.FEATURES_DIR.glob("*/part-*.parquet") if not part_current(f)] + if stale: + return False, f"{len(stale)} feature part(s) from another version (rebuild)" + return True, f"{c.get('built', 0)} markets built" + + +def ticker_strike(ticker: str | None) -> float | None: + """A ladder market's strike from its ticker's last segment: ``T81299.99`` (threshold) or ``B7650`` (bucket); else None.""" + import re + m = re.fullmatch(r"[TB](-?\d+(?:\.\d+)?)", str(ticker or "").rsplit("-", 1)[-1]) + return float(m.group(1)) if m else None + + +def time_anchor(close_time, expected_expiration_time, early_close_condition) -> float | None: + """The close a trader could know at decision time (epoch s), or None when it cannot be told. + + A market with an early-close condition ("after a winner is declared", "if the price criterion is met", "if the event + occurs") may close before its listed close when its event happens; the close recorded after settlement is then the + moment of the event — information from the future, and for a threshold market the outcome itself. Its anchor is the + scheduled ``expected_expiration_time``, set at listing. A market without one closes on its listed schedule, which is + known. Markets whose details were never fetched (no expected expiration recorded) give None: no rows, rather than a + leaky anchor. Until 2026-10-08 every feature, window and trigger used the realized close; the selector learned to buy + favorites in a match's last 17 minutes, which live it can never know (snapshot 188).""" + if expected_expiration_time is None: + return None + if early_close_condition: + return expected_expiration_time.timestamp() + return close_time.timestamp() if close_time is not None else None + + +def load_meta(conn, tickers: list[str]) -> dict[str, MarketMeta]: + rows = conn.execute("""SELECT m.ticker, m.event_ticker, m.open_time, m.close_time, m.expected_expiration_time, m.early_close_condition, + m.floor_strike, m.cap_strike, e.series_ticker, e.category AS ecat, + e.mutually_exclusive, s.category AS scat, s.frequency, s.fee_type, s.fee_multiplier + FROM kalshi_markets m LEFT JOIN kalshi_events e USING (event_ticker) LEFT JOIN kalshi_series s ON s.ticker = e.series_ticker + WHERE m.ticker = ANY(%s)""", (tickers,)).fetchall() + out = {} + for r in rows: + freq = (r["frequency"] or "").lower() + strike = r["floor_strike"] if r["floor_strike"] is not None else r["cap_strike"] + if strike is None: + strike = ticker_strike(r["ticker"]) # the catalogue lacks it (dataset or universe rows): the ticker carries it + out[r["ticker"]] = MarketMeta(ticker=r["ticker"], event_ticker=r["event_ticker"], series_ticker=r["series_ticker"] or D.series_ticker_of(r["ticker"]), + category=D.category_key(r["scat"] or r["ecat"]), is_recurring=1.0 if any(k in freq for k in RECURRING) else 0.0, + open_ts=r["open_time"].timestamp() if r["open_time"] else None, + close_ts=time_anchor(r["close_time"], r["expected_expiration_time"], r["early_close_condition"]), + end_ts=r["close_time"].timestamp() if r["close_time"] else None, + mutually_exclusive=1.0 if r["mutually_exclusive"] else 0.0, + fee_multiplier=float(r["fee_multiplier"]) if r["fee_multiplier"] is not None else 1.0, + maker_fee=1.0 if (r["fee_type"] or "").endswith("maker_fees") else 0.0, strike=float(strike) if strike is not None else None) + return out + + +def bars_from_files(candle_path: str | None, trade_path: str | None) -> list[Bar]: + """Minute bars from a market's candle file, with the taker flow of its trades folded in by minute.""" + if not candle_path or not Path(candle_path).exists(): + return [] + c = pq.read_table(candle_path).to_pandas() + flow: dict[int, list[float]] = defaultdict(lambda: [0.0, 0.0, 0.0, 0.0, 0.0]) # taker_yes, taker_no, n, max, block + if trade_path and Path(trade_path).exists(): + t = pq.read_table(trade_path).to_pandas() + if len(t): + ends = (np.floor(_epoch_s(t["ts"]) / 60.0) * 60 + 60).astype(np.int64) + for e, side, cnt, blk in zip(ends, t["taker_side"].to_numpy(), t["count"].to_numpy(), t["is_block"].to_numpy()): + f = flow[int(e)] + if side == "yes": + f[0] += float(cnt) + else: + f[1] += float(cnt) + f[2] += 1; f[3] = max(f[3], float(cnt)); f[4] += float(cnt) if blk else 0.0 + out = [] + for r in c.itertuples(index=False): + fl = flow.get(int(r.end_ts), [0.0, 0.0, 0.0, 0.0, 0.0]) + out.append(Bar(float(r.end_ts), float(r.yes_bid_close) if r.yes_bid_close == r.yes_bid_close and 0 < r.yes_bid_close < 100 else float("nan"), + float(r.yes_ask_close) if r.yes_ask_close == r.yes_ask_close and 0 < r.yes_ask_close < 100 else float("nan"), + float(r.price_close) if r.price_close == r.price_close else float("nan"), 0.0, 0.0, float(r.volume or 0.0), float(r.open_interest or 0.0), + fl[0], fl[1], fl[2], fl[3], fl[4])) + return out + + +def _side_ask_lows(c_path: str) -> tuple[np.ndarray, np.ndarray, np.ndarray]: + """(end_ts, lowest YES ask of the minute, lowest NO ask of the minute) from a candle file.""" + c = pq.read_table(c_path, columns=["end_ts", "yes_ask_low", "yes_bid_high"]).to_pandas() + ya = c["yes_ask_low"].to_numpy(dtype=float); yb = c["yes_bid_high"].to_numpy(dtype=float) + ya = np.where(np.isfinite(ya) & (ya > 0) & (ya < 100), ya, np.nan); nb = np.where(np.isfinite(yb) & (yb > 0) & (yb < 100), 100.0 - yb, np.nan) + return c["end_ts"].to_numpy(dtype=np.int64), ya, nb + + +def _side_sold_lows(t_path: str | None) -> dict[str, tuple[np.ndarray, np.ndarray]]: + """Per side, (minute end, lowest price a taker sold the side at in that minute) from a trade file: a taker who bought + NO sold YES at the trade's YES price, one who bought YES sold NO at 100 − it.""" + out = {s: (np.zeros(0, np.int64), np.zeros(0)) for s in ("yes", "no")} + if not t_path or not Path(t_path).exists(): + return out + t = pq.read_table(t_path, columns=["ts", "yes_price", "taker_side"]).to_pandas() + if not len(t): + return out + ends = (np.floor(_epoch_s(t["ts"]) / 60.0) * 60 + 60).astype(np.int64); yp = t["yes_price"].to_numpy(dtype=float); who = t["taker_side"].to_numpy() + for side, seller, price in (("yes", "no", yp), ("no", "yes", 100.0 - yp)): + m = (who == seller) & (yp > 0) & (yp < 100) + if m.any(): + s = pd.Series(price[m]).groupby(ends[m]).min() + out[side] = (s.index.to_numpy(np.int64), s.to_numpy(dtype=float)) + return out + + +def _window_lows(ends: np.ndarray, vals: np.ndarray, after: np.ndarray, until: np.ndarray) -> np.ndarray: + """Per query i, the lowest of ``vals`` over the minutes whose end lies in (after[i], until[i]]; NaN where there is none. + ``ends`` ascending; a sparse table answers every query in O(1).""" + out = np.full(len(after), np.nan) + if not len(ends) or not len(after): + return out + tab = [np.where(np.isfinite(vals), vals, np.inf)] + while (1 << len(tab)) <= len(ends): + h = 1 << (len(tab) - 1); tab.append(np.minimum(tab[-1][:-h], tab[-1][h:])) + lo = np.searchsorted(ends, after, side="right"); hi = np.searchsorted(ends, until, side="right"); n = hi - lo; ok = n > 0 + k = np.zeros(len(after), np.int64); k[ok] = np.floor(np.log2(n[ok])).astype(np.int64) + k[ok] -= np.left_shift(1, k[ok]) > n[ok] # float log2 rounding up at a power of two + for kk in np.unique(k[ok]): + m = ok & (k == kk); v = np.minimum(tab[kk][lo[m]], tab[kk][hi[m] - (1 << int(kk))]) + out[m] = np.where(np.isfinite(v), v, np.nan) + return out + + +def fill_lows(candle_path: str, trade_path: str | None, anchor_close: float | None) -> tuple[np.ndarray, dict]: + """(candle minute ends, side -> (fill_ask_low, fill_sold_low) per candle minute) for a maker bid posted at the end of + each candle minute and resting for the order's life: until ``KALSHI_MAKER_TTL_H`` later, ending ``KALSHI_MAKER_QUIET_MIN`` + before the close known at decision time (kalshi/maker.plan's expiry; minutes after an early close have no data).""" + ends, ya, nb = _side_ask_lows(candle_path); sold = _side_sold_lows(trade_path) + o = np.argsort(ends, kind="stable"); ends, ya, nb = ends[o], ya[o], nb[o] + cut = (anchor_close if anchor_close is not None else math.inf) - config.KALSHI_MAKER_QUIET_MIN * 60.0 + until = np.minimum(ends + config.KALSHI_MAKER_TTL_H * 3600.0, cut) + return ends, {side: (_window_lows(ends, asks, ends, until), _window_lows(*sold[side], ends, until)) for side, asks in (("yes", ya), ("no", nb))} + + +def build_event(event_ticker: str, markets: list[dict], metas: dict[str, MarketMeta]) -> list[dict]: + """Rows of every market of one event, stepped minute by minute together.""" + bars = {m["ticker"]: bars_from_files(m["candle_path"], m["trade_path"]) for m in markets} + fills = {m["ticker"]: fill_lows(m["candle_path"], m["trade_path"], metas[m["ticker"]].close_ts) for m in markets if m["candle_path"]} + states = {tk: MarketState(tk) for tk in bars}; ev = EventState() + pos = {tk: 0 for tk in bars} + minutes = sorted({b.t_end for bs in bars.values() for b in bs}) + max_s = config.KALSHI_MAX_DAYS_TO_CLOSE * 86400.0; min_s = config.KALSHI_MIN_MINUTES_TO_CLOSE * 60.0 + out = [] + for t_end in minutes: + for tk, bs in bars.items(): + i = pos[tk] + if i < len(bs) and bs[i].t_end == t_end: + states[tk].append(bs[i]); ev.update(tk, bs[i], metas[tk].strike); pos[tk] = i + 1 + if int(t_end // 60) % STRIDE_MIN: + continue + for m in markets: + tk = m["ticker"]; meta = metas[tk]; st = states[tk] + if st.last is None or st.last.t_end != t_end or meta.close_ts is None: + continue + to_close = meta.close_ts - t_end + if not (min_s <= to_close <= max_s): + continue + ends, fl = fills[tk] + j = int(np.searchsorted(ends, int(t_end))) + for side in ("yes", "no"): + x = st.features(t_end, meta, side, ev) + f_ask, f_sold = fl[side] + row = {"ticker": tk, "side": side, "ts": datetime.fromtimestamp(t_end - 60.0, timezone.utc), "event_ticker": event_ticker, "category": meta.category, + "close_ts": meta.close_ts, "end_ts": meta.end_ts, "settled_ts": (m["settled_ts"].timestamp() if m["settled_ts"] else meta.end_ts), "result": m["result"], + "fill_ask_low": float(f_ask[j]) if j < len(f_ask) else float("nan"), "fill_sold_low": float(f_sold[j]) if j < len(f_sold) else float("nan")} + row.update(zip(K_COLS, x)) + out.append(row) + return out + + +def _events_todo(conn, limit: int) -> dict[str, list[dict]]: + rows = conn.execute("""SELECT c.ticker, c.candle_path, c.trade_path, c.settled_ts, c.result, m.event_ticker FROM kalshi_corpus c JOIN kalshi_markets m USING (ticker) + WHERE c.status = 'done' AND c.candle_path IS NOT NULL AND c.result IN ('yes', 'no') ORDER BY c.close_time DESC LIMIT %s""", (limit,)).fetchall() + by: dict[str, list[dict]] = defaultdict(list) + for r in rows: + by[r["event_ticker"] or r["ticker"]].append(dict(r)) + if not by: + return {} + # an event's markets are built together: pull in its other done markets too + evs = list(by) + more = conn.execute("""SELECT c.ticker, c.candle_path, c.trade_path, c.settled_ts, c.result, m.event_ticker FROM kalshi_corpus c JOIN kalshi_markets m USING (ticker) + WHERE c.status = 'done' AND c.candle_path IS NOT NULL AND c.result IN ('yes', 'no') AND m.event_ticker = ANY(%s)""", (evs,)).fetchall() + for r in more: + if r["ticker"] not in {x["ticker"] for x in by[r["event_ticker"]]}: + by[r["event_ticker"]].append(dict(r)) + return by + + +def build_batch(limit: int = 300) -> int: + """Build the features of up to ``limit`` done markets (whole events); returns markets built.""" + t0 = time.time() + with transaction() as conn: + warm_fees(conn); by = _events_todo(conn, limit) + metas = load_meta(conn, [m["ticker"] for ms in by.values() for m in ms]) + if not by: + return 0 + rows_by_day: dict[str, list[dict]] = defaultdict(list); built = [] + for ev, ms in by.items(): + ms = [m for m in ms if m["ticker"] in metas] + try: + for row in build_event(ev, ms, metas): + rows_by_day[row["ts"].date().isoformat()].append(row) + built.extend(m["ticker"] for m in ms) + except Exception: + log.exception("build %s failed", ev) + with transaction() as conn: + conn.cursor().executemany("UPDATE kalshi_corpus SET status = 'error', last_error = 'feature build failed' WHERE ticker = %s", [(m["ticker"],) for m in ms]) + batch = int(time.time() * 1000); n_rows = 0 + for day, rows in rows_by_day.items(): + path = D.FEATURES_DIR / day / f"part-{batch}.parquet"; path.parent.mkdir(parents=True, exist_ok=True) + tab = pa.Table.from_pylist(rows, schema=SCHEMA).replace_schema_metadata({b"fly_version": str(KALSHI_FEATURE_VERSION).encode(), b"stride_min": str(STRIDE_MIN).encode(), + b"maker_window": MAKER_WINDOW.encode()}) + tmp = path.with_name(path.name + ".tmp"); pq.write_table(tab, tmp, compression="zstd"); tmp.replace(path); n_rows += len(rows) + with transaction() as conn: + conn.cursor().executemany("UPDATE kalshi_corpus SET status = 'built', updated_at = now() WHERE ticker = %s", [(t,) for t in built]) + conn.cursor().executemany("INSERT INTO kalshi_days (day, markets, rows, took_s) VALUES (%s,%s,%s,%s) ON CONFLICT (day) DO UPDATE SET markets = kalshi_days.markets + EXCLUDED.markets, " + "rows = kalshi_days.rows + EXCLUDED.rows, took_s = EXCLUDED.took_s, built_at = now()", + [(datetime.fromisoformat(d).date(), len({r["ticker"] for r in rows}), len(rows), time.time() - t0) for d, rows in rows_by_day.items()]) + log.info("kalshi features: %d markets in %d events -> %d rows over %d days (%.0fs)", len(built), len(by), n_rows, len(rows_by_day), time.time() - t0) + return len(built) + + +def retire_stale() -> int: + """Parts from another feature version are removed and their markets rebuilt.""" + stale = [f for f in D.FEATURES_DIR.glob("*/part-*.parquet") if not part_current(f)] + for f in stale: + f.unlink() + if stale: + with transaction() as conn: + conn.execute("UPDATE kalshi_corpus SET status = 'done' WHERE status = 'built'") + conn.execute("DELETE FROM kalshi_days") + return len(stale) + + +def loop_once(limit: int = 300) -> int: + n = 0 + retire_stale() + while True: + k = build_batch(limit) + if not k: + break + n += k + return n + + +def main() -> None: + from ..logging_setup import setup + setup("kalshi_history") + print(f"built {loop_once()} markets; complete: {build_complete()}") diff --git a/fly_trader/kalshi/paper.py b/fly_trader/kalshi/paper.py new file mode 100644 index 0000000000000000000000000000000000000000..b31102ceec8d1666bbe9e5ec37358bbbb341b5f2 --- /dev/null +++ b/fly_trader/kalshi/paper.py @@ -0,0 +1,215 @@ +"""The Kalshi paper books' shared arithmetic (agent/paper_trading.py for prediction markets): cash and bankroll from the +capital cap, position sizing from the fly's certainty bands and a walk of the order book (vendored ``fill_budget``), the +taker arm's entries, settlement of positions when their market resolves, marks and wealth per minute (``wealth_marks``, +in USD), and the book's own drawdown halt. Every entry, block and settlement is a ``decisions`` row (``mint`` = ticker, +``pool`` = side, ``size_sol`` = dollars). + +Books: ``paper_kalshi_taker`` (IOC at the worst level of the book walk) and ``paper_kalshi_maker`` (kalshi/maker.py); +their live mirrors ``live_kalshi_taker`` / ``live_kalshi_maker`` (kalshi/live.py) reuse ``settle`` and ``mark``. +""" +from __future__ import annotations + +import json +import logging +import math +from datetime import datetime, timezone + +from .. import config +from ..agent import rails, sizing +from .vendor.book_sizing import BookLevel, OrderBook, fill_budget +from .vendor.kalshi_client import effective_price_cents, kalshi_fee_rate_cents + +log = logging.getLogger(__name__) +BOOKS = {"taker": "paper_kalshi_taker", "maker": "paper_kalshi_maker"} +LIVE_BOOKS = {"taker": "live_kalshi_taker", "maker": "live_kalshi_maker"} +MAX_POSITION_FRACTION = 0.10 # of the cap in one position +PAYOUT_CENTS = 100.0 + + +def cap_cents() -> float: + return float(config.KALSHI_CAPITAL_USD) * 100.0 + + +def open_positions(conn, book: str) -> list[dict]: + return [dict(r) for r in conn.execute("SELECT * FROM kalshi_positions WHERE book = %s AND status = 'open' ORDER BY id", (book,)).fetchall()] + + +def realized_cents(conn, book: str) -> float: + r = conn.execute("SELECT COALESCE(sum(realized_cents), 0) AS s FROM kalshi_positions WHERE book = %s AND status <> 'open'", (book,)).fetchone() + return float(r["s"] or 0.0) + + +def open_cost_cents(conn, book: str) -> float: + r = conn.execute("SELECT COALESCE(sum(cost_cents + fee_cents), 0) AS s FROM kalshi_positions WHERE book = %s AND status = 'open'", (book,)).fetchone() + return float(r["s"] or 0.0) + + +def resting_cents(conn, book: str) -> float: + """Collateral tied up by resting orders (price × count, the maker's cost if filled).""" + r = conn.execute("SELECT COALESCE(sum(price_cents * count), 0) AS s FROM kalshi_orders WHERE book = %s AND status = 'resting'", (book,)).fetchone() + return float(r["s"] or 0.0) + + +def cash_cents(conn, book: str) -> float: + """A paper book's cash: the cap, plus what settled, minus what is out in positions and resting orders.""" + return cap_cents() + realized_cents(conn, book) - open_cost_cents(conn, book) - resting_cents(conn, book) + + +def side_quote(quotes: dict, ticker: str, side: str) -> tuple[float, float]: + """(bid, ask) of ``side`` in cents from a YES quote (yes_bid, yes_ask); NaN where missing.""" + q = quotes.get(ticker) + if not q: + return math.nan, math.nan + yb, ya = q[0], q[1] + yb = yb if yb is not None and math.isfinite(yb) else math.nan; ya = ya if ya is not None and math.isfinite(ya) else math.nan + return (yb, ya) if side == "yes" else (100.0 - ya, 100.0 - yb) + + +def top_of_book(quotes: dict, ticker: str, side: str) -> OrderBook: + """The order book from the top of book alone (bid sizes are contracts): what the taker arm walks when no depth is held.""" + q = quotes.get(ticker) or (None, None, 0.0, 0.0) + yb, ya = q[0], q[1]; bs = float(q[2] or 0.0) if len(q) > 2 else 0.0; as_ = float(q[3] or 0.0) if len(q) > 3 else 0.0 + yes_bids = (BookLevel(int(round(yb)), int(max(bs, 1)) if bs else 10_000),) if yb is not None and 1 <= yb <= 99 else () + no_bids = (BookLevel(int(round(100 - ya)), int(max(as_, 1)) if as_ else 10_000),) if ya is not None and 1 <= ya <= 99 else () + return OrderBook(yes_bids=yes_bids, no_bids=no_bids, ticker=ticker) + + +def budget_cents(edge: float, line: float, table: list | None, bankroll: float, cash: float, deployed: float) -> tuple[float, str]: + """Dollars (in cents) to put behind one pick: the certainty band's Kelly fraction of the bankroll (agent/sizing.py), + capped at ``MAX_POSITION_FRACTION`` of the cap, the slate fraction of the cap across every open position, and the cash + above the floor.""" + band = sizing.band_for(table or [], float(edge) - float(line)) + k = float((band or {}).get("kelly") or 0.0) + if k <= 0: + return 0.0, "no certainty band with an edge" if not band else f"band at margin ≥ {band['lo']:.3f} has no edge (kelly 0)" + want = k * bankroll + cap = cap_cents(); room_slate = cap * config.KALSHI_MAX_SLATE_FRACTION - deployed + room_cash = cash - config.KALSHI_CASH_FLOOR_USD * 100.0 + size = min(want, cap * MAX_POSITION_FRACTION, room_slate, room_cash) + if size <= 0: + return 0.0, "slate full" if room_slate <= 0 else "cash at the floor" + return float(size), f"kelly {k:.2f} × bankroll ${bankroll / 100:.2f}" + (" (capped)" if size < want else "") + + +def maker_fee_cents(price_cents: float, ticker: str, charged: bool) -> float: + return config.KALSHI_MAKER_FEE_FACTOR / config.KALSHI_FEE_FACTOR * kalshi_fee_rate_cents(price_cents, ticker) if charged else 0.0 + + +def decision(conn, *, beat_id: int, run_id: str, ts: datetime, ticker: str, side: str, kind: str, edge: float | None, usd: float, reason: str, detail: dict, + rail: str | None = None, book: str | None = None) -> int: + return int(conn.execute("INSERT INTO decisions (beat_id, run_id, ts, mint, pool, kind, m_hat, size_sol, forced, rail, reason, detail, book_targets) " + "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,false,%s,%s,%s,%s) RETURNING id", + (beat_id, run_id, ts, ticker, side, kind, edge, usd, rail, reason, json.dumps({**detail, "book": book}, default=str), [book] if book else None)).fetchone()["id"]) + + +def open_position(conn, *, book: str, ticker: str, side: str, contracts: float, cost_cents: float, fee_cents: float, avg_price: float, decision_id: int | None, + order_id: str | None, strategy: str | None, arm: str, score: float | None, line: float | None, ts: datetime) -> int: + return int(conn.execute("INSERT INTO kalshi_positions (book, ticker, side, contracts, cost_cents, fee_cents, avg_price_cents, opened_at, entry_decision_id, order_id, strategy, arm, score, line) " + "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) RETURNING id", + (book, ticker, side, contracts, cost_cents, fee_cents, avg_price, ts, decision_id, order_id, strategy, arm, score, line)).fetchone()["id"]) + + +def blocked_reason(conn, book: str) -> str | None: + c = rails.load_circuit(conn, rails.KALSHI_CIRCUIT) + if c.kill_switch: + return "kill switch" + if c.entries_paused: + return "paused" + if rails.book_halted(conn, book): + return "book halted" + return None + + +def taker_entries(conn, *, book: str, run_id: str, beat_id: int, ts: datetime, picks: list[dict], quotes: dict, orderbooks: dict | None = None, kind: str = "kalshi_taker") -> dict: + """The taker arm's entries for one minute. ``picks``: [{ticker, side, p, edge, line, table, strategy, hold_s}] at or above the + line; each is sized (``budget_cents``) and walked through its order book (``orderbooks[ticker]`` when held, else the top + of book) at ``p``; a fill is opened at its fee-inclusive VWAP with the walk's worst level as the limit (what the live + mirror submits as an IOC). One position per (market, side); the other side of a held market is skipped.""" + opens = open_positions(conn, book); held = {(p["ticker"], p["side"]) for p in opens}; held_markets = {p["ticker"] for p in opens} + cash = cash_cents(conn, book); deployed = open_cost_cents(conn, book); bankroll = cash + deployed + blocked = blocked_reason(conn, book); entries = []; n = 0 + for pk in picks: + tk, side = pk["ticker"], pk["side"]; edge = float(pk["edge"]); usd0 = 0.0 + if (tk, side) in held or tk in held_markets or blocked: + rail = blocked or ("held" if (tk, side) in held else "other side held") + decision(conn, beat_id=beat_id, run_id=run_id, ts=ts, ticker=tk, side=side, kind="blocked", edge=edge, usd=usd0, reason=f"{kind} pick not taken", rail=rail, + detail={"edge": edge, "line": pk["line"], "strategy": pk.get("strategy"), "p": pk.get("p")}, book=book) + continue + size, why = budget_cents(edge, pk["line"], pk.get("table"), bankroll, cash, deployed) + if size <= 0: + decision(conn, beat_id=beat_id, run_id=run_id, ts=ts, ticker=tk, side=side, kind="blocked", edge=edge, usd=0.0, reason=why, rail="sizing", + detail={"edge": edge, "line": pk["line"], "strategy": pk.get("strategy"), "p": pk.get("p")}, book=book) + continue + ob = (orderbooks or {}).get(tk) or top_of_book(quotes, tk, side) + fill = fill_budget(ob, side, float(pk["p"]), size, min_payout_cents=config.KALSHI_MIN_PAYOUT_CENTS) + if not fill.is_fillable: + decision(conn, beat_id=beat_id, run_id=run_id, ts=ts, ticker=tk, side=side, kind="blocked", edge=edge, usd=size / 100.0, reason=f"book walk: {fill.stopped_reason}", + rail="paper_fill", detail={"edge": edge, "line": pk["line"], "strategy": pk.get("strategy"), "p": pk.get("p"), "budget_cents": size, "best_ask": fill.best_ask_cents}, book=book) + continue + cost = fill.cost_cents - fill.fee_cents; fee = fill.fee_cents + did = decision(conn, beat_id=beat_id, run_id=run_id, ts=ts, ticker=tk, side=side, kind=f"{kind}_enter", edge=edge, usd=fill.cost_cents / 100.0, + reason=f"{pk.get('strategy') or 'ev'}: edge {edge:.3f} ≥ {pk['line']:.3f}; {why}; {fill.quantity} × {fill.vwap_cents:.1f}c (limit {fill.limit_price_cents}c)", + detail={"edge": edge, "line": pk["line"], "strategy": pk.get("strategy"), "p": pk.get("p"), "contracts": fill.quantity, "vwap_cents": fill.vwap_cents, + "limit_price_cents": fill.limit_price_cents, "fee_cents": fee, "levels": fill.levels_taken, "budget_cents": size, "hold_s": pk.get("hold_s")}, book=book) + pid = open_position(conn, book=book, ticker=tk, side=side, contracts=float(fill.quantity), cost_cents=float(cost), fee_cents=float(fee), avg_price=cost / fill.quantity, + decision_id=did, order_id=None, strategy=pk.get("strategy"), arm="taker", score=edge, line=pk["line"], ts=ts) + held.add((tk, side)); held_markets.add(tk); cash -= fill.cost_cents; deployed += fill.cost_cents; n += 1 + entries.append({"position_id": pid, "decision_id": did, "ticker": tk, "side": side, "contracts": fill.quantity, "limit_price_cents": fill.limit_price_cents, + "vwap_cents": fill.vwap_cents, "cost_cents": fill.cost_cents, "p": pk.get("p"), "edge": edge, "line": pk["line"], "strategy": pk.get("strategy")}) + return {"entered": n, "entries": entries, "blocked": blocked, "cash_cents": cash} + + +def settle(conn, *, book: str, results: dict, ts: datetime, run_id: str | None = None, beat_id: int | None = None, payouts: dict | None = None) -> list[dict]: + """Close the book's open positions in markets that resolved (``results``: ticker → 'yes'|'no'); a contract on the + winning side pays 100c. ``payouts`` (live): ticker → (payout_cents, fee_cents) from the exchange's settlement row.""" + out = [] + for p in open_positions(conn, book): + res = results.get(p["ticker"]) + if res not in ("yes", "no"): + continue + won = res == p["side"] + if payouts and p["ticker"] in payouts: + payout, fee_s = payouts[p["ticker"]] + else: + payout, fee_s = (PAYOUT_CENTS * float(p["contracts"]) if won else 0.0), 0.0 + realized = payout - float(p["cost_cents"]) - float(p["fee_cents"]) - fee_s + did = None + if run_id is not None and beat_id is not None: + did = decision(conn, beat_id=beat_id, run_id=run_id, ts=ts, ticker=p["ticker"], side=p["side"], kind="kalshi_settle", edge=p.get("score"), usd=payout / 100.0, + reason=f"settled {res}: {'won' if won else 'lost'} {realized / 100:+.2f} USD", detail={"payout_cents": payout, "realized_cents": realized, "position": int(p["id"])}, book=book) + conn.execute("UPDATE kalshi_positions SET status = 'settled', closed_at = %s, result = %s, payout_cents = %s, realized_cents = %s, exit_decision_id = %s WHERE id = %s", + (ts, res, payout, realized, did, int(p["id"]))) + out.append({**p, "result": res, "payout_cents": payout, "realized_cents": realized}) + return out + + +def exit_value_cents(p: dict, quotes: dict) -> float: + """What the position would fetch now: contracts × the side's bid, net of the taker fee (the training label's exit).""" + bid, _ask = side_quote(quotes, p["ticker"], p["side"]) + if not math.isfinite(bid): + bid = float(p.get("last_mark_cents") or p.get("avg_price_cents") or 0.0) + bid = min(max(bid, 0.0), 100.0) + fee = kalshi_fee_rate_cents(min(max(bid, 1.0), 99.0), p["ticker"]) if 0 < bid < 100 else 0.0 + return float(p["contracts"]) * max(bid - fee, 0.0) + + +def mark(conn, *, book: str, quotes: dict, ts: datetime, beat_id: int, cash: float | None = None) -> dict: + """Marks, the wealth row (USD) and the book's own drawdown halt.""" + opens = open_positions(conn, book); value = 0.0; exposure = 0.0 + for p in opens: + bid, _ = side_quote(quotes, p["ticker"], p["side"]) + if math.isfinite(bid): + conn.execute("UPDATE kalshi_positions SET last_mark_cents = %s, last_mark_ts = %s WHERE id = %s", (bid, ts, int(p["id"]))) + value += exit_value_cents(p, quotes); exposure += float(p["cost_cents"]) + float(p["fee_cents"]) + cash = cash_cents(conn, book) if cash is None else float(cash) + wealth = (cash + value + resting_cents(conn, book)) / 100.0 + pk = conn.execute("SELECT max(wealth) AS pk FROM wealth_marks WHERE book = %s", (book,)).fetchone(); peak = max(float(pk["pk"] or 0.0), wealth) + conn.execute("INSERT INTO wealth_marks (beat_id, book, ts, sol_free, positions_value, exit_cost, wealth, peak, drawdown, exposure, n_open) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) " + "ON CONFLICT (beat_id, book) DO NOTHING", + (beat_id, book, ts, cash / 100.0, value / 100.0, 0.0, wealth, peak, (1.0 - wealth / peak) if peak > 0 else 0.0, exposure / 100.0, len(opens))) + halted = rails.check_book_drawdown(conn, book, wealth, peak, drawdown=config.KALSHI_KILL_SWITCH_DRAWDOWN, unit="USD") + return {"wealth": wealth, "peak": peak, "cash": cash / 100.0, "positions_value": value / 100.0, "open": len(opens), "halted": halted} + + +def eff_ask(ticker: str, ask: float) -> float: + return effective_price_cents(min(max(ask, 1.0), 99.0), ticker) diff --git a/fly_trader/kalshi/pipeline.py b/fly_trader/kalshi/pipeline.py new file mode 100644 index 0000000000000000000000000000000000000000..0f85218d45a3b9e4a15436d7d7f9f2bc5d4bb752 --- /dev/null +++ b/fly_trader/kalshi/pipeline.py @@ -0,0 +1,115 @@ +"""The Kalshi training pipeline — the ``kalshi_train`` worker (train/pipeline.py for prediction markets). Every +``INTERVAL_DAYS`` days, or at once when the operator asks: + +1. The Kalshi selector retrains on the whole corpus (kalshi/selector.py) — walk-forward by settlement day at Kalshi + costs; deployable only if its evaluation half made money and beat random picks. +2. The Kalshi fly is bootstrapped (kalshi/fly.py, the selector teaches it once) when none exists for + ``KALSHI_FLY_VERSION`` or on request; then it learns from settlements through its mushroom body. There is no race and + no handover (decision of 2026-09-22): once a deployable Kalshi fly trades, weekly runs stop and re-bootstraps are on + request (or after three rollbacks, from kalshi/fly_session.py). + +A run starts only when the corpus build is complete (kalshi/mature.build_complete); otherwise it waits an hour. +State for the console: ``ui_settings['kalshi_training_pipeline']``. +""" +from __future__ import annotations + +import json +import logging +import threading +from datetime import datetime, timedelta, timezone + +from ..db.apilog import record_event +from ..db.connection import transaction +from ..logging_setup import setup +from ..train.pipeline import INTERVAL_DAYS, RETRY_S, NotReady + +log = logging.getLogger(__name__) +KEY = "kalshi_training_pipeline" + + +def state() -> dict: + with transaction() as conn: + r = conn.execute("SELECT value FROM ui_settings WHERE key = %s", (KEY,)).fetchone() + v = r["value"] if r else {} + return v if isinstance(v, dict) else json.loads(v or "{}") + + +def _save(**kv) -> dict: + v = {**state(), **kv} + with transaction() as conn: + conn.execute("INSERT INTO ui_settings (key, value) VALUES (%s, %s) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()", + (KEY, json.dumps(v, default=str))) + return v + + +def request_run(fly: bool = False, reason: str | None = None) -> None: + kv = {"run_requested": True, "retry_after": None} + if fly: + kv.update(fly_requested=True, fly_reason=reason or "requested") + _save(**kv) + + +def _now() -> datetime: + return datetime.now(timezone.utc) + + +def _fly_trades() -> bool: + """A deployable Kalshi fly exists and the replay passed: the weekly cadence stops, runs are on request only.""" + from . import fly as KF, fly_replay + with transaction() as conn: + have = KF.latest_deployable(conn) + return have is not None and (fly_replay.verdict() or {}).get("passed") is True + + +def due(st: dict, now: datetime | None = None, fly_trades: bool = False) -> bool: + now = now or _now() + if st.get("retry_after") and now < datetime.fromisoformat(st["retry_after"]): + return False + if fly_trades: + return bool(st.get("run_requested")) + last = st.get("last_run_at") + return bool(st.get("run_requested")) or last is None or now >= datetime.fromisoformat(last) + timedelta(days=INTERVAL_DAYS) + + +def run(stop_event: threading.Event | None = None) -> dict: + from . import fly as KF, mature, selector + ok, why = mature.build_complete() + if not ok: + raise NotReady(why) + st = state() + _save(stage="training the Kalshi selector", started_at=_now().isoformat(), run_requested=False, retry_after=None, last_error=None, waiting=None) + wf = selector.main(stop_event=stop_event) + if stop_event is not None and stop_event.is_set(): + return _save(stage="stopped") + _save(selector_snapshot=wf.get("snapshot_id"), selector_deployable=wf.get("deployable"), selector_reason=wf.get("deploy_reason")) + fly_snapshot = st.get("fly_snapshot") + with transaction() as conn: + have = KF.latest_current(conn) + if st.get("fly_requested") or have is None: + _save(stage="bootstrapping the Kalshi fly (the selector teaches it once)") + boot = KF.main(stop_event=stop_event) + if stop_event is not None and stop_event.is_set(): + return _save(stage="stopped") + fly_snapshot = boot.get("snapshot_id") + now = _now() + out = _save(stage="done", last_run_at=now.isoformat(), next_run_at=(now + timedelta(days=INTERVAL_DAYS)).isoformat(), fly_snapshot=fly_snapshot, + fly_requested=False, fly_reason=None) + record_event("info", "kalshi_pipeline", "kalshi training pipeline finished", {k: out.get(k) for k in ("selector_snapshot", "selector_deployable", "fly_snapshot")}) + return out + + +def main(stop_event: threading.Event | None = None) -> None: + setup("kalshi_train") + record_event("info", "kalshi_pipeline", "kalshi training pipeline worker started", {"interval_days": INTERVAL_DAYS}) + while not (stop_event is not None and stop_event.is_set()): + try: + if due(state(), fly_trades=_fly_trades()): + run(stop_event) + except NotReady as e: + log.info("kalshi training waits for data: %s", e) + _save(stage="waiting for data", waiting=str(e), retry_after=(_now() + timedelta(seconds=RETRY_S)).isoformat()) + except Exception as e: + log.exception("kalshi training pipeline failed") + record_event("error", "kalshi_pipeline", f"kalshi training pipeline failed: {type(e).__name__}: {e}") + _save(stage="failed", last_error=f"{type(e).__name__}: {e}"[:300], retry_after=(_now() + timedelta(seconds=RETRY_S)).isoformat()) + (stop_event or threading.Event()).wait(30.0) diff --git a/fly_trader/kalshi/selector.py b/fly_trader/kalshi/selector.py new file mode 100644 index 0000000000000000000000000000000000000000..4e2ec5158074432e22d0d4027c678503e0c0c6e7 --- /dev/null +++ b/fly_trader/kalshi/selector.py @@ -0,0 +1,162 @@ +"""The Kalshi selector: the strategy stack (kalshi/strategies.py) fitted walk-forward by the shared machinery, judged on +its evaluation half against random picks at the same fee-inclusive prices, and saved as ``brain_snapshots`` kind +'kalshi_selector' (``data/brain/kalshi_selectors/kalshi_.joblib``). Its deployed models are the visual fly's +teacher (kalshi/fly.py); the selector never trades a book of its own (decision of 2026-09-22: no race, no handover). +""" +from __future__ import annotations + +import hashlib +import json +import logging +import threading +import time +from dataclasses import dataclass, field +from pathlib import Path + +import joblib +import numpy as np + +from .. import config +from ..db.apilog import record_event +from ..db.connection import transaction +from ..train import progress as prog +from ..train import strategies as S +from ..train.decisions import DecisionSet, random_trades, summarize +from ..train.selector import MIN_EV, MIN_LINE_TRADES, deploy_decision +from . import decisions as KD +from .features import K_COLS, KALSHI_FEATURE_VERSION +from .mature import MAKER_WINDOW, STRIDE_MIN +from .strategies import KALSHI, effective, probability + +log = logging.getLogger(__name__) +SELECTOR_DIR = config.BRAIN_DIR / "kalshi_selectors" +KIND = "kalshi_selector" +DATA_VERSION = {"features": KALSHI_FEATURE_VERSION, "stride": STRIDE_MIN, "selector": "kalshi-4", "arms": "taker+maker", "maker": MAKER_WINDOW, "gate": "significance", + "cols": hashlib.sha1(",".join(K_COLS).encode()).hexdigest()[:8]} + + +def holdout_passes(h: dict) -> tuple[bool, str]: + """The Kalshi stack trades only if its holdout — the last days, never fitted on — clears the same bars as its two + walk-forward halves (a positive mean, the weekly sign test, the trade count) and beats random picks there. Pass or + fail only: nothing is chosen on it, so a failed holdout is never tuned against (operator decision, 2026-10-09).""" + if not h or not h.get("n"): + return False, "no holdout trades" + m, rm = h.get("mean"), h.get("random_mean") + if not h.get("base_ok"): + return False, (f"its holdout fails the bars: {h.get('n')} trades, mean {(m or 0) * 100:+.2f}%, {h.get('weeks_pos')}/{h.get('weeks')} weeks positive" + f"{' (sign test passed)' if h.get('sign_ok') else ''}") + if rm is not None and m <= rm: + return False, f"its holdout did not beat random picks ({m * 100:+.2f}% vs {rm * 100:+.2f}%)" + return True, f"its holdout made {m * 100:+.2f}% per trade over {h.get('n')} trades, {h.get('weeks_pos')}/{h.get('weeks')} weeks positive" + + + +def is_current(meta: dict | None) -> bool: + d = (meta or {}).get("data") + return bool(meta) and isinstance(d, dict) and all(d.get(k) == v for k, v in DATA_VERSION.items()) + + +def is_deployable(meta: dict | None) -> bool: + return is_current(meta) and bool(meta.get("deployable")) + + +@dataclass +class KalshiSelectorModel: + """The deployed stack (``train/strategies.final_models`` under the Kalshi spec): per strategy its classifier, line + (a minimum edge), arm and sizing; ``decide`` is the shared decision with the Kalshi spec.""" + stack: dict + cols: list[str] + trained_through: str + metrics: dict = field(default_factory=dict) + + @property + def strategies(self) -> list[str]: + return list((self.stack.get("strategies") or {}).keys()) + + def decide(self, X: np.ndarray, cols: list[str], t_start) -> dict: + return S.decide(self.stack, X, cols, t_start, spec=KALSHI) + + def decide_all(self, X: np.ndarray, cols: list[str], t_start) -> dict: + return S.decide_all(self.stack, X, cols, t_start, spec=KALSHI) + + def probabilities(self, X: np.ndarray, cols: list[str]) -> dict[str, np.ndarray]: + """Each strategy's p̂(side pays) on every row (the fly's distillation targets).""" + out = {} + for name, m in (self.stack.get("strategies") or {}).items(): + ci = [cols.index(c) for c in m["cols"]] + out[name] = probability(m, np.atleast_2d(X)[:, ci]) + return out + + +def save(m: KalshiSelectorModel, run_id: str | None = None) -> tuple[Path, int]: + SELECTOR_DIR.mkdir(parents=True, exist_ok=True) + path = SELECTOR_DIR / f"kalshi_{time.strftime('%Y%m%dT%H%M%SZ', time.gmtime())}.joblib" + joblib.dump(m, path); sha = hashlib.sha256(path.read_bytes()).hexdigest() + with transaction() as conn: + row = conn.execute("INSERT INTO brain_snapshots (run_id, path, sha256, kind, note) VALUES (%s,%s,%s,%s,%s) RETURNING id", + (run_id, str(path), sha, KIND, json.dumps({"trained_through": m.trained_through, **m.metrics}, default=str))).fetchone() + return path, int(row["id"]) + + +def latest_current(conn) -> dict | None: + for r in conn.execute("SELECT id, path, note FROM brain_snapshots WHERE kind = %s ORDER BY id DESC", (KIND,)).fetchall(): + try: + meta = json.loads(r["note"] or "{}") + except ValueError: + continue + if is_deployable(meta) and Path(r["path"]).exists(): + return {**dict(r), "meta": meta} + return None + + +def load_snapshot(snapshot_id: int) -> KalshiSelectorModel | None: + with transaction() as conn: + row = conn.execute("SELECT path FROM brain_snapshots WHERE id = %s AND kind = %s", (snapshot_id, KIND)).fetchone() + return joblib.load(row["path"]) if row and Path(row["path"]).exists() else None + + +def load_latest() -> KalshiSelectorModel | None: + with transaction() as conn: + r = latest_current(conn) + return joblib.load(r["path"]) if r else None + + +def main(days: int | None = None, stop_event: threading.Event | None = None, holdout_days: int | None = None) -> dict: + """Fit on ``days`` (all when None), withholding the last ``holdout_days`` (default train/strategies.HOLDOUT_DAYS) from + every fit; the holdout is scored once afterwards, never used to choose anything — but the stack trades only if it + passes there too (``holdout_passes``).""" + from ..ops.reset import reset_training_stats + reset_training_stats("kalshi_selector", reason="kalshi selector training") + prog.set_stop_event(stop_event); prog.clear() + prog.update("kalshi selector: building decision points", 0, 1, force=True) + t0 = time.time(); ds = KD.build(days=days) + log.info("kalshi decision points: %d rows, %d days, taker universe mean %+.2f%%, maker filled %.0f%% (%.0fs)", len(ds.y), len(ds.days), + ds.fwd_pess.mean() * 100, np.isfinite(ds.fwd_h["maker"]).mean() * 100, time.time() - t0) + stack = S.fit_stack(ds, stop_event, holdout_days_n=holdout_days, spec=KALSHI) + for c in stack.components: + log.info("component %-28s %s — %s", c["name"], "PASSED" if c["passed"] else "dropped", c["reason"]) + record_event("info", KIND, f"component {c['name']}: {'passed' if c['passed'] else 'dropped'}", c) + if stop_event is not None and stop_event.is_set(): + return {"stopped": True} + prog.update("kalshi selector: fitting the deployable models on every day", 0, 1, force=True) + models = S.final_models(ds, stack, spec=KALSHI) if stack.fits else {"strategies": {}, "spec": "kalshi"} + holdout = S.score_holdout(ds, S.final_models(ds, stack, exclude_days=stack.holdout_days, spec=KALSHI), stack.holdout_days, spec=KALSHI) if stack.holdout_days and stack.fits else {} + deployable, reason = stack.deployable, stack.reason + if deployable: + h_ok, h_why = holdout_passes(holdout) + deployable = h_ok; reason = f"{reason}; {h_why}" + final = KalshiSelectorModel(stack=models, cols=list(ds.cols), trained_through=str(ds.days[-1])) + final.metrics = {"walk_forward": {**stack.evaluation, "days": None}, "selection": stack.selection, "random_baseline": {"mean": stack.evaluation.get("random_mean")}, + "components": stack.components, + "strategies": {k: {x: v[x] for x in ("line", "hold_min", "thr", "high", "sizing", "selection", "evaluation")} for k, v in models["strategies"].items()}, + "combine": stack.combine, "veto": {k: v for k, v in (stack.veto or {}).items() if k not in ("p",)} or None, "fallback": stack.fallback, + "holdout": holdout, "costs": "Kalshi taker fee (factor x series multiplier x P(1-P)) at the ask; maker one tick above the bid (below the ask) " + "+ maker fee where charged, filled on the ask reaching it or a taker sale into it within the order's life", + "data": DATA_VERSION, "deployable": deployable, "deploy_reason": reason, + "model": {"kind": "kalshi-stack", "max_days_to_close": config.KALSHI_MAX_DAYS_TO_CLOSE, "max_spread": config.KALSHI_MAX_SPREAD_CENTS}, + "rows": int(len(ds.y)), "days": len(ds.days), "first_day": str(ds.days[0]), "last_day": str(ds.days[-1])} + path, sid = save(final) + record_event("info", KIND, f"selector saved (snapshot {sid})", {"path": str(path), "deployable": deployable, "reason": reason, **stack.evaluation}) + prog.update("kalshi selector: saved", 1, 1, force=True, snapshot_id=sid, deployable=deployable, deploy_reason=reason) + log.info("kalshi selector saved: %s (snapshot %d) — %s: %s", path, sid, "put to work" if deployable else "NOT put to work", reason) + return {"snapshot_id": sid, "deployable": deployable, "deploy_reason": reason, "components": stack.components} diff --git a/fly_trader/kalshi/strategies.py b/fly_trader/kalshi/strategies.py new file mode 100644 index 0000000000000000000000000000000000000000..37e8b3aa177144c0aee72d9b58b2ef50f5c0bbd0 --- /dev/null +++ b/fly_trader/kalshi/strategies.py @@ -0,0 +1,116 @@ +"""The Kalshi selector's strategy stack: the literature's candidate sets as a ``train.strategies.StackSpec``, fitted, +filtered and combined by the memecoin machinery (``fit_stack`` / ``final_models`` / ``decide``) unchanged. + +Rows are (market, minute, side); a "hold" is the arm — ``taker`` (IOC at the ask, label ``fwd_pess``) or ``maker`` +(rest one tick inside the ask, label ``fwd_h["maker"]``, NaN when unfilled) — and every position runs to settlement +(``ds.hold_s`` per row). The out-of-sample score of a strategy is the walk-forward classifier's ``p̂(side pays) − +effective price``: the edge per dollar of payout, which is what the lines, the sizing bands and the fly's own head are +calibrated on (kalshi/fly.py). Candidate sets (plan of 2026-09-22, "Literature basis"): +- ``favorite``: the side priced ≥ 70c (Bürgi–Deng–Whelan 2026: positive post-fee returns above 70c, every horizon ≤ 10 d); + fitted thresholds on the side's ask, closeness to close, spread and open interest; +- ``momentum``: the side whose mid rose over 6 h with taker flow behind it (Ottaviani–Sørensen 2015 underreaction; + Reichenbach–Walther 2025 order-flow skill); thresholds on ``ret_6h``, ``taker_imb_1h``, ``log_vol_24h``; +- ``constraint``: in a mutually exclusive event, a side cheaper than its siblings imply, or a strike-ladder violation + (Nunes 2026, Saguillo 2025); thresholds on ``−implied_gap`` and ``ladder_resid``; +- ``ev``: every eligible row. +""" +from __future__ import annotations + +import numpy as np +from sklearn.ensemble import HistGradientBoostingClassifier + +from ..train import strategies as S +from ..train.scaling import RobustScaler +from .decisions import X_COLS, maker_fee_cents, maker_rest +from .features import KIDX + +ARMS = ("taker", "maker") +STRATEGIES = { + "ev": {"holds": ARMS, "vars": (), "highs": (None,)}, + "favorite": {"holds": ARMS, "vars": ("side_ask", "neg_log_h_to_close", "neg_spread", "log_oi"), "highs": (None,)}, + "momentum": {"holds": ARMS, "vars": ("ret_6h", "taker_imb_1h", "log_vol_24h"), "highs": (None,)}, + "constraint": {"holds": ARMS, "vars": ("neg_implied_gap", "ladder_resid"), "highs": (None,)}, +} +FAVORITE_MIN_ASK = 70.0 + + +def _c(cols, X, name): + return X[:, cols.index(name)] + + +def base_mask(name: str, X: np.ndarray, cols: list[str], high: str | None = None) -> np.ndarray: + if name == "ev": + return np.ones(len(X), bool) + if name == "favorite": + return _c(cols, X, "side_ask") >= FAVORITE_MIN_ASK + if name == "momentum": + return (_c(cols, X, "ret_6h") > 0) & (_c(cols, X, "taker_imb_1h") > 0) + if name == "constraint": + return (_c(cols, X, "mutually_exclusive") > 0) & ((_c(cols, X, "implied_gap") < 0) | (_c(cols, X, "ladder_resid") > 0)) + raise KeyError(name) + + +def trigger_value(name: str, var: str, X: np.ndarray, cols: list[str]) -> np.ndarray: + if var.startswith("neg_"): + return -_c(cols, X, var[4:]) + return _c(cols, X, var) + + +def in_universe(X: np.ndarray, cols: list[str]) -> np.ndarray: + X = np.atleast_2d(X); a = X[:, cols.index("side_ask")] + return (a >= 1.0) & (a <= 99.0) + + +def label(ds, H): + if H == "taker": + return ds.fwd_pess + if H == "maker": + return ds.fwd_h.get("maker") + return None + + +def hold_s(ds, H): + return ds.hold_s + + +def effective(X: np.ndarray, cols: list[str], H) -> np.ndarray: + """The arm's fee-inclusive price per contract (cents): the ask for the taker; for the maker its bid (one tick above + the side's bid, below the ask: kalshi/decisions.maker_rest) plus the maker fee.""" + X = np.atleast_2d(X) + if H == "taker": + return X[:, cols.index("eff_price")].astype(np.float64) + rest = maker_rest(X[:, cols.index("side_bid")], X[:, cols.index("side_ask")]) + return rest + maker_fee_cents(rest, X[:, cols.index("fee_mult")], X[:, cols.index("maker_fee")]) + + +def oos(ds, rows: np.ndarray, y: np.ndarray, ci: np.ndarray, H, stop, label_: str) -> np.ndarray: + """Walk-forward P(side pays) on ``rows`` (NaN elsewhere) minus the arm's effective price: the edge per dollar of payout.""" + fin = rows & np.isfinite(y) + p = S.wf_classify(ds, fin, lambda idx: ds.X[np.ix_(idx, ci)], ds.outcome.astype(np.int8), 2, stop)[:, 1] + return np.where(fin & np.isfinite(p), p - effective(ds.X, ds.cols, H) / 100.0, np.nan).astype(np.float32) + + +def fit_final(ds, rows: np.ndarray, ci: np.ndarray, H): + """The deployed model of a strategy: a classifier of the side paying, on its candidate rows (at most ds.train_max_rows).""" + rows = S.cap_rows(ds, np.flatnonzero(rows), 99) + scaler = RobustScaler.fit(ds.X[np.ix_(rows, ci)], seed=99) + m = HistGradientBoostingClassifier(**S._gbm_params(), random_state=99); m.fit(scaler.transform(ds.X[np.ix_(rows, ci)]), ds.outcome[rows].astype(np.int8)) + return m, scaler + + +def score_final(m: dict, Xc: np.ndarray, Xfull: np.ndarray, cols: list[str], H) -> np.ndarray: + """Live: the deployed classifier's p̂ minus the arm's effective price (the same score the walk-forward produced).""" + pr = m["gbm"].predict_proba(m["scaler"].transform(Xc)); p = pr[:, list(m["gbm"].classes_).index(1)] if 1 in m["gbm"].classes_ else np.zeros(len(Xc)) + return p - effective(Xfull, cols, H) / 100.0 + + +def probability(m: dict, Xc: np.ndarray) -> np.ndarray: + pr = m["gbm"].predict_proba(m["scaler"].transform(Xc)) + return pr[:, list(m["gbm"].classes_).index(1)] if 1 in m["gbm"].classes_ else np.zeros(len(Xc)) + + +KALSHI = S.StackSpec(name="kalshi", strategies=STRATEGIES, base_mask=base_mask, trigger_value=trigger_value, groups={}, legacy_cols=list(X_COLS), + ev_hold="taker", in_universe=in_universe, label=label, hold_s=hold_s, oos=oos, random_hold=lambda ds: ds.hold_s) +KALSHI.fit_final = fit_final +KALSHI.score_final = score_final +KALSHI.hold_seconds = lambda H: 10 * 365 * 86400.0 # positions run to settlement; the engine never exits on a timer diff --git a/fly_trader/kalshi/stream.py b/fly_trader/kalshi/stream.py new file mode 100644 index 0000000000000000000000000000000000000000..1e33e4a5643dfeaf45c69de4488a01066f6d4335 --- /dev/null +++ b/fly_trader/kalshi/stream.py @@ -0,0 +1,455 @@ +"""The Kalshi live feed — the ``kalshi_stream`` worker: one authenticated websocket, exchange-wide ``ticker``, ``trade`` +and ``market_lifecycle_v2`` plus the subaccount's ``fill`` and ``user_orders`` channels, aggregated per (market, UTC +minute) into ``kalshi_minutes`` — the fields the corpus builder derives from the candle and trade archive +(kalshi/mature.bars_from_files): the minute's closing YES bid/ask and sizes, its lowest ask and highest bid, the last +price, cumulative volume and open interest, and the taker flow by side. A minute row exists only for minutes with a +message, as a candle exists only for minutes with activity; the engine carries quotes forward from its own state. + +Minutes are written once complete (an event stamped ≥ 2 s after the minute's end has arrived, or 15 s of silence); +``kalshi_stream_status.flushed_through`` publishes the newest complete minute for the engine. Every ticker message also +refreshes ``kalshi_quotes`` (the live top of book the maker arm and the vendored venue read). Lifecycle: ``determined`` / +``settled`` write the result to ``kalshi_markets`` (positions settle and the fly's tags resolve on it); ``created`` / +``activated`` / ``close_date_updated`` queue the market for a REST fetch, so its event and series (category, fee +multiplier) are in the catalogue before the engine scores it. The open-market catalogue is refreshed from REST at start +and hourly. Combo (``KXMVE…``) tickers are not markets of ours and are dropped at the door. +""" +from __future__ import annotations + +import asyncio +import json +import logging +import math +import ssl +import threading +import time +from collections import defaultdict +from datetime import datetime, timezone + +import certifi +import orjson +import pyarrow as pa +import pyarrow.parquet as pq + +from .. import config +from ..db.apilog import record_event +from ..db.connection import transaction +from ..logging_setup import setup +from . import data as D + +log = logging.getLogger(__name__) +STATUS_KEY = "kalshi_stream_status" +FLUSH_GRACE_S, FLUSH_FALLBACK_S = 2.0, 15.0 +QUOTE_FLUSH_S, STATUS_S, CATALOGUE_S, LIFECYCLE_S, ARCHIVE_S = 5.0, 5.0, 3600.0, 30.0, 3600.0 +PUBLIC_CHANNELS = ("ticker", "trade", "market_lifecycle_v2") +PRIVATE_CHANNELS = ("fill", "user_orders") +COMBO_PREFIX = "KXMVE" +SETTLE_EVENTS = ("determined", "settled") +CATALOGUE_EVENTS = ("created", "activated", "close_date_updated", "reactivated") + + +def _cents(v) -> float: + """A ``*_dollars`` string ('0.4600') as cents (46.0); NaN when absent.""" + try: + return float(v) * 100.0 if v is not None and v != "" else math.nan + except (TypeError, ValueError): + return math.nan + + +def _num(v, default=0.0) -> float: + try: + return float(v) if v is not None and v != "" else default + except (TypeError, ValueError): + return default + + +class KMinute: + __slots__ = ("yes_bid", "yes_ask", "last", "bid_size", "ask_size", "volume", "oi", "dollar_volume", "ask_low", "bid_high", "taker_yes", "taker_no", + "n_trades", "max_trade", "block", "sold_low", "bought_high") + + def __init__(self): + self.yes_bid = self.yes_ask = self.last = math.nan; self.bid_size = self.ask_size = 0.0; self.volume = self.oi = self.dollar_volume = math.nan + self.ask_low = math.inf; self.bid_high = -math.inf; self.taker_yes = self.taker_no = 0.0; self.n_trades = 0; self.max_trade = 0.0; self.block = 0.0 + self.sold_low = math.inf; self.bought_high = -math.inf # YES prices of the minute's taker sales of YES (bought NO) / buys of YES + + +class Aggregator: + def __init__(self): + self.minutes: dict[int, dict[str, KMinute]] = defaultdict(dict) # minute start → ticker → KMinute + self.quotes: dict[str, tuple] = {}; self.dirty: set[str] = set() # ticker → (ts, yes_bid, yes_ask, bid_size, ask_size, last) + self.max_event_s = 0.0; self.last_event_wall = 0.0 + self.flushed_through: datetime | None = None + self.settled: list[tuple] = [] # (ticker, result, settled_ts, event_type) + self.catalogue_todo: set[str] = set() # tickers to fetch from REST + self.fills: list[tuple] = []; self.order_updates: list[tuple] = [] + self.stats = {"events": 0, "tickers": 0, "trades": 0, "lifecycle": 0, "fills": 0, "orders": 0, "dropped_late": 0, "dropped_combo": 0, "flushed_minutes": 0, + "flushed_rows": 0, "reconnects": 0, "settled": 0, "catalogued": 0, "started_at": datetime.now(timezone.utc).isoformat()} + + # ---- ingest ---- + def _minute(self, ts_s: float, ticker: str) -> KMinute | None: + m = int(ts_s // 60) * 60 + if self.flushed_through is not None and m <= self.flushed_through.timestamp(): + self.stats["dropped_late"] += 1 + return None + row = self.minutes[m].get(ticker) + if row is None: + row = self.minutes[m][ticker] = KMinute() + return row + + def _seen(self, ts_s: float) -> None: + self.max_event_s = max(self.max_event_s, ts_s); self.last_event_wall = time.time() + + def ingest(self, e: dict) -> None: + self.stats["events"] += 1 + t = e.get("type"); msg = e.get("msg") or {} + if t == "ticker": + tk = msg.get("market_ticker") + if not tk or tk.startswith(COMBO_PREFIX): + self.stats["dropped_combo"] += bool(tk); return + ts_s = _num(msg.get("ts_ms"), 0.0) / 1000.0 or _num(msg.get("ts"), 0.0) + if ts_s <= 0: + return + self._seen(ts_s); self.stats["tickers"] += 1 + yb, ya, px = _cents(msg.get("yes_bid_dollars")), _cents(msg.get("yes_ask_dollars")), _cents(msg.get("price_dollars")) + if "yes_bid_dollars" not in msg and msg.get("yes_bid") is not None: # the older cents fields + yb, ya, px = _num(msg.get("yes_bid"), math.nan), _num(msg.get("yes_ask"), math.nan), _num(msg.get("price"), math.nan) + bs, as_ = _num(msg.get("yes_bid_size_fp")), _num(msg.get("yes_ask_size_fp")) + row = self._minute(ts_s, tk) + if row is not None: + row.yes_bid, row.yes_ask, row.bid_size, row.ask_size = yb, ya, bs, as_ + if math.isfinite(px) and px > 0: + row.last = px + row.volume = _num(msg.get("volume_fp"), row.volume); row.oi = _num(msg.get("open_interest_fp"), row.oi) + row.dollar_volume = _num(msg.get("dollar_volume"), row.dollar_volume) + if math.isfinite(ya): + row.ask_low = min(row.ask_low, ya) + if math.isfinite(yb): + row.bid_high = max(row.bid_high, yb) + self.quotes[tk] = (ts_s, yb, ya, bs, as_, px); self.dirty.add(tk) + elif t == "trade": + tk = msg.get("market_ticker") + if not tk or tk.startswith(COMBO_PREFIX): + return + ts_s = _num(msg.get("ts_ms"), 0.0) / 1000.0 or _num(msg.get("ts"), 0.0) + if ts_s <= 0: + return + self._seen(ts_s); self.stats["trades"] += 1 + px = _cents(msg.get("yes_price_dollars")) if "yes_price_dollars" in msg else _num(msg.get("yes_price"), math.nan) + cnt = _num(msg.get("count_fp")) if "count_fp" in msg else _num(msg.get("count")) + row = self._minute(ts_s, tk) + if row is None: + return + if msg.get("taker_side") == "yes": + row.taker_yes += cnt + if math.isfinite(px) and 0 < px < 100: + row.bought_high = max(row.bought_high, px) + else: + row.taker_no += cnt + if math.isfinite(px) and 0 < px < 100: + row.sold_low = min(row.sold_low, px) + row.n_trades += 1; row.max_trade = max(row.max_trade, cnt) + if msg.get("is_block_trade"): + row.block += cnt + if math.isfinite(px) and px > 0: + row.last = px + elif t == "market_lifecycle_v2": + self.stats["lifecycle"] += 1 + tk = msg.get("market_ticker"); ev = msg.get("event_type") or "" + if not tk or tk.startswith(COMBO_PREFIX): + return + if ev in SETTLE_EVENTS or msg.get("result"): + self.settled.append((tk, msg.get("result"), _num(msg.get("settled_ts") or msg.get("determination_ts") or msg.get("ts"), 0.0) or time.time(), ev or "settled", + _num(msg.get("close_ts"), 0.0) or None)) + elif ev in CATALOGUE_EVENTS or not ev: + self.catalogue_todo.add(tk) + elif t == "fill": + self.stats["fills"] += 1 + self.fills.append((msg.get("trade_id"), msg.get("order_id"), msg.get("market_ticker"), msg.get("side"), + _cents(msg.get("yes_price_dollars")) if "yes_price_dollars" in msg else _num(msg.get("yes_price"), math.nan), + _num(msg.get("count_fp")) if "count_fp" in msg else _num(msg.get("count")), bool(msg.get("is_taker")), msg.get("action"), + _num(msg.get("ts"), time.time()), _num(msg.get("post_position"), math.nan), msg.get("subaccount"), json.dumps(msg, default=str))) + elif t in ("user_order", "user_orders", "order"): + self.stats["orders"] += 1 + o = msg.get("order") or msg + if o.get("order_id"): + self.order_updates.append((o.get("order_id"), o.get("status"), _num(o.get("fill_count_fp") if "fill_count_fp" in o else o.get("fill_count"), math.nan), + _num(o.get("remaining_count_fp") if "remaining_count_fp" in o else o.get("remaining_count"), math.nan), json.dumps(o, default=str))) + + # ---- persistence ---- + def flush(self, before_minute: int) -> int: + done = sorted(m for m in self.minutes if m < before_minute) + if not done: + return 0 + rows = [] + for m in done: + ts = datetime.fromtimestamp(m, timezone.utc) + for tk, r in self.minutes[m].items(): + f = lambda v: (v if math.isfinite(v) else None) + rows.append((tk, ts, f(r.yes_bid), f(r.yes_ask), f(r.last), r.bid_size, r.ask_size, f(r.volume), f(r.oi), f(r.dollar_volume), + r.taker_yes, r.taker_no, r.n_trades, r.max_trade, r.block, f(r.ask_low), f(r.bid_high), f(r.sold_low), f(r.bought_high))) + if rows: + with transaction() as conn: + conn.cursor().executemany( + "INSERT INTO kalshi_minutes (ticker, ts, yes_bid, yes_ask, last, bid_size, ask_size, volume_fp, open_interest_fp, dollar_volume, taker_buy_yes, taker_buy_no, " + "n_trades, max_trade, block_contracts, yes_ask_low, yes_bid_high, yes_sold_low, yes_bought_high) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) " + "ON CONFLICT (ticker, ts) DO UPDATE SET yes_bid = COALESCE(EXCLUDED.yes_bid, kalshi_minutes.yes_bid), yes_ask = COALESCE(EXCLUDED.yes_ask, kalshi_minutes.yes_ask), " + "last = COALESCE(EXCLUDED.last, kalshi_minutes.last), bid_size = EXCLUDED.bid_size, ask_size = EXCLUDED.ask_size, " + "volume_fp = COALESCE(EXCLUDED.volume_fp, kalshi_minutes.volume_fp), open_interest_fp = COALESCE(EXCLUDED.open_interest_fp, kalshi_minutes.open_interest_fp), " + "dollar_volume = COALESCE(EXCLUDED.dollar_volume, kalshi_minutes.dollar_volume), taker_buy_yes = kalshi_minutes.taker_buy_yes + EXCLUDED.taker_buy_yes, " + "taker_buy_no = kalshi_minutes.taker_buy_no + EXCLUDED.taker_buy_no, n_trades = kalshi_minutes.n_trades + EXCLUDED.n_trades, " + "max_trade = greatest(kalshi_minutes.max_trade, EXCLUDED.max_trade), block_contracts = kalshi_minutes.block_contracts + EXCLUDED.block_contracts, " + "yes_ask_low = least(kalshi_minutes.yes_ask_low, EXCLUDED.yes_ask_low), yes_bid_high = greatest(kalshi_minutes.yes_bid_high, EXCLUDED.yes_bid_high), " + "yes_sold_low = least(kalshi_minutes.yes_sold_low, EXCLUDED.yes_sold_low), yes_bought_high = greatest(kalshi_minutes.yes_bought_high, EXCLUDED.yes_bought_high)", rows) + for m in done: + self.minutes.pop(m, None) + self.stats["flushed_minutes"] += len(done); self.stats["flushed_rows"] += len(rows) + newest = datetime.fromtimestamp(done[-1], timezone.utc) + if self.flushed_through is None or newest > self.flushed_through: + self.flushed_through = newest + return len(rows) + + def flush_bound(self, now: float) -> int: + cur = int(now // 60) * 60 + wm = int((self.max_event_s - FLUSH_GRACE_S) // 60) * 60 + if now - self.last_event_wall >= FLUSH_FALLBACK_S: + wm = max(wm, int((now - FLUSH_FALLBACK_S) // 60) * 60) + return min(cur, wm) + + def write_quotes(self) -> int: + todo, self.dirty = self.dirty, set() + rows = [] + for tk in todo: + q = self.quotes.get(tk) + if q is None: + continue + ts_s, yb, ya, bs, as_, px = q + mid = (yb + ya) / 2.0 if math.isfinite(yb) and math.isfinite(ya) else None + rows.append((tk, datetime.fromtimestamp(ts_s, timezone.utc), yb if math.isfinite(yb) else None, ya if math.isfinite(ya) else None, mid, bs, as_, px if math.isfinite(px) else None)) + if not rows: + return 0 + try: + with transaction() as conn: + conn.cursor().executemany("INSERT INTO kalshi_quotes (ticker, ts, yes_bid, yes_ask, mid, bid_size, ask_size, last) VALUES (%s,%s,%s,%s,%s,%s,%s,%s) " + "ON CONFLICT (ticker) DO UPDATE SET ts = EXCLUDED.ts, yes_bid = EXCLUDED.yes_bid, yes_ask = EXCLUDED.yes_ask, mid = EXCLUDED.mid, " + "bid_size = EXCLUDED.bid_size, ask_size = EXCLUDED.ask_size, last = COALESCE(EXCLUDED.last, kalshi_quotes.last), updated_at = now()", rows) + except Exception: + self.dirty |= todo + log.exception("quote upsert failed; %d quotes retried", len(todo)) + return 0 + return len(rows) + + def write_settlements(self) -> int: + rows, self.settled = self.settled, [] + if not rows: + return 0 + try: + with transaction() as conn: + for tk, result, ts_s, ev, close_ts in rows: + conn.execute("INSERT INTO kalshi_markets (ticker, status, result, settlement_ts, source) VALUES (%s,%s,%s,%s,'stream') ON CONFLICT (ticker) DO UPDATE SET " + "status = EXCLUDED.status, result = COALESCE(EXCLUDED.result, kalshi_markets.result), settlement_ts = COALESCE(EXCLUDED.settlement_ts, kalshi_markets.settlement_ts), " + "close_time = COALESCE(kalshi_markets.close_time, %s), updated_at = now()", + (tk, ev, result if result in ("yes", "no") else None, datetime.fromtimestamp(ts_s, timezone.utc), datetime.fromtimestamp(close_ts, timezone.utc) if close_ts else None)) + if result not in ("yes", "no"): + self.catalogue_todo.add(tk) # the result comes with the market fetch when the message lacks it + except Exception: + self.settled = rows + self.settled + log.exception("settlement rows failed; %d retried", len(rows)) + return 0 + self.stats["settled"] += len(rows) + return len(rows) + + def write_private(self) -> int: + fills, self.fills = self.fills, []; orders, self.order_updates = self.order_updates, [] + if not fills and not orders: + return 0 + try: + with transaction() as conn: + if fills: + conn.cursor().executemany( + "INSERT INTO kalshi_fills (trade_id, order_id, book, ticker, side, price_cents, count, fee_cents, is_taker, action, ts, post_position, raw) " + "SELECT %s, %s, o.book, %s, %s, %s, %s, NULL, %s, %s, %s, %s, %s FROM (SELECT %s::text AS oid) q LEFT JOIN kalshi_orders o ON o.order_id = q.oid " + "ON CONFLICT (trade_id) DO NOTHING", + [(tid, oid, tk, side, px, cnt, taker, act, datetime.fromtimestamp(ts_s, timezone.utc), pp if math.isfinite(pp) else None, raw, oid) + for tid, oid, tk, side, px, cnt, taker, act, ts_s, pp, sub, raw in fills if tid and tk]) + if orders: + conn.cursor().executemany("UPDATE kalshi_orders SET status = COALESCE(%s, status), fill_count = COALESCE(%s, fill_count), remaining = COALESCE(%s, remaining), " + "response = %s::jsonb, updated_at = now() WHERE order_id = %s", + [(st, fc if math.isfinite(fc) else None, rem if math.isfinite(rem) else None, raw, oid) for oid, st, fc, rem, raw in orders]) + except Exception: + self.fills = fills + self.fills; self.order_updates = orders + self.order_updates + log.exception("fill/order rows failed; retried") + return 0 + return len(fills) + len(orders) + + +# ---------------------------------------------------------------- the catalogue (REST, in a thread) +def refresh_open_markets(rest=None, stop: threading.Event | None = None) -> int: + """Every open non-combo market into ``kalshi_markets`` with its event and series (the engine's universe and metadata).""" + from .client import rest as _rest + from .history import _known_events, _known_series, ensure_catalogue + rest = rest or _rest(); n = 0 + with transaction() as conn: + known_e, known_s = _known_events(conn), _known_series(conn) + for page in rest.markets(status="open", mve_filter="exclude"): + if stop is not None and stop.is_set(): + break + with transaction() as conn: + D.upsert_markets(conn, page, "stream") + ensure_catalogue(rest, conn, {m["event_ticker"] for m in page if m.get("event_ticker")}, known_e, known_s) + n += len(page) + return n + + +def fetch_markets(tickers: list[str], rest=None) -> int: + """Markets the lifecycle channel announced (or settled without a result): one GET each, with event and series.""" + from .client import KalshiApiError, rest as _rest + from .history import _known_events, _known_series, ensure_catalogue + rest = rest or _rest(); n = 0 + with transaction() as conn: + known_e, known_s = _known_events(conn), _known_series(conn) + for tk in tickers: + try: + m = rest.market(tk) or {} # KalshiRest.market already unwraps {"market": ...} + except KalshiApiError as e: + log.warning("market %s: %s", tk, e); continue + if not m.get("ticker"): + continue + with transaction() as conn: + D.upsert_markets(conn, [m], "stream") + if m.get("event_ticker"): + ensure_catalogue(rest, conn, {m["event_ticker"]}, known_e, known_s) + n += 1 + return n + + +def archive_old(keep_days: int) -> int: + """Minute rows older than ``keep_days`` go to ``data/kalshi/stream/.parquet`` (one file per UTC day), then out of + the table: at ~9,000 quoted markets a minute the table would otherwise grow by ~13 M rows a day.""" + from datetime import timedelta + cutoff = datetime.now(timezone.utc).replace(hour=0, minute=0, second=0, microsecond=0) - timedelta(days=keep_days) + with transaction() as conn: + days = [r["d"] for r in conn.execute("SELECT DISTINCT date_trunc('day', ts) AS d FROM kalshi_minutes WHERE ts < %s", (cutoff,)).fetchall()] + n = 0 + for d in days: + with transaction() as conn: + rows = conn.execute("SELECT * FROM kalshi_minutes WHERE ts >= %s AND ts < %s ORDER BY ticker, ts", (d, d + timedelta(days=1))).fetchall() + if rows: + out = config.KALSHI_DIR / "stream"; out.mkdir(parents=True, exist_ok=True) + path = out / f"{d:%Y-%m-%d}.parquet"; tmp = path.with_name(path.name + ".tmp") + pq.write_table(pa.Table.from_pylist([dict(r) for r in rows]), tmp, compression="zstd"); tmp.replace(path) + conn.execute("DELETE FROM kalshi_minutes WHERE ts >= %s AND ts < %s", (d, d + timedelta(days=1))) + n += len(rows) + return n + + +def _status(agg: Aggregator, extra: dict) -> None: + try: + with transaction() as conn: + conn.execute("INSERT INTO ui_settings (key, value) VALUES (%s, %s) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()", + (STATUS_KEY, json.dumps({**agg.stats, **extra, "updated_at": datetime.now(timezone.utc).isoformat()}, default=str))) + except Exception: + log.debug("kalshi stream status write failed", exc_info=True) + + +def auth_headers() -> dict | None: + from kalshi_python_async.auth import KalshiAuth + from .vendor.kalshi_client import _load_private_key + key = _load_private_key() + if not (config.KALSHI_API_KEY_ID and key): + return None + path = "/" + config.KALSHI_WS_URL.split("/", 3)[3] if config.KALSHI_WS_URL.count("/") >= 3 else "/trade-api/ws/v2" + return KalshiAuth(config.KALSHI_API_KEY_ID, key).create_auth_headers("GET", path) + + +def subscribe_commands(private: bool) -> list[dict]: + chans = list(PUBLIC_CHANNELS) + (list(PRIVATE_CHANNELS) if private else []) + return [{"id": i, "cmd": "subscribe", "params": {"channels": [c]}} for i, c in enumerate(chans, start=1)] + + +def _logged(name: str, fn, *args): + try: + n = fn(*args) + if n: + log.info("%s: %d", name, n) + return n + except Exception: + log.exception("%s failed", name) + return 0 + + +async def _run(stop: threading.Event | None, agg: Aggregator) -> None: + import websockets + ctx = ssl.create_default_context(cafile=certifi.where()); backoff = 1.0 + if auth_headers() is None: + raise RuntimeError("the Kalshi websocket needs KALSHI_API_KEY_ID and a readable KALSHI_PRIVATE_KEY_PATH") + private = config.KALSHI_SUBACCOUNT > 0 or config.KALSHI_LIVE_ENABLED + last_status = last_flush = last_quotes = last_life = 0.0; last_cat = time.time() - CATALOGUE_S + 30; last_msg = 0.0; rate_n = 0; rate_t = time.time() + bg: dict[str, asyncio.Task | None] = {"catalogue": None, "fetch": None, "archive": None}; last_archive = time.time() - ARCHIVE_S + 120 + while not (stop is not None and stop.is_set()): + try: + headers = auth_headers() # signed with the current timestamp: a reconnect with the first handshake's signature is refused (401) + async with websockets.connect(config.KALSHI_WS_URL, additional_headers=headers, ssl=ctx, open_timeout=15, max_queue=None, ping_interval=20, ping_timeout=20) as ws: + for cmd in subscribe_commands(private): + await ws.send(json.dumps(cmd)) + log.info("kalshi stream connected (%s)", "public + private channels" if private else "public channels"); backoff = 1.0 + while not (stop is not None and stop.is_set()): + try: + raw = await asyncio.wait_for(ws.recv(), timeout=30) + except asyncio.TimeoutError: + log.warning("kalshi stream: no messages for 30 s; reconnecting"); break + last_msg = time.time(); rate_n += 1 + try: + e = orjson.loads(raw) + except orjson.JSONDecodeError: + continue + if isinstance(e, dict): + if e.get("type") == "error": + log.warning("kalshi stream error frame: %s", str(e)[:300]) + else: + agg.ingest(e) + now = time.time() + if now - last_flush >= 1.0: + try: + agg.flush(agg.flush_bound(now)) + except Exception: + log.exception("minute flush failed; rows kept for the next attempt") + last_flush = now + if now - last_quotes >= QUOTE_FLUSH_S: + agg.write_quotes(); agg.write_settlements(); agg.write_private(); last_quotes = now + if now - last_life >= LIFECYCLE_S and agg.catalogue_todo and (bg["fetch"] is None or bg["fetch"].done()): + todo = sorted(agg.catalogue_todo)[:200]; agg.catalogue_todo -= set(todo) + bg["fetch"] = asyncio.create_task(asyncio.to_thread(_logged, "markets fetched from the lifecycle channel", fetch_markets, todo)); last_life = now + if now - last_cat >= CATALOGUE_S and (bg["catalogue"] is None or bg["catalogue"].done()): + bg["catalogue"] = asyncio.create_task(asyncio.to_thread(_logged, "open markets refreshed", refresh_open_markets, None, stop)); last_cat = now + if now - last_archive >= ARCHIVE_S and (bg["archive"] is None or bg["archive"].done()): + bg["archive"] = asyncio.create_task(asyncio.to_thread(_logged, "archived old Kalshi minute rows", archive_old, config.KALSHI_MINUTES_KEEP_DAYS)); last_archive = now + if now - last_status >= STATUS_S: + _status(agg, {"events_per_s": rate_n / max(now - rate_t, 1e-9), "pending_minutes": len(agg.minutes), "lag_s": now - last_msg, "connected": True, + "tickers_quoted": len(agg.quotes), "catalogue_todo": len(agg.catalogue_todo), "private": private, + "flushed_through": agg.flushed_through.isoformat() if agg.flushed_through else None, + "event_watermark": datetime.fromtimestamp(agg.max_event_s, timezone.utc).isoformat() if agg.max_event_s else None}) + rate_n = 0; rate_t = now; last_status = now + except Exception as e: + agg.stats["reconnects"] += 1; log.warning("kalshi stream: %s; reconnecting in %.0fs", e, backoff) + _status(agg, {"connected": False, "last_error": str(e)[:200]}) + await asyncio.sleep(backoff); backoff = min(60.0, backoff * 2) + for t in bg.values(): + if t is not None: + try: + await t + except Exception: + pass + try: + agg.flush(int(time.time() // 60) * 60 + 60) + except Exception: + log.exception("final minute flush failed") + agg.write_quotes(); agg.write_settlements(); agg.write_private() + + +def main(stop_event: threading.Event | None = None) -> None: + setup("kalshi_stream"); agg = Aggregator() + record_event("info", "kalshi_stream", "kalshi stream started", {"url": config.KALSHI_WS_URL}) + try: + asyncio.run(_run(stop_event, agg)) + finally: + _status(agg, {"connected": False, "stage": "stopped"}) + record_event("info", "kalshi_stream", "kalshi stream stopped", {k: agg.stats.get(k) for k in ("events", "tickers", "trades", "flushed_rows", "settled", "reconnects")}) diff --git a/fly_trader/kalshi/vendor/__init__.py b/fly_trader/kalshi/vendor/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f52704d3a3c13a8e246cc97ca42034d41caa0de9 --- /dev/null +++ b/fly_trader/kalshi/vendor/__init__.py @@ -0,0 +1,5 @@ +"""better_bot's Kalshi client stack, vendored verbatim minus its sports sections (tools/vendor_kalshi.py). + +The copied modules keep their relative imports; the small modules beside them (``config``, ``gate_register``, +``database``, ``db_connection``, ``prediction_utils``) stand in for better_bot's, reading fly-trader's configuration. +""" diff --git a/fly_trader/kalshi/vendor/book_sizing.py b/fly_trader/kalshi/vendor/book_sizing.py new file mode 100644 index 0000000000000000000000000000000000000000..10b4ddc9319a596d916ca09fc2632335264e5759 --- /dev/null +++ b/fly_trader/kalshi/vendor/book_sizing.py @@ -0,0 +1,374 @@ +# Vendored verbatim from better_bot code/book_sizing.py by tools/vendor_kalshi.py; sports sections removed. Do not edit. +"""Depth-aware position sizing: walk the order book instead of assuming a full +fill at the best ask. + +A level is acceptable exactly while it is still +EV net of fee, so the +threshold falls out of break-even. Pure functions over a levels list; the +Kalshi fetch lives in ``kalshi_client``. +""" +import logging +from dataclasses import dataclass, replace +from typing import Sequence, Tuple + +from .kalshi_client import effective_price_cents, kalshi_fee_rate_cents +from .venue_register import net_odds_at + +logger = logging.getLogger(__name__) + +#: Kalshi prices are whole cents in 1..99. +MIN_PRICE_CENTS = 1 +MAX_PRICE_CENTS = 99 + +@dataclass(frozen=True) +class BookLevel: + """One resting price level.""" + + price_cents: int + quantity: int + + def __post_init__(self): + if not MIN_PRICE_CENTS <= self.price_cents <= MAX_PRICE_CENTS: + raise ValueError( + f"price {self.price_cents} outside Kalshi's 1-99c range") + if self.quantity < 0: + raise ValueError(f"negative quantity {self.quantity}") + + +@dataclass(frozen=True) +class OrderBook: + """Both sides of a Kalshi book, as resting bids. + + Kalshi publishes two bid ladders and no ask ladder: a No bid at ``q`` is a + Yes offer at ``100 - q``. :meth:`ask_ladder` is the only supported way to + ask what a side can be bought at, and :func:`validate_against_quote` catches + an inverted book against the quoted ask. + """ + + yes_bids: Tuple[BookLevel, ...] = () + no_bids: Tuple[BookLevel, ...] = () + ticker: str = "" + + def ask_ladder(self, side): + """Levels at which ``side`` can be BOUGHT, cheapest first. + + Buying Yes consumes No bids and vice versa. + """ + if side not in ("yes", "no"): + raise ValueError(f"side must be 'yes' or 'no', got {side!r}") + opposing = self.no_bids if side == "yes" else self.yes_bids + levels = [BookLevel(100 - lvl.price_cents, lvl.quantity) + for lvl in opposing if lvl.quantity > 0] + return tuple(sorted(levels, key=lambda l: l.price_cents)) + + def best_ask(self, side): + ladder = self.ask_ladder(side) + return ladder[0].price_cents if ladder else None + + def depth(self, side): + return sum(lvl.quantity for lvl in self.ask_ladder(side)) + + @property + def is_empty(self): + return not self.yes_bids and not self.no_bids + + +def validate_against_quote(book, side, quoted_ask_cents, tolerance=1): + """True when the book's best ask agrees with the separately quoted ask, ``None`` + when either is missing. A mismatch means inverted sides or a stale book. + """ + derived = book.best_ask(side) + if derived is None or quoted_ask_cents is None: + return None + return abs(derived - quoted_ask_cents) <= tolerance + + +@dataclass(frozen=True) +class BookFill: + """The result of walking a book: how much to take, and at what real price.""" + + quantity: int + #: Fee-inclusive volume-weighted price per contract. + vwap_cents: float + cost_cents: int + fee_cents: int + stopped_reason: str + levels_taken: int = 0 + best_ask_cents: int = None + #: Worst level price consumed -- the limit to submit an IOC order at, so the + #: order can reach the depth we sized against without paying beyond it. + limit_price_cents: int = None + #: The book's ticker, carried so fee-inclusive figures charge that market's + #: own series multiplier rather than the exchange default. + ticker: str = "" + + @property + def is_fillable(self): + return self.quantity > 0 + + @property + def slippage_cents(self): + """Fee-inclusive VWAP minus the fee-inclusive best ask. + + The measured cost of depth, per contract. Logged per wager so the + slippage figure can be re-measured against fee-adjusted fills rather + than inherited from a study that predates the fee correction. + """ + if self.best_ask_cents is None or not self.is_fillable: + return 0.0 + return self.vwap_cents - effective_price_cents(self.best_ask_cents, + self.ticker) + + @property + def expected_profit_cents(self): + """Profit if the contract settles in our favour.""" + if not self.is_fillable: + return 0 + return int(self.quantity * (100.0 - self.vwap_cents)) + + +NO_FILL = BookFill(quantity=0, vwap_cents=0.0, cost_cents=0, fee_cents=0, + stopped_reason="no_liquidity") + + +def kelly_fraction_at(p, vwap_cents): + """Raw Kelly fraction at an already fee-inclusive fill price; zero without + edge. Distinct from ``kelly_sizing._kelly_bet_fraction``, which adds the + spread and fee itself. + """ + if vwap_cents <= 0 or vwap_cents >= 100: + return 0.0 + b = net_odds_at(vwap_cents) + return max((p * b - (1.0 - p)) / b, 0.0) + + +def kelly_contracts(p, vwap_cents, bankroll_cents, kelly_fraction): + """Fractional-Kelly position size, in contracts, at a given fill price. + + ``vwap_cents`` is already fee-inclusive, so this is Kelly on the true cost + basis rather than on a quoted price we would not actually pay. + """ + raw = kelly_fraction_at(p, vwap_cents) + if raw <= 0: + return 0.0 + return raw * kelly_fraction * bankroll_cents / vwap_cents + + +def _contracts_at_level(taken, cost, level_price_eff, level_qty, p, + bankroll_cents, kelly_fraction): + """How many more contracts to take at one level before Kelly is satisfied. + ``taken + k - kelly_target(k)`` is increasing in ``k``, so a bisection finds + the crossing. + """ + def surplus(k): + vwap = (cost + k * level_price_eff) / (taken + k) if (taken + k) else \ + level_price_eff + return (taken + k) - kelly_contracts(p, vwap, bankroll_cents, + kelly_fraction) + + if surplus(level_qty) <= 0: + return level_qty # the whole level fits inside Kelly + if surplus(0) >= 0: + return 0 # already at or past the target + + lo, hi = 0, level_qty + while hi - lo > 1: + mid = (lo + hi) // 2 + if surplus(mid) <= 0: + lo = mid + else: + hi = mid + return lo + + +def fill_budget(book, side, p, budget_cents, *, min_payout_cents, + max_contracts=None): + """Spend a pre-decided allocation against real depth. + + Unlike :func:`walk_book`, Kelly has already run upstream. This answers how + much of the allocation can be spent at a price still +EV, and what it will + cost. Levels past break-even are refused even with budget left, and a fill + whose profit is under ``min_payout_cents`` is refused whole. + """ + ladder = book.ask_ladder(side) + if not ladder: + return NO_FILL + if budget_cents <= 0: + # Zero allocation is the allocator's answer, not an exchange problem. + return replace(NO_FILL, stopped_reason="no_allocation", + ticker=book.ticker) + + best_ask = ladder[0].price_cents + taken, cost, fee, levels_taken = 0, 0.0, 0.0, 0 + limit_price = None + reason = "depth_exhausted" + + for level in ladder: + price_eff = effective_price_cents(level.price_cents, book.ticker) + if price_eff >= 100.0 * p: + reason = "price_exceeds_edge" + break + + affordable = int((budget_cents - cost) // price_eff) + if affordable <= 0: + reason = "budget_spent" + break + + available = min(level.quantity, affordable) + if max_contracts is not None: + available = min(available, max_contracts - taken) + if available <= 0: + reason = "max_contracts" + break + + taken += available + cost += available * price_eff + fee += available * kalshi_fee_rate_cents(level.price_cents, book.ticker) + limit_price = level.price_cents + levels_taken += 1 + + if available < level.quantity: + reason = "budget_spent" if max_contracts is None else "max_contracts" + break + + if taken <= 0: + # Carry the real stop reason: a price the model declines is not an + # empty market. + return replace(NO_FILL, stopped_reason=reason, best_ask_cents=best_ask, + ticker=book.ticker) + + vwap = cost / taken + fill = BookFill( + quantity=int(taken), vwap_cents=vwap, cost_cents=int(round(cost)), + fee_cents=int(round(fee)), stopped_reason=reason, + levels_taken=levels_taken, best_ask_cents=best_ask, + limit_price_cents=limit_price, ticker=book.ticker) + + if fill.expected_profit_cents < min_payout_cents: + logger.info( + f"Skipping {book.ticker or side}: fillable size {fill.quantity} at " + f"{vwap:.2f}c yields ${fill.expected_profit_cents / 100:.2f} profit, " + f"below the ${min_payout_cents / 100:.2f} minimum") + return BookFill(quantity=0, vwap_cents=vwap, cost_cents=0, fee_cents=0, + stopped_reason="below_min_payout", + best_ask_cents=best_ask, ticker=book.ticker) + return fill + + +def kelly_allocation_cents(p, price_cents, bankroll_cents, kelly_fraction): + """What one prop position may spend: fractional Kelly at the fee-inclusive + best ask, capped by the prop sub-budget's concentration ceiling. The + tighter bound binds, as in ``prop_betting._position_caps``. + """ + from .config import PROP_BUDGET_FRACTION, PROP_MAX_POSITION_FRACTION + + raw = kelly_fraction_at(p, price_cents) + if raw <= 0: + return 0 + ceiling = PROP_BUDGET_FRACTION * PROP_MAX_POSITION_FRACTION + return int(min(ceiling, raw * kelly_fraction) * bankroll_cents) + + +def walk_book(book, side, p, bankroll_cents, *, kelly_fraction, + min_payout_cents, max_contracts=None): + """Size a position against real depth, solving the Kelly size and the fill + price together: ascend the ladder while each level is +EV net of fee and + stop at whichever binds first -- Kelly, depth, or break-even. Returns + :data:`NO_FILL` when nothing is fillable at a +EV price or the fill cannot + clear ``min_payout_cents``. + """ + ladder = book.ask_ladder(side) + if not ladder: + return NO_FILL + + best_ask = ladder[0].price_cents + taken = 0 + cost = 0.0 + fee = 0.0 + levels_taken = 0 + reason = "depth_exhausted" + + for level in ladder: + price_eff = effective_price_cents(level.price_cents, book.ticker) + if price_eff >= 100.0 * p: + reason = "price_exceeds_edge" + break + + available = level.quantity + if max_contracts is not None: + available = min(available, max_contracts - taken) + if available <= 0: + reason = "max_contracts" + break + + take = _contracts_at_level(taken, cost, price_eff, available, p, + bankroll_cents, kelly_fraction) + if take <= 0: + reason = "kelly_satisfied" + break + + taken += take + cost += take * price_eff + # The fee is quadratic in price, so each rung pays its own rate. + fee += take * kalshi_fee_rate_cents(level.price_cents, book.ticker) + levels_taken += 1 + + if take < available: + reason = "kelly_satisfied" + break + + if taken <= 0: + # Carry the real stop reason: a price the model declines is not an + # empty market. + return replace(NO_FILL, stopped_reason=reason, best_ask_cents=best_ask, + ticker=book.ticker) + + vwap = cost / taken + fill = BookFill( + quantity=int(taken), vwap_cents=vwap, cost_cents=int(round(cost)), + fee_cents=int(round(fee)), stopped_reason=reason, + levels_taken=levels_taken, best_ask_cents=best_ask, + ticker=book.ticker) + + if fill.expected_profit_cents < min_payout_cents: + logger.info( + f"Skipping {book.ticker or side}: fillable size {fill.quantity} at " + f"{vwap:.2f}c yields ${fill.expected_profit_cents / 100:.2f} profit, " + f"below the ${min_payout_cents / 100:.2f} minimum") + return BookFill(quantity=0, vwap_cents=vwap, cost_cents=0, fee_cents=0, + stopped_reason="below_min_payout", + best_ask_cents=best_ask, ticker=book.ticker) + return fill + + +def parse_kalshi_orderbook(payload, ticker=""): + """Build an :class:`OrderBook` from Kalshi's ``orderbook_fp`` payload. Levels + arrive as ``[price_dollars, count]`` string pairs; ``"0.47"`` is 47c. + """ + book = (payload or {}).get("orderbook_fp") or (payload or {}).get("orderbook") or {} + + def levels(key_dollars, key_cents): + """Read one side, converting by which key it came from. + + Not by sniffing the value: 1 could be one cent or one dollar, and + guessing wrong is a silent 100x. The key name is the authority. + """ + raw, in_dollars = book.get(key_dollars), True + if raw is None: + raw, in_dollars = book.get(key_cents) or [], False + + out = [] + for entry in raw: + try: + price, count = entry[0], entry[1] + cents = int(round(float(price) * 100)) if in_dollars else int(price) + quantity = int(float(count)) + except (TypeError, ValueError, IndexError): + logger.warning(f"unparseable orderbook level {entry!r} for {ticker}") + continue + if quantity <= 0 or not MIN_PRICE_CENTS <= cents <= MAX_PRICE_CENTS: + continue + out.append(BookLevel(cents, quantity)) + return tuple(out) + + return OrderBook(yes_bids=levels("yes_dollars", "yes"), + no_bids=levels("no_dollars", "no"), ticker=ticker) diff --git a/fly_trader/kalshi/vendor/config.py b/fly_trader/kalshi/vendor/config.py new file mode 100644 index 0000000000000000000000000000000000000000..334a56607cf86bda927370a4ec24578d2fb7dbf7 --- /dev/null +++ b/fly_trader/kalshi/vendor/config.py @@ -0,0 +1,16 @@ +"""Shim for better_bot's ``code/config.py``: the names the vendored Kalshi modules import, from fly-trader's config.""" +from __future__ import annotations + +from ... import config as _c + +KALSHI_API_KEY_ID = _c.KALSHI_API_KEY_ID +KALSHI_PRIVATE_KEY_PATH = _c.KALSHI_PRIVATE_KEY_PATH +KALSHI_API_BASE_URL = _c.KALSHI_API_BASE_URL +KALSHI_FEE_FACTOR = _c.KALSHI_FEE_FACTOR +ENABLE_KALSHI_CALIBRATION = True +ENABLE_VENUE_KALSHI = True +KELLY_BET_FRACTION = _c.KELLY_FRACTION +KELLY_MAX_SLATE_FRACTION = _c.KALSHI_MAX_SLATE_FRACTION +KELLY_POSITION_CAP_FRACTION = _c.MAX_POSITION_FRACTION +PROP_BUDGET_FRACTION = 1.0 +PROP_MAX_POSITION_FRACTION = _c.MAX_POSITION_FRACTION diff --git a/fly_trader/kalshi/vendor/database.py b/fly_trader/kalshi/vendor/database.py new file mode 100644 index 0000000000000000000000000000000000000000..79c479613851cac76f362abc12a4f57b098217d8 --- /dev/null +++ b/fly_trader/kalshi/vendor/database.py @@ -0,0 +1,27 @@ +"""Shim for better_bot's ``database.get_connection`` (SQLite, ``?`` placeholders): a Postgres connection whose +``execute`` rewrites ``?`` to ``%s``. The vendored code reaches it only from ``venue_kalshi._cached_half_spread``, +which reads ``venue_market_cache`` — a view over fly-trader's ``kalshi_quotes`` (db/schema.py).""" +from __future__ import annotations + +from ...db.connection import connect + + +class _Conn: + def __init__(self, conn): + self._c = conn + + def execute(self, sql: str, params=()): + return self._c.execute(sql.replace("?", "%s"), tuple(params)) + + def commit(self) -> None: + self._c.commit() + + def rollback(self) -> None: + self._c.rollback() + + def close(self) -> None: + self._c.close() + + +def get_connection(create: bool = False) -> _Conn: + return _Conn(connect()) diff --git a/fly_trader/kalshi/vendor/db_connection.py b/fly_trader/kalshi/vendor/db_connection.py new file mode 100644 index 0000000000000000000000000000000000000000..1d06f883bdbca1ce3a2d60e46718b975bbf0ddf1 --- /dev/null +++ b/fly_trader/kalshi/vendor/db_connection.py @@ -0,0 +1,3 @@ +"""Shim for better_bot's ``db_connection``: ``kalshi_stream.flush``/``top_structures`` are never called by fly-trader +(its own stream handlers live in fly_trader/kalshi/stream.py), so the connection is the same adapter as ``database``.""" +from .database import get_connection # noqa: F401 diff --git a/fly_trader/kalshi/vendor/gate_register.py b/fly_trader/kalshi/vendor/gate_register.py new file mode 100644 index 0000000000000000000000000000000000000000..df0917bf9b4df4d6febeb37e4a79cee75a472caf --- /dev/null +++ b/fly_trader/kalshi/vendor/gate_register.py @@ -0,0 +1,6 @@ +"""Shim for better_bot's gate register: fly-trader has no capability gates; the client is always allowed to run.""" +from __future__ import annotations + + +def can_use(*_args, **_kwargs) -> bool: + return True diff --git a/fly_trader/kalshi/vendor/kalshi_client.py b/fly_trader/kalshi/vendor/kalshi_client.py new file mode 100644 index 0000000000000000000000000000000000000000..b5194283dcfa786886de551663ff8b7b6e54e47f --- /dev/null +++ b/fly_trader/kalshi/vendor/kalshi_client.py @@ -0,0 +1,1097 @@ +# Vendored verbatim from better_bot code/kalshi_client.py by tools/vendor_kalshi.py; sports sections removed. Do not edit. +import asyncio +import json +import logging +import math +import os +import re +from dataclasses import dataclass +from datetime import datetime, timedelta + +from .config import ( + KALSHI_API_KEY_ID, KALSHI_PRIVATE_KEY_PATH, KALSHI_API_BASE_URL, + KALSHI_FEE_FACTOR, +) +from .gate_register import can_use + +logger = logging.getLogger(__name__) + + +#: Per-series fee configuration, read from the exchange and memoized for the +#: process: ``series_ticker -> {"fee_type": str, "fee_multiplier": float}``. +#: +#: Never populated lazily from inside the fee arithmetic. The fee functions run +#: inside orderbook walks, wager writes and the coverage gate's replay, and a +#: network call in any of those would make sizing depend on reachability and the +#: gate's verdict depend on the day it ran. Warm it explicitly with +#: :func:`prefetch_series_fees` before a cycle that prices anything. +_series_fee_cache = {} + +#: The multiplier the exchange itself defaults to, and therefore what an +#: unknown series is charged at. Deliberately the CONSERVATIVE direction: every +#: real multiplier we have seen is 1.0 or 0.5, so assuming 1.0 can only +#: over-state the fee and under-state edge. A missing lookup must never make a +#: wager look cheaper than it is. +DEFAULT_FEE_MULTIPLIER = 1.0 + + +def series_ticker_of(ticker): + """The series segment of a Kalshi ticker, or ``None``. + + Kalshi tickers are ``SERIES-EVENT-OUTCOME``, and a combo market's is + ``KXMVECROSSCATEGORY-S2026...-F5F8...``, so the first dash-part is the + series for market, event and multivariate tickers alike. + """ + if not ticker: + return None + return str(ticker).split("-")[0] or None + + +def series_fee_multiplier(ticker=None): + """The fee multiplier for a market's series, defaulting to 1.0. + + MLB is the reason this exists: every ``KXMLB*`` series -- game, spread, + total, first five, and all eight player-prop ladders -- carries + ``fee_multiplier: 0.5``, so charging the flat factor over-stated the fee by + 2x and discarded ~0.8pp of edge on every MLB wager, including at the + selection gate that rejects candidates for falling short of break-even. + """ + entry = _series_fee_cache.get(series_ticker_of(ticker)) + if not entry: + return DEFAULT_FEE_MULTIPLIER + return entry["fee_multiplier"] + + +def series_fee_type(ticker=None): + """``'quadratic'`` or ``'quadratic_with_maker_fees'`` for a market's series. + + Only ``quadratic_with_maker_fees`` charges a resting order. Nothing sizes on + this yet -- every order the bot places is IOC, so it is always the taker -- + but it is the difference between a free and a charged quote on the RFQ path, + and it arrives in the same response as the multiplier. + """ + entry = _series_fee_cache.get(series_ticker_of(ticker)) + return entry["fee_type"] if entry else None + + +def kalshi_fee_rate_cents(price_cents, ticker=None): + """Marginal Kalshi entry fee per contract, in cents, NOT rounded. + + ``factor * multiplier * p * (1-p) * 100``. Quadratic in price and maximal at + 50c (1.75c/contract at the default 7% factor and multiplier 1, versus 1.12c + at 80c) -- so a market trading near even money costs ~3.5% of stake to + enter, against ~1.4% at 80c. Fee as a fraction of STAKE is + ``factor * multiplier * (1-p)``, which falls monotonically as price rises: + a 5c contract pays 6.7% of stake, a 50c one 3.5%. + + ``ticker`` is a market, event or series ticker and selects the series' + multiplier; omitting it charges the exchange default of 1.0. Pass it + wherever it is known -- an MLB market charged without it pays double. + + Use this for **sizing** (Kelly, edge, break-even), where what matters is the + marginal cost of the next contract. Use :func:`kalshi_trading_fee_cents` for + **accounting** an actual order, where the exchange ceils the whole order to a + cent. Sizing must not use the ceiled single-contract figure: the exchange + ceils per order, so ``kalshi_trading_fee_cents(price, 1)`` is 2c at every + price and overstates the true marginal rate by 14% at 50c and 79% at 80c. + """ + p = price_cents / 100.0 + return KALSHI_FEE_FACTOR * series_fee_multiplier(ticker) * p * (1 - p) * 100 + + +def effective_price_cents(price_cents, ticker=None): + """Fee-inclusive cost basis per contract, in cents. + + The entry fee is part of what a contract costs, so every profitability + question -- break-even, edge, Kelly -- is asked against price + fee, never + the raw ask. Break-even follows directly: EV = p(100 - P) - (1 - p)P - f + = 100p - P - f, so EV > 0 iff p > (P + f)/100. + + At 50c and multiplier 1 that is p* = 0.5175, not 0.50: 1.75 points of edge + are consumed by the fee before a wager breaks even. On MLB's half-rate + series the same contract breaks even at 0.5088. + """ + return price_cents + kalshi_fee_rate_cents(price_cents, ticker) + + +def kalshi_trading_fee_cents(price_cents, quantity, ticker=None): + """Modeled Kalshi trading fee in cents: ceil(factor * mult * C * p * (1-p)). + + Real fills report the fee via the API; --paper simulated fills have no + exchange record, so this models the entry fee for realistic paper P&L. + """ + # Round before ceil so float artifacts don't bump an exact-cent fee up a cent + # (e.g. 0.07*100*0.5*0.5*100 = 175.0 must not ceil to 176). + return math.ceil( + round(kalshi_fee_rate_cents(price_cents, ticker) * quantity, 6)) + + +_client_instance = None + + +def _kalshi_available(): + return bool( + can_use("kalshi_calibration") + and KALSHI_API_KEY_ID + and KALSHI_PRIVATE_KEY_PATH + ) + + +def _load_private_key(): + key_path = KALSHI_PRIVATE_KEY_PATH + if not key_path or not os.path.exists(key_path): + return None + with open(key_path, "r") as f: + return f.read() + + +def _build_client(): + global _client_instance + if _client_instance is not None: + return _client_instance + if not _kalshi_available(): + return None + try: + from kalshi_python_async import KalshiClient, Configuration + private_key_pem = _load_private_key() + if not private_key_pem: + logger.warning("Kalshi private key file not found or empty") + return None + config = Configuration() + config.host = KALSHI_API_BASE_URL + config.api_key_id = KALSHI_API_KEY_ID + config.private_key_pem = private_key_pem + try: + import certifi + config.ssl_ca_cert = certifi.where() + except ImportError: + logger.debug("certifi not available, using default CA bundle") + _client_instance = KalshiClient(configuration=config) + return _client_instance + except Exception as e: + logger.warning(f"Failed to initialize Kalshi client: {e}") + return None + + +async def _close_client(): + global _client_instance + if _client_instance is not None: + try: + await _client_instance.close() + except Exception as e: + logger.debug("kalshi client close failed: %s", e) + _client_instance = None + + +def _market_implied_probability(market): + try: + yes_bid = float(market.yes_bid_dollars or "0") + yes_ask = float(market.yes_ask_dollars or "0") + if yes_bid > 0 and yes_ask > 0: + midpoint = (yes_bid + yes_ask) / 2.0 + return round(midpoint * 100.0, 2) + last_price = float(market.last_price_dollars or "0") + if last_price > 0: + return round(last_price * 100.0, 2) + except (ValueError, TypeError): + pass + return None + + +def market_quote(market): + """``{implied_prob, volume, ticker, yes_ask}`` for one market, or ``None``. + + One reader for the four values every consumer of a Kalshi market wants, so + the ``volume``/``volume_fp`` fallback and the dollars-to-cents conversion + live in one place instead of being repeated at each call site. + """ + implied = _market_implied_probability(market) + if implied is None: + return None + quote = { + "market_prob": implied, + "volume": int(float(getattr(market, "volume", None) + or getattr(market, "volume_fp", None) or 0)), + "market_id": market.ticker, + "yes_ask_cents": None, + } + try: + ask = float(market.yes_ask_dollars or "0") + if ask > 0: + quote["yes_ask_cents"] = round(ask * 100.0) + except (ValueError, TypeError): + pass + return quote + + +def _run_sync(coro): + """Run an async coroutine synchronously, handling existing event loops.""" + try: + try: + loop = asyncio.get_running_loop() + except RuntimeError: + loop = None + if loop is not None: + import concurrent.futures + with concurrent.futures.ThreadPoolExecutor() as pool: + future = pool.submit(asyncio.run, coro) + return future.result(timeout=30) + return asyncio.run(coro) + except Exception as e: + logger.warning(f"Kalshi sync wrapper failed: {e}") + return None + + +def _kalshi_sync(async_fn, default=None, error_msg="Kalshi operation failed", + log_level="error"): + """Run an async Kalshi operation with standard availability check and error handling. + + async_fn receives the built client as its sole argument and returns the result. + """ + if not _kalshi_available(): + return default + + async def _run(): + client = _build_client() + if not client: + return default + try: + return await async_fn(client) + except Exception as e: + getattr(logger, log_level)(f"{error_msg}: {e}") + return default + finally: + await _close_client() + + result = _run_sync(_run()) + return result if result is not None else default + + +def get_orderbook_sync(ticker, depth=0): + """Full resting depth for one market, or ``None`` when unavailable. + + ``depth`` of 0 requests every level. Returns the raw payload; parsing and the + Yes/No side convention live in ``book_sizing`` so the network boundary stays + thin and the sizing logic stays a pure function. + + ``None`` (rather than an empty book) on failure is deliberate: the caller + must be able to tell "the market has no depth" from "we could not ask", and + fall back to top-of-book sizing rather than skipping a wager because a fetch + errored. + """ + async def _fetch(client): + # Called through call_api rather than the SDK's MarketApi class, matching + # how every other endpoint here reaches the API and avoiding a dependency + # on the generated model layer for a two-field payload. + url = f"{KALSHI_API_BASE_URL}/markets/{ticker}/orderbook" + if depth: + url = f"{url}?depth={int(depth)}" + raw = await client.call_api("GET", url) + return _read_json_body(await raw.read()) + + return _kalshi_sync(_fetch, default=None, + error_msg=f"Kalshi orderbook fetch failed for {ticker}", + log_level="warning") + + +#: Kalshi caps a /markets page at 1000; 200 keeps a single prop series to one +#: or two round trips without risking a rejected limit. +_MARKETS_PAGE_LIMIT = 200 +#: Refuses to walk a series forever if the cursor never terminates. A prop +#: series for one league-day is a few hundred markets, so this is far above any +#: real answer and only bounds a pathological response. +_MARKETS_MAX_PAGES = 25 + + +def get_markets_sync(series_ticker=None, event_ticker=None, status="open", + max_pages=_MARKETS_MAX_PAGES): + """Every market in a series (or event), following Kalshi's cursor. + + This is the endpoint prop discovery needs and the one that was missing: only + ``GET /events?with_nested_markets`` was wired, which reaches the single + winner market per game and cannot enumerate a prop ladder. + + Returns ``[]`` on failure rather than ``None`` -- unlike an orderbook, an + empty series and an unreachable one lead to the same action (price nothing), + and every caller would otherwise repeat the same None check. + """ + if not series_ticker and not event_ticker: + raise ValueError("get_markets_sync needs a series_ticker or an " + "event_ticker; an unfiltered /markets walk would page " + "through the whole exchange") + + async def _fetch(client): + collected = [] + cursor = None + for _ in range(max_pages): + params = [f"limit={_MARKETS_PAGE_LIMIT}"] + if series_ticker: + params.append(f"series_ticker={series_ticker}") + if event_ticker: + params.append(f"event_ticker={event_ticker}") + if status: + params.append(f"status={status}") + if cursor: + params.append(f"cursor={cursor}") + raw = await client.call_api( + "GET", f"{KALSHI_API_BASE_URL}/markets?" + "&".join(params)) + if raw.status >= 400: + break + data = _read_json_body(await raw.read()) + page = data.get("markets") or [] + collected.extend(page) + cursor = data.get("cursor") + # Kalshi returns the same cursor with an empty page at the end of a + # series, so stop on either signal rather than on the cursor alone. + if not cursor or not page: + break + return collected + + label = series_ticker or event_ticker + return _kalshi_sync(_fetch, default=[], + error_msg=f"Kalshi market listing failed for {label}", + log_level="warning") + + +#: Kalshi's ``/markets?tickers=`` accepts a comma-joined list; keep the batch +#: well inside any URL length limit rather than discovering it in production. +_TICKERS_PER_REQUEST = 40 + + +def get_markets_by_ticker_sync(tickers): + """``{ticker: market}`` for specific markets, batched. + + The complement to :func:`get_markets_sync`: that one enumerates a series, + this one reads a known set. Combo legs come as tickers scattered across many + series, so listing each series to find one market would be hundreds of + requests for a handful of rows. + + A ticker the exchange does not return is simply absent from the result. The + caller decides what a missing market means -- for a combo leg it means the + whole structure is unmeasurable, which is not an error. + """ + wanted = sorted({t for t in tickers or () if t}) + if not wanted: + return {} + + async def _fetch(client): + collected = {} + for start in range(0, len(wanted), _TICKERS_PER_REQUEST): + batch = wanted[start:start + _TICKERS_PER_REQUEST] + raw = await client.call_api( + "GET", f"{KALSHI_API_BASE_URL}/markets" + f"?limit={_MARKETS_PAGE_LIMIT}&tickers=" + ",".join(batch)) + if raw.status >= 400: + continue + for market in _read_json_body(await raw.read()).get("markets") or []: + collected[market.get("ticker")] = market + return collected + + return _kalshi_sync(_fetch, default={}, + error_msg=f"Kalshi market lookup failed for " + f"{len(wanted)} ticker(s)", + log_level="warning") + + +def prefetch_series_fees(tickers): + """Warm the per-series fee cache for every series among ``tickers``. + + Accepts market, event or series tickers interchangeably and fetches each + distinct series once. Returns the number of series newly resolved. + + A series that cannot be read is left ABSENT from the cache rather than + recorded at the default, so a later call can retry it. Recording the + fallback would make one unlucky request permanently charge that series the + full rate for the rest of the process. + """ + wanted = {series_ticker_of(t) for t in tickers or ()} + wanted.discard(None) + missing = sorted(wanted - _series_fee_cache.keys()) + if not missing: + return 0 + + async def _fetch(client): + resolved = {} + for series in missing: + raw = await client.call_api( + "GET", f"{KALSHI_API_BASE_URL}/series/{series}") + if raw.status >= 400: + continue + body = _read_json_body(await raw.read()).get("series") or {} + multiplier = body.get("fee_multiplier") + if multiplier is None: + continue + resolved[series] = {"fee_type": body.get("fee_type"), + "fee_multiplier": float(multiplier)} + return resolved + + resolved = _kalshi_sync(_fetch, default={}, + error_msg="Kalshi series fee lookup failed", + log_level="warning") + _series_fee_cache.update(resolved) + discounted = sorted(s for s, v in resolved.items() + if v["fee_multiplier"] != DEFAULT_FEE_MULTIPLIER) + if discounted: + logger.info("kalshi fees: %d series below the default multiplier (%s)", + len(discounted), ", ".join(discounted[:8])) + return len(resolved) + + +def clear_series_fee_cache(): + """Forget every memoized series fee. For tests and long-lived processes.""" + _series_fee_cache.clear() + + +def market_ask_cents(market, side): + """Best quoted ask for ``side`` on a raw ``/markets`` row, or ``None``. + + Kalshi publishes both ``yes_ask`` (cent integer) and ``yes_ask_dollars`` + (fixed-point string). Which key the value came from decides the scale -- + never the value itself, since ``1`` is a legitimate reading of both one cent + and one dollar and guessing wrong is a silent 100x. Same discipline as + ``book_sizing.parse_kalshi_orderbook``. + + This is top-of-book only, and is a *filter* rather than a sizing input: the + order book proper is fetched for the few candidates that survive it, because + fetching depth for every rung of every ladder on a slate is thousands of + round trips to answer a question a quote already answers. + """ + if side not in ("yes", "no"): + raise ValueError(f"side must be 'yes' or 'no', got {side!r}") + dollars, cents = market.get(f"{side}_ask_dollars"), market.get(f"{side}_ask") + try: + if dollars not in (None, ""): + value = int(round(float(dollars) * 100)) + elif cents not in (None, ""): + value = int(cents) + else: + return None + except (TypeError, ValueError): + return None + return value if 1 <= value <= 99 else None + + +def get_market_sync(ticker): + """One market's current row, or ``None``. + + Settlement reads the exchange's own ``result`` through this. A prop resolves + on the exchange's rules -- which player was credited with the assist, how a + postponement is handled -- and re-deriving that from our box scores would be + inventing a second, disagreeing authority for something Kalshi publishes. + """ + async def _fetch(client): + raw = await client.call_api( + "GET", f"{KALSHI_API_BASE_URL}/markets/{ticker}") + if raw.status >= 400: + return None + return (_read_json_body(await raw.read()) or {}).get("market") + + return _kalshi_sync(_fetch, default=None, + error_msg=f"Kalshi market fetch failed for {ticker}", + log_level="warning") + + +#: Candlestick periods Kalshi accepts, in minutes. Not a free integer. +CANDLE_PERIODS = (1, 60, 1440) + + +def get_candlesticks_sync(series_ticker, ticker, start_ts, end_ts, + period_interval=1): + """Quote history for one market, oldest first, or ``[]``. + + The only way to recover what the exchange was asking at a moment that has + passed. A finalized market reports ``yes_ask`` as ``None`` on ``/markets`` + -- it has settled, so there is no quote left to read -- which is why the + calibration log has to capture the price when it prices a rung, or come + back here for it. + + Note the path: candlesticks hang off the SERIES, not off ``/markets``, and + ``/markets/{ticker}/candlesticks`` is a plain 404 rather than an error that + explains itself. The ``/historical/`` prefix in the current API reference + resolves the ticker and then reports it not found for a settled prop. + + Prices come back as fixed-point dollar strings (``close_dollars``), never + cent integers, so the scale is decided by the key -- see + :func:`market_ask_cents` for why guessing it is a silent 100x. + """ + if period_interval not in CANDLE_PERIODS: + raise ValueError(f"period_interval must be one of {CANDLE_PERIODS}, " + f"got {period_interval}") + + async def _fetch(client): + raw = await client.call_api( + "GET", f"{KALSHI_API_BASE_URL}/series/{series_ticker}/markets/" + f"{ticker}/candlesticks?start_ts={int(start_ts)}" + f"&end_ts={int(end_ts)}&period_interval={period_interval}") + if raw.status >= 400: + return [] + return (_read_json_body(await raw.read()) or {}).get("candlesticks") or [] + + return _kalshi_sync(_fetch, default=[], + error_msg=f"Kalshi candlesticks failed for {ticker}", + log_level="debug") + + +def candle_ask_cents(candle, side="yes"): + """Closing ask for one side of a candlestick, in cents, or ``None``. + + A candle with no trades still carries a quote, so this reads the book rather + than ``price`` -- the latter is null whenever nothing changed hands and would + drop exactly the quiet rungs the calibration log is full of. + + A candle quotes only the Yes side, and **an ask on one side is a bid on the + other**: buying No consumes Yes bids, so a No ask is ``100 - yes_bid``. Same + identity ``prop_betting._quoted_book`` uses to build a book from a listing's + two quotes, and reading ``yes_ask`` for both sides instead is an error worth + most of a dollar on a rung the market prices at 0.94. + """ + if side not in ("yes", "no"): + raise ValueError(f"side must be 'yes' or 'no', got {side!r}") + key = "yes_ask" if side == "yes" else "yes_bid" + dollars = (candle.get(key) or {}).get("close_dollars") + if dollars in (None, ""): + return None + try: + value = int(round(float(dollars) * 100)) + except (TypeError, ValueError): + return None + if side == "no": + value = 100 - value + return value if 1 <= value <= 99 else None + + +async def _get_balance(client): + response = await client.get_balance() + if not response: + return None + return { + "balance": response.balance, + "portfolio_value": response.portfolio_value, + } + + +def get_balance_sync(): + """Return dict with balance and portfolio_value in cents, or None on failure.""" + return _kalshi_sync(_get_balance, error_msg="Kalshi balance fetch failed") + + +async def _paginate_raw(client, method_name, result_key, limit=200): + """Fetch all pages from a paginated Kalshi endpoint using raw responses.""" + all_items = [] + cursor = None + while True: + kwargs = {"limit": limit} + if cursor: + kwargs["cursor"] = cursor + method = getattr(client, method_name) + raw = await method(**kwargs) + data = json.loads(await raw.read()) + items = data.get(result_key) or [] + if not items: + break + all_items.extend(items) + cursor = data.get("cursor") + if not cursor: + break + return all_items + + +def get_settlements_sync(): + """Return list of all settlement dicts from Kalshi.""" + return _kalshi_sync( + lambda c: _paginate_raw(c, "get_settlements_without_preload_content", "settlements"), + default=[], error_msg="Kalshi settlements fetch failed", + ) + + +def get_fills_sync(): + """Return list of all fill dicts from Kalshi.""" + return _kalshi_sync( + lambda c: _paginate_raw(c, "get_fills_without_preload_content", "fills"), + default=[], error_msg="Kalshi fills fetch failed", + ) + + +def get_transfers_sync(): + """Fetch deposit/withdrawal history via GET /portfolio/transfers. + + This endpoint isn't exposed in the SDK, so we call it directly. + """ + async def _fetch(client): + raw = await client.call_api("GET", KALSHI_API_BASE_URL + "/portfolio/transfers") + body = await raw.read() + text = body.decode("utf-8") if isinstance(body, bytes) else str(body) + # API sometimes returns concatenated responses; find the JSON object + start = text.find("{") + if start < 0: + return [] + data = json.loads(text[start:]) + return data.get("transfers") or [] + + return _kalshi_sync(_fetch, default=[], error_msg="Kalshi transfers fetch failed", + log_level="warning") + + +# Kalshi's V2 order endpoint quotes everything from the YES leg: "bid" buys +# YES, "ask" sells YES. There is one book per market, so an ask at p from an +# account holding no Yes IS a No position bought at 1 - p, and the positions +# endpoint reports it as a negative contract count (docs.kalshi.com, Create +# Order V2 and Get Positions). +_V2_SIDE_FOR_BUY = {"yes": "bid", "no": "ask"} + + +def _v2_yes_price_cents(side, price_cents): + """The YES-leg price the V2 endpoint quotes buying ``side`` at ``price_cents`` in.""" + return price_cents if side == "yes" else 100 - price_cents + + +def _v2_order_result(data, side="yes"): + """Map a Kalshi V2 order response into the internal result dict. + + V2 returns fixed-point strings (count "10.00", price "0.55") and reports + fills via average_fill_price rather than a taker/maker cost split, so the + contract cost in cents is derived here. average_fill_price/average_fee_paid + are only present when at least one contract filled. + + ``average_fill_price`` is the YES-leg price whichever side was bought, so a + No fill's cost per contract is its complement. ``yes_price`` stays the YES + price: it is what the exchange reports and what the fee is computed on. + """ + fill_count = int(float(data.get("fill_count") or 0)) + avg_price_dollars = float(data.get("average_fill_price") or 0) + avg_fee_dollars = float(data.get("average_fee_paid") or 0) + paid_dollars = avg_price_dollars if side == "yes" else 1.0 - avg_price_dollars + return { + "order_id": data.get("order_id", ""), + "fill_count": fill_count, + "remaining_count": int(float(data.get("remaining_count") or 0)), + "taker_fill_cost": round(fill_count * paid_dollars * 100) if fill_count else 0, + "maker_fill_cost": 0, + # What the exchange ACTUALLY charged, per contract, rather than what the + # series multiplier says it should have. The two differ: KXMLBHRR is one + # of the thirteen series below the default multiplier, and a fee + # recomputed at the default is roughly double what was paid. + "fee_cents": round(fill_count * avg_fee_dollars * 100), + "yes_price": round(avg_price_dollars * 100), + } + + +@dataclass(frozen=True) +class OrderFill: + """The part of an order that actually executed. + + **An ``order_id`` means the exchange ACCEPTED the order, not that it + filled**, and that distinction cost real money before this existed. Every + order this repo places is immediate-or-cancel, so a response reading + ``fill_count 0`` with ``remaining_count 0`` is an order that was cancelled + unfilled -- and it carries an id exactly like one that traded. Two of four + real prop orders on 2026-08-15 came back that way and were written to + ``prop_wagers`` as filled positions at their requested size, 670c of + exposure the account never had. + + Deliberately shaped like :class:`code.book_sizing.BookFill`, which is what + the caller sized the order against: the pair are the intent and the outcome + of the same trade, and reading one against the other should not require + translating between two vocabularies. + + ``quantity`` and ``cost_cents`` are what the exchange executed, never what + was asked for. A partial fill is a real position of a different size, and + recording the request instead overstates the book by the unfilled part. + """ + + order_id: str + quantity: int + cost_cents: int + fee_cents: int + #: Average price actually paid per contract. + vwap_cents: float + #: How much of the request went unfilled, for the log. + requested: int + + @property + def is_filled(self): + return self.quantity > 0 + + @property + def status(self): + """``filled`` or ``partial`` -- the vocabulary the wager tables store.""" + return "filled" if self.quantity >= self.requested else "partial" + + +def order_fill(result, requested): + """What ``result`` executed, or ``None`` if the order did not fill at all. + + ``None`` is the "no position was opened" answer, and every caller placing a + real order must treat it as one: no row, no count, no exposure. Returning a + zero-quantity object instead would let a caller that forgot to check write a + position of size zero, which settles and books P/L just as wrongly. + """ + if not result or not result.get("order_id"): + return None + quantity = int(result.get("fill_count") or 0) + if quantity < 1: + return None + cost = int(result.get("taker_fill_cost") or 0) + int( + result.get("maker_fill_cost") or 0) + return OrderFill(order_id=result["order_id"], quantity=quantity, + cost_cents=cost, + fee_cents=int(result.get("fee_cents") or 0), + vwap_cents=cost / quantity if quantity else 0.0, + requested=int(requested)) + + +#: Kalshi split its matching engine into shards ("Exchange Sharding" in the API +#: docs): 0 is the default, 1 combos, 2 crypto, 3 tennis and baseball. A +#: market's ``exchange_index`` is the authority. An order that omits it is +#: routed to shard 0, and a shard-3 market answers 404 -- which is what every +#: baseball order did on 2026-09-06, the first real order after the split. +#: Collateral is held PER SHARD, so a correctly routed order still needs a +#: balance on that shard (a target allocation or an intra-exchange transfer). +DEFAULT_EXCHANGE_INDEX = 0 +_exchange_index_by_ticker = {} + + +def _api_error(status, data): + """An ``ApiException`` whose reason carries Kalshi's own code and message. + + Kalshi nests them -- ``{"error": {"code", "message", "details"}}`` -- and a + reason read from the top level logged a bare ``(404)`` for a mis-routed + order, which took an afternoon to trace to the shard. + """ + from kalshi_python_async.exceptions import ApiException + error = data.get("error") if isinstance(data.get("error"), dict) else {} + parts = [data.get("message") or error.get("message"), error.get("code"), + error.get("details")] + reason = " | ".join(str(p) for p in parts if p) or str(data) + return ApiException(status=status, reason=reason) + + +async def _market_exchange_index(client, ticker): + """The shard ``ticker`` trades on, read from the market itself. + + Cached per process -- a market never changes shard. Falls back to the + default shard with a warning when the market cannot be read, so the order + still goes out and the exchange, not this client, gives the final answer. + """ + if ticker in _exchange_index_by_ticker: + return _exchange_index_by_ticker[ticker] + raw = await client.call_api("GET", KALSHI_API_BASE_URL + f"/markets/{ticker}") + data = _read_json_body(await raw.read()) + index = (data.get("market") or {}).get("exchange_index") + if raw.status >= 400 or index is None: + logger.warning("Kalshi market %s has no readable exchange_index " + "(status %s); routing to shard %d", ticker, raw.status, + DEFAULT_EXCHANGE_INDEX) + return DEFAULT_EXCHANGE_INDEX + _exchange_index_by_ticker[ticker] = int(index) + return int(index) + + +async def _create_order_safe(client, ticker, side, price_cents, count): + """Submit an immediate-or-cancel buy order via Kalshi's V2 order endpoint. + + ``price_cents`` is what one contract of ``side`` costs; it is converted to + the YES-leg price the endpoint quotes in. The order is routed to the shard + the market lives on -- see :data:`DEFAULT_EXCHANGE_INDEX`. + + The legacy POST /portfolio/orders endpoint now returns 410 (deprecated); + V2 lives at POST /portfolio/events/orders, quotes from the YES leg, and + expects fixed-point dollar/count strings. Uses call_api directly to skip + the SDK's pydantic models, which still encode the deprecated V1 schema. + """ + v2_side = _V2_SIDE_FOR_BUY.get(side) + if v2_side is None: + raise ValueError( + f"Kalshi V2 orders buy 'yes' or 'no'; got side={side!r}" + ) + body = { + "ticker": ticker, + "side": v2_side, + "count": str(count), + "price": f"{_v2_yes_price_cents(side, price_cents) / 100:.2f}", + "time_in_force": "immediate_or_cancel", + "self_trade_prevention_type": "taker_at_cross", + "exchange_index": await _market_exchange_index(client, ticker), + } + raw = await client.call_api( + "POST", KALSHI_API_BASE_URL + "/portfolio/events/orders", body=body, + ) + data = json.loads(await raw.read()) + if raw.status >= 400: + raise _api_error(raw.status, data) + if not data.get("order_id"): + return None + logger.info(f"Kalshi V2 order response for {ticker}: {data}") + return _v2_order_result(data, side) + + +def order_collateral_cents(ticker, price_cents, count): + """What one IOC order locks up on its shard: cost plus the taker fee.""" + return int(count) * int(price_cents) + kalshi_trading_fee_cents( + price_cents, count, ticker) + + +async def _shard_balances(client): + """``{exchange_index: cents}`` from the balance breakdown. + + A pre-sharding response carries only ``balance``; it is read as shard 0 so + the arithmetic below still holds. + """ + raw = await client.call_api("GET", KALSHI_API_BASE_URL + "/portfolio/balance") + data = _read_json_body(await raw.read()) + if raw.status >= 400: + raise _api_error(raw.status, data) + balances = {int(row["exchange_index"]): int(round(float(row["balance"]) * 100)) + for row in data.get("balance_breakdown") or []} + return balances or {DEFAULT_EXCHANGE_INDEX: int(data.get("balance") or 0)} + + +async def _transfer_between_shards(client, source, destination, cents): + """Move ``cents`` of event-contract collateral from one shard to another. + + Kalshi prices the body in centicents. Cross-shard transfers run in up to + three non-atomic steps on the exchange side; a failure part-way leaves the + money in the primary account, never lost, which is why the caller re-reads + balances rather than trusting its own arithmetic afterwards. + """ + body = {"source": "event_contract", "destination": "event_contract", + "amount": int(cents) * 100, + "source_exchange_shard": int(source), + "destination_exchange_shard": int(destination)} + raw = await client.call_api( + "POST", KALSHI_API_BASE_URL + "/portfolio/intra_exchange_instance_transfer", + body=body) + data = _read_json_body(await raw.read()) + if raw.status >= 400: + raise _api_error(raw.status, data) + return data.get("transfer_id") + + +def shard_transfers(demand, balances): + """``[(source, destination, cents)]`` that tops each shard up to its demand. + + Pure. ``demand`` is what the real orders about to be placed need per shard, + ``balances`` what each shard holds. A shard short of its demand draws from + whichever shard has the largest surplus over its OWN demand, so collateral + a shard's own orders need is never lent out. When the account as a whole + cannot cover the plan, the shortest shard is served first and the rest is + left for the exchange to refuse -- an order the account cannot fund is not + a position, and the log names the gap. + """ + surplus = {shard: balances.get(shard, 0) - demand.get(shard, 0) + for shard in set(balances) | set(demand)} + plan = [] + for shard in sorted((s for s, v in surplus.items() if v < 0), + key=lambda s: surplus[s]): + while surplus[shard] < 0: + donors = [s for s, v in surplus.items() if v > 0] + if not donors: + break + donor = max(donors, key=lambda s: surplus[s]) + move = min(-surplus[shard], surplus[donor]) + plan.append((donor, shard, move)) + surplus[donor] -= move + surplus[shard] += move + return plan + + +def fund_shards_for_orders(orders): + """Move collateral so every shard holds what its real orders are about to spend. + + ``orders`` is an iterable of ``(ticker, cents)`` -- the real order plan, not + the paper book. Each market's shard is read from the exchange, demand is + summed per shard, and the shortfalls are transferred from the shard with + the largest surplus (deposits land on shard 0). Returns the transfers made. + Nothing moves when every shard already covers its demand. + """ + orders = [(ticker, int(cents)) for ticker, cents in orders if cents > 0] + if not orders or not _kalshi_available(): + return [] + + async def _fund(client): + demand = {} + for ticker, cents in orders: + shard = await _market_exchange_index(client, ticker) + demand[shard] = demand.get(shard, 0) + cents + balances = await _shard_balances(client) + moved = [] + for source, destination, cents in shard_transfers(demand, balances): + transfer_id = await _transfer_between_shards(client, source, + destination, cents) + logger.info("Kalshi collateral: $%.2f shard %d -> shard %d for " + "$%.2f of real orders there (transfer %s)", + cents / 100, source, destination, + demand[destination] / 100, transfer_id) + moved.append((source, destination, cents)) + for shard, need in demand.items(): + held = balances.get(shard, 0) + sum(c for _s, d, c in moved if d == shard) + if held < need: + logger.warning("Kalshi collateral: shard %d holds $%.2f against " + "$%.2f of real orders; the account cannot cover " + "the plan", shard, held / 100, need / 100) + return moved + + return _kalshi_sync(_fund, default=[], + error_msg="Kalshi collateral transfer failed") or [] + + +def place_order_sync(ticker, side, price_cents, count): + """Place an IOC order buying ``count`` of ``side`` at ``price_cents`` each. Returns result dict or None. + + Funds the market's shard for this one order first. Callers with a whole + slate to place fund it in one pass with :func:`fund_shards_for_orders`; + this covers the single-order paths (moneyline, combos) and costs a + balance read when the shard is already funded. + """ + fund_shards_for_orders([(ticker, order_collateral_cents(ticker, price_cents, + count))]) + + async def _place(client): + result = await _create_order_safe(client, ticker, side, price_cents, count) + if not result: + logger.error(f"Kalshi returned empty response for {ticker}") + return None + return result + + return _kalshi_sync(_place, error_msg=f"Kalshi order failed for {ticker}") + + +def get_portfolio_positions_sync(): + """Return list of open positions from Kalshi.""" + async def _fetch(client): + raw = await client.get_positions_without_preload_content(limit=200) + data = json.loads(await raw.read()) + return data.get("market_positions") or data.get("positions") or [] + + return _kalshi_sync(_fetch, default=[], error_msg="Failed to fetch Kalshi positions") + + +def get_open_orders_sync(): + """Return list of resting orders from Kalshi.""" + async def _fetch(client): + raw = await client.get_orders_without_preload_content(status="resting", limit=200) + data = json.loads(await raw.read()) + return data.get("orders") or [] + + return _kalshi_sync(_fetch, default=[], error_msg="Failed to fetch Kalshi orders") + + +# --------------------------------------------------------------------------- +# Multivariate "combo"/parlay markets (real multi-leg placement). The rest of +# this client trades single-leg markets only; these wrap Kalshi's multivariate +# event collections against the same trade-api/v2 host + key-pair auth. A combo +# market is created from a set of leg (event_ticker, market_ticker) pairs and +# then trades as an ordinary single YES market, so placement reuses +# place_order_sync. Real placement stays dormant behind ENABLE_REAL_PARLAYS + +# the promotion gate + parlay_combos_available() (see code/parlay.py), so this +# untested-against-live path can never fire unintentionally. +# --------------------------------------------------------------------------- + +_MULTIVARIATE_PATH = "/multivariate_event_collections" +_parlay_combos_available_cache = None # None=unknown, then bool + + +def _read_json_body(text): + """Parse a Kalshi raw body, tolerating the occasional concatenated response + (as in get_transfers_sync).""" + if isinstance(text, (bytes, bytearray)): + text = text.decode("utf-8") + start = text.find("{") + return json.loads(text[start:]) if start >= 0 else {} + + +async def _list_multivariate_collections(client, status="open"): + raw = await client.call_api( + "GET", KALSHI_API_BASE_URL + _MULTIVARIATE_PATH + f"?status={status}", + ) + if raw.status >= 400: + return [] + data = _read_json_body(await raw.read()) + return data.get("multivariate_contracts") or [] + + +def list_multivariate_collections_sync(status="open"): + """Return Kalshi's multivariate (combo/parlay) event collections, or [].""" + return _kalshi_sync(lambda c: _list_multivariate_collections(c, status), + default=[], error_msg="Failed to list multivariate collections", + log_level="warning") + + +def parlay_combos_available(): + """True only if Kalshi currently exposes at least one multivariate collection + (cached per process). The hard availability gate for real parlays — when + False, every league stays paper-only regardless of the --parlay flag or + promotion state.""" + global _parlay_combos_available_cache + if _parlay_combos_available_cache is not None: + return _parlay_combos_available_cache + if not _kalshi_available(): + _parlay_combos_available_cache = False + return False + _parlay_combos_available_cache = bool(list_multivariate_collections_sync()) + return _parlay_combos_available_cache + + +def find_parlay_collection_sync(event_tickers): + """Return a collection_ticker whose associated events cover ALL of the given + leg event_tickers, or None. The collection is the combo market that can host + the parlay's legs.""" + wanted = {t for t in event_tickers if t} + if not wanted: + return None + for col in list_multivariate_collections_sync(): + assoc = set(col.get("associated_event_tickers") or []) + if wanted <= assoc: + return col.get("collection_ticker") + return None + + +async def _create_combo_market(client, collection_ticker, legs): + """The combo market for ``legs``, created or found (Kalshi is idempotent). + + Each leg carries its own ``side``: the collection endpoint takes ``yes`` or + ``no`` per selected market and records both in the market's strike + (verified live 2026-09-06, "Associated Market Sides: no,yes"), so a No rung + is as expressible in a combo as in a single order. The payload's + ``exchange_index`` is cached for the order that follows -- collections and + their markets live on shard 1, not on any leg's shard. + """ + selected = [ + {"market_ticker": lg["market_ticker"], "event_ticker": lg["event_ticker"], + "side": lg.get("side", "yes")} + for lg in legs + ] + body = {"selected_markets": selected, "with_market_payload": True} + raw = await client.call_api( + "POST", KALSHI_API_BASE_URL + _MULTIVARIATE_PATH + f"/{collection_ticker}", body=body, + ) + data = _read_json_body(await raw.read()) + if raw.status >= 400: + raise _api_error(raw.status, data) + ticker = data.get("market_ticker") + shard = (data.get("market") or {}).get("exchange_index") + if ticker and shard is not None: + _exchange_index_by_ticker[ticker] = int(shard) + return ticker + + +def create_combo_market_sync(collection_ticker, legs): + """Create (idempotent per Kalshi's "hit once before trading") the combo market + for a set of legs and return its market_ticker, or None. Each leg needs + market_ticker + event_ticker.""" + return _kalshi_sync(lambda c: _create_combo_market(c, collection_ticker, legs), + error_msg="Failed to create Kalshi combo market") + + +def place_combo_order_sync(collection_ticker, legs, yes_price_cents, count): + """Create the combo market for `legs`, then place an IOC buy on it. Returns + the order result dict (with the combo market_ticker attached) or None.""" + combo_ticker = create_combo_market_sync(collection_ticker, legs) + if not combo_ticker: + return None + result = place_order_sync(combo_ticker, "yes", yes_price_cents, count) + if result: + result["combo_ticker"] = combo_ticker + return result + + diff --git a/fly_trader/kalshi/vendor/kalshi_stream.py b/fly_trader/kalshi/vendor/kalshi_stream.py new file mode 100644 index 0000000000000000000000000000000000000000..118acc815c773dbde36303823fb37e3c1b638312 --- /dev/null +++ b/fly_trader/kalshi/vendor/kalshi_stream.py @@ -0,0 +1,330 @@ +# Vendored verbatim from better_bot code/kalshi_stream.py by tools/vendor_kalshi.py; sports sections removed. Do not edit. +"""The two broadcast channels: who wants which combo, and what the board is doing. + +Kalshi broadcasts every RFQ any member creates to every authenticated +subscriber, and ``rfq_created`` carries ``mve_selected_legs`` -- so the demand +side of the combo market is public, structure and size included, before anyone +quotes it. That is the training set for a combo pricing model and the map of +what would be worth quoting, and it is the one input that cannot be recovered +retroactively: nothing on the REST API remembers an RFQ that was never filled. + +**What is NOT public, and the correction that matters.** Only ``rfq_created`` +and ``rfq_deleted`` reach every subscriber. ``quote_created``, +``quote_accepted`` and ``quote_executed`` go solely to the RFQ's creator and the +quoting maker, so this feed shows demand and never the prices makers answered +with. Reading it as "we can see the whole RFQ book" would be wrong. + +**It is a firehose, and that shapes everything here.** Measured over a 40-second +window on 2026-08-09: 30,570 ``rfq_created``, 30,161 ``rfq_deleted`` and 12,989 +multivariate lifecycle messages -- roughly 760 RFQs a second, created and +cancelled in near-equal numbers by bots probing for prices. Persisting raw +messages would write ~65 million rows a day of almost pure churn. So this +aggregates in memory by STRUCTURE -- the sorted set of ``(market_ticker, +side)`` legs -- and flushes counts periodically. The same combo requested five +hundred times is one row with a count of five hundred, which is also the more +useful shape: it says what the market keeps asking for. + +Endpoint note, because the published docs are wrong about it: the per-channel +URLs in Kalshi's websocket reference (``wss://external-api-ws.kalshi.com/ +communications``) return HTTP 404. Both channels are served on the ordinary v2 +socket and selected with a ``subscribe`` command, which is what +:data:`WS_PATH` and :func:`subscribe_commands` implement. + +Run it with ``scripts/kalshi_stream.py``. It is deliberately NOT wired into the +daily cycle: the cycle is a batch job that exits, and a feed whose whole value +is continuity does not belong inside one. +""" + +import asyncio +import json +import logging +import time + +from .config import KALSHI_API_KEY_ID +from .prediction_utils import utc_now +from .db_connection import get_connection +from .kalshi_client import _kalshi_available, _load_private_key + +logger = logging.getLogger(__name__) + +#: The v2 socket, not the per-channel URLs the docs advertise -- those 404. +WS_HOST = "wss://api.elections.kalshi.com" +WS_PATH = "/trade-api/ws/v2" + +#: Both broadcast channels. ``communications`` carries the RFQ flow; +#: ``multivariate_market_lifecycle`` announces combo markets as they are created +#: and settled, which is how a ticker becomes visible intraday rather than at +#: the next daily listing walk. +CHANNELS = ("communications", "multivariate_market_lifecycle") + +#: Lifecycle events worth recording. The channel is dominated by +#: ``close_date_updated`` churn -- 12,989 messages in 40 seconds, nearly all of +#: it re-stamping close times -- and none of that says anything we act on. +LIFECYCLE_EVENTS = frozenset({"created", "determined", "settled"}) + +#: How often the in-memory aggregate is written. Long enough that the firehose +#: collapses into a few hundred rows, short enough that a crash loses minutes +#: rather than a session. +FLUSH_SECONDS = 60 + +#: Reconnect backoff, doubling to this ceiling. A long-lived socket WILL drop. +RECONNECT_MIN_SECONDS = 2 +RECONNECT_MAX_SECONDS = 120 + + +def subscribe_commands(channels=CHANNELS): + """The subscribe frames to send once the socket is open.""" + return [{"id": index, "cmd": "subscribe", "params": {"channels": [channel]}} + for index, channel in enumerate(channels, start=1)] + + +def structure_fingerprint(legs): + """A stable id for a combo's shape, order-independent. + + The RFQ id is useless as a key -- every probe mints a new one -- and the + market ticker is nearly as bad, since a combo market is created per request. + What repeats, and what is worth counting, is the SET of legs. + """ + return "|".join(sorted( + f"{leg.get('market_ticker')}:{leg.get('side') or 'yes'}" + for leg in legs)) + + +def _number(value): + try: + return float(value) + except (TypeError, ValueError): + return 0.0 + + +class DemandAggregate: + """In-memory tally of requested combo structures, flushed periodically. + + Deliberately not a cache in front of the database: at ~760 RFQs a second an + upsert per message would spend the whole budget on write amplification for + rows that are re-written milliseconds later. + """ + + def __init__(self): + self.structures = {} + self.channel_counts = {} + + def note_message(self, message_type): + self.channel_counts[message_type] = ( + self.channel_counts.get(message_type, 0) + 1) + + def note_rfq(self, msg): + """Record one ``rfq_created``. Ignores RFQs that are not combos. + + A single-market RFQ is somebody asking for a price on one contract; it + carries no joint structure and is not what this feed is for. + """ + legs = msg.get("mve_selected_legs") or [] + if len(legs) < 2: + return False + fingerprint = structure_fingerprint(legs) + entry = self.structures.get(fingerprint) + if entry is None: + entry = self.structures[fingerprint] = { + "fingerprint": fingerprint, + "collection_ticker": msg.get("mve_collection_ticker"), + "num_legs": len(legs), + "legs_json": json.dumps( + [{"market_ticker": leg.get("market_ticker"), + "event_ticker": leg.get("event_ticker"), + "side": leg.get("side") or "yes"} for leg in legs], + sort_keys=True), + # A combo whose legs all name one game is the population the + # dependence experiment cares about; carrying the flag here + # saves re-deriving it from legs_json later. + "is_same_game": int(len({leg.get("event_ticker") + for leg in legs}) == 1), + "rfq_count": 0, + "total_contracts": 0.0, + "total_target_cost": 0.0, + } + entry["rfq_count"] += 1 + entry["total_contracts"] += _number(msg.get("contracts_fp")) + entry["total_target_cost"] += _number(msg.get("target_cost_dollars")) + return True + + @property + def is_empty(self): + return not self.structures and not self.channel_counts + + def drain(self): + """Take everything accumulated so far and reset.""" + structures = list(self.structures.values()) + counts = dict(self.channel_counts) + self.structures = {} + self.channel_counts = {} + return structures, counts + + +def flush(aggregate, conn=None): + """Persist one window. Returns ``{structures, messages}``. + + Counts ADD on conflict rather than replace, so a structure seen across many + windows accumulates one total instead of keeping only the last window's. + """ + structures, counts = aggregate.drain() + if not structures and not counts: + return {"structures": 0, "messages": 0} + owned = conn is None + conn = conn or get_connection() + try: + seen_at = utc_now() + conn.executemany( + """INSERT INTO kalshi_rfq_demand + (fingerprint, collection_ticker, num_legs, legs_json, + is_same_game, rfq_count, total_contracts, + total_target_cost, first_seen, last_seen) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(fingerprint) DO UPDATE SET + rfq_count = rfq_count + excluded.rfq_count, + total_contracts = total_contracts + excluded.total_contracts, + total_target_cost = + total_target_cost + excluded.total_target_cost, + last_seen = excluded.last_seen""", + [(s["fingerprint"], s["collection_ticker"], s["num_legs"], + s["legs_json"], s["is_same_game"], s["rfq_count"], + s["total_contracts"], s["total_target_cost"], seen_at, seen_at) + for s in structures]) + conn.executemany( + """INSERT INTO kalshi_stream_stats + (window_end, message_type, message_count) + VALUES (?, ?, ?)""", + [(seen_at, name, count) for name, count in sorted(counts.items())]) + conn.commit() + return {"structures": len(structures), "messages": sum(counts.values())} + finally: + if owned: + conn.close() + + +def _auth_headers(): + """Handshake headers, signed by the SDK's own signer. + + Reused rather than reimplemented: the signature is over + ``timestamp + METHOD + path`` with RSA-PSS, and a hand-rolled second copy + would be one upstream change away from silently failing to authenticate. + """ + from kalshi_python_async.auth import KalshiAuth + + private_key = _load_private_key() + if not (KALSHI_API_KEY_ID and private_key): + return None + return KalshiAuth(KALSHI_API_KEY_ID, private_key).create_auth_headers( + "GET", WS_PATH) + + +async def _consume(aggregate, *, deadline=None, flush_seconds=FLUSH_SECONDS, + conn=None, channels=CHANNELS): + """One connection's lifetime: subscribe, aggregate, flush on a timer.""" + import websockets + + headers = _auth_headers() + if headers is None: + raise RuntimeError("Kalshi websocket needs KALSHI_API_KEY_ID and a " + "readable KALSHI_PRIVATE_KEY_PATH") + async with websockets.connect(WS_HOST + WS_PATH, + additional_headers=headers, + open_timeout=15, + max_queue=None) as socket: + for command in subscribe_commands(channels): + await socket.send(json.dumps(command)) + logger.info("kalshi stream: subscribed to %s", ", ".join(channels)) + next_flush = time.monotonic() + flush_seconds + while deadline is None or time.monotonic() < deadline: + timeout = max(0.5, next_flush - time.monotonic()) + try: + raw = await asyncio.wait_for(socket.recv(), timeout) + except asyncio.TimeoutError: + raw = None + if raw is not None: + handle(aggregate, raw) + if time.monotonic() >= next_flush: + written = flush(aggregate, conn=conn) + if written["messages"]: + logger.info("kalshi stream: %s", written) + next_flush = time.monotonic() + flush_seconds + + +def handle(aggregate, raw): + """Route one frame into the aggregate. Malformed frames are counted, not raised.""" + try: + message = json.loads(raw) + except (TypeError, ValueError): + aggregate.note_message("unparseable") + return + message_type = message.get("type") + payload = message.get("msg") or {} + if message_type == "multivariate_market_lifecycle": + event_type = payload.get("event_type") + if event_type not in LIFECYCLE_EVENTS: + return + aggregate.note_message(f"lifecycle:{event_type}") + return + if message_type == "rfq_created": + aggregate.note_message( + "rfq_created:combo" if aggregate.note_rfq(payload) + else "rfq_created:single") + return + if message_type in ("rfq_deleted", "subscribed", "error"): + aggregate.note_message(message_type) + + +async def stream(duration_seconds=None, *, flush_seconds=FLUSH_SECONDS, + conn=None, channels=CHANNELS): + """Listen until ``duration_seconds`` elapses, reconnecting on drop. + + ``None`` runs forever, which is the intended production mode. + """ + if not _kalshi_available(): + raise RuntimeError("Kalshi is not configured; nothing to subscribe to") + aggregate = DemandAggregate() + deadline = None if duration_seconds is None else ( + time.monotonic() + duration_seconds) + backoff = RECONNECT_MIN_SECONDS + try: + while deadline is None or time.monotonic() < deadline: + try: + await _consume(aggregate, deadline=deadline, + flush_seconds=flush_seconds, conn=conn, + channels=channels) + backoff = RECONNECT_MIN_SECONDS + except asyncio.CancelledError: + raise + except Exception as exc: # noqa: BLE001 + # Any drop is survivable and expected on a socket meant to stay + # up for days. Flush first so the window's counts are not lost + # to the reconnect. + flush(aggregate, conn=conn) + logger.warning("kalshi stream: %s: %s; reconnecting in %ss", + type(exc).__name__, exc, backoff) + await asyncio.sleep(backoff) + backoff = min(backoff * 2, RECONNECT_MAX_SECONDS) + finally: + # Whatever the reason for stopping -- deadline, Ctrl-C, cancellation -- + # the partial window is worth more written than discarded. + flush(aggregate, conn=conn) + + +def top_structures(limit=25, *, same_game_only=False, conn=None): + """The most-requested combo structures seen so far. + + What the market keeps asking to be priced, which is where quoting would + matter and which structures the dependence experiment should cover first. + """ + owned = conn is None + conn = conn or get_connection() + try: + return [dict(row) for row in conn.execute( + f"""SELECT fingerprint, collection_ticker, num_legs, is_same_game, + rfq_count, total_contracts, first_seen, last_seen + FROM kalshi_rfq_demand + {'WHERE is_same_game = 1' if same_game_only else ''} + ORDER BY rfq_count DESC LIMIT ?""", (limit,))] + finally: + if owned: + conn.close() diff --git a/fly_trader/kalshi/vendor/prediction_utils.py b/fly_trader/kalshi/vendor/prediction_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..f72275e5be030614f35dfd5e3fb2ff27a8bfe684 --- /dev/null +++ b/fly_trader/kalshi/vendor/prediction_utils.py @@ -0,0 +1,8 @@ +"""Shim for better_bot's ``prediction_utils``: only ``utc_now`` is imported by the vendored modules.""" +from __future__ import annotations + +from datetime import datetime, timezone + + +def utc_now() -> datetime: + return datetime.now(timezone.utc) diff --git a/fly_trader/kalshi/vendor/venue_kalshi.py b/fly_trader/kalshi/vendor/venue_kalshi.py new file mode 100644 index 0000000000000000000000000000000000000000..d865ae98bb172ac1f70d59419cb9173fd815b706 --- /dev/null +++ b/fly_trader/kalshi/vendor/venue_kalshi.py @@ -0,0 +1,347 @@ +# Vendored verbatim from better_bot code/venue_kalshi.py by tools/vendor_kalshi.py; sports sections removed. Do not edit. +"""Kalshi as a :class:`code.venue_register.Venue`. + +A thin adapter, deliberately. ``code.kalshi_client`` is 1,931 lines with a +1,975-line test file pointed at it, and none of that moves: the seam is this +class, not a relocation. Those tests staying green *unchanged* is what proves +the fee model was not perturbed while the abstraction was built around it. + +Everything here delegates. The one piece of real logic is +:meth:`KalshiVenue.simulate`, which reproduces the paper arm's arithmetic +exactly -- see its docstring for why that matters more than it looks. +""" + +import logging + +from . import kalshi_client as kc +from .book_sizing import (MAX_PRICE_CENTS, MIN_PRICE_CENTS, fill_budget, + parse_kalshi_orderbook, validate_against_quote) +from .config import ENABLE_VENUE_KALSHI +from .venue_register import Fill, MarketRef, NO_QUOTE, Quote, TopOfBook + +logger = logging.getLogger(__name__) + +#: A Kalshi contract settles at one dollar. This is the venue's payout algebra +#: and the only place the literal belongs. +CONTRACT_PAYOUT_CENTS = 100 + +#: What a half-spread can credibly be, in cents. Kalshi quotes whole cents, so a +#: one-cent book puts the mid at x.5 and 0.5c is the tightest a real two-sided +#: quote can show; a gap past the ceiling is a one-sided or stale row rather than +#: a spread, and taking it at face value would price that market off the slate. +SPREAD_BAND = (0.0, 5.0) + +#: Used only when there is neither a quote to read nor a settled record to +#: measure -- a cold start. Deliberately the mean of the 26 fills the frozen +#: constant was provenanced from, so an unmeasured venue behaves as this one did +#: before any of this existed. +DECLARED_HALF_SPREAD_CENTS = 0.52 + +#: Below this many settled fills the realised mean is noise, not a measurement. +MIN_SPREAD_SAMPLE = 30 + +_REALISED_HALF_SPREAD = None +_SPREAD_BY_MARKET = {} + + +def _in_band(value): + return SPREAD_BAND[0] <= float(value) <= SPREAD_BAND[1] + + +def reset_spread_cache(): + """Drop the memoised spreads. Tests, and after an attach or settlement pass.""" + global _REALISED_HALF_SPREAD + _REALISED_HALF_SPREAD = None + _SPREAD_BY_MARKET.clear() + + +class KalshiVenue: + """The CFTC-designated contract market this system has always traded.""" + + name = "kalshi" + + # -- activation --------------------------------------------------------- + + def is_available(self): + """Credentials load, the calibration gate is on, and the flag is set. + + ``_kalshi_available`` already answers the first two; the flag is the + venue-level switch every venue carries, so "which venues are active" is + one question with one shape rather than a special case per venue. + """ + return bool(ENABLE_VENUE_KALSHI) and kc._kalshi_available() + + # -- identity ----------------------------------------------------------- + + def market_group_of(self, market_id): + return kc.series_ticker_of(market_id) + + + # -- pricing: pure, no network ------------------------------------------ + + def fee_rate_cents(self, price_cents, market_id=None): + return kc.kalshi_fee_rate_cents(price_cents, market_id) + + def effective_price_cents(self, price_cents, market_id=None): + return kc.effective_price_cents(price_cents, market_id) + + def order_fee_cents(self, price_cents, quantity, market_id=None): + return kc.kalshi_trading_fee_cents(price_cents, quantity, market_id) + + def order_cost_cents(self, price_cents, quantity, market_id=None): + return kc.order_collateral_cents(market_id, price_cents, quantity) + + def payout_cents(self, quantity, market_id=None): + """A winning position returns a dollar a contract.""" + return int(quantity) * CONTRACT_PAYOUT_CENTS + + def half_spread_cents(self, market_id=None, top=None): + """What crossing this market costs, per contract, in cents. + + Three tiers, in order, and the order is the point: + + 1. **Observed** -- ``ask - mid`` for this market. Live if the caller + already holds a quote, otherwise from the attach pass's stored pair. + This is per-market and needs no fill history. + 2. **Realised** -- the mean of ``price - mid`` over settled fills at + this venue. What a caller without a market gets, because there is no + book to read. + 3. **Declared** -- a floor, when neither exists. + + Read rather than fetched: the stored pair was written by the attach pass, + so sizing a slate twice sizes it identically. An observed value outside + :data:`SPREAD_BAND` is refused and falls to tier 2 -- a one-sided or + stale cache row can show an arbitrary gap, and trusting it would silently + price a market out of the slate. + """ + if top is not None: + observed = top.half_spread_cents + if observed is not None and _in_band(observed): + return observed + if market_id: + observed = self._cached_half_spread(market_id) + if observed is not None: + if _in_band(observed): + return observed + logger.warning( + "%s: cached half-spread %.2fc for %s outside %s -- using the " + "realised mean instead", self.name, observed, market_id, + SPREAD_BAND) + return self._realised_half_spread() + + def _cached_half_spread(self, market_id): + """``ask - mid`` from the attach pass's stored quote, or ``None``. + + Memoised per market: sizing asks this once per candidate, and re-reading + halfway through a slate could price two candidates on the same card + against two different snapshots. + """ + key = (self.name, str(market_id)) + if key in _SPREAD_BY_MARKET: + return _SPREAD_BY_MARKET[key] + from .database import get_connection + conn = get_connection() + try: + row = conn.execute( + "SELECT yes_ask_cents, market_prob FROM venue_market_cache " + "WHERE venue = ? AND market_id = ?", + (self.name, str(market_id))).fetchone() + except Exception as exc: # noqa: BLE001 + logger.warning("%s: could not read the quote cache (%s)", self.name, exc) + return None + finally: + conn.close() + observed = None + if row and row["yes_ask_cents"] is not None and row["market_prob"] is not None: + observed = float(row["yes_ask_cents"]) - float(row["market_prob"]) + _SPREAD_BY_MARKET[key] = observed + return observed + + def _realised_half_spread(self): + """The mean gap between what was paid and the mid it was sized on. + + Memoised per process: it is a property of a settled record, not of the + moment it is asked for, and re-reading it mid-slate would let two + candidates on one card be sized against different constants. + """ + global _REALISED_HALF_SPREAD + if _REALISED_HALF_SPREAD is None: + from .kelly_sizing import measure_half_spread_cents + try: + mean, n = measure_half_spread_cents() + except Exception as exc: # noqa: BLE001 + logger.warning("%s: could not measure the realised spread (%s)", + self.name, exc) + mean, n = None, 0 + if mean is None or n < MIN_SPREAD_SAMPLE or not _in_band(mean): + logger.info("%s: realised spread unusable (n=%s, mean=%s); " + "using the declared floor %.2fc", + self.name, n, mean, DECLARED_HALF_SPREAD_CENTS) + mean = DECLARED_HALF_SPREAD_CENTS + _REALISED_HALF_SPREAD = float(mean) + return _REALISED_HALF_SPREAD + + def devig(self, yes_ask_cents, no_ask_cents): + """The market's BELIEF for the yes side, overround removed. + + Proportional normalisation, which is the standard two-sided treatment + for a central limit order book: with both sides quoted, the asks sum to + more than 100 by the overround, and each side's share of that sum is its + implied probability once the overround is divided out. + + A venue operation rather than a general one: this is right for a + two-sided book, but a parimutuel pool's de-vig is its takeout rate and + an AMM's is its curve mid. ``None`` when only one side is quoted, since + a one-sided book carries no overround to remove. + """ + if yes_ask_cents is None or no_ask_cents is None: + return None + total = float(yes_ask_cents) + float(no_ask_cents) + if total <= 0: + return None + return float(yes_ask_cents) / total + + def price_bounds(self): + """Kalshi quotes whole cents in 1..99; outside that is not a contract.""" + return MIN_PRICE_CENTS, MAX_PRICE_CENTS + + # -- quoting and execution: network ------------------------------------- + + + def top_of_book(self, market_id, side="yes"): + market = kc.get_market_sync(market_id) + if not market: + return None + quote = kc.market_quote(market) or {} + return TopOfBook( + market=MarketRef(venue=self.name, market_id=market_id, + group_id=self.market_group_of(market_id)), + side=side, + mid_cents=quote.get("market_prob"), + ask_cents=kc.market_ask_cents(market, side), + volume=quote.get("volume")) + + def quote_for_size(self, market_id, side, *, budget_cents, p_win, + min_payout_cents, max_quantity=None, + quoted_ask_cents=None): + """Walk the book and price the size actually reachable. + + One call covers fetch, parse, sanity-check and walk, because those are + one question -- "what would this cost here" -- and splitting them across + the interface would force every caller to know that this venue answers + it with a ladder. + + Two refusals are deliberately distinguishable by ``stopped_reason``, and + the caller must keep treating them differently: ``no_book`` and + ``stale_book`` mean this venue could not be asked, so the caller falls + back to top-of-book rather than dropping the position, while anything + else means it WAS asked and offered nothing worth taking. + """ + payload = kc.get_orderbook_sync(market_id) + if payload is None: + logger.info("no orderbook for %s; caller sizes off top-of-book", + market_id) + return Quote(market=self._ref(market_id), side=side, quantity=0, + cost_cents=0.0, payout_cents=0.0, + stopped_reason="no_book") + book = parse_kalshi_orderbook(payload, ticker=market_id) + if validate_against_quote(book, side, quoted_ask_cents) is False: + # Inverted sides or a stale snapshot: fall back rather than trust it. + logger.warning( + "orderbook for %s implies best ask %sc against quoted %sc -- " + "ignoring depth", market_id, book.best_ask(side), + quoted_ask_cents) + return Quote(market=self._ref(market_id), side=side, quantity=0, + cost_cents=0.0, payout_cents=0.0, + stopped_reason="stale_book") + fill = fill_budget(book, side, p_win, budget_cents, + min_payout_cents=min_payout_cents, + max_contracts=max_quantity) + if fill.is_fillable: + logger.info( + "depth sizing %s: %d contracts across %d level(s), vwap %.2fc " + "vs best %sc, slippage %+.2fc/contract, cost $%.2f of $%.2f " + "allocated (%s)", market_id, fill.quantity, fill.levels_taken, + fill.vwap_cents, fill.best_ask_cents, fill.slippage_cents, + fill.cost_cents / 100, budget_cents / 100, fill.stopped_reason) + return self._quote_from_fill(market_id, side, fill) + + def place(self, quote): + """Submit ``quote`` as an IOC order; ``None`` when nothing filled.""" + if not quote or not quote.is_fillable: + return None + market_id = quote.market.market_id + price = quote.limit_price_cents or quote.best_price_cents + result = kc.place_order_sync(market_id, quote.side, price, quote.quantity) + if not result or not result.get("order_id"): + logger.error("order rejected by %s for %s at %sc", + self.name, market_id, price) + return None + executed = kc.order_fill(result, quote.quantity) + if executed is None: + # Accepted and filled nothing. An IOC that crossed no resting size + # comes back carrying an order id exactly like one that traded, so + # this is logged apart from a rejection: they are the same outcome + # for the caller and very different things to debug. + logger.warning("%s accepted then filled 0 of %d for %s at %sc", + self.name, quote.quantity, market_id, price) + return None + return Fill(order_id=executed.order_id, market=quote.market, + quantity=executed.quantity, + cost_cents=executed.cost_cents, + fee_cents=executed.fee_cents, + payout_cents=self.payout_cents(executed.quantity, + market_id), + requested=executed.requested) + + def simulate(self, quote): + """The paper fill for ``quote``, priced exactly as the paper book is. + + **Deliberately the worst level consumed, not the volume-weighted price.** + The real arm books the VWAP the exchange reported; paper charges + ``limit_price_cents``, which is the most expensive rung the walk + touched. That is conservative by construction, and it is load-bearing: + paper P/L feeds ``league_paper_gate``'s strategy comparison, which + decides which leagues trade real money. Pricing paper at the VWAP would + make it cheaper, raise its measured ROI, and move that verdict -- so any + change here is a change to capital routing and belongs in its own commit + with its own measurement. + """ + if not quote or not quote.is_fillable: + return None + market_id = quote.market.market_id + price = quote.limit_price_cents or quote.best_price_cents + fee = self.order_fee_cents(price, quote.quantity, market_id) + return Fill(order_id="", market=quote.market, quantity=quote.quantity, + cost_cents=int(quote.quantity) * int(price), + fee_cents=fee, + payout_cents=self.payout_cents(quote.quantity, market_id), + requested=quote.quantity) + + def balance_cents(self): + balance = kc.get_balance_sync() + return balance.get("balance") if balance else None + + # -- internals ---------------------------------------------------------- + + def _ref(self, market_id): + return MarketRef(venue=self.name, market_id=market_id, + group_id=self.market_group_of(market_id)) + + def _quote_from_fill(self, market_id, side, fill): + """Carry a :class:`code.book_sizing.BookFill` into venue vocabulary.""" + if fill is None or not fill.is_fillable: + return Quote(market=self._ref(market_id), side=side, quantity=0, + cost_cents=0.0, payout_cents=0.0, + stopped_reason=getattr(fill, "stopped_reason", + NO_QUOTE.stopped_reason)) + return Quote(market=self._ref(market_id), side=side, + quantity=fill.quantity, + cost_cents=float(fill.cost_cents), + payout_cents=float(self.payout_cents(fill.quantity, + market_id)), + fee_cents=float(fill.fee_cents), + best_price_cents=fill.best_ask_cents, + limit_price_cents=fill.limit_price_cents, + stopped_reason=fill.stopped_reason, + levels_taken=fill.levels_taken) diff --git a/fly_trader/kalshi/vendor/venue_register.py b/fly_trader/kalshi/vendor/venue_register.py new file mode 100644 index 0000000000000000000000000000000000000000..39603efbfa2770a8b80e5a38c5903b777ad021e1 --- /dev/null +++ b/fly_trader/kalshi/vendor/venue_register.py @@ -0,0 +1,341 @@ +# Vendored verbatim from better_bot code/venue_register.py by tools/vendor_kalshi.py; sports sections removed. Do not edit. +"""What a trading venue is, and which ones this process may reach. + +One protocol, one registry, and the four value types every caller passes around: +a market's identity, its top of book, what buying a size there would cost, and +what an order actually executed. The Kalshi implementation lives in +``code.venue_kalshi``; routing between venues lives in ``code.venue_routing``. + +The reason this exists is that "the fee" and "the payout" are venue facts and +were written as global ones. ``effective_price_cents`` had twenty-five direct +importers and ``quantity * 100`` was spelled out at fifteen sites, so a second +venue could not be added without editing arithmetic in fifteen files. Here they +are asked of the venue that quoted the price. + +Named ``venue_register`` rather than ``venue`` on purpose: ``code.venue_geo`` +already exists and ``teams.venue_name`` is a stadium. The word is taken. +""" + +import logging +from dataclasses import dataclass, field +from typing import Optional, Protocol, runtime_checkable + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class MarketRef: + """One tradeable contract at one venue. + + ``market_id`` is whatever that venue calls a market -- a Kalshi ticker, an + exchange's market id, an on-chain condition id -- and is opaque above this + layer. ``group_id`` is what the venue's FEE SCHEDULE keys on (Kalshi's + series), which is why it is carried separately rather than parsed out of the + market id by every caller that needs a fee. + """ + + venue: str + market_id: str + group_id: Optional[str] = None + event_id: Optional[str] = None + title: str = "" + + +@dataclass(frozen=True) +class TopOfBook: + """The best quote on one side, and the spread implied by it. + + ``mid_cents`` and ``ask_cents`` are both carried because sizing and + execution ask different questions of the same quote: sizing is done on the + mid plus a modelled half-spread, while the order is submitted at the ask. + """ + + market: MarketRef + side: str + mid_cents: Optional[float] = None + ask_cents: Optional[int] = None + volume: Optional[int] = None + + @property + def half_spread_cents(self): + """``ask - mid``, or ``None`` when either is missing. + + The observed cost of crossing, per contract, at this market right now -- + as opposed to a constant measured once across a whole book of fills. + """ + if self.ask_cents is None or self.mid_cents is None: + return None + return float(self.ask_cents) - float(self.mid_cents) + + +@dataclass(frozen=True) +class Quote: + """What buying ``quantity`` at one venue would cost, and what it returns. + + This is the primitive the whole abstraction rests on, because it is the only + shape that fits every venue we might add. A central limit order book answers + it by walking its ask ladder; a fixed-odds book by reading one price and its + posted maximum; an automated market maker by walking its own curve, where the + price genuinely depends on the size asked for; a parimutuel pool cannot + answer it before the round closes and says so via ``stopped_reason``. + + ``cost_cents`` is ALL-IN for the whole quantity -- fee included -- and + ``payout_cents`` is what the position returns if it wins. Every existing + formula in the codebase is expressible in those two numbers, which is what + makes the venues interchangeable without rewriting the arithmetic. + """ + + market: MarketRef + side: str + quantity: int + cost_cents: float + payout_cents: float + fee_cents: float = 0.0 + best_price_cents: Optional[int] = None + #: Worst level consumed: the limit an IOC order is submitted at, so it can + #: reach the depth it was sized against without paying beyond it. + limit_price_cents: Optional[int] = None + stopped_reason: str = "" + levels_taken: int = 0 + + @property + def is_fillable(self): + return self.quantity > 0 + + @property + def effective_price_cents(self): + """Cost per 100c of payout: the venue-neutral all-in basis. + + This is the bridge that lets the existing formulas stand unchanged. On a + binary $1 contract it equals price + fee, because payout is + ``quantity * 100``; on a commission book it is the stake per 100c of + return once commission is taken. ``p - effective_price/100`` is the edge + at either. + """ + if not self.payout_cents: + return 0.0 + return 100.0 * self.cost_cents / self.payout_cents + + @property + def net_odds(self): + """``b`` in Kelly: profit per unit staked if the position wins. + + Written out longhand as ``(100 - p_eff) / p_eff`` at three sites before + this existed, which is exactly where the literal 100 was wrong for any + venue that does not settle at a dollar. + """ + if not self.cost_cents: + return 0.0 + return (self.payout_cents - self.cost_cents) / self.cost_cents + + +@dataclass(frozen=True) +class Fill: + """What an order actually executed -- never what was requested. + + **An order id means the venue ACCEPTED the order, not that it filled.** Every + order this repo places is immediate-or-cancel, so a response reading zero + fills carries an id exactly like one that traded; treating that as a position + booked exposure the account never had. A caller that gets ``None`` from + :meth:`Venue.place` has no position, and must write no row. + """ + + order_id: str + market: MarketRef + quantity: int + cost_cents: float + fee_cents: float + payout_cents: float + requested: int = 0 + + @property + def is_filled(self): + return self.quantity >= self.requested > 0 + + @property + def status(self): + return "filled" if self.is_filled else "partial" + + @property + def vwap_cents(self): + return self.cost_cents / self.quantity if self.quantity else 0.0 + + +def net_odds_at(effective_price_cents): + """``b`` in Kelly at an all-in basis: profit per unit staked on a win. + + **Venue-independent, and that is the finding.** It was written longhand at + three sites -- ``book_sizing``, ``kelly_sizing`` and ``ev_ledger`` -- and the + literal 100 in each looks like Kalshi's dollar settlement. It is not. An + effective price is defined as cost per 100c of PAYOUT, so the 100 is the + normalisation constant of the basis itself, and the formula is already right + at a commission book or an AMM. What was duplicated is the expression, not a + venue assumption; the venue-dependence lives entirely upstream, in how the + basis was computed. + """ + basis = float(effective_price_cents) + if basis <= 0: + return 0.0 + return (100.0 - basis) / basis + + +#: A quote that cannot be filled. ``stopped_reason`` says why, because "no +#: depth" and "this venue cannot price a size before the round closes" are +#: different answers and the router treats them differently. +NO_QUOTE = Quote(market=None, side="", quantity=0, cost_cents=0.0, + payout_cents=0.0, stopped_reason="no_liquidity") + + +@runtime_checkable +class Venue(Protocol): + """A place this system can price and take a position. + + Split into three bands, and the split is load-bearing: + + * **identity** -- what a market is called here, and which of ours it is. + * **pricing** -- PURE. No network, no lazy fetch, no database. Sizing runs + through these on every candidate on the slate, and a fetch inside the fee + arithmetic would make a slate's allocation depend on whether the venue was + reachable at the moment it was priced. Warm caches explicitly instead. + * **quoting and execution** -- network. Called once per candidate, not once + per arithmetic step. + """ + + name: str + + # -- activation --------------------------------------------------------- + def is_available(self) -> bool: ... + + # -- identity ----------------------------------------------------------- + def market_group_of(self, market_id): ... + def series_for_league(self, league_id): ... + def series_for_slug(self, league_slug): ... + def tradable_leagues(self): ... + def market_for_event(self, league_slug, home_team, away_team, *, + predicted_winner=None, event_date=None, + home_abbrev=None, away_abbrev=None): ... + + # -- pricing: pure ------------------------------------------------------ + def fee_rate_cents(self, price_cents, market_id=None): ... + def effective_price_cents(self, price_cents, market_id=None): ... + def order_fee_cents(self, price_cents, quantity, market_id=None): ... + def order_cost_cents(self, price_cents, quantity, market_id=None): ... + def payout_cents(self, quantity, market_id=None): ... + def half_spread_cents(self, market_id=None, top=None): ... + def devig(self, yes_ask_cents, no_ask_cents): ... + def price_bounds(self): ... + + # -- quoting and execution: network ------------------------------------- + def cached_odds(self, event_id, predicted_winner, home_team, away_team, *, + league_slug=None, home_abbrev=None, away_abbrev=None, + conn=None): ... + def top_of_book(self, market_id, side="yes"): ... + def quote_for_size(self, market_id, side, *, budget_cents, p_win, + min_payout_cents, max_quantity=None): ... + def place(self, quote): ... + def simulate(self, quote): ... + def balance_cents(self): ... + + +# --------------------------------------------------------------------------- +# Registry +# +# Lazy, and mirroring code.gate_register: the module that owns the mechanism +# imports the module that declares the content, not the other way round, so a +# venue implementation can import these types without a cycle. +# --------------------------------------------------------------------------- + +#: The venue whose balance anchors Kelly sizing, for every book, whatever venue +#: a position is finally routed to. +#: +#: **Sizing does not read the venue a position lands on, and that is deliberate.** +#: A second exchange is funded ahead of integration -- KYC has to clear before it +#: can hold anything -- and carries only enough to cover what routes to it. Its +#: local balance is a settlement constraint, not a statement of how much capital +#: the book has, and sizing against it would shrink every stake to the float +#: sitting at whichever venue happened to quote best. +#: +#: One bankroll, one solve: the same edge earns the same stake wherever it is +#: finally taken, and adding an exchange moves where a position rests without +#: moving what the optimiser thinks it is playing with. +SIZING_ANCHOR_VENUE = "kalshi" + + +_REGISTRY = None + + +def _build_registry(): + from .venue_kalshi import KalshiVenue + return {v.name: v for v in (KalshiVenue(),)} + + +def venues(): + """Every venue this build knows how to reach, declared or not.""" + global _REGISTRY + if _REGISTRY is None: + _REGISTRY = _build_registry() + return _REGISTRY + + +def venue_for(name): + """The venue called *name*, or ``KeyError`` naming what is registered. + + A position row carries its venue as a string, so a missing implementation + surfaces here rather than as an attribute error three frames deeper. + """ + registry = venues() + if name not in registry: + raise KeyError( + f"no venue named {name!r}; registered: {sorted(registry)}") + return registry[name] + + +def active_venues(): + """Registered venues whose credentials load and whose flag is set.""" + return [v for v in venues().values() if v.is_available()] + + +def default_venue(): + """The single active venue. + + Raises when none is active -- that is a misconfiguration, not an empty + slate -- and when several are, because choosing between them is + ``code.venue_routing``'s job and doing it implicitly here would hide the + choice from the ledger. + """ + active = active_venues() + if not active: + raise RuntimeError( + "no venue is active: check credentials and ENABLE_VENUE_* flags") + if len(active) > 1: + raise RuntimeError( + "several venues are active (%s); route explicitly via " + "code.venue_routing" % ", ".join(sorted(v.name for v in active))) + return active[0] + + +def sizing_bankroll_cents(): + """The balance every Kelly solve sizes against, or ``None`` when unfunded. + + Always the anchor venue's, never the routed one's -- see + :data:`SIZING_ANCHOR_VENUE` for why. Callers own the fallback when this is + ``None``: an unfunded account still has to size a paper slate. + """ + return venue_for(SIZING_ANCHOR_VENUE).balance_cents() + + +def reset(): + """Drop the registry. Tests only.""" + global _REGISTRY + _REGISTRY = None + + +def register_for_test(venue): + """Add *venue* to the registry. The only supported way in for a fake. + + Production code must never call this: ``tests/unit/test_venue_isolation.py`` + greps ``code/`` for it, because pytest puts the repo root on the path and a + test double is therefore importable from production without this guard. + """ + venues()[venue.name] = venue + return venue diff --git a/fly_trader/market/exit_cost.py b/fly_trader/market/exit_cost.py index 8188bc2c10fccb044a6e7818450b4f3810cbc25a..70a940714eccfd2ed3c79c69182add75f3d9c1d7 100644 --- a/fly_trader/market/exit_cost.py +++ b/fly_trader/market/exit_cost.py @@ -40,6 +40,13 @@ def pool_fee_rate(mcap_sol): return float(out) if out.ndim == 0 else out +PONS_LABEL = "Pons v4" # a graduated Pons launch on Robinhood Chain (fly_trader/markets.RH): amounts in ETH + + +def is_pons(program_label: str | None) -> bool: + return program_label == PONS_LABEL + + def impact_fraction(value_sol: float, res_quote_sol: float | None, program_label: str | None = None) -> float: if value_sol <= 0: return 0.0 @@ -50,10 +57,15 @@ def impact_fraction(value_sol: float, res_quote_sol: float | None, program_label return value_sol / (value_sol + res_quote_sol) -def fee_fraction(value_sol: float, mcap_sol: float | None, pool_fee: float | None = None) -> float: - """Fees of one swap of ``value_sol`` as a fraction of it: pool fee + Jupiter's platform fee + the network fee.""" +def fee_fraction(value_sol: float, mcap_sol: float | None, pool_fee: float | None = None, program_label: str | None = None) -> float: + """Fees of one swap of ``value_sol`` as a fraction of it: pool fee + Jupiter's platform fee + the network fee. + A Pons pool (Robinhood Chain, amounts in ETH): the hook's fee + tax (``pool_fee``, measured per minute; else + ``RH_HOOK_FEE``) + the gas of one swap (``RH_TX_FEE_ETH``); KyberSwap charges nothing (rh/kyber.check_build).""" if value_sol <= 0: return 0.0 + if is_pons(program_label): + hf = pool_fee if pool_fee is not None and math.isfinite(pool_fee) and pool_fee > 0 else config.RH_HOOK_FEE + return hf + config.RH_TX_FEE_ETH / value_sol pf = pool_fee if pool_fee is not None and math.isfinite(pool_fee) and pool_fee > 0 else pool_fee_rate(mcap_sol) return pf + JUPITER_FEE_BPS / 1e4 + TX_FEE_LAMPORTS / config.LAMPORTS_PER_SOL / value_sol @@ -63,25 +75,37 @@ def exit_cost_fraction(value_sol: float, res_quote_sol: float | None, mcap_sol: """Everything a swap of ``value_sol`` loses, as a fraction of it: fees plus price impact.""" if value_sol <= 0: return 0.0 - return min(1.0, fee_fraction(value_sol, mcap_sol, pool_fee) + impact_fraction(value_sol, res_quote_sol, program_label)) + return min(1.0, fee_fraction(value_sol, mcap_sol, pool_fee, program_label) + impact_fraction(value_sol, res_quote_sol, program_label)) + + +def label_size(res_quote_sol, market=None) -> float: + """The size every label is priced at (``cost_at_size``'s default) and so the largest position any edge was measured + at: the market's label size (``LABEL_SIZE_SOL``; RH ``RH_LABEL_SIZE_ETH``), or ``MAX_POOL_SHARE`` of a shallower pool. + Unknown liquidity: the label size.""" + ls = float(config.LABEL_SIZE_SOL) if market is None else market.label_size() + r = float(res_quote_sol) if res_quote_sol is not None and math.isfinite(float(res_quote_sol)) and float(res_quote_sol) > 0 else None + return ls if r is None else float(min(ls, config.MAX_POOL_SHARE * r)) -def cost_at_size(ec_0p1, res_quote_sol, size_sol: float | None = None): +def cost_at_size(ec_0p1, res_quote_sol, size_sol: float | None = None, market=None): """``exit_cost_0p1`` (priced for a fixed 0.1 SOL in market/features.py) repriced for the size the book actually trades, as a fraction of one side. - Positions are Kelly-sized up to ``MAX_POSITION_FRACTION`` of the bankroll and impact is ``value/(value+reserves)``, - so the stored feature understates a real exit. The fee part (pool, platform, network) is recovered from the stored - value and only impact recomputed, so a feature part never has to be rebuilt. ``size_sol`` defaults to the sizing - rule's own ceiling and is capped by ``MAX_POOL_SHARE`` of the pool, exactly as agent/sizing.py caps a live buy. + Impact is ``value/(value+reserves)``, so the stored feature understates a real exit. The fee part (pool, platform, + network) is recovered from the stored value and only impact recomputed, so a feature part never has to be rebuilt. + ``size_sol`` defaults to ``LABEL_SIZE_SOL`` and is capped by ``MAX_POOL_SHARE`` of the pool (``label_size``), which + is also the largest buy agent/sizing.py makes. ``market`` (fly_trader/markets.RH): the stored value was priced at + 0.1 SOL's worth of ETH (0.1·K) with one swap's gas ``RH_TX_FEE_ETH``, and the label size is ``RH_LABEL_SIZE_ETH``. Unknown liquidity costs 1.0: a position that cannot be exited. Takes scalars or arrays.""" ec = np.clip(np.asarray(ec_0p1, dtype=float), 0.0, 1.0) r = np.asarray(res_quote_sol, dtype=float) ok = np.isfinite(r) & (r > 0) - want = size_sol if size_sol is not None else (config.LABEL_SIZE_SOL or config.MAX_POSITION_FRACTION * config.CAPITAL_SOL) + rh = market is not None and market.chain == "rh" + want = size_sol if size_sol is not None else (market.label_size() if rh else config.LABEL_SIZE_SOL) sz = np.maximum(np.minimum(want, config.MAX_POOL_SHARE * np.where(ok, r, 0.0)), 1e-9) - tx = TX_FEE_LAMPORTS / config.LAMPORTS_PER_SOL - fees01 = np.clip(ec - np.where(ok, 0.1 / (0.1 + r), 1.0), 0.0, 1.0) # the stored value minus its own impact term - fees = np.clip(fees01 - tx / 0.1 + tx / sz, 0.0, 1.0) # the network fee is per trade, not per SOL + tx = config.RH_TX_FEE_ETH if rh else TX_FEE_LAMPORTS / config.LAMPORTS_PER_SOL + ref = 0.1 * market.k() if rh else 0.1 # the size the stored feature was priced at + fees01 = np.clip(ec - np.where(ok, ref / (ref + r), 1.0), 0.0, 1.0) # the stored value minus its own impact term + fees = np.clip(fees01 - tx / ref + tx / sz, 0.0, 1.0) # the network fee is per trade, not per unit out = np.where(ok, np.clip(fees + np.where(ok, sz / (sz + r), 1.0), 0.0, 1.0), 1.0) return float(out) if np.ndim(ec_0p1) == 0 else out diff --git a/fly_trader/market/features.py b/fly_trader/market/features.py index 0411b87da6cd4aa47b3700c1d0c75d0e5ea88300..6c7196ce0061bd2bde74d07b1cc7c810529fe834 100644 --- a/fly_trader/market/features.py +++ b/fly_trader/market/features.py @@ -54,6 +54,8 @@ class TokenMeta: token2022: bool = False stats: dict | None = None # latest token_stats row (dict) or None supply: float = PUMP_SUPPLY # token supply: market cap = price × supply sets the pool fee tier + pool_fee: float | None = None # the fee a Pons pool charged this minute (rh_minutes.fee_rate); Solana: None (tier by market cap) + ec_size: float = 0.1 # exit_cost_0p1's size in the market's unit: 0.1 SOL, or 0.1 SOL's worth of ETH on RH (0.1·K) @dataclass @@ -250,7 +252,7 @@ class TokenState: f[FIDX["vol_divergence"]] = math.log1p((self.cum_vol[-1] - self.cum_vol[idx["5m"]]) / 5.0) - \ math.log1p((self.cum_vol[-1] - self.cum_vol[idx["1h"]]) / 60.0) mask |= (1 << FIDX["mom_decel"]) | (1 << FIDX["vol_divergence"]) - f[FIDX["exit_cost_0p1"]] = exit_cost_fraction(EXIT_COST_SIZE_SOL, liq, p_now * meta.supply, meta.program_label) # real fees at this market cap + f[FIDX["exit_cost_0p1"]] = exit_cost_fraction(meta.ec_size, liq, p_now * meta.supply, meta.program_label, pool_fee=meta.pool_fee) # real fees at this market cap mask |= 1 << FIDX["exit_cost_0p1"] # Jupiter token stats st = meta.stats diff --git a/fly_trader/markets.py b/fly_trader/markets.py new file mode 100644 index 0000000000000000000000000000000000000000..a490ba7d5ea6c69bbd475d6ccec01615980f1f1f --- /dev/null +++ b/fly_trader/markets.py @@ -0,0 +1,104 @@ +"""The two memecoin markets the one selector and fly trade: Solana (pump.fun graduates, SOL) and Robinhood Chain (Pons +graduates, native ETH). + +A ``MarketSpec`` is everything that differs between them in the trading and accounting path: the unit its books keep +their money in, the book names, the rails circuit, the ui_settings keys, and the money settings (capital, the label size +every edge was measured at, the minimum position, the gas reserve, the kill-switch drawdown). Money settings are read from +``config`` at call time, so a test that monkeypatches config sees its value and Solana's are exactly today's settings. + +``k`` converts the few fixed-unit thresholds (eligibility, unit-bearing strategy triggers): SOL 1, RH = ETH per SOL at +build time (``config.RH_ETH_PER_SOL``). Shared functions take ``market=SOL`` as their default, so no Solana call site +changes. +""" +from __future__ import annotations + +from dataclasses import dataclass + +from . import config + +SOLANA_CIRCUIT, KALSHI_CIRCUIT, RH_CIRCUIT = 1, 2, 3 + + +@dataclass(frozen=True) +class MarketSpec: + chain: str # 'sol' | 'rh' + name: str # human name + unit: str # 'SOL' | 'ETH' + circuit_id: int + selector_book: str + fly_book: str + live_book: str + handover_key: str # ui_settings key of the fly's live seat on this chain + fly_status_key: str + selector_status_key: str + stream_status_key: str # ui_settings key whose flushed_through gates this chain's minutes + program_label: str # TokenMeta.program_label of this market's pools (the cost model dispatches on it) + _capital: str + _label_size: str + _min_position: str + _gas_reserve: str + _kill_drawdown: str + _k: str | None # config name of ETH-per-SOL, None for SOL (k = 1) + + @property + def books(self) -> tuple[str, str, str]: + return self.selector_book, self.fly_book, self.live_book + + @property + def paper_books(self) -> tuple[str, str]: + return self.selector_book, self.fly_book + + def k(self) -> float: + return 1.0 if self._k is None else float(getattr(config, self._k)) + + def capital(self) -> float: + return float(getattr(config, self._capital)) + + def label_size(self) -> float: + return float(getattr(config, self._label_size)) + + def min_position(self) -> float: + return float(getattr(config, self._min_position)) + + def gas_reserve(self) -> float: + return float(getattr(config, self._gas_reserve)) + + def kill_drawdown(self) -> float: + return float(getattr(config, self._kill_drawdown)) + + def min_resq(self) -> float: + from .train.decisions import MIN_RESQ_SOL + return MIN_RESQ_SOL * self.k() + + def min_vol_15m(self) -> float: + from .train.decisions import MIN_VOL_15M_SOL + return MIN_VOL_15M_SOL * self.k() + + +SOL = MarketSpec(chain="sol", name="Solana", unit="SOL", circuit_id=SOLANA_CIRCUIT, selector_book="paper_selector", fly_book="paper_fly", + live_book="live", handover_key="handover", fly_status_key="fly_status", selector_status_key="selector_status", + stream_status_key="pumpstream_status", program_label="Pump.fun Amm", _capital="CAPITAL_SOL", _label_size="LABEL_SIZE_SOL", + _min_position="MIN_POSITION_SOL", _gas_reserve="GAS_RESERVE_SOL", _kill_drawdown="KILL_SWITCH_DRAWDOWN", _k=None) +RH = MarketSpec(chain="rh", name="Robinhood Chain", unit="ETH", circuit_id=RH_CIRCUIT, selector_book="paper_rh_selector", fly_book="paper_rh_fly", + live_book="live_rh", handover_key="handover_rh", fly_status_key="fly_status_rh", selector_status_key="selector_status_rh", + stream_status_key="rh_stream_status", program_label="Pons v4", _capital="RH_CAPITAL_ETH", _label_size="RH_LABEL_SIZE_ETH", + _min_position="RH_MIN_POSITION_ETH", _gas_reserve="RH_GAS_RESERVE_ETH", _kill_drawdown="RH_KILL_SWITCH_DRAWDOWN", _k="RH_ETH_PER_SOL") +MARKETS = {"sol": SOL, "rh": RH} + + +def for_chain(chain: str | None) -> MarketSpec: + """The market of a chain id ('sol' | 'rh'); None is Solana (rows and positions from before the second chain).""" + return MARKETS[chain or "sol"] + + +def for_book(book: str) -> MarketSpec: + """The market a book belongs to: an RH book by name, every other (Solana books, replay_* test books) Solana.""" + for m in MARKETS.values(): + if book in m.books: + return m + return SOL + + +def enabled() -> list[MarketSpec]: + """The markets this install trades: Solana always, RH when ``RH_ENABLED``.""" + return [SOL] + ([RH] if config.RH_ENABLED else []) diff --git a/fly_trader/ops/autoupdate.py b/fly_trader/ops/autoupdate.py index 7cb8618cadab345b84105ff20ca8f10ffcbef05d..9bdab63725b3d31d1fc99b2e32231fceae31c768 100644 --- a/fly_trader/ops/autoupdate.py +++ b/fly_trader/ops/autoupdate.py @@ -1,15 +1,25 @@ -"""Keep a fire-and-forget install current. Every ``UPDATE_INTERVAL_S`` (an hour) ask Hugging Face for the head commit -of the model repo this install came from; when it has moved, stop the console, download the new snapshot over ``/app`` -(code, models, seed -- the repo is the whole distribution), reinstall the package, rerun the bootstrap steps and start -the console again. The previous tree is kept beside ``/app``: if the new console is not answering within -``HEALTH_WAIT_S`` it is put back, and that revision is never tried again. - -``docker/entrypoint.sh`` runs this as pid 1 when AUTO_UPDATE=1; the console is its child and is restarted if it dies. -A restart takes about a minute; the paper books and the fly's learned state live in the database and the data volume, -and open live positions stay in the wallet, sold on schedule by the restarted engine (README: "Stop everything"). +"""Keep a fire-and-forget install current, with SIGNED releases only. Every ``UPDATE_INTERVAL_S`` (an hour) ask Hugging +Face for the head commit of the repo this install came from; when it has moved: + +1. fetch its ``release.json`` + ``release.json.sig`` and verify the signature against ``release/allowed_signers`` + of the INSTALLED tree (the release keys this install trusts; only a signed release can change them), with the + same code the vault server's updater uses (``deploy/fly_update.py``: namespace, principals, strict paths, exactly + the listed files, each file's size and sha256) +2. refuse a release whose ``seq`` is not larger than the installed one (no rollback to an old signed release) +3. stop the console, keep the current tree beside ``/app``, copy the verified files over it, reinstall exactly + ``uv.lock`` (``uv sync --frozen``), rerun the bootstrap steps and start the console again; if it is not answering + within ``HEALTH_WAIT_S`` the previous tree is put back and that revision is never tried again + +An unsigned or wrongly signed commit is logged once and never installed. With no release keys in +``release/allowed_signers`` the updater installs nothing. ``docker/entrypoint.sh`` runs this as pid 1 when +AUTO_UPDATE=1; the console is its child and is restarted if it dies. The vault server never runs it (AUTO_UPDATE=0 +there: its host updater, deploy/fly_update.py, applies releases with a 24 h hold on code). """ from __future__ import annotations +import hashlib +import importlib.util +import json import logging import os import shutil @@ -22,7 +32,7 @@ from pathlib import Path log = logging.getLogger("autoupdate") logging.basicConfig(level=logging.INFO, format="%(asctime)s autoupdate %(levelname)s %(message)s", stream=sys.stdout) -for _noisy in ("httpx", "huggingface_hub", "urllib3"): # the download's every request is not news +for _noisy in ("httpx", "huggingface_hub", "urllib3"): logging.getLogger(_noisy).setLevel(logging.WARNING) APP = Path(os.environ.get("APP_DIR", "/app")) @@ -31,9 +41,18 @@ REPO = os.environ.get("HF_REPO", "bryceweiner/fly-trader") INTERVAL_S = float(os.environ.get("UPDATE_INTERVAL_S", "3600")) PORT = os.environ.get("PORT", "8501") HEALTH_WAIT_S = float(os.environ.get("UPDATE_HEALTH_WAIT_S", "300")) +TORCH_INDEX = os.environ.get("TORCH_INDEX", "cpu") REV_FILE = APP / "data" / "hf_revision" # the revision this install runs; on the data volume, survives recreates BAD_FILE = APP / "data" / "hf_bad_revisions" +RELEASE_FILE = APP / "data" / "hf_release.json" # {"seq", "manifest_sha256"} of the installed signed release +STAGE = APP / "data" / "updates" # verified downloads, before they replace the tree KEEP_OUT = ("data", "logs", ".venv", ".cache", ".env") # never part of a backup or a restore: the install's own state +PRINCIPALS = ["bryce", "bryce-recovery"] +NAMESPACE = "fly-trader-release" + + +class Refused(RuntimeError): + pass def head_sha(repo: str = REPO) -> str | None: @@ -70,15 +89,66 @@ def mark_bad(sha: str) -> None: f.write(sha + "\n") +def installed_release() -> dict: + try: + return json.loads(RELEASE_FILE.read_text()) + except (FileNotFoundError, ValueError): + return {"seq": 0} + + def wants_update(head: str | None, current: str | None, skip: set[str]) -> bool: """A new head that is not the installed one and has not already failed.""" return bool(head) and head != current and head not in skip +def signers() -> Path | None: + """The installed tree's release keys, or None when it holds none (then nothing is ever installed).""" + p = APP / "release" / "allowed_signers" + try: + lines = [l for l in p.read_text().splitlines() if l.strip() and not l.lstrip().startswith("#")] + except FileNotFoundError: + return None + return p if lines else None + + +_FU = None # tests put a stand-in here + + +def _verifier(): + """deploy/fly_update.py of the INSTALLED tree: the vault server's own release checks.""" + if _FU is not None: + return _FU + spec = importlib.util.spec_from_file_location("fly_update_installed", APP / "deploy" / "fly_update.py") + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +def fetch_verified(sha: str) -> tuple[dict, Path, bytes, str]: + """Download and verify the release at ``sha``; returns (manifest, directory, manifest bytes, signer).""" + keys = signers() + if keys is None: + raise Refused("no release keys in release/allowed_signers: this install accepts no update") + fu = _verifier() + c = {"repo": REPO, "allowed_signers": str(keys), "principals": PRINCIPALS, "namespace": NAMESPACE, "root": str(STAGE)} + try: + m, d, raw, who = fu.fetch(c, sha) # signature, manifest, paths, sizes, sha256 + except SystemExit as e: # fly_update reports refusals as SystemExit + raise Refused(str(e)) from None + have = int(installed_release().get("seq") or 0) + if int(m["seq"]) <= have: + raise Refused(f"release {m['seq']} is not newer than the installed {have}") + return m, d, raw, who + + def sh(*cmd: str, cwd: Path = APP) -> None: subprocess.run(list(cmd), cwd=str(cwd), check=True) +def reinstall() -> None: + sh("uv", "sync", "--frozen", "--no-dev", "--extra", TORCH_INDEX, "-q") + + def bootstrap() -> None: sh("sh", str(APP / "docker" / "bootstrap.sh")) @@ -129,45 +199,60 @@ def restore() -> None: (shutil.copytree if p.is_dir() else shutil.copy2)(p, APP / p.name) -def download(sha: str) -> None: - from huggingface_hub import snapshot_download - snapshot_download(REPO, revision=sha, local_dir=str(APP)) +def install_tree(d: Path) -> None: + """Replace the code, models and seed with exactly the verified release (the install's state is kept).""" + for p in _code_files(): + shutil.rmtree(p) if p.is_dir() else p.unlink() + for p in d.iterdir(): + if p.name in KEEP_OUT: + continue + (shutil.copytree if p.is_dir() else shutil.copy2)(p, APP / p.name) -def update(sha: str, proc: subprocess.Popen | None) -> subprocess.Popen: - """Stop, replace, reinstall, bootstrap, start; on any failure put the previous tree back and start that.""" - log.info("Hugging Face %s moved to %s: updating", REPO, sha[:12]) +def update(sha: str, proc: subprocess.Popen | None) -> subprocess.Popen | None: + """Verify, stop, replace, reinstall, bootstrap, start; on a failure after the swap put the previous tree back.""" + try: + m, d, raw, who = fetch_verified(sha) + except Refused as e: + log.error("Hugging Face %s @ %s refused: %s", REPO, sha[:12], e) + mark_bad(sha) + return proc + except Exception: + log.exception("could not fetch %s; will retry", sha[:12]) + return proc + log.info("release %s (Hugging Face %s), signed by %s: updating", m["seq"], sha[:12], who) stop_console(proc); backup() try: - download(sha); sh("uv", "pip", "install", "-e", ".", "-q"); bootstrap() + install_tree(d); reinstall(); bootstrap() p = start_console() if healthy(): - record(sha); log.info("updated to %s", sha[:12]); return p + record(sha) + RELEASE_FILE.write_text(json.dumps({"seq": int(m["seq"]), "manifest_sha256": hashlib.sha256(raw).hexdigest()}) + "\n") + shutil.rmtree(d, ignore_errors=True) + log.info("updated to release %s", m["seq"]); return p log.error("the new console did not answer within %.0fs", HEALTH_WAIT_S); stop_console(p) except Exception: log.exception("update to %s failed", sha[:12]) mark_bad(sha); log.error("restoring the previous install; %s will not be tried again", sha[:12]) - restore(); sh("uv", "pip", "install", "-e", ".", "-q"); bootstrap() + restore(); reinstall(); bootstrap() return start_console() def main() -> None: bootstrap() - if installed() is None: - head = head_sha() - if head: - record(head); log.info("this install is taken to be Hugging Face %s @ %s", REPO, head[:12]) + if signers() is None: + log.warning("release/allowed_signers holds no release key: automatic updates are off until it does") proc = start_console() stopping = {"now": False} def _term(signum, frame): stopping["now"] = True signal.signal(signal.SIGTERM, _term); signal.signal(signal.SIGINT, _term) - last = time.time() + last = 0.0 # a new install checks at once: it installs the newest signed release while not stopping["now"]: time.sleep(5) - if proc.poll() is not None: - log.error("console exited with %s; restarting it", proc.returncode); proc = start_console(); continue + if proc is None or proc.poll() is not None: + log.error("console exited; restarting it"); proc = start_console(); continue if time.time() - last < INTERVAL_S: continue last = time.time(); head = head_sha() diff --git a/fly_trader/ops/procs.py b/fly_trader/ops/procs.py index c71c66e1019de891b65aca99382e06520e84ca50..9d0039fdba42a4b93a57d787f21b2df2ce739d54 100644 --- a/fly_trader/ops/procs.py +++ b/fly_trader/ops/procs.py @@ -23,6 +23,11 @@ WORKERS: dict[str, list[str]] = { "train": ["train-selector"], "replay": ["replay-pull"], "pumpstream": ["pumpstream"], + "kalshi_stream": ["kalshi-stream"], + "rh_stream": ["rh-backfill"], + "kalshi_runner": ["kalshi-run"], + "kalshi_history": ["kalshi-history"], + "kalshi_train": ["kalshi-train"], } PYTHON = str(config.REPO_ROOT / ".venv" / "bin" / "python") diff --git a/fly_trader/ops/release.py b/fly_trader/ops/release.py index c7b8d3bb6d793dcf81f9b714463c87ec982ad964..1960f9bb90be75bc8d3197bc62915b8d90d07a84 100644 --- a/fly_trader/ops/release.py +++ b/fly_trader/ops/release.py @@ -43,6 +43,16 @@ def _set(conn, key: str, value) -> None: (key, json.dumps(value, default=str))) +def remap_teacher(note: dict, new_ids: dict[int, int]) -> dict | None: + """The fly names its teacher by the snapshot id it had where it was trained (``teacher_snapshot``); an install gives + the shipped selector a fresh id. The note naming the teacher's id here, or None when nothing changes. Without it a + fresh install's fly found no teacher, and its gate (agent/fly_session) had nothing to apply (2026-10-10 boot test).""" + t = note.get("teacher_snapshot") + if t is None or int(t) not in new_ids or new_ids[int(t)] == int(t): + return None + return {**note, "teacher_snapshot": new_ids[int(t)]} + + def apply_release(release_dir: Path, bootstrap: bool = False) -> dict: """Install the release at ``release_dir``. ``bootstrap``: first boot (also writes autostart if absent).""" import sys @@ -55,6 +65,14 @@ def apply_release(release_dir: Path, bootstrap: bool = False) -> dict: seed.setdefault("seq", man.get("seq")); seed.setdefault("git_commit", man.get("git_commit")) models = rel / "models" brain = config.BRAIN_DIR + # 0. nothing in models/ may run code when loaded: a model release installs without a human step + from ..train import model_io + for f in sorted(models.rglob("*")): + if f.is_file(): + try: + model_io.check_file(f) + except model_io.UnsafeModel as e: + raise ReleaseError(str(e)) from e # 1. files (copy only what is missing or different; the hashes were verified by the updater, re-checked here) placed: dict[str, Path] = {} for snap in seed["snapshots"]: @@ -84,7 +102,7 @@ def apply_release(release_dir: Path, bootstrap: bool = False) -> dict: if not ok: raise ReleaseError(f"the release's fly is not deployable on this code: {why}") v = seed["verdict"] - if not v.get("passed") or v.get("data") != fly_selector.FLY_VERSION: + if not v.get("passed") or v.get("data") != fly_selector.verdict_version(): raise ReleaseError("the release's replay verdict did not pass on this code's definitions") ids: dict[str, int] = {} with transaction() as conn: @@ -96,12 +114,28 @@ def apply_release(release_dir: Path, bootstrap: bool = False) -> dict: r = conn.execute("INSERT INTO brain_snapshots (path, sha256, kind, note, promoted_by) VALUES (%s,%s,%s,%s,'release') RETURNING id", (str(placed[snap["sha256"]]), snap["sha256"], snap["kind"], note)).fetchone() ids[snap["sha256"]] = int(r["id"]) + new_ids = {int(s["source_id"]): ids[s["sha256"]] for s in seed["snapshots"] if s.get("source_id") is not None} + for snap in seed["snapshots"]: + if snap["kind"] != "fly_selector": + continue + r = conn.execute("SELECT note FROM brain_snapshots WHERE id = %s", (ids[snap["sha256"]],)).fetchone() + note = r["note"] if isinstance(r["note"], dict) else json.loads(r["note"] or "{}") + fixed = remap_teacher(note, new_ids) + if fixed is not None: + conn.execute("UPDATE brain_snapshots SET note = %s WHERE id = %s", (json.dumps(fixed, default=str), ids[snap["sha256"]])) _set(conn, "fly_replay", v) pin = seed.get("pinned_selector_sha256") if pin: _set(conn, "pinned_selector_snapshot", {"id": ids[pin], "why": f"release {seed.get('seq')}", "pinned_at": time.strftime("%Y-%m-%d")}) if bootstrap and seed.get("autostart"): conn.execute("INSERT INTO ui_settings (key, value) VALUES ('autostart', %s) ON CONFLICT (key) DO NOTHING", (json.dumps(seed["autostart"]),)) + for a in seed.get("rh_assets") or []: # Robinhood Chain reference rows (tools/make_seed.py); an existing row is kept + conn.execute("INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, " + "ref_fee_bps, trading_hours, note, ref_path) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT (asset) DO NOTHING", + (str(a["asset"]).lower(), a.get("symbol"), a["class"], int(a["decimals"]), bool(a.get("approved", True)), + a.get("graduation_threshold_raw"), a.get("ref_pool"), a.get("ref_kind"), a.get("ref_quote"), a.get("ref_fee_bps"), + None if a.get("trading_hours") is None else json.dumps(a["trading_hours"]), a.get("note"), + None if a.get("ref_path") is None else json.dumps(a["ref_path"]))) _set(conn, "release", {"seq": seed.get("seq"), "git_commit": seed.get("git_commit"), "applied_at": int(time.time()), "fly": ids.get(next((s["sha256"] for s in seed["snapshots"] if s["kind"] == "fly_selector"), ""))}) _set(conn, "release_applied_at", int(time.time())) @@ -136,4 +170,10 @@ def health() -> tuple[bool, str]: return False, f"database: {type(e).__name__}" if not w: return False, "no worker threads running" + if config.SIGNER_SOCKET: # the vault server: a brain that cannot reach its signer cannot trade or pay + try: + from ..signer.client import SocketSigner + SocketSigner(config.SIGNER_SOCKET, timeout=5).call("pubkeys") + except Exception as e: + return False, f"signer: {type(e).__name__}" return True, f"{w} worker threads; last event {r['t']}" diff --git a/fly_trader/ops/reset.py b/fly_trader/ops/reset.py index 0b082d3966a4a53759e9acce5303d00533069c78..2d0256e320431b1c7025bff4f777922e477ecab0 100644 --- a/fly_trader/ops/reset.py +++ b/fly_trader/ops/reset.py @@ -28,14 +28,17 @@ from ..db.connection import transaction log = logging.getLogger(__name__) -PRESERVED_BOOKS = ("live", "paper_selector", "paper_fly") -RACE_RUN_KINDS = ("selector", "fly") -MODEL_KINDS = ("selector", "fly_selector", "fly_plastic") +PRESERVED_BOOKS = ("live", "paper_selector", "paper_fly", "live_rh", "paper_rh_selector", "paper_rh_fly", "paper_kalshi_taker", "paper_kalshi_maker", "live_kalshi_taker", "live_kalshi_maker") +RACE_RUN_KINDS = ("selector", "fly", "kalshi_fly", "selector_rh", "fly_rh") +MODEL_KINDS = ("selector", "fly_selector", "fly_plastic", "kalshi_selector", "kalshi_fly_selector", "kalshi_fly_plastic") FLY_BOOK, FLY_RUN_KIND = "paper_fly", "fly" +FLY_BOOKS, FLY_RUN_KINDS = ("paper_fly", "paper_rh_fly"), ("fly", "fly_rh") # one brain: both chains' fly books reset together FLY_TABLES = ["fly_scored", "fly_calibrations", "fly_updates", "fly_rollbacks"] BOOK_TABLES = ["positions", "orders", "fills"] TRAINING_EVENTS = {"selector": "(source = 'selector' AND (message LIKE 'walk-forward%' OR message LIKE 'selector saved%'))", # per regimen: a fly - "fly_selector": "source = 'fly_selector'"} # run keeps the selector's + "fly_selector": "source = 'fly_selector'", # run keeps the selector's + "kalshi_selector": "(source = 'kalshi_selector' AND (message LIKE 'walk-forward%' OR message LIKE 'selector saved%'))", + "kalshi_fly_selector": "source = 'kalshi_fly_selector'"} def _sql_list(xs) -> str: @@ -48,9 +51,9 @@ _KEPT_DECISIONS = (f"SELECT decision_id FROM orders WHERE book IN {_BOOKS} AND d f"UNION SELECT entry_decision_id FROM positions WHERE book IN {_BOOKS} AND entry_decision_id IS NOT NULL " f"UNION SELECT exit_decision_id FROM positions WHERE book IN {_BOOKS} AND exit_decision_id IS NOT NULL " f"UNION SELECT id FROM decisions WHERE run_id IN ({_RACE_RUNS})") -_LIVE_DECISIONS = ("SELECT decision_id FROM orders WHERE book = 'live' AND decision_id IS NOT NULL " - "UNION SELECT entry_decision_id FROM positions WHERE book = 'live' AND entry_decision_id IS NOT NULL " - "UNION SELECT exit_decision_id FROM positions WHERE book = 'live' AND exit_decision_id IS NOT NULL") +_LIVE_DECISIONS = ("SELECT decision_id FROM orders WHERE book IN ('live', 'live_rh') AND decision_id IS NOT NULL " + "UNION SELECT entry_decision_id FROM positions WHERE book IN ('live', 'live_rh') AND entry_decision_id IS NOT NULL " + "UNION SELECT exit_decision_id FROM positions WHERE book IN ('live', 'live_rh') AND exit_decision_id IS NOT NULL") def _archive(conn, table: str, where: str, out_dir: Path) -> int: @@ -88,7 +91,7 @@ def reset_training_state(reason: str = "runner start", archive: bool = True) -> counts = _wipe(conn, plan, out_dir if archive else None) conn.execute(f"UPDATE brain_state SET live_snapshot_id = CASE WHEN (SELECT kind FROM brain_snapshots WHERE id = live_snapshot_id) IN {_KINDS} THEN live_snapshot_id END, " f"pending_snapshot_id = CASE WHEN (SELECT kind FROM brain_snapshots WHERE id = pending_snapshot_id) IN {_KINDS} THEN pending_snapshot_id END, updated_at = now() WHERE singleton") - conn.execute("DELETE FROM ui_settings WHERE key = 'selector_status'") + conn.execute("DELETE FROM ui_settings WHERE key IN ('selector_status', 'selector_status_rh')") conn.execute("INSERT INTO circuit_events (kind, detail) VALUES ('training_reset', %s)", (f'{{"reason": "{reason}"}}',)) record_event("info", "reset", f"training state reset ({reason})", {"archived_to": str(out_dir) if archive else None, "rows": counts}) log.info("training state reset (%s): %s", reason, counts) @@ -99,6 +102,47 @@ def fly_state_dir() -> Path: return config.BRAIN_DIR / "plastic" +def kalshi_fly_state_dir() -> Path: + return config.BRAIN_DIR / "plastic_kalshi" + + +def kalshi_activity_dir() -> Path: + return config.BRAIN_DIR / "activity_kalshi" + + +KALSHI_BOOKS = ("paper_kalshi_taker", "paper_kalshi_maker") +KALSHI_FLY_TABLES = ["kalshi_fly_scored", "kalshi_fly_calibrations", "kalshi_fly_updates", "kalshi_fly_rollbacks"] +KALSHI_BOOK_TABLES = ["kalshi_positions", "kalshi_orders", "kalshi_fills"] + + +def reset_kalshi_fly(reason: str = "operator", archive: bool = True) -> dict: + """The visual (Kalshi) fly starts again from its bootstrap: its paper books, sessions, learned state, activity and + statistics are archived and cleared. The live Kalshi books and the bootstrap snapshots are kept.""" + out_dir = config.PG_ARCHIVE_DIR / f"reset_kalshi_fly_{datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')}" + runs = "SELECT run_id FROM runs WHERE kind = 'kalshi_fly'" + books = _sql_list(KALSHI_BOOKS) + plan = [("decisions", f"WHERE run_id IN ({runs}) AND (detail->>'book') IN {books}"), + ("beats", f"WHERE run_id IN ({runs})"), + ("wealth_marks", f"WHERE book IN {books}"), + *[(t, f"WHERE book IN {books}") for t in KALSHI_BOOK_TABLES], + ("brain_snapshots", "WHERE kind = 'kalshi_fly_plastic'"), + ("runs", "WHERE kind = 'kalshi_fly'"), + *[(t, "") for t in KALSHI_FLY_TABLES]] + with transaction() as conn: + counts = _wipe(conn, plan, out_dir if archive else None) + conn.execute("DELETE FROM book_state WHERE book = ANY(%s)", (list(KALSHI_BOOKS),)) + conn.execute("DELETE FROM ui_settings WHERE key = 'kalshi_fly_status'") + for d, name in ((kalshi_fly_state_dir(), "plastic_kalshi"), (kalshi_activity_dir(), "activity_kalshi")): + if d.exists(): + if archive: + out_dir.mkdir(parents=True, exist_ok=True); shutil.move(str(d), str(out_dir / name)) + else: + shutil.rmtree(d) + record_event("warning", "reset", f"kalshi fly reset to its bootstrap ({reason})", {"archived_to": str(out_dir) if archive else None, "rows": counts}) + log.info("kalshi fly reset (%s): %s", reason, counts) + return {"archived_to": str(out_dir) if archive else None, "rows": counts} + + def activity_dir() -> Path: """The fly's per-minute network activity for the console's brain view (``brain/activity.py``).""" return config.BRAIN_DIR / "activity" @@ -108,18 +152,19 @@ def reset_fly(reason: str = "operator", archive: bool = True) -> dict: """The plastic fly starts again from its bootstrap: its paper book, sessions, learned state and statistics are archived and cleared. The live book, the selector's book and the bootstrap snapshots are kept.""" out_dir = config.PG_ARCHIVE_DIR / f"reset_fly_{datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')}" - fly_runs = f"SELECT run_id FROM runs WHERE kind = '{FLY_RUN_KIND}'" + kinds, books = _sql_list(FLY_RUN_KINDS), _sql_list(FLY_BOOKS) + fly_runs = f"SELECT run_id FROM runs WHERE kind IN {kinds}" plan = [("decisions", f"WHERE run_id IN ({fly_runs}) AND id NOT IN ({_LIVE_DECISIONS})"), ("beats", f"WHERE run_id IN ({fly_runs})"), - ("wealth_marks", f"WHERE book = '{FLY_BOOK}'"), - *[(t, f"WHERE book = '{FLY_BOOK}'") for t in BOOK_TABLES], + ("wealth_marks", f"WHERE book IN {books}"), + *[(t, f"WHERE book IN {books}") for t in BOOK_TABLES], ("brain_snapshots", "WHERE kind = 'fly_plastic'"), - ("runs", f"WHERE kind = '{FLY_RUN_KIND}'"), + ("runs", f"WHERE kind IN {kinds}"), *[(t, "") for t in FLY_TABLES]] with transaction() as conn: counts = _wipe(conn, plan, out_dir if archive else None) - conn.execute("DELETE FROM book_state WHERE book = %s", (FLY_BOOK,)) - conn.execute("DELETE FROM ui_settings WHERE key = 'fly_status'") + conn.execute("DELETE FROM book_state WHERE book = ANY(%s)", (list(FLY_BOOKS),)) + conn.execute("DELETE FROM ui_settings WHERE key IN ('fly_status', 'fly_status_rh')") for d, name in ((fly_state_dir(), "plastic"), (activity_dir(), "activity")): # its learned state and its recorded activity if d.exists(): if archive: diff --git a/fly_trader/ops/supervisor.py b/fly_trader/ops/supervisor.py index 9fcf5ab1c6ff627d6a815f17b3f6b5eaac34b3fb..f5c30289918dc85a86e8560321b4642b818fb479 100644 --- a/fly_trader/ops/supervisor.py +++ b/fly_trader/ops/supervisor.py @@ -41,10 +41,25 @@ def _entry(name: str): if name == "vault": # the $FLY vault: flows, settlement, claims, public stats (fly_trader/vault) from ..vault import worker return worker.main + if name == "kalshi_history": # the Kalshi corpus: settled markets, candles, trades, feature rows (kalshi/history.py) + from ..kalshi import history + return history.main + if name == "rh_stream": # Robinhood Chain: the Pons / Uniswap v4 index, marks and minutes (rh/stream.py) + from ..rh import stream as rh_stream + return rh_stream.main + if name == "kalshi_stream": # the Kalshi live feed (kalshi/stream.py) + from ..kalshi import stream + return stream.main + if name == "kalshi_runner": # the Kalshi trading engine (kalshi/engine.py) + from ..kalshi import engine + return engine.main + if name == "kalshi_train": # the Kalshi training pipeline (kalshi/pipeline.py) + from ..kalshi import pipeline + return pipeline.main raise KeyError(name) -WORKERS = ("discover", "runner", "train", "replay", "pumpstream", "vault") +WORKERS = ("discover", "runner", "train", "replay", "pumpstream", "rh_stream", "vault", "kalshi_stream", "kalshi_runner", "kalshi_history", "kalshi_train") REPLACE_WAIT_S = 20.0 # how long a new console waits for the one it replaces to finish winding down (its workers flush on stop) @@ -186,6 +201,9 @@ class Supervisor: import json val = r["value"] if isinstance(r["value"], dict) else json.loads(r["value"] or "{}") started = [] + from .. import config + if "rh_stream" not in val and config.RH_ENABLED: + val["rh_stream"] = True # an install whose autostart predates Robinhood Chain starts its feed with the others for n in WORKERS: if val.get(n) and not self.alive(n): try: diff --git a/fly_trader/ops/wallets.py b/fly_trader/ops/wallets.py new file mode 100644 index 0000000000000000000000000000000000000000..830b4082fcc72463797c0d92c4144366e539c674 --- /dev/null +++ b/fly_trader/ops/wallets.py @@ -0,0 +1,142 @@ +"""The bot's two memecoin wallets as the console shows them (System → Safety & wallet): each chain's address, native +balance and open live positions, in the chain's coin and in USD (SOL from Jupiter, ETH from a KyberSwap quote into USDG: +vault/prices.py), their withdrawals, and the withdrawal itself (chain/withdraw.py, rh/withdraw.py).""" +from __future__ import annotations + +from datetime import datetime, timezone + +from .. import config +from ..chain import withdraw as sol_withdraw +from ..chain.cluster_guard import SigningRefused +from ..chain.keys import WalletKeyError +from ..db.connection import transaction +from ..logging_setup import scrub +from ..markets import MARKETS +from ..rh import guard as rh_guard, withdraw as rh_withdraw +from ..rh.wallet import FeeTooHigh, SendFailed +from ..vault import prices + +# a withdrawal that raises one of these sent nothing +REFUSALS = (sol_withdraw.WithdrawRefused, SigningRefused, WalletKeyError, rh_withdraw.WithdrawRefused, rh_guard.SigningRefused, FeeTooHigh, SendFailed) +SETTINGS = {"sol": "FUNDING_ADDRESSES", "rh": "RH_FUNDING_ADDRESSES"} + + +def explorer(chain: str, kind: str, value: str) -> str: + """The block explorer's page of an address (``kind`` 'address') or a transaction ('tx').""" + if chain == "sol": + return f"https://solscan.io/{'account' if kind == 'address' else 'tx'}/{value}" + base = "https://explorer.testnet.chain.robinhood.com" if config.RH_TESTNET else "https://robinhoodchain.blockscout.com" + return f"{base}/{kind}/{value}" + + +def address(chain: str) -> str | None: + """The bot wallet's address: Solana's from its key (else the last one created), RH's pinned RH_BOT_ADDRESS.""" + if chain == "rh": + return (config.RH_BOT_ADDRESS or "").lower() or None + from ..chain.keys import bot_pubkey + try: + return bot_pubkey() + except WalletKeyError: + with transaction() as conn: + r = conn.execute("SELECT pubkey FROM wallet_events WHERE kind = 'created' ORDER BY id DESC LIMIT 1").fetchone() + return r["pubkey"] if r else None + + +def _positions(book: str) -> dict: + """The live book's open positions, valued at its last wealth mark (net of exit cost), or at cost before any mark.""" + with transaction() as conn: + r = conn.execute("SELECT count(*) AS n, COALESCE(sum(cost_sol), 0) AS cost FROM positions WHERE book = %s AND status = 'open'", (book,)).fetchone() + m = conn.execute("SELECT ts, positions_value FROM wealth_marks WHERE book = %s ORDER BY ts DESC LIMIT 1", (book,)).fetchone() if r["n"] else None + if m is None or m["positions_value"] is None: + return {"n_open": int(r["n"]), "positions": float(r["cost"]), "marked_at": None} + return {"n_open": int(r["n"]), "positions": float(m["positions_value"]), "marked_at": m["ts"]} + + +def summary(chain: str) -> dict: + """One chain's bot wallet: {chain, unit, address, native, n_open, positions, marked_at, price_usd, price_at, native_usd, + positions_usd, total_usd, error}. Amounts in the chain's coin; a USD figure is None without a price. Never raises: a + failed read is ``error``.""" + spec = MARKETS[chain] + out = {"chain": chain, "unit": spec.unit, "address": None, "native": None, "error": None, **_positions(spec.live_book)} + try: + out["address"] = address(chain) + if out["address"] and chain == "sol": + from ..chain.rpc import HttpSolanaRpc + out["native"] = HttpSolanaRpc().get_balance(out["address"]) / config.LAMPORTS_PER_SOL + elif out["address"]: + from ..rh.rpc import RhRpc + out["native"] = RhRpc().get_balance(out["address"]) / rh_withdraw.WEI + except Exception as e: # noqa: BLE001 — the page shows it, the rest still renders + out["error"] = f"{type(e).__name__}: {scrub(str(e))[:200]}" + px, at = prices.sol_usd() if chain == "sol" else prices.eth_usd() + usd = (lambda v: v * px if px and v is not None else None) + out.update(price_usd=px or None, price_at=datetime.fromtimestamp(at, timezone.utc) if at else None, native_usd=usd(out["native"]), + positions_usd=usd(out["positions"]), total_usd=usd(None if out["native"] is None else out["native"] + out["positions"])) + return out + + +def destinations(chain: str) -> list[str]: + return sol_withdraw.destinations() if chain == "sol" else rh_withdraw.destinations() + + +def withdraw_blockers(chain: str) -> list[str]: + """Why the console cannot withdraw from ``chain``'s bot wallet (empty: it can). Reads no chain.""" + out = [] + if config.VAULT_ENABLED: + out.append("the vault fly's money leaves through its treasury and claims, not from the console") + if chain == "sol": + if config.SOLANA_CLUSTER != "mainnet-beta": + out.append(f"SOLANA_CLUSTER is {config.SOLANA_CLUSTER}, the bot wallet is on mainnet-beta") + from ..chain.keys import key_available + if not key_available(): + out.append("no bot key in this process (BOT_PRIVATE_KEY)") + else: + try: + rh_guard.pinned_key() + except rh_guard.SigningRefused as e: + out.append(f"no usable RH key: {e}") + if not destinations(chain): + out.append(f"list your own wallet in {SETTINGS[chain]} (.env), then restart the console") + return out + + +def limits(chain: str, to: str) -> dict: + """Fresh chain reads for a withdrawal to ``to``: the balance, the fee, what must stay and the most that can go out.""" + if chain == "sol": + from ..chain.keys import bot_pubkey + from ..chain.rpc import HttpSolanaRpc + return sol_withdraw.limits(HttpSolanaRpc(), bot_pubkey(), to) + return rh_withdraw.limits(rh_withdraw.wallet(to), to.lower()) + + +def plan(chain: str, lim: dict, amount: float | None) -> tuple[int, str | None]: + """(base units to send, why not or None) for ``amount`` of the coin, or everything withdrawable when None.""" + return (sol_withdraw if chain == "sol" else rh_withdraw).plan(lim, amount) + + +def withdraw(chain: str, amount: float | None, to: str) -> dict: + """Send ``amount`` of the chain's coin (None: everything withdrawable) to ``to``, one of your own wallets. Returns + {status, tx, amount, to}: status 'confirmed', 'failed' (only the fee moved), 'expired' (nothing moved) or 'pending' + (sent, no receipt yet). Raises one of ``REFUSALS`` when nothing was sent.""" + if chain == "sol": + r = sol_withdraw.withdraw(amount, to) + return {"status": {"failed_on_chain": "failed"}.get(r["status"], r["status"]), "tx": r["signature"], + "amount": r["lamports"] / config.LAMPORTS_PER_SOL, "to": to} + r = rh_withdraw.withdraw(amount, to) + return {"status": {"mined_ok": "confirmed", "reverted": "failed"}.get(r["status"], r["status"]), "tx": r["hash"], + "amount": r["wei"] / rh_withdraw.WEI, "to": r["to"]} + + +def history(chain: str, addr: str | None, limit: int = 20) -> list[dict]: + """The wallet's latest withdrawals: {when, amount, to, status, tx}.""" + if not addr: + return [] + with transaction() as conn: + if chain == "sol": + rows = conn.execute("SELECT ts AS \"when\", (detail->>'sol')::float AS amount, detail->>'to' AS \"to\", detail->>'status' AS status, " + "detail->>'signature' AS tx FROM wallet_events WHERE kind = 'withdrawal' AND pubkey = %s ORDER BY id DESC LIMIT %s", + (addr, limit)).fetchall() + else: + rows = conn.execute("SELECT created_at AS \"when\", value_wei / 1e18 AS amount, to_addr AS \"to\", status, hash AS tx FROM rh_txs " + "WHERE kind = 'transfer' AND from_addr = %s ORDER BY id DESC LIMIT %s", (addr.lower(), limit)).fetchall() + return [dict(r) for r in rows] diff --git a/fly_trader/rh/__init__.py b/fly_trader/rh/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ba79c88b2f4e7058046f82b3fa313d338c4ef9e5 --- /dev/null +++ b/fly_trader/rh/__init__.py @@ -0,0 +1,2 @@ +"""Robinhood Chain memecoins: Pons launches graduated into Uniswap v4 pools, indexed from our own RPC reads, scored by the +same selector and fly as Solana (fly_trader/markets.py), traded in ETH books (paper_rh_*, live_rh).""" diff --git a/fly_trader/rh/abi.py b/fly_trader/rh/abi.py new file mode 100644 index 0000000000000000000000000000000000000000..0b8f740de812328f441d38dc32af3d698962ca8f --- /dev/null +++ b/fly_trader/rh/abi.py @@ -0,0 +1,242 @@ +"""Solidity ABI encoding and decoding: every type the RH trading path meets (uintN/intN, address, bool, bytesN, bytes, +string, T[], T[k], nested tuples), for calldata we build (UniversalRouter, Permit2, ERC-20, V4Quoter) and calldata we +must check before signing (a KyberSwap build). No web3 dependency, like fly_trader/vault/evm.py whose keccak it reuses. + +Types are canonical strings: ``"uint256"``, ``"address[]"``, ``"(address,uint24,int24,address)"``, +``"(address,(bytes32,uint256)[])"``. Addresses decode to lowercase 0x-hex; bytes to ``bytes``; ints to ``int``. +""" +from __future__ import annotations + +from ..vault.evm import keccak256 + +WORD = 32 + + +# ---------------------------------------------------------------- type parsing +def _split_top(s: str) -> list[str]: + """Split a comma list at depth 0: ``"a,(b,c)[],d"`` → ``["a", "(b,c)[]", "d"]``.""" + out, depth, cur = [], 0, [] + for ch in s: + if ch == "(": + depth += 1 + elif ch == ")": + depth -= 1 + if ch == "," and depth == 0: + out.append("".join(cur)); cur = [] + else: + cur.append(ch) + if cur or out: + out.append("".join(cur)) + return [x.strip() for x in out if x.strip()] + + +def _array_suffix(t: str) -> tuple[str, int | None] | None: + """``"T[]"`` → (T, None), ``"T[3]"`` → (T, 3); not an array → None.""" + if not t.endswith("]"): + return None + i = t.rindex("[") + inner, n = t[:i], t[i + 1:-1] + return inner, (int(n) if n else None) + + +def _tuple_parts(t: str) -> list[str] | None: + if t.startswith("(") and t.endswith(")"): + return _split_top(t[1:-1]) + return None + + +def is_dynamic(t: str) -> bool: + arr = _array_suffix(t) + if arr is not None: + inner, n = arr + return n is None or is_dynamic(inner) + parts = _tuple_parts(t) + if parts is not None: + return any(is_dynamic(p) for p in parts) + return t in ("bytes", "string") + + +def head_size(t: str) -> int: + if is_dynamic(t): + return WORD + arr = _array_suffix(t) + if arr is not None: + return arr[1] * head_size(arr[0]) + parts = _tuple_parts(t) + if parts is not None: + return sum(head_size(p) for p in parts) + return WORD + + +# ---------------------------------------------------------------- encoding +def _enc_int(t: str, v: int) -> bytes: + v = int(v) + if t.startswith("uint"): + bits = int(t[4:] or 256) + if not 0 <= v < 2 ** bits: + raise ValueError(f"{v} out of range for {t}") + return v.to_bytes(WORD, "big") + bits = int(t[3:] or 256) + if not -(2 ** (bits - 1)) <= v < 2 ** (bits - 1): + raise ValueError(f"{v} out of range for {t}") + return (v % 2 ** 256).to_bytes(WORD, "big") + + +def _enc_address(v: str) -> bytes: + h = v[2:] if v.startswith("0x") else v + if len(h) != 40: + raise ValueError(f"not an address: {v!r}") + return bytes(12) + bytes.fromhex(h) + + +def _pad_right(b: bytes) -> bytes: + return b + bytes((-len(b)) % WORD) + + +def encode_single(t: str, v) -> bytes: + arr = _array_suffix(t) + if arr is not None: + inner, n = arr + vals = list(v) + if n is not None and len(vals) != n: + raise ValueError(f"{t} needs {n} values, got {len(vals)}") + body = encode([inner] * len(vals), vals) + return (len(vals).to_bytes(WORD, "big") if n is None else b"") + body + parts = _tuple_parts(t) + if parts is not None: + return encode(parts, list(v)) + if t == "address": + return _enc_address(v) + if t == "bool": + return (1 if v else 0).to_bytes(WORD, "big") + if t.startswith("uint") or t.startswith("int"): + return _enc_int(t, v) + if t == "bytes": + b = bytes(v) + return len(b).to_bytes(WORD, "big") + _pad_right(b) + if t == "string": + b = v.encode() + return len(b).to_bytes(WORD, "big") + _pad_right(b) + if t.startswith("bytes"): + n = int(t[5:]); b = bytes(v) + if len(b) != n: + raise ValueError(f"{t} needs {n} bytes, got {len(b)}") + return _pad_right(b) + raise ValueError(f"unsupported ABI type {t!r}") + + +def encode(types: list[str], values: list) -> bytes: + """``abi.encode(values...)`` for ``types`` (heads then tails).""" + if len(types) != len(values): + raise ValueError(f"{len(types)} types, {len(values)} values") + heads, tails = [], [] + offset = sum(head_size(t) for t in types) + for t, v in zip(types, values): + if is_dynamic(t): + enc = encode_single(t, v) + heads.append(offset.to_bytes(WORD, "big")); tails.append(enc); offset += len(enc) + else: + heads.append(encode_single(t, v)) + return b"".join(heads) + b"".join(tails) + + +def selector(signature: str) -> bytes: + return keccak256(signature.encode())[:4] + + +def signature_types(signature: str) -> list[str]: + """``"swap(address,(uint256,bytes))"`` → ``["address", "(uint256,bytes)"]``.""" + return _split_top(signature[signature.index("(") + 1:signature.rindex(")")]) + + +def encode_call(signature: str, *values) -> bytes: + return selector(signature) + encode(signature_types(signature), list(values)) + + +def encode_packed_path(parts: list[tuple[str, object]]) -> bytes: + """``abi.encodePacked`` for (type, value) pairs of static types (a Uniswap v3 path: address, uint24, address, …).""" + out = b"" + for t, v in parts: + if t == "address": + out += bytes.fromhex(v[2:]) + elif t.startswith("uint"): + out += int(v).to_bytes(int(t[4:]) // 8, "big") + else: + raise ValueError(f"unsupported packed type {t!r}") + return out + + +# ---------------------------------------------------------------- decoding +def _word(data: bytes, pos: int) -> bytes: + if pos + WORD > len(data): + raise ValueError("ABI data too short") + return data[pos:pos + WORD] + + +def decode_single(t: str, data: bytes, pos: int): + """Decode the value of type ``t`` whose head is at ``pos`` within ``data`` (the enclosing tuple's encoding).""" + if is_dynamic(t): + off = int.from_bytes(_word(data, pos), "big") + return _decode_at(t, data, off) + return _decode_at(t, data, pos) + + +def _decode_at(t: str, data: bytes, pos: int): + arr = _array_suffix(t) + if arr is not None: + inner, n = arr + if n is None: + n = int.from_bytes(_word(data, pos), "big"); pos += WORD + if n > 1_000_000: + raise ValueError("ABI array length absurd") + return decode([inner] * n, data[pos:]) + parts = _tuple_parts(t) + if parts is not None: + return tuple(decode(parts, data[pos:])) + w = _word(data, pos) + if t == "address": + if any(w[:12]): + raise ValueError("dirty address word") + return "0x" + w[12:].hex() + if t == "bool": + v = int.from_bytes(w, "big") + if v > 1: + raise ValueError("dirty bool word") + return bool(v) + if t.startswith("uint"): + v = int.from_bytes(w, "big"); bits = int(t[4:] or 256) + if v >= 2 ** bits: + raise ValueError(f"dirty {t} word") + return v + if t.startswith("int"): + v = int.from_bytes(w, "big", signed=True); bits = int(t[3:] or 256) + if not -(2 ** (bits - 1)) <= v < 2 ** (bits - 1): + raise ValueError(f"dirty {t} word") + return v + if t in ("bytes", "string"): + n = int.from_bytes(w, "big") + if pos + WORD + n > len(data): + raise ValueError("ABI bytes run past the data") + b = data[pos + WORD:pos + WORD + n] + return b.decode() if t == "string" else b + if t.startswith("bytes"): + return w[:int(t[5:])] + raise ValueError(f"unsupported ABI type {t!r}") + + +def decode(types: list[str], data: bytes) -> list: + out, pos = [], 0 + for t in types: + out.append(decode_single(t, data, pos)); pos += head_size(t) + return out + + +def decode_call(signature: str, calldata: bytes) -> list: + """Decode calldata of ``signature``; refuses another selector.""" + if calldata[:4] != selector(signature): + raise ValueError(f"calldata is not {signature.split('(')[0]}") + return decode(signature_types(signature), calldata[4:]) + + +def event_topic(signature: str) -> str: + return "0x" + keccak256(signature.encode()).hex() diff --git a/fly_trader/rh/accounting.py b/fly_trader/rh/accounting.py new file mode 100644 index 0000000000000000000000000000000000000000..02862f0e1b2b5edb985280b518dfe8a82e80d3ae --- /dev/null +++ b/fly_trader/rh/accounting.py @@ -0,0 +1,255 @@ +"""The live RH book's accounting (book ``live_rh``, amounts in ETH): what a finished swap intent books, the wallet's +wealth, and the reconciler. + +What an intent books (every figure measured, never assumed): +- ETH moved by a swap = the native balance delta around the intent (one transaction in flight per wallet, so the delta + is that intent's) plus the gas its transactions paid (``rh_txs.fee_wei``); token amounts = balanceOf deltas, checked + against the receipt's Transfer logs; +- a buy opens a position whose ``cost_sol`` (ETH) is the ETH spent + the gas of every transaction of the intent + (approvals and reverted attempts included) + the basis of a consumed base lot; ``gas_q`` keeps the gas part; +- a buy that produced no tokens is booked as an aborted position (qty 0, closed at once): realized = −gas − base loss; +- a sale's proceeds are the ETH received − the gas of the exit intent; realized = proceeds − cost; +- two-leg routes (rh/exec.py fallback) pass through ``rh_base_lots``: an entry leg's base is a lot consumed by the + meme buy; an exit leg's base is a lot valued at the mark when the position closes, and its liquidation books the + difference onto that same position; +- every leg is an ``rh_legs`` row in raw units (``asset`` 'eth' = native ETH). + +``wealth``: native ETH − reserved + open positions at the ETH close net of the Pons exit cost + open lots at their base +mark. ``reconcile`` compares the chain with the books: token balances vs open positions and lots; the native balance vs +the last mark plus every booked native flow since (legs, gas, recorded payouts/deposits). Tokens missing, or native ETH +leaving unexplained, pause entries on circuit 3 (unknown outbound also trips its kill switch). +""" +from __future__ import annotations + +import json +import logging +from datetime import datetime, timedelta, timezone + +from ..db.apilog import record_event +from ..execution import ledger +from ..market.exit_cost import PONS_LABEL, exit_cost_fraction +from ..markets import RH, RH_CIRCUIT +from . import abi + +log = logging.getLogger(__name__) +BOOK = RH.live_book +NATIVE = "eth" # rh_legs / rh_base_lots asset name of native ETH +WEI = 10 ** 18 +TRANSFER = abi.event_topic("Transfer(address,address,uint256)").lower() +DUST_ETH = 1e-6 # reconciliation tolerance on native ETH + + +def _addr_topic(a: str) -> str: + return "0x" + "0" * 24 + a.lower()[2:] + + +def transfer_delta(receipt: dict | None, token: str, owner: str) -> int: + """Net ``token`` units into ``owner`` from a receipt's Transfer logs.""" + if not receipt: + return 0 + t, me = token.lower(), _addr_topic(owner) + n = 0 + for lg in receipt.get("logs") or []: + tp = [x.lower() for x in lg.get("topics") or []] + if lg.get("address", "").lower() != t or len(tp) < 3 or tp[0] != TRANSFER: + continue + v = int(lg.get("data") or "0x0", 16) + n += v if tp[2] == me else 0 + n -= v if tp[1] == me else 0 + return n + + +def intent_gas_wei(conn, intent_id: int) -> int: + r = conn.execute("SELECT COALESCE(sum(fee_wei), 0) AS g FROM rh_txs WHERE intent_id = %s AND fee_wei IS NOT NULL", (intent_id,)).fetchone() + return int(r["g"]) + + +def add_leg(conn, *, intent_id: int, tx_id: int | None, position_id: int | None, leg_no: int, asset_in: str, asset_out: str, + amount_in: int, amount_out: int, eth_in: float | None, eth_out: float | None, verified_by: str = "balance_delta") -> None: + conn.execute("INSERT INTO rh_legs (intent_id, tx_id, position_id, leg_no, asset_in, asset_out, amount_in_raw, amount_out_raw, eth_in, eth_out, verified_by) " + "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", + (intent_id, tx_id, position_id, leg_no, asset_in, asset_out, int(amount_in), int(amount_out), eth_in, eth_out, verified_by)) + + +# ---- positions +def book_buy(conn, *, intent_id: int, token: str, pool: str | None, quote_asset: str, eth_spent_wei: int, tokens_raw: int, decision_id: int | None, + ts: datetime, hold_s: float | None, strategy: str | None, lot_basis_eth: float = 0.0, decimals: int = 18) -> tuple[int, float]: + """Book a finished buy intent. Returns (position id, realized) — realized is 0 for an opened position and −cost for an + aborted one (no tokens).""" + gas = intent_gas_wei(conn, intent_id) + spent = eth_spent_wei / WEI + lot_basis_eth + cost = spent + gas / WEI + if tokens_raw <= 0: # nothing bought: the gas and any base loss are the loss + pid = ledger.open_position(conn, book=BOOK, mint=token, pool=pool, qty_raw=0, cost_sol=cost, entry_price=0.0, decision_id=decision_id, + fees_sol=gas / WEI, ts=ts, hold_s=hold_s, strategy=strategy) + conn.execute("UPDATE positions SET quote_asset = %s, gas_q = %s WHERE id = %s", (quote_asset, gas / WEI, pid)) + realized = ledger.close_position(conn, position_id=pid, exit_price=0.0, proceeds_sol=0.0, fees_sol=0.0, decision_id=None, forced_kind="aborted", ts=ts) + return pid, float(realized or -cost) + price = spent / (tokens_raw / 10 ** decimals) + pid = ledger.open_position(conn, book=BOOK, mint=token, pool=pool, qty_raw=tokens_raw, cost_sol=cost, entry_price=price, decision_id=decision_id, + fees_sol=gas / WEI, ts=ts, hold_s=hold_s, strategy=strategy) + conn.execute("UPDATE positions SET quote_asset = %s, gas_q = %s WHERE id = %s", (quote_asset, gas / WEI, pid)) + conn.execute("UPDATE rh_intents SET position_id = %s WHERE id = %s", (pid, intent_id)) + conn.execute("UPDATE rh_legs SET position_id = %s WHERE intent_id = %s", (pid, intent_id)) + return pid, 0.0 + + +def book_sell(conn, *, intent_id: int, position: dict, eth_in_wei: int, tokens_sold_raw: int, decision_id: int | None, ts: datetime, + forced_kind: str | None = None, lot_value_eth: float = 0.0) -> float | None: + """Book a finished sell intent on ``position``: proceeds = ETH in (+ the mark value of an exit lot) − the intent's gas. + A partial fill shrinks the position pro rata. Returns this sale's realized P&L (None: position not open).""" + gas = intent_gas_wei(conn, intent_id) + proceeds = eth_in_wei / WEI + lot_value_eth - gas / WEI + qty = int(position["qty"]); dec = int(position.get("decimals") or 18) + conn.execute("UPDATE positions SET gas_q = gas_q + %s WHERE id = %s", (gas / WEI, int(position["id"]))) + conn.execute("UPDATE rh_intents SET position_id = %s WHERE id = %s", (int(position["id"]), intent_id)) + if tokens_sold_raw <= 0: # nothing sold: only the gas is lost, charged to the position + conn.execute("UPDATE positions SET cost_sol = cost_sol + %s, fees_sol = fees_sol + %s WHERE id = %s AND status = 'open'", + (gas / WEI, gas / WEI, int(position["id"]))) + return -gas / WEI + px = (eth_in_wei / WEI + lot_value_eth) / (tokens_sold_raw / 10 ** dec) + if tokens_sold_raw < qty: + frac = tokens_sold_raw / qty + return ledger.realize_partial(conn, position_id=int(position["id"]), qty_raw=tokens_sold_raw, cost_part=float(position["cost_sol"]) * frac, + proceeds_sol=proceeds, fees_sol=gas / WEI, price=px, ts=ts) + return ledger.close_position(conn, position_id=int(position["id"]), exit_price=px, proceeds_sol=proceeds, fees_sol=gas / WEI, + decision_id=decision_id, forced_kind=forced_kind, ts=ts) + + +def charge_gas(conn, intent_id: int, position_id: int | None) -> float: + """A failed intent's gas: onto its open position's cost when it has one (an exit that reverted), else the book's + overhead as an aborted zero-quantity position (a failed entry). Returns the ETH charged.""" + gas = intent_gas_wei(conn, intent_id) / WEI + if gas <= 0: + return 0.0 + if position_id is not None: + conn.execute("UPDATE positions SET cost_sol = cost_sol + %s, fees_sol = fees_sol + %s, gas_q = gas_q + %s WHERE id = %s", + (gas, gas, gas, position_id)) + return gas + it = conn.execute("SELECT token, decision_id FROM rh_intents WHERE id = %s", (intent_id,)).fetchone() + now = datetime.now(timezone.utc) + pid = ledger.open_position(conn, book=BOOK, mint=it["token"] or "gas", pool=None, qty_raw=0, cost_sol=gas, entry_price=0.0, decision_id=None, + fees_sol=gas, ts=now) + conn.execute("UPDATE positions SET gas_q = %s WHERE id = %s", (gas, pid)) + ledger.close_position(conn, position_id=pid, exit_price=0.0, proceeds_sol=0.0, fees_sol=0.0, decision_id=None, forced_kind="aborted", ts=now) + return gas + + +# ---- base lots (two-leg fallback, stray balances) +def open_lot(conn, *, asset: str, qty_raw: int, basis_eth: float, source: str, position_id: int | None = None, intent_id: int | None = None) -> int: + return int(conn.execute("INSERT INTO rh_base_lots (asset, qty_raw, basis_eth, position_id, intent_id, source) VALUES (%s,%s,%s,%s,%s,%s) RETURNING id", + (asset.lower(), int(qty_raw), float(basis_eth), position_id, intent_id, source)).fetchone()["id"]) + + +def consume_lot(conn, lot_id: int, position_id: int | None) -> float: + r = conn.execute("UPDATE rh_base_lots SET status = 'consumed', position_id = COALESCE(%s, position_id), closed_at = now() WHERE id = %s AND status = 'open' " + "RETURNING basis_eth", (position_id, lot_id)).fetchone() + return float(r["basis_eth"]) if r else 0.0 + + +def liquidate_lot(conn, lot_id: int, proceeds_eth: float, fee_eth: float) -> float: + """Close a lot sold for ``proceeds_eth`` (after ``fee_eth`` gas). The difference to its basis lands on the position it + came from (an exit lot) — so a two-leg sale books exactly what the two legs returned. Returns that difference.""" + r = conn.execute("UPDATE rh_base_lots SET status = 'liquidated', proceeds_eth = %s, fee_eth = %s, closed_at = now() WHERE id = %s AND status = 'open' " + "RETURNING basis_eth, position_id", (proceeds_eth, fee_eth, lot_id)).fetchone() + if r is None: + return 0.0 + diff = proceeds_eth - fee_eth - float(r["basis_eth"]) + if r["position_id"] is not None: + conn.execute("UPDATE positions SET realized_sol = COALESCE(realized_sol, 0) + %s, gas_q = gas_q + %s WHERE id = %s", (diff, fee_eth, r["position_id"])) + return diff + + +def open_lots(conn) -> list[dict]: + return [dict(r) for r in conn.execute("SELECT * FROM rh_base_lots WHERE status = 'open' ORDER BY id").fetchall()] + + +# ---- marks +def base_mark(conn, asset: str, stale_after_s: float = 600.0) -> tuple[float | None, bool]: + """(ETH per whole unit of ``asset``, stale). Native ETH is 1 and never stale.""" + if asset in (NATIVE, "0x0000000000000000000000000000000000000000"): + return 1.0, False + r = conn.execute("SELECT ts, price_eth, stale FROM rh_base_prices WHERE asset = %s AND price_eth IS NOT NULL ORDER BY ts DESC LIMIT 1", + (asset.lower(),)).fetchone() + if r is None: + return None, True + age = (datetime.now(timezone.utc) - r["ts"]).total_seconds() + return float(r["price_eth"]), bool(r["stale"]) or age > stale_after_s + + +def wealth(conn, native_wei: int, reserved_wei: int, prices: dict, resqs: dict, fees: dict) -> dict: + """The live RH book's wealth in ETH (native − reserved + positions net of exit cost + lots at their base mark).""" + gross_v = exit_cost = exposure = 0.0; n = 0 + for p in ledger.open_positions(conn, BOOK): + px = prices.get(p["mint"]) or float(p.get("last_mark_price") or p["entry_price"] or 0.0) + g = int(p["qty"]) / 10 ** int(p.get("decimals") or 18) * px + gross_v += g; exposure += float(p["cost_sol"]); n += 1 + exit_cost += g * exit_cost_fraction(g, resqs.get(p["mint"]), None, PONS_LABEL, fees.get(p["mint"])) + lots_v = 0.0 + for lot in open_lots(conn): + dec = _decimals(conn, lot["asset"]) + mark, _ = base_mark(conn, lot["asset"]) + lots_v += int(lot["qty_raw"]) / 10 ** dec * (mark if mark is not None else 0.0) + native = native_wei / WEI; reserved = reserved_wei / WEI + return {"native": native, "reserved": reserved, "positions": gross_v - exit_cost, "exit_cost": exit_cost, "lots": lots_v, "exposure": exposure, "n_open": n, + "wealth": native - reserved + gross_v - exit_cost + lots_v} + + +def _decimals(conn, asset: str) -> int: + r = conn.execute("SELECT decimals FROM rh_assets WHERE asset = %s", (asset.lower(),)).fetchone() + return int(r["decimals"]) if r else 18 + + +# ---- reconciliation +def booked_native_since(conn, since: datetime) -> int: + """Native wei the books say moved since ``since``: legs out of/into native ETH, all gas paid, recorded wallet flows.""" + legs = conn.execute("SELECT COALESCE(sum(CASE WHEN asset_out = %s THEN amount_out_raw ELSE 0 END), 0) - " + "COALESCE(sum(CASE WHEN asset_in = %s THEN amount_in_raw ELSE 0 END), 0) AS d FROM rh_legs WHERE ts > %s", + (NATIVE, NATIVE, since)).fetchone()["d"] + gas = conn.execute("SELECT COALESCE(sum(fee_wei), 0) AS g FROM rh_txs WHERE fee_wei IS NOT NULL AND fee_at > %s", (since,)).fetchone()["g"] + flows = conn.execute("SELECT COALESCE(sum(CASE WHEN direction = 'in' THEN wei ELSE -wei END), 0) AS f FROM rh_wallet_flows " + "WHERE ts > %s AND kind <> 'unknown_outbound' AND kind <> 'unexplained_in'", (since,)).fetchone()["f"] + return int(legs) - int(gas) + int(flows) + + +def reconcile(conn, wallet, now: datetime | None = None) -> dict: + """Chain vs books. Records a wallet mark; returns {ok, native_gap_eth, token_gaps}. Tokens short of the books or + native ETH gone unexplained pause circuit 3 (unexplained outbound also trips its kill switch); an unexplained native + increase is recorded as a deposit.""" + now = now or datetime.now(timezone.utc) + before = now - timedelta(microseconds=1) # flows found now belong before this mark, never after it + native = wallet.balance() + gaps = [] + held: dict[str, int] = {} + for p in ledger.open_positions(conn, BOOK): + held[p["mint"].lower()] = held.get(p["mint"].lower(), 0) + int(p["qty"]) + for lot in open_lots(conn): + if lot["asset"] != NATIVE: + held[lot["asset"]] = held.get(lot["asset"], 0) + int(lot["qty_raw"]) + for token, want in held.items(): + have = wallet.balance_of(token) + if have < want: + gaps.append({"token": token, "books": want, "chain": have}) + last = conn.execute("SELECT ts, native_wei FROM rh_wallet_marks ORDER BY ts DESC LIMIT 1").fetchone() + gap_eth = 0.0 + if last is None: # the first check: what the wallet holds was funded + conn.execute("INSERT INTO rh_wallet_flows (ts, direction, kind, wei, note) VALUES (%s, 'in', 'deposit', %s, 'reconciler: opening balance')", (before, native)) + elif last["native_wei"] is not None: + expected = int(last["native_wei"]) + booked_native_since(conn, last["ts"]) + gap_eth = (native - expected) / WEI + if gap_eth > DUST_ETH: + conn.execute("INSERT INTO rh_wallet_flows (ts, direction, kind, wei, note) VALUES (%s, 'in', 'deposit', %s, 'reconciler: unexplained increase')", + (before, native - expected)) + elif gap_eth < -DUST_ETH: + conn.execute("INSERT INTO rh_wallet_flows (ts, direction, kind, wei, note) VALUES (%s, 'out', 'unknown_outbound', %s, 'reconciler')", (before, expected - native)) + ok = not gaps and gap_eth >= -DUST_ETH + conn.execute("INSERT INTO rh_wallet_marks (ts, native_wei, consistent) VALUES (%s, %s, %s) ON CONFLICT (ts) DO NOTHING", (now, native, ok)) + if not ok: + detail = {"token_gaps": gaps, "native_gap_eth": gap_eth} + conn.execute("UPDATE circuit_state SET entries_paused = true, updated_at = now() WHERE id = %s", (RH_CIRCUIT,)) + if gap_eth < -DUST_ETH: + conn.execute("UPDATE circuit_state SET kill_switch = true, kill_reason = %s, updated_at = now() WHERE id = %s", + (f"{-gap_eth:.6f} ETH left the RH wallet unexplained", RH_CIRCUIT)) + conn.execute("INSERT INTO circuit_events (kind, detail) VALUES ('rh_reconcile', %s)", (json.dumps({**detail, "circuit": RH_CIRCUIT}, default=str),)) + record_event("error", "rh_live", "RH wallet does not match the books: entries paused", detail) + return {"ok": ok, "native_gap_eth": gap_eth, "token_gaps": gaps, "native_eth": native / WEI} diff --git a/fly_trader/rh/blocktime.py b/fly_trader/rh/blocktime.py new file mode 100644 index 0000000000000000000000000000000000000000..a9db464779eafa3c780b6d3e0d19a6da943ecc0f --- /dev/null +++ b/fly_trader/rh/blocktime.py @@ -0,0 +1,117 @@ +"""Block → time on Robinhood Chain without one lookup per block. + +The RPC's ``blockTimestamp`` on logs reads 0x0 (2026-09-28) and the chain makes 10–20 blocks a second, so timing every +swap's block cost thousands of lookups per chain-minute. Minute bars only need each event's exact minute: block times never +decrease, so sample one block in every ``SAMPLE`` (batched), and where two neighbouring samples fall in different minutes +find each minute's first block by binary search (all open searches advance together, one batch per round). An event's +minute is then exact; its seconds are interpolated between the samples around it. +""" +from __future__ import annotations + +import bisect + +from . import scan + +SAMPLE = 256 +BATCH = 100 +HDR_MAX = 400_000 # header timestamps kept per endpoint before the cache resets +# Per header endpoint: {"starts": {minute_s: first block with ts >= minute_s}, "hdr": {block: ts}}. The index and every +# price scan time the same block ranges; a minute's first block found once is never searched for again. +_CACHE: dict[str, dict] = {} + + +def _cache_for(rpc) -> dict | None: + src = getattr(rpc, "blocks", rpc) + url = getattr(src, "url", None) + if not url: + return None + c = _CACHE.setdefault(url, {"starts": {}, "hdr": {}}) + if len(c["hdr"]) > HDR_MAX: + c["hdr"].clear() + return c + + +def _fetch(rpc, blocks: list[int], cache: dict[int, int]) -> None: + need = [b for b in sorted(set(blocks)) if b not in cache] + src = getattr(rpc, "blocks", rpc) # the header endpoint (RH_RPC_URL_BLOCKS) when there is one + for i in range(0, len(need), BATCH): + chunk = need[i:i + BATCH] + res = src.batch([("eth_getBlockByNumber", [hex(b), False]) for b in chunk]) + if src is rpc: + scan.pause() + for b, r in zip(chunk, res): + if not isinstance(r, dict) or not r.get("timestamp"): + raise RuntimeError(f"no block {b} from the RPC") + cache[b] = int(r["timestamp"], 16) + + +def times(rpc, lo: int, hi: int, wanted: list[int], sample: int = SAMPLE) -> dict[int, float]: + """Seconds since epoch for each block in ``wanted`` (all within lo..hi): exact minute, interpolated seconds.""" + if not wanted: + return {} + shared = _cache_for(rpc) + cache: dict[int, int] = shared["hdr"] if shared else {} + if shared is not None: + _fetch(rpc, [lo, hi], cache) + mlo, mhi = cache[lo] // 60, cache[hi] // 60 + st = shared["starts"] + if all(m * 60 in st for m in range(mlo + 1, mhi + 1)): + return _from_starts(cache, lo, hi, [(st[m * 60], m * 60) for m in range(mlo + 1, mhi + 1)], wanted) + grid = list(range(lo, hi + 1, sample)) + if grid[-1] != hi: + grid.append(hi) + _fetch(rpc, grid, cache) + # the first block of every minute that starts inside the range + searches = [] # [lo_block, hi_block, minute_start_s] + for a, b in zip(grid, grid[1:]): + ma, mb = cache[a] // 60, cache[b] // 60 + for m in range(ma + 1, mb + 1): + searches.append([a, b, m * 60]) # ts(a) < m*60 <= ts(b) + while any(s[1] - s[0] > 1 for s in searches): + mids = [(s[0] + s[1]) // 2 for s in searches if s[1] - s[0] > 1] + _fetch(rpc, mids, cache) + for s in searches: + if s[1] - s[0] > 1: + mid = (s[0] + s[1]) // 2 + if cache[mid] >= s[2]: + s[1] = mid + else: + s[0] = mid + starts = sorted((s[1], s[2]) for s in searches) # (first block of the minute, minute start) + if shared is not None: + for blk, m in starts: + shared["starts"][m] = blk + start_blocks = [b for b, _ in starts] + known = sorted(cache.items()); kb = [b for b, _ in known] + out = {} + for blk in wanted: + if blk in cache: + out[blk] = float(cache[blk]); continue + i = bisect.bisect_right(start_blocks, blk) - 1 + # exact minute: the last minute whose first block is at or before blk (else the minute of the range start) + minute = starts[i][1] if i >= 0 else (cache[lo] // 60) * 60 + j = bisect.bisect_right(kb, blk) + (b0, t0), (b1, t1) = known[max(0, j - 1)], known[min(len(known) - 1, j)] + t = t0 + (t1 - t0) * ((blk - b0) / (b1 - b0)) if b1 != b0 else float(t0) + out[blk] = min(max(t, float(minute)), minute + 59.999) + return out + + +def _from_starts(hdr: dict[int, int], lo: int, hi: int, starts: list[tuple[int, int]], wanted: list[int]) -> dict[int, float]: + """``times`` from known minute starts: the exact minute by bisection, seconds interpolated between the anchors + (range ends, minute starts, any header already seen).""" + starts = sorted(starts) + start_blocks = [b for b, _ in starts] + anchors = sorted({(lo, hdr[lo]), (hi, hdr[hi]), *((b, m) for b, m in starts), *((b, t) for b, t in hdr.items() if lo <= b <= hi)}) + ab = [b for b, _ in anchors] + out = {} + for blk in wanted: + if blk in hdr: + out[blk] = float(hdr[blk]); continue + i = bisect.bisect_right(start_blocks, blk) - 1 + minute = starts[i][1] if i >= 0 else (hdr[lo] // 60) * 60 + j = bisect.bisect_right(ab, blk) + (b0, t0), (b1, t1) = anchors[max(0, j - 1)], anchors[min(len(anchors) - 1, j)] + t = t0 + (t1 - t0) * ((blk - b0) / (b1 - b0)) if b1 != b0 else float(t0) + out[blk] = min(max(t, float(minute)), minute + 59.999) + return out diff --git a/fly_trader/rh/corpus.py b/fly_trader/rh/corpus.py new file mode 100644 index 0000000000000000000000000000000000000000..cb4daceb79ed75242526ac99bcad5f9aee037969 --- /dev/null +++ b/fly_trader/rh/corpus.py @@ -0,0 +1,99 @@ +"""The Robinhood Chain training corpus, day by day, beside the Solana one (data/corpus/rh/...): + +1. ``rh/wallet_skill.write_table(D)``: D's skill table, from wallet days ending 3 days before D (earlier days only); +2. ``aggregate(D)``: D's minutes recomputed from the indexed swaps with that table (rh/minutes.aggregate: the same bands, + ETH conversion and wallet columns as live) → data/corpus/rh/mature/.parquet, in train/mature's aggregate columns + (the ``*_sol`` columns hold ETH on RH rows: native units + the chain input); +3. ``train/mature.build_day(D)`` with the RH directories, program label "Pons v4" (the Pons cost model), each launch's + graduation time, the 1e9 supply and its measured hook fee → data/corpus/rh/features_mature//part.parquet; +4. ``rh/wallet_skill.build_wallet_day(D − 1)`` once D's aggregate and part exist. + +A day is built once complete (the minutes cursor past its end); a part records ``fly_chain = rh`` and ``fly_rh_agg``. +""" +from __future__ import annotations + +import logging +from datetime import date, datetime, timedelta, timezone + +import pyarrow as pa +import pyarrow.parquet as pq + +from .. import config +from ..db.connection import transaction +from ..market.exit_cost import PONS_LABEL +from ..train import mature +from . import minutes, wallet_skill +from .wallet_skill import FEAT_DIR, MATURE_DIR + +log = logging.getLogger(__name__) +RH_AGG_VERSION = 1 +AGG_COLS = ["mint", "ts", "open", "high", "low", "close", "buy_sol", "sell_sol", "n_buys", "n_sells", "n_traders", "resq_sol", "fee_rate", + "n_buyers", "wash_sol", "wash_buy_sol", "top_sell_sol", "insider_sell_sol", "skill_buy"] + + +def aggregate(d: date) -> int: + t0 = int(datetime(d.year, d.month, d.day, tzinfo=timezone.utc).timestamp()) + skill = wallet_skill.table_for(d) + with transaction() as conn: + rows = minutes.aggregate(conn, t0, t0 + 86400, skill) + recs = [{"mint": r[0], "ts": r[1], "open": r[5], "high": r[6], "low": r[7], "close": r[8], "buy_sol": r[11], "sell_sol": r[12], "n_buys": r[13], + "n_sells": r[14], "n_traders": r[15], "resq_sol": r[16], "fee_rate": r[18], "n_buyers": r[19], "wash_sol": r[20], "wash_buy_sol": r[21], + "top_sell_sol": r[22], "insider_sell_sol": r[23], "skill_buy": r[24]} for r in rows] + MATURE_DIR.mkdir(parents=True, exist_ok=True) + schema = pa.schema([("mint", pa.string()), ("ts", pa.timestamp("us", tz="UTC"))] + [(c, pa.float64()) for c in AGG_COLS[2:-1]] + [("skill_buy", pa.list_(pa.float64()))]) + t = pa.Table.from_pylist(recs, schema=schema).replace_schema_metadata({b"fly_version": str(mature.AGG_VERSION).encode(), b"fly_chain": b"rh", + b"fly_rh_agg": str(RH_AGG_VERSION).encode(), b"fly_skill": wallet_skill.skill_version().encode()}) + tmp = MATURE_DIR / f"{d.isoformat()}.parquet.tmp"; pq.write_table(t, tmp, compression="zstd"); tmp.replace(MATURE_DIR / f"{d.isoformat()}.parquet") + return len(recs) + + +def _meta_extra() -> tuple[dict, dict, dict]: + """(graduations, supplies, TokenMeta extras) for every graduated launch.""" + k = config.RH_ETH_PER_SOL + with transaction() as conn: + g = conn.execute("SELECT mint, graduated_at FROM rh_meta").fetchall() + fees = conn.execute("SELECT mint, percentile_cont(0.5) WITHIN GROUP (ORDER BY fee_rate) AS f FROM rh_minutes WHERE fee_rate IS NOT NULL GROUP BY mint").fetchall() + grads = {r["mint"]: r["graduated_at"] for r in g} + fee = {r["mint"]: float(r["f"]) for r in fees} + extra = {m: {"ec_size": 0.1 * k, "pool_fee": fee.get(m, config.RH_HOOK_FEE)} for m in grads} + return grads, {m: 1e9 for m in grads}, extra + + +def build_day(d: date) -> int: + grads, supplies, extra = _meta_extra() + return mature.build_day(d, grads=grads, supplies=supplies, src_dir=MATURE_DIR, out_dir=FEAT_DIR, program_label=PONS_LABEL, meta_extra=extra, + extra_md={"fly_chain": "rh", "fly_rh_agg": RH_AGG_VERSION}) + + +def part_current(path) -> bool: + if not mature.part_current(path): + return False + md = pq.read_schema(path).metadata or {} + return md.get(b"fly_chain") == b"rh" and int(md.get(b"fly_rh_agg", b"0")) == RH_AGG_VERSION + + +def complete_through() -> date | None: + """The last UTC day whose minutes are all written (the minutes cursor past its end).""" + with transaction() as conn: + r = conn.execute("SELECT detail FROM rh_scan WHERE name = 'rh_minutes'").fetchone() + t = ((r["detail"] or {}) if r else {}).get("through") + if not t: + return None + return datetime.fromtimestamp(int(t), timezone.utc).date() - timedelta(days=1) + + +def run_once(max_days: int = 3) -> dict: + last = complete_through() + if last is None: + return {"days": 0} + d = date.fromisoformat(config.RH_START_DAY); done = 0; out = {} + while d <= last and done < max_days: + part = FEAT_DIR / d.isoformat() / "part.parquet"; agg = MATURE_DIR / f"{d.isoformat()}.parquet" + if not agg.exists() or not part_current(part): + wallet_skill.write_table(d) + n_agg = aggregate(d) + n_part = build_day(d) if n_agg else 0 + wallet_skill.build_wallet_day(d - timedelta(days=1)) + out[d.isoformat()] = {"minutes": n_agg, "rows": n_part}; done += 1 + d += timedelta(days=1) + return {"days": done, "built": out} diff --git a/fly_trader/rh/exec.py b/fly_trader/rh/exec.py new file mode 100644 index 0000000000000000000000000000000000000000..fba42eed60223444906439489a9458e04e3a38fd --- /dev/null +++ b/fly_trader/rh/exec.py @@ -0,0 +1,412 @@ +"""The RH executor: swap intents for the live RH book, one at a time, on the bot's ETH wallet. + +An intent (``rh_intents``) is one decision's buy or sell, a sweep of stray tokens, or the liquidation of a base lot. +``execute`` runs it under the wallet lock, so exactly one intent — and one transaction — is in flight: + +1. the intent row is written with the request and a snapshot of the wallet's balances (native ETH, the token, the + quote's base asset) and committed before anything is signed; +2. route: rh/router.plan_swap — KyberSwap atomic (a USDG- or stock-quoted coin is bought ETH→base→meme in one + transaction), the Uniswap v4 single-pool swap when Kyber has none (ETH-quoted pools only: the pool is the whole + route). A non-ETH-quoted coin Kyber cannot route takes two legs: ETH→base (Kyber), then base→meme (Kyber or the + pool); a sale meme→base, then base→ETH; +3. exact approvals and pre-calls, then the swap is simulated (eth_call as the wallet) before it is signed; a sale that + reverts climbs the slippage ladder (``SLIPPAGE_STEP_BPS`` up to the request's maximum); +4. the result is measured: native and token balance deltas against the snapshot (+ the gas every transaction of the + intent paid), token deltas cross-checked with the receipt's Transfer logs; rh/accounting.py books it. Base left + over after a two-leg route that stopped half way becomes a lot (an exit leg's lot stays with its position and is + liquidated later; an entry leg's is a stray lot). + +``recover`` (on start, before the first new intent): every transaction without a receipt applied is settled — mined +(its receipt recorded), superseded at its nonce, rebroadcast (the same bytes, under 5 minutes old) or cancelled at its +nonce — then every unfinished intent is finished from its snapshot: nothing else moved the wallet in between, so the +deltas are that intent's. A crash at any step books each intent exactly once. +""" +from __future__ import annotations + +import json +import logging +import queue +import threading +from dataclasses import asdict, dataclass, field +from datetime import datetime, timezone + +from .. import config +from ..db.apilog import record_event +from ..db.connection import transaction +from ..execution import ledger +from ..vault.evm import EvmRpcError +from . import accounting as A, kyber as K, router + +log = logging.getLogger(__name__) +ZERO = "0x" + "00" * 20 +REBROADCAST_S = 300.0 +TERMINAL = ("done", "failed", "aborted") + + +class SwapReverted(RuntimeError): + pass + + +class NoReceipt(RuntimeError): + """Sent, no receipt within the wait: left to ``recover`` (the intent stays open, its token pending).""" + + +@dataclass +class RhRequest: + kind: str # 'buy' | 'sell' | 'sweep' | 'liquidate' + token: str # the memecoin (a base asset for 'liquidate') + quote_asset: str = ZERO # the pool's quote (ZERO: native ETH) + amount_in: int = 0 # wei for a buy; raw token units otherwise + slippage_bps: float = 150.0 + max_slippage_bps: float = 300.0 + pool: dict | None = None # rh_pools row: currency0/1, fee, tick_spacing, hooks, pool_id + decision_id: int | None = None + position_id: int | None = None + lot_id: int | None = None + hold_s: float | None = None + strategy: str | None = None + forced_kind: str | None = None + decimals: int = 18 + + +@dataclass +class RhResult: + request: RhRequest + ok: bool + error: str | None = None + position_id: int | None = None + realized: float | None = None + route: str | None = None + detail: dict = field(default_factory=dict) + + +def _native(a: str | None) -> bool: + return (a or ZERO).lower() in (ZERO, K.NATIVE, A.NATIVE) + + +class RhExecutor: + def __init__(self, wallet, planner=None, start: bool = True, recover: bool = True): + self.wallet = wallet; self.planner = planner or router.plan_swap + self.q: queue.Queue = queue.Queue(); self.results: list[RhResult] = []; self._pending: dict[str, RhRequest] = {} + self._mu = threading.Lock(); self._stop = threading.Event(); self.thread = None + if recover: + self.recover() + if start: + self.thread = threading.Thread(target=self._loop, name="rh-exec", daemon=True); self.thread.start() + + # ---- queue + def submit(self, req: RhRequest) -> bool: + key = req.token.lower() + with self._mu: + if key in self._pending: + return False + self._pending[key] = req + self.q.put(req) + return True + + def pending(self) -> set[str]: + with self._mu: + return set(self._pending) + + def committed_wei(self) -> int: + """ETH promised to buys queued or in flight (not yet in the balance).""" + with self._mu: + return sum(r.amount_in for r in self._pending.values() if r.kind == "buy") + + def drain(self) -> list[RhResult]: + with self._mu: + out, self.results = self.results, [] + return out + + def _loop(self) -> None: + while not self._stop.is_set(): + try: + req = self.q.get(timeout=0.5) + except queue.Empty: + continue + res = self.execute(req) + if (res.error or "").startswith("no receipt"): # sent, unconfirmed: settle it now (rebroadcast, cancel, or book) + try: + self.recover() + except Exception: + log.exception("rh recovery after a missing receipt failed; the intent stays open until the next start") + continue + with self._mu: + self.results.append(res); self._pending.pop(req.token.lower(), None) + + def stop(self) -> None: + self._stop.set() + if self.thread is not None: + self.thread.join(timeout=30) + + # ---- one intent + def execute(self, req: RhRequest) -> RhResult: + with self.wallet.lock: + iid = self._open_intent(req) + if iid is None: + return RhResult(req, False, "this decision already has an intent") + try: + if req.kind == "buy": + return self._buy(req, iid) + return self._sell(req, iid) + except NoReceipt as e: + return RhResult(req, False, f"no receipt: {e}") + except Exception as e: # noqa: BLE001 — every failure is booked, then reported + log.warning("rh %s %s failed: %s", req.kind, req.token, e) + with transaction() as conn: + mined = conn.execute("SELECT count(*) AS n FROM rh_txs WHERE intent_id = %s AND kind = 'swap' AND status = 'mined_ok'", (iid,)).fetchone()["n"] + if mined: # part of the route ran: book what the balances show + res = self._finish(req, iid, "partial") + res.error = res.error or str(e)[:300] + return res + with transaction() as conn: + conn.execute("UPDATE rh_intents SET state = 'failed', error = %s, updated_at = now() WHERE id = %s", (str(e)[:300], iid)) + gas = A.charge_gas(conn, iid, req.position_id if req.kind == "sell" else None) + conn.execute("UPDATE rh_txs SET applied_at = now() WHERE intent_id = %s AND applied_at IS NULL AND status IN ('mined_ok', 'reverted')", (iid,)) + return RhResult(req, False, str(e)[:300], detail={"gas_eth": gas}) + + def _open_intent(self, req: RhRequest) -> int | None: + assets = [req.token] + ([] if _native(req.quote_asset) else [req.quote_asset]) + pre = {"native": self.wallet.balance(), **{a.lower(): self.wallet.balance_of(a) for a in assets}} + committed = req.amount_in if req.kind == "buy" else 0 + with transaction() as conn: + r = conn.execute("INSERT INTO rh_intents (book, decision_id, position_id, kind, token, quote_asset, amount_in_raw, committed_wei, plan) " + "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT (decision_id, kind) WHERE decision_id IS NOT NULL DO NOTHING RETURNING id", + (A.BOOK, req.decision_id, req.position_id, req.kind, req.token.lower(), (req.quote_asset or ZERO).lower(), req.amount_in, committed, + json.dumps({"req": asdict(req), "pre": {k: str(v) for k, v in pre.items()}}))).fetchone() + return int(r["id"]) if r else None + + def _route(self, iid: int, route: str, state: str | None = None) -> None: + with transaction() as conn: + conn.execute("UPDATE rh_intents SET route = %s, state = COALESCE(%s, state), attempts = attempts + 1, updated_at = now() WHERE id = %s", + (route, state, iid)) + + def _run_plan(self, plan, iid: int, position_id: int | None) -> dict: + for token, spender, amt in plan.approvals: + self.wallet.approve_exact(token, spender, amt, intent_id=iid, position_id=position_id) + for c in plan.pre_calls: + r = self.wallet.send_and_wait(to=c.to, data=c.data, value=c.value, kind=c.kind, intent_id=iid, position_id=position_id) + if r.get("ok") is None: + raise NoReceipt(r["hash"]) + if not r["ok"]: + raise SwapReverted(f"{c.kind} before the swap reverted") + try: + self.wallet.simulate(plan.swap.to, plan.swap.data, plan.swap.value) + except EvmRpcError as e: + raise SwapReverted(f"simulation: {str(e)[:200]}") from None + r = self.wallet.send_and_wait(to=plan.swap.to, data=plan.swap.data, value=plan.swap.value, kind="swap", intent_id=iid, position_id=position_id, + router=plan.route, quote=plan.quote, build=plan.build) + if r.get("ok") is None: + raise NoReceipt(r["hash"]) + if not r["ok"]: + raise SwapReverted(f"swap {r['hash']} reverted") + return r + + def _plan(self, token_in: str, token_out: str, amount: int, slip: float, pool: dict | None): + return self.planner(self.wallet.address, token_in, token_out, amount, slip, rpc=self.wallet.rpc, pool=pool) + + def _ladder(self, req: RhRequest) -> list[float]: + out, s = [], float(req.slippage_bps) + while s < req.max_slippage_bps: + out.append(s); s += config.SLIPPAGE_STEP_BPS + return out + [float(req.max_slippage_bps)] + + def _buy(self, req: RhRequest, iid: int) -> RhResult: + eth_q = _native(req.quote_asset); route = "atomic" + try: + plan = self._plan(K.NATIVE, req.token, req.amount_in, req.slippage_bps, req.pool if eth_q else None) + except router.NoRoute: + if eth_q: + raise + plan = None + if plan is not None: + self._route(iid, plan.route, "leg1"); route = plan.route + self._run_plan(plan, iid, None) + else: # two legs: ETH → base, base → meme + self._route(iid, "two_leg", "leg1"); route = "two_leg" + p1 = self._plan(K.NATIVE, req.quote_asset, req.amount_in, req.slippage_bps, None) + self._run_plan(p1, iid, None) + got = self.wallet.balance_of(req.quote_asset) - self._pre(iid).get(req.quote_asset.lower(), 0) + self._route(iid, "two_leg", "leg2") + try: + self._run_plan(self._plan(req.quote_asset, req.token, got, req.slippage_bps, req.pool), iid, None) + except (SwapReverted, router.NoRoute, EvmRpcError) as e: # the meme leg failed: the base goes back to ETH + log.warning("rh two-leg buy of %s stopped after the base leg: %s", req.token, e) + try: + self._run_plan(self._plan(req.quote_asset, K.NATIVE, got, config.SLIPPAGE_FORCED_BPS, None), iid, None) + except (SwapReverted, router.NoRoute, EvmRpcError) as e2: + log.warning("rh: returning the base of %s to ETH failed too (%s): it stays a lot", req.token, e2) + return self._finish(req, iid, route) + + def _sell(self, req: RhRequest, iid: int) -> RhResult: + eth_q = _native(req.quote_asset); last = None + if req.kind == "liquidate": # a base lot back to ETH (Kyber) + eth_q = True + for slip in self._ladder(req): + try: + plan = self._plan(req.token, K.NATIVE, req.amount_in, slip, req.pool if eth_q and req.kind != "liquidate" else None) + except router.NoRoute: + if eth_q: + raise + return self._sell_two_leg(req, iid) + self._route(iid, plan.route, "leg1") + try: + self._run_plan(plan, iid, req.position_id) + return self._finish(req, iid, plan.route) + except SwapReverted as e: + last = e; log.info("rh sell %s reverted at %.0f bps: %s", req.token, slip, e) + raise last or SwapReverted("no attempt") + + def _sell_two_leg(self, req: RhRequest, iid: int) -> RhResult: + self._route(iid, "two_leg", "leg1") + self._run_plan(self._plan(req.token, req.quote_asset, req.amount_in, req.max_slippage_bps, req.pool), iid, req.position_id) + got = self.wallet.balance_of(req.quote_asset) - self._pre(iid).get(req.quote_asset.lower(), 0) + self._route(iid, "two_leg", "leg2") + try: + self._run_plan(self._plan(req.quote_asset, K.NATIVE, got, req.max_slippage_bps, None), iid, req.position_id) + except (SwapReverted, router.NoRoute, EvmRpcError) as e: # the base stays an exit lot of this position + log.warning("rh two-leg sale of %s: the base leg failed (%s); its base is kept as a lot", req.token, e) + return self._finish(req, iid, "two_leg") + + def _pre(self, iid: int) -> dict: + with transaction() as conn: + r = conn.execute("SELECT plan FROM rh_intents WHERE id = %s", (iid,)).fetchone() + return {k: int(v) for k, v in (r["plan"] or {}).get("pre", {}).items()} + + # ---- measure and book + def _finish(self, req: RhRequest, iid: int, route: str | None) -> RhResult: + pre = self._pre(iid); tok = req.token.lower(); base = None if _native(req.quote_asset) or req.kind == "liquidate" else req.quote_asset.lower() + post_native = self.wallet.balance(); post_tok = self.wallet.balance_of(tok) + post_base = self.wallet.balance_of(base) if base else 0 + now = datetime.now(timezone.utc) + with transaction() as conn: + gas = A.intent_gas_wei(conn, iid) + eth_delta = post_native - pre["native"] + gas # ETH the swaps moved (gas put back) + tok_delta = post_tok - pre.get(tok, 0) + base_left = (post_base - pre.get(base, 0)) if base else 0 + swaps = conn.execute("SELECT id, hash FROM rh_txs WHERE intent_id = %s AND kind = 'swap' AND status = 'mined_ok' ORDER BY id", (iid,)).fetchall() + tx_id = int(swaps[-1]["id"]) if swaps else None + left_eth = 0.0 + if base_left > 0: + mark, _ = A.base_mark(conn, base) + left_eth = base_left / 10 ** A._decimals(conn, base) * (mark or 0.0) + realized = None; pid = req.position_id + if req.kind == "buy": + spent = -eth_delta + A.add_leg(conn, intent_id=iid, tx_id=tx_id, position_id=None, leg_no=1, asset_in=A.NATIVE, asset_out=tok, amount_in=spent, amount_out=max(tok_delta, 0), + eth_in=spent / A.WEI, eth_out=None) + if base_left > 0: # an entry leg's leftover base: a stray lot, not position cost + A.open_lot(conn, asset=base, qty_raw=base_left, basis_eth=left_eth, source="stray", intent_id=iid) + pid, realized = A.book_buy(conn, intent_id=iid, token=tok, pool=(req.pool or {}).get("pool_id"), quote_asset=(req.quote_asset or ZERO).lower(), + eth_spent_wei=int(spent - left_eth * A.WEI), tokens_raw=max(tok_delta, 0), decision_id=req.decision_id, ts=now, + hold_s=req.hold_s, strategy=req.strategy, decimals=req.decimals) + else: + sold = -tok_delta + A.add_leg(conn, intent_id=iid, tx_id=tx_id, position_id=req.position_id, leg_no=1, asset_in=tok, asset_out=A.NATIVE, amount_in=max(sold, 0), + amount_out=max(eth_delta, 0), eth_in=None, eth_out=max(eth_delta, 0) / A.WEI) + if req.kind == "liquidate" and req.lot_id is not None: + realized = A.liquidate_lot(conn, req.lot_id, max(eth_delta, 0) / A.WEI, gas / A.WEI) + elif req.position_id is not None: + pos = next((p for p in ledger.open_positions(conn, A.BOOK) if int(p["id"]) == req.position_id), None) + if pos is not None: + if base_left > 0: # an exit leg's leftover base stays with the position as a lot + A.open_lot(conn, asset=base, qty_raw=base_left, basis_eth=left_eth, source="exit_leg", position_id=req.position_id, intent_id=iid) + realized = A.book_sell(conn, intent_id=iid, position=pos, eth_in_wei=max(eth_delta, 0), tokens_sold_raw=max(sold, 0), + decision_id=req.decision_id, ts=now, forced_kind=req.forced_kind, lot_value_eth=left_eth) + ok = (tok_delta > 0) if req.kind == "buy" else (tok_delta < 0 or req.kind == "liquidate") + conn.execute("UPDATE rh_intents SET state = %s, position_id = COALESCE(%s, position_id), updated_at = now() WHERE id = %s", + ("done" if ok else "aborted", pid, iid)) + conn.execute("UPDATE rh_txs SET applied_at = now() WHERE intent_id = %s AND applied_at IS NULL AND status IN ('mined_ok', 'reverted')", (iid,)) + receipts_tok = self._receipt_tokens(conn, iid, tok) + if receipts_tok is not None and receipts_tok != tok_delta and req.kind != "liquidate": + record_event("warning", "rh_live", f"{tok}: balance moved {tok_delta} but the receipts' Transfer logs say {receipts_tok} (fee-on-transfer or a foreign transfer)", + {"intent": iid}) + return RhResult(req, ok, None if ok else "no tokens moved", position_id=pid, realized=realized, route=route, + detail={"eth_delta_wei": eth_delta, "token_delta": tok_delta, "gas_wei": gas, "base_left": base_left}) + + def _receipt_tokens(self, conn, iid: int, token: str) -> int | None: + rows = conn.execute("SELECT hash FROM rh_txs WHERE intent_id = %s AND kind = 'swap' AND status = 'mined_ok'", (iid,)).fetchall() + if not rows: + return None + n = 0 + for r in rows: + rc = self.wallet.rpc.receipt(r["hash"]) + if rc is None: + return None + n += A.transfer_delta(rc, token, self.wallet.address) + return n + + # ---- crash recovery + def recover(self) -> dict: + """Settle every transaction without an applied receipt, then finish every open intent from its snapshot.""" + out = {"txs": 0, "intents": 0} + with self.wallet.lock: + with transaction() as conn: + txs = [dict(r) for r in conn.execute("SELECT * FROM rh_txs WHERE from_addr = %s AND applied_at IS NULL AND status IN ('signed', 'sent') ORDER BY nonce, id", + (self.wallet.address,)).fetchall()] + for tx in txs: + self._settle_tx(tx); out["txs"] += 1 + with transaction() as conn: + intents = [dict(r) for r in conn.execute("SELECT * FROM rh_intents WHERE book = %s AND state NOT IN ('done', 'failed', 'aborted') ORDER BY id", + (A.BOOK,)).fetchall()] + for it in intents: + req = RhRequest(**(it["plan"] or {}).get("req", {"kind": it["kind"], "token": it["token"]})) + with transaction() as conn: + mined = conn.execute("SELECT count(*) AS n FROM rh_txs WHERE intent_id = %s AND kind = 'swap' AND status = 'mined_ok'", (it["id"],)).fetchone()["n"] + if mined: + res = self._finish(req, int(it["id"]), it.get("route")) + else: + with transaction() as conn: + conn.execute("UPDATE rh_intents SET state = 'failed', error = 'recovered: no swap mined', updated_at = now() WHERE id = %s", (it["id"],)) + A.charge_gas(conn, int(it["id"]), req.position_id if req.kind == "sell" else None) + conn.execute("UPDATE rh_txs SET applied_at = now() WHERE intent_id = %s AND applied_at IS NULL AND status IN ('mined_ok', 'reverted')", (it["id"],)) + res = RhResult(req, False, "recovered: no swap mined") + with self._mu: + self.results.append(res); self._pending.pop(req.token.lower(), None) + out["intents"] += 1 + if out["txs"] or out["intents"]: + record_event("warning", "rh_live", "recovered unfinished RH transactions/intents", out) + return out + + def _settle_tx(self, tx: dict) -> None: + settle_tx(self.wallet, tx) + + +def settle_tx(w, tx: dict) -> None: + """Settle one of ``w``'s transactions that has no applied receipt (``recover``; rh/withdraw.settle_pending): its + receipt recorded once mined; 'replaced' when something else took its nonce; under ``REBROADCAST_S`` old, the same + bytes rebroadcast; otherwise cancelled at its nonce (the original recorded if it wins the race). Under the wallet's lock.""" + rpc = w.rpc + rc = rpc.receipt(tx["hash"]) + if rc is not None and rc.get("blockNumber"): + with transaction() as conn: + w.record_receipt(conn, int(tx["id"]), rc) + return + if rpc.tx_count(w.address, "latest") > int(tx["nonce"]): # something else took the nonce (a replacement or a cancel) + with transaction() as conn: + conn.execute("UPDATE rh_txs SET status = 'replaced', applied_at = now(), updated_at = now() WHERE id = %s", (tx["id"],)) + return + age = (datetime.now(timezone.utc) - tx["created_at"]).total_seconds() + if age < REBROADCAST_S: + try: + rpc.send_raw(tx["raw"]) + except EvmRpcError as e: + if "known" not in (e.message or "").lower(): + log.warning("rebroadcast of %s failed: %s", tx["hash"], e) + rc = w.wait(tx["hash"], timeout_s=60.0) + if rc is not None: + with transaction() as conn: + w.record_receipt(conn, int(tx["id"]), rc) + return + c = w.cancel(int(tx["nonce"]), int(tx["max_fee_wei"] or 0)) + with transaction() as conn: + conn.execute("UPDATE rh_txs SET intent_id = %s, position_id = %s WHERE id = %s", (tx["intent_id"], tx["position_id"], c["id"])) + rc_c = w.wait(c["hash"], timeout_s=120.0) + rc = rpc.receipt(tx["hash"]) + with transaction() as conn: + if rc is not None and rc.get("blockNumber"): # the original won the race after all + w.record_receipt(conn, int(tx["id"]), rc) + else: + conn.execute("UPDATE rh_txs SET status = 'replaced', applied_at = now(), updated_at = now() WHERE id = %s", (tx["id"],)) + if rc_c is not None: + w.record_receipt(conn, int(c["id"]), rc_c) diff --git a/fly_trader/rh/guard.py b/fly_trader/rh/guard.py new file mode 100644 index 0000000000000000000000000000000000000000..320deb1cd286dc9e9432edf472dc151450a8572b --- /dev/null +++ b/fly_trader/rh/guard.py @@ -0,0 +1,69 @@ +"""Nothing signs an RH transaction unless every check here passes, every time (not once at start): + +- ``RH_LIVE_ENABLED`` is on — except a withdrawal to one of the operator's own addresses (``check_withdrawal``): it takes + money out of the fly's reach and never trades; +- the RPC answers the chain id this install expects (4663, or 46630 with ``RH_TESTNET``) — a testnet key never signs + on mainnet and the reverse; +- ``RH_BOT_PRIVATE_KEY`` is set and derives to ``RH_BOT_ADDRESS`` (a pasted wrong key never trades). +""" +from __future__ import annotations + +from .. import config +from .tx import address_of + + +class SigningRefused(RuntimeError): + pass + + +def load_key() -> bytes: + raw = (config.env_str("RH_BOT_PRIVATE_KEY") or "").strip() + if not raw: + raise SigningRefused("RH_BOT_PRIVATE_KEY is not set") + h = raw[2:] if raw.lower().startswith("0x") else raw + try: + key = bytes.fromhex(h) + except ValueError: + raise SigningRefused("RH_BOT_PRIVATE_KEY is not hex") from None + if len(key) != 32: + raise SigningRefused("RH_BOT_PRIVATE_KEY must be 32 bytes") + return key + + +def pinned_key() -> tuple[bytes, str]: + """(private key, address) when RH_BOT_PRIVATE_KEY is set and derives to RH_BOT_ADDRESS (no RPC needed).""" + key = load_key(); addr = address_of(key) + want = (config.RH_BOT_ADDRESS or "").lower() + if not want: + raise SigningRefused("RH_BOT_ADDRESS is not set: the key's address must be pinned") + if addr != want: + raise SigningRefused(f"RH_BOT_PRIVATE_KEY derives to {addr}, not RH_BOT_ADDRESS") + return key, addr + + +def _on_expected_chain(rpc) -> None: + cid = rpc.chain() + if cid != config.RH_EXPECTED_CHAIN_ID: + raise SigningRefused(f"the RPC is chain {cid}; this install signs only on {config.RH_EXPECTED_CHAIN_ID}") + + +def check(rpc) -> tuple[bytes, str]: + """(private key, address) when signing is allowed; raises ``SigningRefused`` with the reason otherwise.""" + if not config.RH_LIVE_ENABLED: + raise SigningRefused("RH_LIVE_ENABLED is off") + key, addr = pinned_key() + _on_expected_chain(rpc) + return key, addr + + +def check_withdrawal(rpc, dest: str) -> tuple[bytes, str]: + """(private key, address) for a withdrawal to ``dest`` (rh/withdraw.py): every check but RH_LIVE_ENABLED, and ``dest`` + must be listed in RH_FUNDING_ADDRESSES — the list lives in .env and is never typed into the console, so whoever reaches + the console can at most send the bot's ETH to its owner. Never on the hosted vault fly: its ETH pays lockers' claims.""" + if config.VAULT_ENABLED: + raise SigningRefused("refusing: the vault fly's ETH pays lockers' claims; it is not withdrawn from the console") + if (dest or "").lower() not in {a.lower() for a in config.RH_FUNDING_ADDRESSES}: + raise SigningRefused(f"refusing: {dest} is not one of your RH_FUNDING_ADDRESSES (.env)") + key, addr = pinned_key() + _on_expected_chain(rpc) + return key, addr diff --git a/fly_trader/rh/hypersync.py b/fly_trader/rh/hypersync.py new file mode 100644 index 0000000000000000000000000000000000000000..8043d2964d95f83d39d3c34defb86410d6daa06d --- /dev/null +++ b/fly_trader/rh/hypersync.py @@ -0,0 +1,174 @@ +"""Log reads from Envio HyperSync (``ENVIO_API_TOKEN``): the chain's full history in bulk, instead of the public RPC's +~1 eth_getLogs a second under a 10,000-log cap. ``HyperRpc`` is an ``RhRpc`` whose ``get_logs_multi`` asks HyperSync and +returns the same log dicts as eth_getLogs (plus ``blockTimestamp``, which the RPC leaves at 0x0). Blocks HyperSync has +not reached yet are read from the RPC. + +Query API (measured 2026-09-30): POST {url}/query {from_block, to_block (exclusive), logs: [{address, topics}], +field_selection} → {data: [{logs, blocks}], next_block, archive_height}; a response may stop early at ``next_block``. +""" +from __future__ import annotations + +import threading +import time + +import httpx + +from .. import config +from ..db.apilog import record_api_call +from .rpc import RhRpc + +LOG_FIELDS = ["block_number", "log_index", "transaction_hash", "block_hash", "address", "data", "topic0", "topic1", "topic2", "topic3"] +TRIES = 8 +LIVE_TIMEOUT_S = 15.0 +LIVE_SPAN = 20_000 # a read of at most this many blocks is a live read (the stream near the head) +_PACE_LOCK = threading.Lock() +_LAST = [0.0] + + +def _pace() -> None: + """At most ``RH_HYPERSYNC_RPM`` queries a minute across threads (the free tier is fair-use; Starter 100 rpm).""" + gap = 60.0 / max(1.0, float(config.RH_HYPERSYNC_RPM)) + with _PACE_LOCK: + wait = _LAST[0] + gap - time.monotonic() + if wait > 0: + time.sleep(wait) + _LAST[0] = time.monotonic() + + +class HyperSyncError(RuntimeError): + pass + + +def _topics(topics: list | None) -> list[list[str]]: + out = [] + for t in topics or []: + out.append([] if t is None else ([x.lower() for x in t] if isinstance(t, (list, tuple)) else [t.lower()])) + return out + + +def to_rpc_log(g: dict, ts: dict[int, int]) -> dict: + b = int(g["block_number"]) + return {"address": (g.get("address") or "").lower(), "topics": [g[k] for k in ("topic0", "topic1", "topic2", "topic3") if g.get(k)], + "data": g.get("data") or "0x", "blockNumber": hex(b), "logIndex": hex(int(g["log_index"])), "transactionHash": g.get("transaction_hash"), + "blockHash": g.get("block_hash"), "blockTimestamp": hex(ts[b]) if b in ts else "0x0"} + + +class HyperSync: + def __init__(self, token: str, url: str | None = None, client: httpx.Client | None = None): + self.url = (url or config.RH_HYPERSYNC_URL).rstrip("/") + # connect=5: one of robinhood.hypersync.xyz's six addresses (193.32.220.135) never answers from here (2026-10-07), and a + # fresh connection that drew it hung ~76 s; with a short connect timeout the next address is tried after 5 s. + # keepalive 120 s: httpx closes idle connections after 5 s by default, so nearly every live pass connected anew. + self.http = client or httpx.Client(timeout=httpx.Timeout(120.0, connect=5.0), limits=httpx.Limits(keepalive_expiry=120.0), + headers={"Authorization": f"Bearer {token}", "Content-Type": "application/json"}) + self.height = 0 + + def _post(self, body: dict, live: bool = False, tries: int = TRIES) -> dict: + err = None + for i in range(tries): + _pace() + t0 = time.monotonic(); status = None; err = None; retry_after = None + try: + # a live read answers in well under a second; one stuck on a dropped connection (seen 20-62 s, 2026-10-07) + # is abandoned after LIVE_TIMEOUT_S and sent again on a fresh connection + r = self.http.post(f"{self.url}/query", json=body, timeout=httpx.Timeout(LIVE_TIMEOUT_S, connect=5.0) if live else httpx.USE_CLIENT_DEFAULT) + status = r.status_code + if r.status_code == 200: + return r.json() + err = f"HTTP {r.status_code}: {r.text[:200]}" + if r.status_code not in (429, 500, 502, 503, 504): + raise HyperSyncError(err) + ra = r.headers.get("retry-after") + retry_after = float(ra) if ra and ra.replace(".", "", 1).isdigit() else None + except httpx.HTTPError as e: + err = type(e).__name__ + finally: + record_api_call("rh_hypersync", "/query", "POST", status, int((time.monotonic() - t0) * 1000), err is None, err) + if retry_after is None and err in ("ReadTimeout", "ConnectTimeout", "ConnectError", "RemoteProtocolError", "ReadError", "WriteError"): + retry_after = 0.5 if i == 0 else min(10.0, 1.0 * 2 ** i) # a dead connection: retry at once on a new one + if i == tries - 1: + break + time.sleep(retry_after if retry_after is not None else min(60.0, 5.0 * 2 ** i)) # a rate-limit window resets within a minute + raise HyperSyncError(f"HyperSync failed {tries} times: {err}") + + def archive_height(self) -> int: + err = None + for i in range(TRIES): + t0 = time.monotonic(); status = None; err = None + try: + r = self.http.get(f"{self.url}/height"); status = r.status_code; r.raise_for_status() + self.height = int(r.json()["height"]) + return self.height + except httpx.HTTPError as e: + err = type(e).__name__ + finally: + record_api_call("rh_hypersync", "/height", "GET", status, int((time.monotonic() - t0) * 1000), err is None, err) + time.sleep(min(10.0, 1.0 * 2 ** i)) + raise HyperSyncError(f"HyperSync height failed {TRIES} times: {err}") + + def get_logs(self, addresses: list[str] | None, lo: int, hi: int, topics: list | None = None, upto: bool = False): + """Logs in blocks lo..hi (inclusive), eth_getLogs-shaped, in chain order. With ``upto`` the read stops at + HyperSync's archive height (each response reports it) and (logs, last block covered) is returned; the caller + reads the rest elsewhere, so the hot path never asks /height.""" + logs, through, _ = self.get_logs_sel([selection(addresses, topics)], lo, hi, upto=upto) + return (logs, through) if upto else logs + + def get_logs_sel(self, sels: list[dict], lo: int, hi: int, upto: bool = False, tries: int = TRIES, all_blocks: bool = False): + """(logs, last block covered, {block: timestamp}) for several selections (``selection``) in one query: the union + of their logs, each once, in chain order. ``all_blocks``: every block's timestamp in the range, not only those + with a selected log (a live pass learns its last block's time without asking the RPC).""" + out, ts, frm = [], {}, lo + live = hi - lo + 1 <= LIVE_SPAN + while frm <= hi: + body = {"from_block": frm, "to_block": hi + 1, "logs": sels, "field_selection": {"log": LOG_FIELDS, "block": ["number", "timestamp"]}} + if all_blocks: + body["include_all_blocks"] = True + d = self._post(body, live=live, tries=tries) + for part in d.get("data") or []: + for b in part.get("blocks") or []: + ts[int(b["number"])] = int(b["timestamp"], 16) if isinstance(b["timestamp"], str) else int(b["timestamp"]) + out += part.get("logs") or [] + nxt = int(d.get("next_block") or hi + 1) + ah = d.get("archive_height") + if ah is not None: + self.height = max(self.height, int(ah)) + if nxt <= frm: + if upto: + break # at the archive tip: nothing more here yet + raise HyperSyncError(f"HyperSync made no progress at block {frm}") + frm = nxt + if upto and ah is not None and frm > int(ah): + break # covered everything HyperSync has + seen = set(); logs = [] + for g in out: # a log two selections both match comes back twice + k = (int(g["block_number"]), int(g["log_index"])) + if k not in seen: + seen.add(k); logs.append(to_rpc_log(g, ts)) + logs.sort(key=lambda x: (int(x["blockNumber"], 16), int(x["logIndex"], 16))) + return logs, (min(hi, frm - 1) if upto else hi), ts + + +def selection(addresses: list[str] | None, topics: list | None = None) -> dict: + """One HyperSync log selection from eth_getLogs' (addresses, topics).""" + sel: dict = {} + if addresses: + sel["address"] = [a.lower() for a in addresses] + if topics: + sel["topics"] = _topics(topics) + return sel + + +class HyperRpc(RhRpc): + """An RhRpc whose log reads come from HyperSync up to its archive height, and from the RPC beyond it.""" + + def __init__(self, hs: HyperSync, **kw): + super().__init__(**kw) + self.hs = hs + + def get_logs_multi(self, addresses, from_block: int, to_block: int, topics=None) -> list[dict]: + if config.RH_RPC_URL_LOGS and self.hs.height and from_block > self.hs.height - LIVE_SPAN: + return super().get_logs_multi(addresses, from_block, to_block, topics) # near the head: the dedicated RPC + out, through = self.hs.get_logs(addresses, from_block, to_block, topics, upto=True) + if through >= to_block: + return out + return out + super().get_logs_multi(addresses, max(from_block, through + 1), to_block, topics) diff --git a/fly_trader/rh/index.py b/fly_trader/rh/index.py new file mode 100644 index 0000000000000000000000000000000000000000..7e1e323c572b0ddb369a4248658b8eef75410d28 --- /dev/null +++ b/fly_trader/rh/index.py @@ -0,0 +1,338 @@ +"""The Robinhood Chain indexer: Pons launches, curve trades and graduations, and the swaps of every graduated Pons pool, +read from our own RPC (one cursor ``rh``; backfill from Pons V2's deployment, then live near the head). + +Per block range, in order: +1. the factory / launch router / meme hook logs (one request): launches, dev buys, graduations, pool registrations, and + the hook's fee on every Pons swap; +2. those events applied in (block, log index) order → rh_tokens, rh_pools, rh_insiders; a graduation also pulls that + launch's curve trades from its own curve contract (launch → graduation, address-filtered): only the ~1 % of launches + that graduate are ever traded, and their curve phase is all the features need (curve facts, time to graduate); +3. the v4 PoolManager's Initialize / Swap / ModifyLiquidity logs for the Pons pools known by then (topic1 = pool id, + 200 ids per request), each swap matched to its hook fee by (tx, pool) in log order; +4. each swap's trader from the memecoin's own Transfer logs in that transaction (address-filtered, one request per 200 + tokens): the wallet whose token balance rose on a buy (fell on a sell), not the pool manager or a router — the swap + event itself names the router (measured 2026-09-28: per-swap tx lookups made a 55-day backfill take ~43 days); +5. rh_swaps rows priced in the pool's quote (price_q, virtual quote reserve resq_q, fee_frac); the ETH conversion of + non-ETH quotes happens per minute in rh/minutes.py with the base asset's mark. + +Everything a range writes is committed with its cursor, so a crash replays at most one range (inserts are idempotent). +""" +from __future__ import annotations + +import logging + +from .. import config +from ..db.connection import transaction +from . import abi, blocktime, pons, scan, v4 +from .rpc import RhRpc, logs_rpc + +log = logging.getLogger(__name__) + +CURSOR = "rh" +IDS_PER_REQUEST = 200 +TOKENS_PER_REQUEST = 200 +FEE_OBS = 20 # hook-fee logs read per pool before its fee is taken as known +TOKENS_PER_REQUEST_BULK = 5000 # HyperSync: bounded by its 2 MB request body, not a log cap +IDS_PER_REQUEST_BULK = 5000 # 66-char pool ids: ~0.35 MB of filter per request +LIVE_SPAN_BULK = 20_000 # an index range this small is live: PoolManager events unfiltered (one query) +TX_BATCH = 100 +TRANSFER = abi.event_topic("Transfer(address,address,uint256)") +PONS_TOPICS = [[pons.TOPIC[k] for k in (*pons.FACTORY_EVENTS, "Launched", *pons.HOOK_EVENTS) if k != "HookFeeCollected"]] +V4_TOPICS0 = [v4.TOPIC["Initialize"], v4.TOPIC["Swap"], v4.TOPIC["ModifyLiquidity"]] +FEE_TOPIC = pons.TOPIC["HookFeeCollected"] +PONS_DECIMALS = 18 # every Pons launch mints 1e9 × 1e18 raw +STABLES = ("USDG", "USDC", "USDT", "PYUSD", "USDE") + + +# ---------------------------------------------------------------- quote assets +def ensure_asset(conn, rpc, asset: str) -> dict: + """The rh_assets row of a quote asset (decimals, symbol, class), created on first sight.""" + asset = asset.lower() + r = conn.execute("SELECT asset, symbol, class, decimals FROM rh_assets WHERE asset = %s", (asset,)).fetchone() + if r: + return dict(r) + if int(asset, 16) == 0: + row = {"asset": asset, "symbol": "ETH", "class": "eth", "decimals": 18} + else: + dec = abi.decode(["uint8"], bytes.fromhex(rpc.eth_call(asset, "0x" + abi.selector("decimals()").hex())[2:]))[0] + try: + sym = abi.decode(["string"], bytes.fromhex(rpc.eth_call(asset, "0x" + abi.selector("symbol()").hex())[2:]))[0] + except Exception: + sym = asset[:10] + s = sym.upper() + cls = "stable" if s in STABLES else "btc" if "BTC" in s else "eth" if s in ("WETH", "ETH") else "stock" + row = {"asset": asset, "symbol": sym, "class": cls, "decimals": int(dec)} + conn.execute("INSERT INTO rh_assets (asset, symbol, class, decimals) VALUES (%(asset)s, %(symbol)s, %(class)s, %(decimals)s) ON CONFLICT (asset) DO NOTHING", row) + return row + + +def block_at(rpc, ts: float) -> int: + """The first block whose timestamp is at or after ``ts`` (binary search over block headers: the public RPC keeps every + header and log but no historical state, so a getCode search for the factory's deployment cannot run).""" + lo, hi = 0, rpc.block_number() + while lo < hi: + mid = (lo + hi) // 2 + if int(rpc.block(mid)["timestamp"], 16) >= ts: + hi = mid + else: + lo = mid + 1 + return lo + + +# ---------------------------------------------------------------- one range +def _get_logs(rpc, addresses, lo, hi, topics): + """eth_getLogs, splitting the range in halves while the RPC's 10,000-logs-per-query cap is hit.""" + try: + out = rpc.get_logs_multi(addresses, lo, hi, topics) + except Exception as e: + msg = str(e).lower() + if ("exceeds limit" in msg or "too many" in msg or "timed out" in msg or "timeout" in msg) and hi > lo: # too big a query: halve it + mid = (lo + hi) // 2 + return _get_logs(rpc, addresses, lo, mid, topics) + _get_logs(rpc, addresses, mid + 1, hi, topics) + raise + if not getattr(rpc, "served_last", False): # a read answered from a live pass's memory needs no pause + scan.pause() + return out + + +def _order(logs: list[dict]) -> list[dict]: + return sorted(logs, key=lambda x: (int(x["blockNumber"], 16), int(x["logIndex"], 16))) + + +def _not_trader() -> set[str]: + return {a.lower() for a in (config.V4_POOL_MANAGER, config.UNIVERSAL_ROUTER, config.KYBER_ROUTER, config.PERMIT2, config.PONS_HOOK, + config.PONS_ROUTER, "0x" + "00" * 20)} + + +def traders_from_transfers(transfers: list[dict], swaps: list[dict], tokens: dict[str, str]) -> dict[tuple, str]: + """(tx, pool) → the swapping wallet: the address with the largest net memecoin gain (buy) or loss (sell) in that tx, + excluding the pool manager, routers and the hook. ``tokens``: pool id → memecoin.""" + skip = _not_trader(); net: dict[tuple, dict[str, int]] = {} + for t in transfers: + tok = t["address"].lower(); tx = t["transactionHash"] + a, b = "0x" + t["topics"][1][-40:].lower(), "0x" + t["topics"][2][-40:].lower(); v = int(t["data"], 16) + d = net.setdefault((tx, tok), {}) + d[a] = d.get(a, 0) - v; d[b] = d.get(b, 0) + v + out = {} + for s in swaps: + tok = tokens[s["pool_id"]]; d = net.get((s["tx_hash"], tok)) or {} + cands = [(v, a) for a, v in d.items() if a not in skip and a != tok] + if not cands: + continue + v, a = (max(cands) if s["side"] > 0 else min(cands)) + if (s["side"] > 0 and v > 0) or (s["side"] < 0 and v < 0): + out[(s["tx_hash"], s["pool_id"])] = a + return out + + +def fill_timestamps(rpc, events: list[dict], lo: int, hi: int) -> None: + """The RPC's ``blockTimestamp`` on logs is usually 0x0 (measured 2026-09-28): every event that lands in a minute bar + gets its time from rh/blocktime (exact minute, interpolated seconds), never one lookup per block.""" + need = [e for e in events if not e.get("ts")] + if not need: + return + t = blocktime.times(rpc, lo, hi, sorted({e["block"] for e in need})) + for e in need: + e["ts"] = t[e["block"]] + + +def index_curve(conn, rpc, token: str, until_block: int) -> int: + """A graduating launch's curve trades (its own curve contract, launch → graduation). Idempotent; returns rows written.""" + r = conn.execute("SELECT curve, launch_block, curve_indexed FROM rh_tokens WHERE token = %s", (token,)).fetchone() + if not r or not r["curve"] or r["launch_block"] is None or r["curve_indexed"]: + return 0 + router = config.PONS_ROUTER.lower(); n = 0; lo = int(r["launch_block"]); step = scan.MAX_RANGE + while lo <= until_block: + hi = min(until_block, lo + step - 1) + logs = _get_logs(rpc, [r["curve"]], lo, hi, [[pons.TOPIC["CurveBuy"], pons.TOPIC["CurveSell"]]]) + evs = [pons.decode(x) for x in _order(logs)]; evs = [e for e in evs if e] + fill_timestamps(rpc, evs, lo, hi) + for e in evs: + trader = e["recipient"] if e["trader"] == router else e["trader"] + conn.execute("INSERT INTO rh_curve_trades (block, log_index, tx_hash, ts, token, side, trader, recipient, quote_raw, tokens_raw, fee_raw, tax_raw) " + "VALUES (%s, %s, %s, to_timestamp(%s), %s, %s, %s, %s, %s, %s, %s, %s) ON CONFLICT DO NOTHING", + (e["block"], e["log_index"], e["tx_hash"], e["ts"], token, e["side"], trader, e["recipient"], e["quote_raw"], e["tokens_raw"], e["fee_raw"], e["tax_raw"])) + n += 1 + lo = hi + 1 + conn.execute("UPDATE rh_tokens SET curve_indexed = true WHERE token = %s", (token,)) + return n + + +def apply_pons(conn, rpc, events: list[dict]) -> dict: + """Pons events of one range, in order. Returns counts.""" + n = {"launches": 0, "curve_trades": 0, "graduations": 0, "pools": 0} + for e in events: + ev = e["event"] + if ev == "TokenLaunched": + a = ensure_asset(conn, rpc, e["pair_token"]) + conn.execute("INSERT INTO rh_tokens (token, launch_block, launch_ts, launch_tx, creator, curve, quote_asset, quote_class, decimals, supply_raw) " + "VALUES (%s, %s, to_timestamp(%s), %s, %s, %s, %s, %s, %s, %s) ON CONFLICT (token) DO NOTHING", + (e["token"], e["block"], e["ts"], e["tx_hash"], e["deployer"], e["curve"], a["asset"], a["class"], PONS_DECIMALS, 10 ** 27)) + conn.execute("INSERT INTO rh_assets (asset, class, decimals, graduation_threshold_raw) VALUES (%s, %s, %s, %s) " + "ON CONFLICT (asset) DO UPDATE SET graduation_threshold_raw = EXCLUDED.graduation_threshold_raw", (a["asset"], a["class"], a["decimals"], e["threshold_raw"])) + conn.execute("INSERT INTO rh_insiders (mint, wallet, kind) VALUES (%s, %s, 'dev') ON CONFLICT DO NOTHING", (e["token"], e["deployer"])) + n["launches"] += 1 + elif ev == "Launched": + conn.execute("UPDATE rh_tokens SET dev_quote_in_raw = COALESCE(dev_quote_in_raw, 0) + %s, dev_tokens_raw = COALESCE(dev_tokens_raw, 0) + %s WHERE token = %s", + (e["quote_in_raw"], e["tokens_out_raw"], e["token"])) + conn.execute("INSERT INTO rh_insiders (mint, wallet, kind) VALUES (%s, %s, 'dev') ON CONFLICT DO NOTHING", (e["token"], e["recipient"])) + elif ev == "PoolGraduated": + conn.execute("UPDATE rh_tokens SET graduated_at = to_timestamp(%s), grad_block = %s, grad_position_id = %s, grad_token_raw = %s, grad_quote_raw = %s, " + "status = 'graduated', updated_at = now() WHERE token = %s", + (e["ts"], e["block"], e["position_id"], e["token_amount_raw"], e["pair_amount_raw"], e["token"])) + n["graduations"] += 1 + elif ev == "PoolRegistered": + q = e["quote_asset"]; tok = e["token"] + c0, c1 = sorted([q, tok], key=lambda a: int(a, 16)) + conn.execute("INSERT INTO rh_pools (pool_id, token, currency0, currency1, token_is_0, fee, tick_spacing, hooks, quote_asset, init_block, init_ts, is_pons) " + "VALUES (%s, %s, %s, %s, %s, 0, 200, %s, %s, %s, to_timestamp(%s), true) ON CONFLICT (pool_id) DO UPDATE SET is_pons = true, token = EXCLUDED.token", + (e["pool_id"], tok, c0, c1, c0 == tok, config.PONS_HOOK.lower(), q, e["block"], e["ts"])) + conn.execute("UPDATE rh_tokens SET pool_id = %s, status = 'graduated', graduated_at = COALESCE(graduated_at, to_timestamp(%s)) WHERE token = %s", + (e["pool_id"], e["ts"], tok)) + n["pools"] += 1 + n["curve_trades"] += index_curve(conn, rpc, tok, e["block"]) + return n + + +def apply_v4(conn, rpc, logs: list[dict], fees: list[dict]) -> dict: + """v4 logs of Pons pools (+ their hook fees) → rh_swaps / rh_liquidity / rh_pools state.""" + pools = {r["pool_id"]: dict(r) for r in conn.execute( + "SELECT p.pool_id, p.token, p.token_is_0, p.quote_asset, p.hook_fee, a.decimals AS qdec FROM rh_pools p JOIN rh_assets a ON a.asset = p.quote_asset WHERE p.is_pons").fetchall()} + fee_q: dict[tuple, list] = {}; observed: dict[str, int] = {} + for f in fees: + fee_q.setdefault((f["tx_hash"], f["pool_id"]), []).append(f) + evs = [v4.decode(x) for x in _order(logs)] + evs = [e for e in evs if e is not None] + if evs: + fill_timestamps(rpc, evs, min(e["block"] for e in evs), max(e["block"] for e in evs)) + swaps = [e for e in evs if e and e["event"] == "Swap" and e["pool_id"] in pools] + for s_ in swaps: + ti0 = bool(pools[s_["pool_id"]]["token_is_0"]); s_["side"] = 1 if (s_["amount0"] if ti0 else s_["amount1"]) > 0 else -1 + toks = sorted({pools[s_["pool_id"]]["token"] for s_ in swaps}) + transfers = [] + if swaps: + lo_b, hi_b = min(s_["block"] for s_ in swaps), max(s_["block"] for s_ in swaps) + per = TOKENS_PER_REQUEST_BULK if hasattr(rpc, "hs") else TOKENS_PER_REQUEST + for i in range(0, len(toks), per): + transfers += _get_logs(rpc, toks[i:i + per], lo_b, hi_b, [TRANSFER]) + senders = traders_from_transfers(transfers, swaps, {k: v["token"] for k, v in pools.items()}) + n = {"swaps": 0, "liquidity": 0} + last_state: dict[str, dict] = {} + for e in evs: + if e is None or e["pool_id"] not in pools: + continue + p = pools[e["pool_id"]] + if e["event"] == "Initialize": + conn.execute("UPDATE rh_pools SET sqrt_price_x96 = %s, tick = %s, fee = %s, tick_spacing = %s, hooks = %s, updated_block = %s WHERE pool_id = %s", + (e["sqrt_price_x96"], e["tick"], e["fee"], e["tick_spacing"], e["hooks"], e["block"], e["pool_id"])) + elif e["event"] == "ModifyLiquidity": + conn.execute("INSERT INTO rh_liquidity (block, log_index, tx_hash, ts, pool_id, sender, tick_lower, tick_upper, liquidity_delta, salt) " + "VALUES (%s, %s, %s, to_timestamp(%s), %s, %s, %s, %s, %s, %s) ON CONFLICT DO NOTHING", + (e["block"], e["log_index"], e["tx_hash"], e["ts"], e["pool_id"], e["sender"], e["tick_lower"], e["tick_upper"], e["liquidity_delta"], e["salt"])) + n["liquidity"] += 1 + elif e["event"] == "Swap": + ti0 = bool(p["token_is_0"]); qdec = int(p["qdec"]) + tok_amt, q_amt = (e["amount0"], e["amount1"]) if ti0 else (e["amount1"], e["amount0"]) + side = 1 if tok_amt > 0 else -1 # the swapper received tokens: a buy + fq = fee_q.get((e["tx_hash"], e["pool_id"])) or [] + fee = fq.pop(0) if fq else None + price_q = v4.token_price_in_quote(e["sqrt_price_x96"], ti0, PONS_DECIMALS, qdec) + fee_frac = None + if fee: + net = abs(tok_amt) if fee["currency"] == p["token"] else abs(q_amt) # the fee is taken from the incoming currency + taken = fee["fee_raw"] + fee["tax_raw"] + fee_frac = taken / (net + taken) if net + taken > 0 else None + observed[e["pool_id"]] = observed.get(e["pool_id"], 0) + 1 + elif p.get("hook_fee") is not None: + fee_frac = float(p["hook_fee"]) # the pool's measured, fixed fee + resq_q = v4.quote_reserve(e["sqrt_price_x96"], e["liquidity"], ti0, qdec) + conn.execute( + "INSERT INTO rh_swaps (block, log_index, tx_hash, block_hash, ts, pool_id, token, trader, side, token_raw, quote_raw, hook_fee_raw, hook_tax_raw, " + "sqrt_price_x96, liquidity, tick, price_q, resq_q, fee_frac) VALUES (%s,%s,%s,%s,to_timestamp(%s),%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING", + (e["block"], e["log_index"], e["tx_hash"], e["block_hash"], e["ts"], e["pool_id"], p["token"], senders.get((e["tx_hash"], e["pool_id"])), side, abs(tok_amt), + abs(q_amt), fee["fee_raw"] if fee else None, fee["tax_raw"] if fee else None, e["sqrt_price_x96"], e["liquidity"], e["tick"], price_q, resq_q, fee_frac)) + last_state[e["pool_id"]] = e; n["swaps"] += 1 + for pid, k in observed.items(): # a pool's fee is known once FEE_OBS swaps showed it + r = conn.execute("UPDATE rh_pools SET fee_obs = fee_obs + %s WHERE pool_id = %s RETURNING fee_obs, hook_fee", (k, pid)).fetchone() + if r and r["fee_obs"] >= FEE_OBS and r["hook_fee"] is None: + conn.execute("UPDATE rh_pools SET hook_fee = (SELECT percentile_cont(0.5) WITHIN GROUP (ORDER BY fee_frac) FROM rh_swaps " + "WHERE pool_id = %s AND hook_fee_raw IS NOT NULL AND fee_frac IS NOT NULL) WHERE pool_id = %s", (pid, pid)) + for pid, e in last_state.items(): + conn.execute("UPDATE rh_pools SET sqrt_price_x96 = %s, liquidity = %s, tick = %s, updated_block = %s WHERE pool_id = %s", + (e["sqrt_price_x96"], e["liquidity"], e["tick"], e["block"], pid)) + return n + + +def step(conn, rpc, lo: int, hi: int) -> dict: + """Index blocks lo..hi (inclusive) into ``conn`` (the caller commits with the cursor).""" + topics0 = V4_TOPICS0 + bulk = hasattr(rpc, "hs") # HyperSync: no per-query cap, so no pool-id chunks + main = _get_logs(rpc, [config.PONS_FACTORY, config.PONS_ROUTER, config.PONS_HOOK], lo, hi, PONS_TOPICS) + evs = [pons.decode(x) for x in _order(main)] + evs = [e for e in evs if e is not None] + fill_timestamps(rpc, evs, lo, hi) + n = apply_pons(conn, rpc, evs) + ids = [r["pool_id"] for r in conn.execute("SELECT pool_id FROM rh_pools WHERE is_pons ORDER BY pool_id").fetchall()] + need = [r["pool_id"] for r in conn.execute("SELECT pool_id FROM rh_pools WHERE is_pons AND hook_fee IS NULL ORDER BY pool_id").fetchall()] + fee_topic = [FEE_TOPIC] + v4_logs, fee_logs = [], [] + if bulk and hi - lo + 1 <= LIVE_SPAN_BULK: # a live range: every PoolManager event, matched to Pons pools here + mine = set(ids) # (10k+ pool ids make a ~0.7 MB filter; HyperSync caps bodies at 2 MB) + v4_logs = [g for g in _get_logs(rpc, [config.V4_POOL_MANAGER], lo, hi, [topics0]) if len(g["topics"]) > 1 and g["topics"][1].lower() in mine] + else: + per = IDS_PER_REQUEST_BULK if bulk else IDS_PER_REQUEST + for i in range(0, len(ids), per): # swaps of every Pons pool, in id chunks + v4_logs += _get_logs(rpc, [config.V4_POOL_MANAGER], lo, hi, [topics0, ids[i:i + per]]) + per = IDS_PER_REQUEST_BULK if bulk else IDS_PER_REQUEST + for i in range(0, len(need), per): # hook fees only of pools whose fee is not known yet + fee_logs += _get_logs(rpc, [config.PONS_HOOK], lo, hi, [fee_topic, need[i:i + per]]) + fees = [e for e in (pons.decode(x) for x in _order(fee_logs)) if e is not None and e["event"] == "HookFeeCollected"] + n.update(apply_v4(conn, rpc, v4_logs, fees)) + n["hook_fees"] = len(fees) + return n + + +def start_block(rpc) -> int: + """``RH_START_BLOCK``, else the first block of ``RH_START_DAY`` (Pons V2 went live 2026-08-04).""" + if config.RH_START_BLOCK: + return config.RH_START_BLOCK + from datetime import datetime, timezone + return block_at(rpc, datetime.fromisoformat(config.RH_START_DAY).replace(tzinfo=timezone.utc).timestamp()) + + +def run_once(rpc: RhRpc | None = None, *, head_margin: int | None = None, max_ranges: int = 1, head: int | None = None) -> dict: + """Advance the cursor by up to ``max_ranges`` ranges, each ending at most ``head − head_margin`` (RH_CONFIRMATIONS); + ``head``: the last block to index, already settled (a live pass's fetched range, rh/prefetch.py). + Returns {through, head, ranges, counts}.""" + rpc = rpc or logs_rpc() + margin = config.RH_CONFIRMATIONS if head_margin is None else head_margin + head = rpc.block_number() - margin if head is None else head + total: dict = {}; done = 0 + with transaction() as conn: + at, rng = scan.cursor(conn, CURSOR, -1) + if at < 0: + sb = start_block(rpc) + with transaction() as conn: + scan.set_cursor(conn, CURSOR, sb - 1, rng, {"start_block": sb}); at = sb - 1 + for _ in range(max_ranges): + r = scan.next_range(at, rng, head) + if r is None: + break + lo, hi = r + try: + with transaction() as conn: + c = step(conn, rpc, lo, hi) + rng = scan.grow(rng); at = hi + scan.set_cursor(conn, CURSOR, at, rng) + for k, v in c.items(): + total[k] = total.get(k, 0) + v + done += 1 + except Exception as e: + rng = scan.shrink(rng) if not scan.is_rate_limit(e) else rng + with transaction() as conn: + scan.set_cursor(conn, CURSOR, at, rng) + log.warning("rh index %d..%d: %s (range now %d)", lo, hi, str(e)[:200], rng) + import time + time.sleep(scan.BACKOFF_S) + break + return {"through": at, "head": head, "ranges": done, "counts": total} diff --git a/fly_trader/rh/kyber.py b/fly_trader/rh/kyber.py new file mode 100644 index 0000000000000000000000000000000000000000..f1c7c29a1a31a8bc7d204bdfcf2602d0af6b54d7 --- /dev/null +++ b/fly_trader/rh/kyber.py @@ -0,0 +1,146 @@ +"""KyberSwap aggregator on Robinhood Chain for the bot: quote (routes) and calldata (route/build), with every check the +website's client makes (web/src/lib/kyber.ts) before anything is signed: + +- the router is the allowlisted MetaAggregationRouterV2 (``config.KYBER_ROUTER``), in the quote and in the build; +- the quote answers the swap asked for (tokens, amount); +- the built calldata (``swap`` 0xe21fd0e9 / ``swapSimpleMode`` 0x8af033fb, decoded here) pays this wallet, swaps exactly + the quoted tokens and amount, lists no fee, and carries a minReturnAmount of at least the slippage floor; +- the build delivers at most ``BUILD_DRIFT_BPS`` less than the quote; +- the bot's own addition: the transaction's native value equals the amount when the input is native ETH, and is 0 otherwise. +""" +from __future__ import annotations + +import time + +import httpx + +from .. import config +from ..db.apilog import record_api_call +from . import abi + +NATIVE = "0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" +MIN_SLIPPAGE_BPS, MAX_SLIPPAGE_BPS = 20, 1000 +BUILD_DRIFT_BPS = 50 +DEADLINE_S = 20 * 60 +_DESC = "(address,address,address[],uint256[],address[],uint256[],address,uint256,uint256,uint256,bytes)" +SWAP_SIG = f"swap((address,address,bytes,{_DESC},bytes))" +SWAP_SIMPLE_SIG = f"swapSimpleMode(address,{_DESC},bytes,bytes)" + + +class KyberError(RuntimeError): + pass + + +def _same(a: str | None, b: str | None) -> bool: + return bool(a) and bool(b) and a.lower() == b.lower() + + +def _big(v) -> int | None: + return int(v) if isinstance(v, str) and v.isdigit() else (v if isinstance(v, int) and v >= 0 else None) + + +def clamp_slippage(bps: float) -> int: + try: + return int(min(MAX_SLIPPAGE_BPS, max(MIN_SLIPPAGE_BPS, round(float(bps))))) + except (TypeError, ValueError): + return MIN_SLIPPAGE_BPS + + +def min_received(amount_out: int, slippage_bps: float) -> int: + return amount_out * (10_000 - clamp_slippage(slippage_bps)) // 10_000 + + +def _allowed_router(addr: str | None) -> str: + if not _same(addr, config.KYBER_ROUTER): + raise KyberError(f"refusing: KyberSwap returned router {addr or '(none)'}, not the allowlisted {config.KYBER_ROUTER}") + return addr.lower() + + +class Kyber: + def __init__(self, base: str | None = None, client: httpx.Client | None = None): + self.base = base or config.KYBER_API + self.http = client or httpx.Client(timeout=15.0, headers={"x-client-id": config.KYBER_CLIENT_ID}) + + def _req(self, method: str, path: str, **kw) -> dict: + t0 = time.monotonic(); status, ok, err = None, False, None + try: + r = self.http.request(method, f"{self.base}{path}", **kw) + status = r.status_code + body = r.json() if r.content else None + if r.status_code != 200 or not body or body.get("code") != 0 or not body.get("data"): + err = (body or {}).get("message") or f"HTTP {r.status_code}" + raise KyberError(f"KyberSwap: {err}") + ok = True + return body["data"] + except httpx.HTTPError as e: + err = type(e).__name__ + raise KyberError("KyberSwap could not be reached") from None + finally: + record_api_call("kyber", path.split("?")[0], method, status, int((time.monotonic() - t0) * 1000), ok, err) + + def get_route(self, token_in: str, token_out: str, amount_in: int, included_sources: str | None = None) -> dict: + """{routeSummary, routerAddress} for exactly this swap, or KyberError (incl. 'no route'). ``included_sources``: only + these venues (a comma list of KyberSwap source ids).""" + params = {"tokenIn": token_in, "tokenOut": token_out, "amountIn": str(int(amount_in)), "gasInclude": "true"} + if included_sources: + params["includedSources"] = included_sources + d = self._req("GET", "/routes", params=params) + router = _allowed_router(d.get("routerAddress")) + rs = d.get("routeSummary") or {} + if not _same(rs.get("tokenIn"), token_in) or not _same(rs.get("tokenOut"), token_out) or _big(rs.get("amountIn")) != int(amount_in) \ + or _big(rs.get("amountOut")) is None: + raise KyberError("refusing the quote: KyberSwap answered for a different swap than the one asked for") + return {"routeSummary": rs, "routerAddress": router} + + def build_route(self, route: dict, account: str, slippage_bps: float, now_s: int | None = None) -> dict: + now_s = int(time.time()) if now_s is None else now_s + d = self._req("POST", "/route/build", json={"routeSummary": route["routeSummary"], "sender": account, "recipient": account, + "slippageTolerance": clamp_slippage(slippage_bps), "deadline": now_s + DEADLINE_S, + "source": config.KYBER_CLIENT_ID}) + built = {**d, "routerAddress": _allowed_router(d.get("routerAddress"))} + check_build(built, route, account, slippage_bps) + return built + + +def decode_desc(data: str) -> dict: + """The SwapDescriptionV2 of a built call, as a dict.""" + cd = bytes.fromhex(data[2:] if data.startswith("0x") else data) + if cd[:4] == abi.selector(SWAP_SIG): + desc = abi.decode_call(SWAP_SIG, cd)[0][3] + elif cd[:4] == abi.selector(SWAP_SIMPLE_SIG): + desc = abi.decode_call(SWAP_SIMPLE_SIG, cd)[1] + else: + raise KyberError("refusing the swap: calldata is neither swap nor swapSimpleMode") + keys = ("srcToken", "dstToken", "srcReceivers", "srcAmounts", "feeReceivers", "feeAmounts", "dstReceiver", "amount", "minReturnAmount", "flags", "permit") + return dict(zip(keys, desc)) + + +def check_build(built: dict, route: dict, account: str, slippage_bps: float) -> None: + rs = route["routeSummary"] + refuse = lambda why: (_ for _ in ()).throw(KyberError(f"refusing the swap: KyberSwap built {why}")) + amount_in, quoted_out, built_out = _big(rs.get("amountIn")), _big(rs.get("amountOut")), _big(built.get("amountOut")) + if amount_in is None or quoted_out is None or built_out is None: + refuse("a transaction without readable amounts") + if _big(built.get("amountIn")) != amount_in: + refuse("a transaction for a different amount") + if built_out * 10_000 < quoted_out * (10_000 - BUILD_DRIFT_BPS): + refuse("a transaction that delivers much less than quoted") + try: + desc = decode_desc(built["data"]) + except KyberError: + raise + except Exception: + refuse("calldata the bot cannot verify") + if not _same(desc["dstReceiver"], account): + refuse("a transaction that pays another address") + if not _same(desc["srcToken"], rs.get("tokenIn")) or not _same(desc["dstToken"], rs.get("tokenOut")): + refuse("a transaction for other tokens") + if desc["amount"] != amount_in: + refuse("a transaction for a different amount") + if desc["feeReceivers"] or any(desc["feeAmounts"]): + refuse("a transaction that pays a fee") + if desc["minReturnAmount"] < min_received(built_out, slippage_bps): + refuse("a transaction without the slippage limit asked for") + value = _big(str(built.get("transactionValue") or "0")) + if value != (amount_in if _same(rs.get("tokenIn"), NATIVE) else 0): + refuse("a transaction whose ETH value does not match the swap") diff --git a/fly_trader/rh/live.py b/fly_trader/rh/live.py new file mode 100644 index 0000000000000000000000000000000000000000..deafd2e5bfed8cad7a7a827a59f16fb548c26d85 --- /dev/null +++ b/fly_trader/rh/live.py @@ -0,0 +1,206 @@ +"""The Robinhood Chain fly's live book (``live_rh``): the paper RH fly's minute mirrored on the bot's ETH wallet once +the fly holds the RH seat (handover_rh) and signing is allowed (config.rh_live_prerequisites_missing, rh/guard.py). + +``RhLiveMirror.minute`` — agent/fly_live.LiveMirror's shape, in ETH, through rh/exec.RhExecutor: +- drains the executor (one intent at a time; each booked by rh/accounting.py from measured balance deltas); +- exits: positions held their hold are sold (exit slippage, the forced ladder once ``OVERDUE_S`` late); ``DEAD_BAG_S`` + past the hold an unsellable position is written off and its tokens swept hourly; +- entries: this minute's paper RH entries, sized from the wallet (native ETH + open positions at cost as the bankroll, + native − committed buys as cash, the RH gas reserve never spent), one position per token; blocked by circuit 3 + (kill switch, trip, paused entries) or a stale mark of the pool's base asset (a stock outside its trading hours); +- wealth: native + positions net of the Pons exit cost + base lots → ``wealth_marks`` (book live_rh); the kill switch + on its own peak (circuit 3, ``RH_KILL_SWITCH_DRAWDOWN``), liquidating with ``KILL_SWITCH_LIQUIDATE``; +- gap: live trailing its paper mirror by more than ``GAP_MAX`` per trade pauses circuit 3's entries; +- hourly: dead-bag tokens swept, base lots sold back to ETH; every ``RECONCILE_S`` the chain is reconciled with the + books (rh/accounting.reconcile) while no intent is in flight. +""" +from __future__ import annotations + +import json +import logging +from datetime import timedelta + +import numpy as np + +from .. import config +from ..agent import rails, sizing +from ..db.apilog import record_event +from ..execution import ledger +from ..markets import RH, RH_CIRCUIT +from . import accounting as A +from .exec import ZERO, RhExecutor, RhRequest + +log = logging.getLogger(__name__) +BOOK, MIRROR = RH.live_book, RH.fly_book +DEAD_BAG_S, OVERDUE_S, SWEEP_S, RECONCILE_S = 3 * 3600.0, 600.0, 3600.0, 600.0 +GAP_TRADES, GAP_MAX = 50, 0.02 + + +def minute(book, ctx, st: dict) -> dict: + """One minute of the live RH mirror for ``book`` (agent/fly_session.FlyBook on chain 'rh').""" + missing = config.rh_live_prerequisites_missing() + if missing: + return {"stage": "live gated: " + ", ".join(missing)} + if book.mirror is None: + book.mirror = RhLiveMirror(book.H) + record_event("info", "rh_live", "the RH fly trades the ETH wallet", {"run_id": book.run_id}) + return book.mirror.minute(ctx, run_id=book.run_id, beat_id=st["beat_id"], entries=st["entries"]) + + +def reserved_wei(conn) -> int: + """ETH owed to vault lockers and held in the trading wallet (never traded): the vault's ETH pot, 0 until it exists.""" + try: + from ..vault import settle_eth + except ImportError: + return 0 + return int(settle_eth.reserved_in_trading(conn)) + + +def pool_of(conn, pool_id: str | None) -> dict | None: + if not pool_id: + return None + r = conn.execute("SELECT pool_id, token, currency0, currency1, fee, tick_spacing, hooks, quote_asset FROM rh_pools WHERE pool_id = %s", (pool_id,)).fetchone() + return dict(r) if r else None + + +class RhLiveMirror: + def __init__(self, horizon_s: float, executor: RhExecutor | None = None): + if executor is None: + from .rpc import RhRpc + from .wallet import RhWallet + executor = RhExecutor(RhWallet(RhRpc())) + self.ex, self.H = executor, float(horizon_s) + self.last_sweep = self.last_reconcile = 0.0 + + @property + def wallet(self): + return self.ex.wallet + + def _decision(self, conn, ctx, run_id, beat_id, p, reason: str, forced: bool) -> int: + return int(conn.execute("INSERT INTO decisions (beat_id, run_id, ts, mint, pool, kind, size_sol, forced, reason, detail) VALUES (%s,%s,%s,%s,%s,'fly_exit',%s,%s,%s,%s) RETURNING id", + (beat_id, run_id, ctx.m1, p["mint"], p["pool"], float(p["cost_sol"]), forced, reason, json.dumps({"book": BOOK}))).fetchone()["id"]) + + def _sell(self, conn, p: dict, did: int | None, slip: float, forced_kind: str | None = None) -> bool: + pool = pool_of(conn, p.get("pool")) + return self.ex.submit(RhRequest(kind="sell", token=p["mint"], quote_asset=(p.get("quote_asset") or (pool or {}).get("quote_asset") or ZERO), amount_in=int(p["qty"]), + slippage_bps=slip, max_slippage_bps=config.SLIPPAGE_FORCED_BPS, pool=pool, decision_id=did, position_id=int(p["id"]), + forced_kind=forced_kind, decimals=int(p.get("decimals") or 18))) + + def minute(self, ctx, *, run_id: str, beat_id: int, entries: list) -> dict: + conn, m1 = ctx.conn, ctx.m1 + drained = self.ex.drain() + for r in drained: + if not r.ok: + log.warning("live rh %s %s failed: %s", r.request.kind, r.request.token, r.error) + inflight = self.ex.pending() + native = self.wallet.balance(); reserved = reserved_wei(conn) + n_exit = n_dead = 0 + for p in ledger.open_positions(conn, BOOK): + held = (m1 - p["opened_at"]).total_seconds(); H = float(p.get("hold_s") or self.H) + if held < H or p["mint"].lower() in inflight: + continue + if held >= H + DEAD_BAG_S: + did = self._decision(conn, ctx, run_id, beat_id, p, f"dead-bag: unsold {held / 3600:.1f} h after entry", True) + ledger.close_position(conn, position_id=int(p["id"]), exit_price=0.0, proceeds_sol=0.0, fees_sol=0.0, decision_id=did, forced_kind="dead_bag", ts=m1) + record_event("warning", "rh_live", f"dead-bag: {p['mint']} could not be sold; written off, its tokens are swept hourly", {"position": int(p["id"])}) + n_dead += 1 + continue + overdue = held >= H + OVERDUE_S + did = self._decision(conn, ctx, run_id, beat_id, p, f"held {held / 60:.0f} min" + (" (retrying)" if overdue else ""), False) + n_exit += self._sell(conn, p, did, config.SLIPPAGE_FORCED_BPS if overdue else config.SLIPPAGE_EXIT_BPS) + opens = ledger.open_positions(conn, BOOK); held_mints = {p["mint"].lower() for p in opens} | inflight + c = rails.load_circuit(conn, RH_CIRCUIT) + blocked = "kill switch" if c.kill_switch else "circuit tripped" if c.tripped else "paused" if c.entries_paused else None + native_eth = native / A.WEI; committed = self.ex.committed_wei() / A.WEI + bankroll = native_eth + sum(float(p["cost_sol"]) for p in opens) - reserved / A.WEI + cash = native_eth - reserved / A.WEI - committed; n_enter = 0; skipped = [] + for e in entries: + mint = e["mint"].lower() + if mint in held_mints: + skipped.append((mint, "held")); continue + pool = pool_of(conn, e["info"].get("pool")); quote = ((pool or {}).get("quote_asset") or ZERO).lower() + _, stale = A.base_mark(conn, quote) + if stale: + skipped.append((mint, "base mark stale")); continue + size, why = sizing.size_position(e["score"], e.get("threshold", 0.0), e.get("table") or [], bankroll, cash, e["info"]["resq"], + flat=config.FLY_SIZING == "flat", market=RH) + if blocked or size <= 0: + skipped.append((mint, blocked or why)); continue + if self.ex.submit(RhRequest(kind="buy", token=mint, quote_asset=quote, amount_in=int(size * A.WEI), slippage_bps=config.SLIPPAGE_ENTRY_BPS, + max_slippage_bps=config.MAX_SLIPPAGE_ENTRY_BPS, pool=pool, decision_id=e["decision_id"], hold_s=e.get("hold_s"), + strategy=e.get("strategy"), decimals=int(e["info"].get("decimals") or 18))): + n_enter += 1; cash -= size; held_mints.add(mint) + conn.execute("UPDATE decisions SET book_targets = array_append(COALESCE(book_targets, '{}'), %s) WHERE id = %s", (BOOK, e["decision_id"])) + ledger.mark_positions(conn, BOOK, ctx.prices, {}, m1) + w = A.wealth(conn, native, reserved, ctx.prices, {p["mint"]: (ctx.resqs.get(p["mint"]) or ctx.last_resq(p["mint"])) for p in opens}, ctx.fees) + rebase = rails.kill_rebase_at(conn, RH_CIRCUIT) + pk = conn.execute("SELECT max(wealth) AS pk FROM wealth_marks WHERE book = %s AND (%s::timestamptz IS NULL OR ts > %s)", (BOOK, rebase, rebase)).fetchone() + peak = max(float(pk["pk"] or 0.0), w["wealth"]) + conn.execute("INSERT INTO wealth_marks (beat_id, book, ts, sol_free, positions_value, exit_cost, wealth, peak, drawdown, exposure, n_open) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) " + "ON CONFLICT (beat_id, book) DO NOTHING", + (beat_id, BOOK, m1, native_eth, w["positions"] + w["lots"], w["exit_cost"], w["wealth"], peak, (1.0 - w["wealth"] / peak) if peak > 0 else 0.0, + w["exposure"], w["n_open"])) + killed = rails.check_drawdown(conn, w["wealth"], peak, circuit_id=RH_CIRCUIT, drawdown=RH.kill_drawdown(), unit="ETH") + n_liq = self.liquidate(conn, ctx, run_id, beat_id) if killed and config.KILL_SWITCH_LIQUIDATE else 0 + gap = self.gap(conn) + if ctx.m1_epoch - self.last_sweep >= SWEEP_S: + self.last_sweep = ctx.m1_epoch; self.sweep(conn, m1) + rec = None + if ctx.m1_epoch - self.last_reconcile >= RECONCILE_S and not self.ex.pending() and self.wallet.lock.acquire(blocking=False): + try: + self.last_reconcile = ctx.m1_epoch; rec = A.reconcile(conn, self.wallet) + finally: + self.wallet.lock.release() + return {"stage": "trading live", "wallet_eth": native_eth, "wealth": w["wealth"], "entered": n_enter, "exited": n_exit, "dead_bags": n_dead, + "in_flight": len(inflight), "blocked": blocked, "skipped": skipped[:10], "gap": gap, "failed_orders": sum(1 for r in drained if not r.ok), + "liquidated": n_liq, "reconcile": rec} + + def liquidate(self, conn, ctx, run_id: str, beat_id: int) -> int: + n = 0; inflight = self.ex.pending() + for p in ledger.open_positions(conn, BOOK): + if p["mint"].lower() in inflight: + continue + did = self._decision(conn, ctx, run_id, beat_id, p, "kill switch: liquidating", True) + n += self._sell(conn, p, did, config.SLIPPAGE_FORCED_BPS) + if n: + record_event("error", "rh_live", f"kill switch: liquidating {n} open live RH position(s)", {"positions": n}) + return n + + def gap(self, conn) -> dict: + rows = conn.execute("""SELECT l.realized_sol / NULLIF(l.cost_sol, 0) AS rl, p.realized_sol / NULLIF(p.cost_sol, 0) AS rp + FROM positions l JOIN positions p ON p.entry_decision_id = l.entry_decision_id AND p.book = %s AND p.status = 'closed' + WHERE l.book = %s AND l.status = 'closed' AND l.entry_decision_id IS NOT NULL ORDER BY l.closed_at DESC LIMIT %s""", + (MIRROR, BOOK, GAP_TRADES)).fetchall() + pairs = [(r["rl"], r["rp"]) for r in rows if r["rl"] is not None and r["rp"] is not None] + rl = np.array([a for a, _ in pairs], dtype=np.float64); rp = np.array([b for _, b in pairs], dtype=np.float64) + out = {"trades": int(len(rl)), "live_mean": float(rl.mean()) if len(rl) else None, "mirror_mean": float(rp.mean()) if len(rp) else None} + if len(rl) >= GAP_TRADES and rl.mean() - rp.mean() < -GAP_MAX: + if not rails.load_circuit(conn, RH_CIRCUIT).entries_paused: + conn.execute("UPDATE circuit_state SET entries_paused = true, updated_at = now() WHERE id = %s", (RH_CIRCUIT,)) + conn.execute("INSERT INTO circuit_events (kind, detail) VALUES ('gap_pause', %s)", (json.dumps({**out, "circuit": RH_CIRCUIT}),)) + record_event("error", "rh_live", f"RH entries paused: live trails its paper mirror by {(rp.mean() - rl.mean()) * 100:.1f} points per trade", out) + out["paused"] = True + return out + + def sweep(self, conn, m1) -> int: + """Sell the last week's dead-bag tokens and every open base lot (exit legs that stopped half way, stray base).""" + n = 0; inflight = self.ex.pending() + for r in conn.execute("SELECT DISTINCT mint, pool, quote_asset FROM positions WHERE book = %s AND forced_exit_kind = 'dead_bag' AND closed_at > %s", + (BOOK, m1 - timedelta(days=7))).fetchall(): + amt = self.wallet.balance_of(r["mint"]) + if amt > 0 and r["mint"].lower() not in inflight: + n += self.ex.submit(RhRequest(kind="sweep", token=r["mint"], quote_asset=r["quote_asset"] or ZERO, amount_in=amt, slippage_bps=config.SLIPPAGE_FORCED_BPS, + max_slippage_bps=config.SLIPPAGE_FORCED_BPS, pool=pool_of(conn, r["pool"]), forced_kind="dead_bag_sweep")) + for lot in A.open_lots(conn): + if lot["asset"] == A.NATIVE or lot["asset"].lower() in inflight: + continue + mark, _ = A.base_mark(conn, lot["asset"]) + value = int(lot["qty_raw"]) / 10 ** A._decimals(conn, lot["asset"]) * (mark or 0.0) + if value < config.RH_MIN_SWEEP_ETH: + conn.execute("UPDATE rh_base_lots SET status = 'dust', closed_at = now() WHERE id = %s", (lot["id"],)); continue + n += self.ex.submit(RhRequest(kind="liquidate", token=lot["asset"], amount_in=int(lot["qty_raw"]), slippage_bps=config.SLIPPAGE_EXIT_BPS, + max_slippage_bps=config.SLIPPAGE_FORCED_BPS, lot_id=int(lot["id"]))) + return n + + def stop(self) -> None: + self.ex.stop() diff --git a/fly_trader/rh/meta.py b/fly_trader/rh/meta.py new file mode 100644 index 0000000000000000000000000000000000000000..9d06453f367e0df506b1721e4861dc29c26a9824 --- /dev/null +++ b/fly_trader/rh/meta.py @@ -0,0 +1,125 @@ +"""Per-launch facts for graduated Pons launches (``rh_meta``): corpus_meta's columns and definitions (train/corpus_meta.py), +so both chains feed the same model inputs; amounts in ETH. + +- ttg_min: launch → graduation, minutes; dev_sol / dev_tokens / dev_share: the deployer's own first buy (the launch + router's ``Launched``: quote in ETH at the launch's base mark, tokens, share of the 1e9 supply); mayhem: 0 (Pons has none); +- rq0: the quote the graduation seeded the pool with, in ETH (Solana: the ~79 SOL reserve at migration); +- prior_launches / prior_grads / prior_known / prior_rug_share / prior_moon_share: the deployer's earlier launches and + graduations, and among the graduations whose 60-minute outcome was known by this graduation, the share that fell 90 % + (rug) or doubled (moon) — corpus_meta.creator_history's definitions on Pons rows; +- own_dd60 / own_max60: the launch's own first-hour low / high after graduation (from rh_minutes, filled once 61 minutes old); +- bundle_share / dev_hold_share / dev_sold_frac / grad_hhi: corpus_meta.curve_facts on the curve trades (the launch block is + the "create slot"; the deployer's first buy is a curve leg on Pons, so no separate initial buy), and its insiders + (deployer, bundle wallets) → rh_insiders. +""" +from __future__ import annotations + +import logging +from datetime import timedelta + +from ..db.connection import transaction +from ..train.corpus_meta import curve_facts +from . import prices + +log = logging.getLogger(__name__) +SUPPLY = 1e9 +TOKEN = 1e18 + + +def _qty(raw, dec: int) -> float: + return float(raw or 0) / 10 ** dec + + +def build(conn, token: str) -> dict | None: + t = conn.execute("SELECT t.*, a.decimals AS qdec FROM rh_tokens t JOIN rh_assets a ON a.asset = t.quote_asset WHERE t.token = %s AND t.status = 'graduated'", + (token,)).fetchone() + if not t or t["graduated_at"] is None: + return None + qdec = int(t["qdec"]) + b_launch, _ = prices.base_eth(conn, t["quote_asset"], t["launch_ts"]) if t["launch_ts"] else (None, True) + b_grad, _ = prices.base_eth(conn, t["quote_asset"], t["graduated_at"]) + dev_tokens = _qty(t["dev_tokens_raw"], 18) + out = {"mint": token, "graduated_at": t["graduated_at"], "create_ts": t["launch_ts"], "creator": t["creator"], "quote_asset": t["quote_asset"], + "quote_class": t["quote_class"], "supply": SUPPLY, "pool_id": t["pool_id"], "mayhem": False, + "ttg_min": (t["graduated_at"] - t["launch_ts"]).total_seconds() / 60.0 if t["launch_ts"] else None, + "dev_sol": _qty(t["dev_quote_in_raw"], qdec) * b_launch if b_launch is not None and t["dev_quote_in_raw"] is not None else None, + "dev_tokens": dev_tokens, "dev_share": dev_tokens / SUPPLY, + "rq0": _qty(t["grad_quote_raw"], qdec) * b_grad if b_grad is not None and t["grad_quote_raw"] is not None else None} + # the deployer's history, as corpus_meta.creator_history + c, g = t["creator"], t["graduated_at"] + out["prior_launches"] = int(conn.execute("SELECT count(*) AS n FROM rh_tokens WHERE creator = %s AND launch_ts < %s", (c, t["launch_ts"])).fetchone()["n"]) \ + if t["launch_ts"] else None + r = conn.execute("""SELECT count(*) AS grads, + count(*) FILTER (WHERE own_dd60 IS NOT NULL AND graduated_at + interval '60 minutes' <= %(g)s) AS known, + avg(CASE WHEN own_dd60 <= -0.9 THEN 1.0 ELSE 0.0 END) FILTER (WHERE own_dd60 IS NOT NULL AND graduated_at + interval '60 minutes' <= %(g)s) AS rug, + avg(CASE WHEN own_max60 >= 1.0 THEN 1.0 ELSE 0.0 END) FILTER (WHERE own_dd60 IS NOT NULL AND graduated_at + interval '60 minutes' <= %(g)s) AS moon + FROM rh_meta WHERE creator = %(c)s AND graduated_at < %(g)s AND mint <> %(m)s""", {"c": c, "g": g, "m": token}).fetchone() + out.update(prior_grads=int(r["grads"]), prior_known=int(r["known"]), prior_rug_share=float(r["rug"]) if r["rug"] is not None else None, + prior_moon_share=float(r["moon"]) if r["moon"] is not None else None) + # curve facts from the curve trades up to graduation + wallets: dict[str, list] = {} + for x in conn.execute("SELECT block, trader, side, tokens_raw FROM rh_curve_trades WHERE token = %s AND ts <= %s ORDER BY block, log_index", (token, g)).fetchall(): + w = wallets.setdefault(x["trader"], [None, 0.0, 0.0]); q = float(x["tokens_raw"] or 0) / TOKEN + if x["side"] > 0: + w[0] = x["block"] if w[0] is None else w[0]; w[1] += q + else: + w[2] += q + if t["curve_indexed"]: + facts, insiders = curve_facts(t["launch_block"], c, SUPPLY, 0.0, wallets) + out.update(facts); out["curve_known"] = 1.0 + for w, kind in insiders: + conn.execute("INSERT INTO rh_insiders (mint, wallet, kind) VALUES (%s, %s, %s) ON CONFLICT DO NOTHING", (token, w, kind)) + else: + out.update(bundle_share=None, dev_hold_share=None, dev_sold_frac=None, grad_hhi=None, curve_known=0.0) + return out + + +def outcomes(conn, token: str, graduated_at) -> tuple[float, float] | None: + """(own_dd60, own_max60) from the first hour of rh_minutes after graduation, or None when it has no minute.""" + rows = conn.execute("SELECT close FROM rh_minutes WHERE mint = %s AND ts >= %s AND ts < %s ORDER BY ts", (token, graduated_at, graduated_at + timedelta(minutes=60))).fetchall() + cl = [float(r["close"]) for r in rows if r["close"]] + if not cl or cl[0] <= 0: + return None + return min(cl) / cl[0] - 1.0, max(cl) / cl[0] - 1.0 + + +COLS = ("mint", "graduated_at", "create_ts", "creator", "quote_asset", "quote_class", "supply", "ttg_min", "dev_sol", "dev_tokens", "dev_share", "mayhem", "rq0", + "pool_id", "prior_launches", "prior_grads", "prior_known", "prior_rug_share", "prior_moon_share", "bundle_share", "dev_hold_share", "dev_sold_frac", + "grad_hhi", "curve_known") + + +def run_once(limit: int = 500) -> dict: + """New graduations → rh_meta (in graduation order, so a deployer's earlier launches are there first); first-hour + outcomes for those 61+ minutes past graduation.""" + n_new = n_out = 0 + with transaction() as conn: + todo = conn.execute("SELECT t.token FROM rh_tokens t LEFT JOIN rh_meta m ON m.mint = t.token WHERE t.status = 'graduated' AND t.graduated_at IS NOT NULL " + "AND m.mint IS NULL ORDER BY t.graduated_at LIMIT %s", (limit,)).fetchall() + for r in todo: + row = build(conn, r["token"]) + if row is None: + continue + conn.execute(f"INSERT INTO rh_meta ({', '.join(COLS)}) VALUES ({', '.join(['%s'] * len(COLS))}) ON CONFLICT (mint) DO NOTHING", [row.get(k) for k in COLS]) + n_new += 1 + due = conn.execute("SELECT m.mint, m.graduated_at FROM rh_meta m WHERE m.own_dd60 IS NULL AND m.graduated_at + interval '61 minutes' <= " + "(SELECT max(ts) FROM rh_minutes) LIMIT %s", (limit,)).fetchall() + for r in due: + o = outcomes(conn, r["mint"], r["graduated_at"]) + if o is not None: + conn.execute("UPDATE rh_meta SET own_dd60 = %s, own_max60 = %s, updated_at = now() WHERE mint = %s", (o[0], o[1], r["mint"])); n_out += 1 + return {"meta": n_new, "outcomes": n_out} + + +def load_features(): + """rh_meta in corpus_meta.load_features' shape (mint + FEATURE_COLS; mayhem and curve_known as 0/1 floats).""" + import pandas as pd + + from ..train.corpus_meta import FEATURE_COLS + with transaction() as conn: + rows = conn.execute("SELECT mint, " + ", ".join(FEATURE_COLS) + " FROM rh_meta").fetchall() + df = pd.DataFrame(rows) + if df.empty: + return pd.DataFrame(columns=["mint"] + FEATURE_COLS) + df["mayhem"] = df["mayhem"].map({True: 1.0, False: 0.0}) + df["curve_known"] = df["curve_known"].astype(float) + return df diff --git a/fly_trader/rh/minutes.py b/fly_trader/rh/minutes.py new file mode 100644 index 0000000000000000000000000000000000000000..e9a5c6f6d7c94fde3ae07ccb651eca12cebed608 --- /dev/null +++ b/fly_trader/rh/minutes.py @@ -0,0 +1,150 @@ +"""RH minutes (``rh_minutes``): the twin of ``pump_minutes``, in ETH, built from the indexed swaps. + +The same rules as the Solana stream (ingest/pumpstream.Aggregator) and the archive (train/mature.aggregate_table): +- a swap counts only while its pool's reserve is plausible (``RESQ_BAND`` scaled by K = ETH per SOL) and its price is + within ``PRICE_BAND`` (50×) of the median of the mint's previous ``REF_LEGS`` (200) prices that UTC day; +- open/high/low/close = the pool price after each swap (sqrtPriceX96 of the Swap event), in ETH per token: the quote price + times the quote asset's ETH value that minute (rh/prices.py; ETH-quoted pools: 1). A minute whose non-ETH quote has no + mark yet is not written (it cannot be expressed in ETH); +- buy/sell volume in ETH; resq = the pool's virtual quote reserve in ETH (full-range locked liquidity: constant product); +- fee_rate = the volume-weighted Pons hook fee + tax fraction (the analogue of pump_minutes.fee_rate, the pool fee charged); +- the wallet columns (buyers, wash, top seller, insider sells, buys per skill decile) by train/flow.minute_wallet_cols. + +Minutes are recomputed from the swaps, never accumulated, so a live pass and a backfill write identical rows. +""" +from __future__ import annotations + +import json +import statistics +from collections import deque +from datetime import datetime, timezone + +from .. import config +from ..db.connection import transaction +from ..train.flow import minute_wallet_cols +from . import prices + +RESQ_BAND_SOL = (0.001, 100000.0) # ingest/pumpstream.RESQ_BAND +PRICE_BAND = 50.0 +REF_LEGS = 200 +GRACE_S = 2 # a minute is complete once the index is this far past its end: block times never +# decrease, so every swap before the minute's end is indexed once a later block is (ingest/pumpstream.FLUSH_GRACE_S; 20 until 2026-10-09) + + +class _Row: + __slots__ = ("pool", "asset", "cls", "open", "high", "low", "close", "close_q", "base", "buy", "sell", "nb", "ns", "traders", "wallets", + "resq_eth", "resq_q", "fee_w", "fee_v") + + def __init__(self, pool, asset, cls, base): + self.pool, self.asset, self.cls, self.base = pool, asset, cls, base + self.open = None; self.high = -1e308; self.low = 1e308; self.close = None; self.close_q = None + self.buy = self.sell = 0.0; self.nb = self.ns = 0; self.traders = set(); self.wallets: dict[str, list[float]] = {} + self.resq_eth = self.resq_q = None; self.fee_w = 0.0; self.fee_v = 0.0 + + +def aggregate(conn, t0: int, t1: int, skill: dict | None = None) -> list[tuple]: + """rh_minutes rows for every minute in [t0, t1) (epoch s, minute-aligned).""" + k = config.RH_ETH_PER_SOL; lo_band, hi_band = RESQ_BAND_SOL[0] * k, RESQ_BAND_SOL[1] * k + day0 = (t0 // 86400) * 86400 + swaps = conn.execute( + "SELECT s.ts, s.token, s.pool_id, s.trader, s.side, s.quote_raw, s.price_q, s.resq_q, s.fee_frac, p.quote_asset, a.decimals AS qdec, a.class AS cls " + "FROM rh_swaps s JOIN rh_pools p ON p.pool_id = s.pool_id JOIN rh_assets a ON a.asset = p.quote_asset " + "WHERE s.ts >= to_timestamp(%s) AND s.ts < to_timestamp(%s) ORDER BY s.ts, s.block, s.log_index", (day0, t1)).fetchall() + insiders: dict[str, frozenset] = {} + base_cache: dict[tuple, tuple] = {} + ref: dict[str, deque] = {}; ref_day = None + rows: dict[tuple, _Row] = {} + for s in swaps: + ts = s["ts"].timestamp(); day = int(ts // 86400) + if ref_day is None or day != ref_day: + ref = {}; ref_day = day + p = float(s["price_q"] or 0.0) + if p <= 0: + continue + dq = ref.setdefault(s["token"], deque(maxlen=REF_LEGS)) + ok = not dq or (statistics.median(dq) / PRICE_BAND <= p <= statistics.median(dq) * PRICE_BAND) + dq.append(p) + if not ok or ts < t0: + continue # before t0: only the median's history + m = int(ts // 60) * 60 + key_b = (s["quote_asset"], m) + if key_b not in base_cache: + base_cache[key_b] = prices.base_eth(conn, s["quote_asset"], datetime.fromtimestamp(m + 59, timezone.utc)) + base, _stale = base_cache[key_b] + if base is None: + continue + resq_eth = float(s["resq_q"] or 0.0) * base + if not (lo_band <= resq_eth <= hi_band): + continue + r = rows.get((s["token"], m)) + if r is None: + r = rows[(s["token"], m)] = _Row(s["pool_id"], s["quote_asset"], s["cls"], base) + pe = p * base + r.open = pe if r.open is None else r.open; r.high = max(r.high, pe); r.low = min(r.low, pe); r.close = pe; r.close_q = p + vol = float(s["quote_raw"] or 0) / 10 ** int(s["qdec"]) * base + if s["side"] > 0: + r.buy += vol; r.nb += 1 + else: + r.sell += vol; r.ns += 1 + if s["trader"]: + r.traders.add(s["trader"]); r.wallets.setdefault(s["trader"], [0.0, 0.0])[0 if s["side"] > 0 else 1] += vol + r.resq_eth, r.resq_q = resq_eth, float(s["resq_q"] or 0.0) + if s["fee_frac"] is not None: + r.fee_w += float(s["fee_frac"]) * vol; r.fee_v += vol + out = [] + for (mint, m), r in sorted(rows.items(), key=lambda kv: (kv[0][1], kv[0][0])): + if mint not in insiders: + insiders[mint] = frozenset(x["wallet"] for x in conn.execute("SELECT wallet FROM rh_insiders WHERE mint = %s", (mint,)).fetchall()) + wc = minute_wallet_cols(r.wallets, insiders[mint], skill) + out.append((mint, datetime.fromtimestamp(m, timezone.utc), r.pool, r.asset, r.cls, r.open, r.high, r.low, r.close, r.close_q, r.base, r.buy, r.sell, + r.nb, r.ns, len(r.traders), r.resq_eth, r.resq_q, (r.fee_w / r.fee_v) if r.fee_v > 0 else None, + wc["n_buyers"], wc["wash_sol"], wc["wash_buy_sol"], wc["top_sell_sol"], wc["insider_sell_sol"], wc["skill_buy"])) + return out + + +def write(conn, rows: list[tuple]) -> int: + if rows: + conn.cursor().executemany( + "INSERT INTO rh_minutes (mint, ts, pool_id, quote_asset, quote_class, open, high, low, close, close_q, base_eth, buy_eth, sell_eth, n_buys, n_sells, " + "n_traders, resq_eth, resq_q, fee_rate, n_buyers, wash_eth, wash_buy_eth, top_sell_eth, insider_sell_eth, skill_buy) " + "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT (mint, ts) DO UPDATE SET " + "pool_id = EXCLUDED.pool_id, open = EXCLUDED.open, high = EXCLUDED.high, low = EXCLUDED.low, close = EXCLUDED.close, close_q = EXCLUDED.close_q, " + "base_eth = EXCLUDED.base_eth, buy_eth = EXCLUDED.buy_eth, sell_eth = EXCLUDED.sell_eth, n_buys = EXCLUDED.n_buys, n_sells = EXCLUDED.n_sells, " + "n_traders = EXCLUDED.n_traders, resq_eth = EXCLUDED.resq_eth, resq_q = EXCLUDED.resq_q, fee_rate = EXCLUDED.fee_rate, n_buyers = EXCLUDED.n_buyers, " + "wash_eth = EXCLUDED.wash_eth, wash_buy_eth = EXCLUDED.wash_buy_eth, top_sell_eth = EXCLUDED.top_sell_eth, insider_sell_eth = EXCLUDED.insider_sell_eth, " + "skill_buy = EXCLUDED.skill_buy, revised = rh_minutes.revised OR (rh_minutes.close IS DISTINCT FROM EXCLUDED.close)", rows) + return len(rows) + + +def indexed_through_s(conn, rpc) -> float | None: + """The chain time the index has reached (the cursor block's timestamp).""" + r = conn.execute("SELECT block FROM rh_scan WHERE name = 'rh'").fetchone() + if not r or int(r["block"]) <= 0: + return None + b = rpc.block(int(r["block"])) + return float(int(b["timestamp"], 16)) if b else None + + +def run_once(rpc, skill: dict | None = None, max_minutes: int = 240) -> dict: + """Write every complete minute since the last flush (up to ``max_minutes``), set rh_stream_status.flushed_through.""" + with transaction() as conn: + top = indexed_through_s(conn, rpc) + st = conn.execute("SELECT detail FROM rh_scan WHERE name = 'rh_minutes'").fetchone() + if top is None: + return {"minutes": 0} + bound = int((top - GRACE_S) // 60) * 60 + done = int(((st["detail"] or {}) if st else {}).get("through") or 0) + if not done: + first = conn.execute("SELECT min(ts) AS t FROM rh_swaps").fetchone()["t"] + if first is None: + return {"minutes": 0} + done = int(first.timestamp() // 60) * 60 + t1 = min(bound, done + max_minutes * 60) + if t1 <= done: + return {"minutes": 0, "through": done} + n = write(conn, aggregate(conn, done, t1, skill)) + conn.execute("INSERT INTO rh_scan (name, block, detail) VALUES ('rh_minutes', 0, %s) ON CONFLICT (name) DO UPDATE SET detail = EXCLUDED.detail, updated_at = now()", + (json.dumps({"through": t1}),)) + conn.execute("INSERT INTO ui_settings (key, value) VALUES ('rh_stream_status', %s) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()", + (json.dumps({"flushed_through": datetime.fromtimestamp(t1 - 60, timezone.utc).isoformat(), "rows": n}),)) + return {"minutes": (t1 - done) // 60, "rows": n, "through": t1} diff --git a/fly_trader/rh/pons.py b/fly_trader/rh/pons.py new file mode 100644 index 0000000000000000000000000000000000000000..cd9136862c085e1147255d406097b7888650d27d --- /dev/null +++ b/fly_trader/rh/pons.py @@ -0,0 +1,78 @@ +"""Pons V2 events (Robinhood Chain), pinned from chain data on 2026-09-28: topic0 = keccak of each signature below matched +the logs the factory, launch router, curves, meme hook and launch locker emit (the explorer's verified sources sit behind a +bot wall). Argument roles were read from real transactions (a launch: mint Transfer 0 → curve; the router's buy for the +deployer; the curve's buy/sell with a 1 % fee word). + +Emitters: TokenLaunched / LaunchSwept / PoolGraduated / GraduationTokensPermanentlyLocked — the V2 factory; Launched — the +launch router (the deployer's own first buy); CurveBuy / CurveSell — each launch's own curve contract; PoolRegistered / +HookFeeCollected — the meme hook; TokenSupplyLocked / PositionLocked — the launch locker. +""" +from __future__ import annotations + +from . import abi + +ZERO = "0x" + "00" * 20 +SIGS = { + "TokenLaunched": "TokenLaunched(address,address,address,address,uint256,uint256)", # token, curve, deployer (indexed); pairToken, ?, graduationThreshold + "Launched": "Launched(address,address,address,address,uint256,uint256)", # token, curve, deployer (indexed); recipient, quoteIn, tokensOut + "CurveBuy": "CurveBuy(address,address,uint256,uint256,uint256,uint256)", # buyer, recipient (indexed); quoteIn, tokensOut, fee, tax + "CurveSell": "CurveSell(address,address,uint256,uint256,uint256,uint256)", # seller, recipient (indexed); tokensIn, quoteOut, fee, tax + "LaunchSwept": "LaunchSwept(address,uint256,uint256)", # token (indexed); quoteOut, tokenOut + "PoolGraduated": "PoolGraduated(address,uint256,uint256,uint256)", # token (indexed); positionId, tokenAmount, pairTokenAmount + "GraduationTokensPermanentlyLocked": "GraduationTokensPermanentlyLocked(address,uint256)", + "PoolRegistered": "PoolRegistered(bytes32,address,address,address)", # poolId (indexed); memecoin, quoteToken, creator + "HookFeeCollected": "HookFeeCollected(bytes32,address,uint256,uint256)", # poolId (indexed); currency, feeAmount, taxAmount + "TokenSupplyLocked": "TokenSupplyLocked(address,uint256)", + "PositionLocked": "PositionLocked(address,uint256)", +} +TOPIC = {k: abi.event_topic(v) for k, v in SIGS.items()} +NAME = {v: k for k, v in TOPIC.items()} +FACTORY_EVENTS = ("TokenLaunched", "LaunchSwept", "PoolGraduated", "GraduationTokensPermanentlyLocked") +CURVE_EVENTS = ("CurveBuy", "CurveSell") +HOOK_EVENTS = ("PoolRegistered", "HookFeeCollected") + + +def _addr_topic(t: str) -> str: + return "0x" + t[-40:].lower() + + +def _words(data: str, types: list[str]) -> list: + return abi.decode(types, bytes.fromhex(data[2:] if data.startswith("0x") else data)) + + +def decode(log: dict) -> dict | None: + """One Pons log → a dict with ``event`` and its fields (addresses lowercase), or None for an event we do not use.""" + name = NAME.get(log["topics"][0]) + if name is None: + return None + t, d = log["topics"], log["data"] + base = {"event": name, "emitter": log["address"].lower(), "block": int(log["blockNumber"], 16), "log_index": int(log["logIndex"], 16), + "tx_hash": log["transactionHash"], "ts": int(log.get("blockTimestamp") or "0x0", 16) or None} + if name == "TokenLaunched": + pair, x, thr = _words(d, ["address", "uint256", "uint256"]) + return {**base, "token": _addr_topic(t[1]), "curve": _addr_topic(t[2]), "deployer": _addr_topic(t[3]), "pair_token": pair, "word1": x, "threshold_raw": thr} + if name == "Launched": + rcpt, qin, tout = _words(d, ["address", "uint256", "uint256"]) + return {**base, "token": _addr_topic(t[1]), "curve": _addr_topic(t[2]), "deployer": _addr_topic(t[3]), "recipient": rcpt, "quote_in_raw": qin, "tokens_out_raw": tout} + if name in CURVE_EVENTS: + a, b, fee, tax = _words(d, ["uint256"] * 4) + side = 1 if name == "CurveBuy" else -1 + return {**base, "side": side, "trader": _addr_topic(t[1]), "recipient": _addr_topic(t[2]), + "quote_raw": a if side > 0 else b, "tokens_raw": b if side > 0 else a, "fee_raw": fee, "tax_raw": tax} + if name == "LaunchSwept": + q, tok = _words(d, ["uint256", "uint256"]) + return {**base, "token": _addr_topic(t[1]), "quote_out_raw": q, "token_out_raw": tok} + if name == "PoolGraduated": + pos, tok, q = _words(d, ["uint256", "uint256", "uint256"]) + return {**base, "token": _addr_topic(t[1]), "position_id": pos, "token_amount_raw": tok, "pair_amount_raw": q} + if name in ("GraduationTokensPermanentlyLocked", "TokenSupplyLocked"): + return {**base, "token": _addr_topic(t[1]), "amount_raw": _words(d, ["uint256"])[0]} + if name == "PoolRegistered": + meme, quote, creator = _words(d, ["address", "address", "address"]) + return {**base, "pool_id": t[1].lower(), "token": meme, "quote_asset": quote, "creator": creator} + if name == "HookFeeCollected": + cur, fee, tax = _words(d, ["address", "uint256", "uint256"]) + return {**base, "pool_id": t[1].lower(), "currency": cur, "fee_raw": fee, "tax_raw": tax} + if name == "PositionLocked": + return {**base, "owner": _addr_topic(t[1]), "token_id": int(t[2], 16)} + return None diff --git a/fly_trader/rh/prefetch.py b/fly_trader/rh/prefetch.py new file mode 100644 index 0000000000000000000000000000000000000000..c61009602792bfe7eb017d8f106ff5b3d1981b1c --- /dev/null +++ b/fly_trader/rh/prefetch.py @@ -0,0 +1,185 @@ +"""One log query per live pass of the RH stream (rh/stream.py), and the RPC behind HyperSync when HyperSync fails. + +Until 2026-10-09 a live pass asked HyperSync 8-12 times, one query after another (Pons events, PoolManager events, hook +fees, the swappers' Transfers, every reference-pool venue, twice over for two price ranges, each paced and each on a fresh +connection for the price scan): ~21 s a pass, so RH decisions landed 45-70 s after their minute. + +``fetch`` reads every log the pass's index step and price scan will ask for over the pass's block range at once: from +HyperSync as one multi-selection query (capped at its archive height, every block's time included), or, while HyperSync +is failing, from the RPC as one eth_getLogs over every selection's addresses (any topic, at most ``rpc_span`` blocks). +``Prefetch`` wraps the stream's rpc for the pass: ``get_logs_multi`` answers from memory a request inside the fetched +range that a fetched selection covers (applying the request's own filter, so a broad fetch answers a narrower request +exactly) and passes anything else on — to HyperSync, or to the RPC while HyperSync is down. ``block`` answers the times +the fetch saw. A request the selections do not cover is only slower, never wrong. + +HyperSync is asked ``LIVE_TRIES`` times in a pass; when it still fails the RPC serves the logs for ``COOLDOWN_S`` before +HyperSync is tried again, so the RH books keep trading through a HyperSync outage instead of holding until it returns. +""" +from __future__ import annotations + +import json +import logging +import time + +from .. import config +from ..db.apilog import record_event +from ..db.connection import transaction +from . import hypersync as H +from .rpc import RhRpc + +log = logging.getLogger(__name__) + +LIVE_TRIES = 3 # HyperSync attempts in a live pass before the RPC takes the pass over +COOLDOWN_S = 60.0 # after HyperSync failed a pass: this long on the RPC before HyperSync is asked again +RPC_SPAN = 400 # blocks per pass from the RPC (its 10,000-log cap; ~800 PoolManager logs in 400 blocks, 2026-10-09) +RPC_SPAN_MIN = 25 +_STATE: dict = {"down_until": 0.0, "down_since": None, "last_error": None, "rpc_span": RPC_SPAN} + + +def hypersync_down() -> bool: + """HyperSync failed a pass less than ``COOLDOWN_S`` ago: the RPC serves the live logs.""" + return time.monotonic() < _STATE["down_until"] + + +def status() -> dict: + return {"hypersync_down_since": _STATE["down_since"], "last_error": _STATE["last_error"], "rpc_span": _STATE["rpc_span"]} if _STATE["down_since"] else {} + + +def _failed(e: Exception) -> None: + _STATE["down_until"] = time.monotonic() + COOLDOWN_S; _STATE["last_error"] = str(e)[:200] + if _STATE["down_since"] is None: + _STATE["down_since"] = time.time() + log.warning("HyperSync failed a live pass (%s): the RPC serves the RH logs for now", str(e)[:200]) + record_event("warning", "rh_stream", "HyperSync is failing: the public RPC serves the RH logs", {"error": str(e)[:300]}) + + +def _recovered() -> None: + if _STATE["down_since"] is not None: + down = time.time() - _STATE["down_since"] + log.info("HyperSync is back after %.0f s", down) + record_event("info", "rh_stream", "HyperSync is back: it serves the RH logs again", {"down_s": round(down)}) + _STATE.update(down_until=0.0, down_since=None, last_error=None) + + +# ---------------------------------------------------------------- coverage and matching (eth_getLogs semantics) +def _sets(topics) -> list: + """Topic positions as sets (None: any), from eth_getLogs' or HyperSync's form.""" + out = [] + for t in topics or []: + if t is None or (isinstance(t, (list, tuple)) and not t): + out.append(None) + else: + out.append({x.lower() for x in (t if isinstance(t, (list, tuple)) else [t])}) + return out + + +def covered(addresses, topics, sels: list[dict]) -> bool: + """Whether some selection's logs include every log the request (``addresses``, ``topics``) can match.""" + if not addresses: + return False # an any-address request is never answered from memory + want_a = {a.lower() for a in addresses}; want_t = _sets(topics) + for s in sels: + if not want_a <= {a.lower() for a in (s.get("address") or [])}: + continue + ok = True + for i, have in enumerate(_sets(s.get("topics"))): + if have is None: + continue + want = want_t[i] if i < len(want_t) else None + if want is None or not want <= have: + ok = False; break + if ok: + return True + return False + + +def matches(g: dict, want_a: set, want_t: list) -> bool: + if g["address"].lower() not in want_a: + return False + tp = g.get("topics") or [] + for i, want in enumerate(want_t): + if want is not None and (i >= len(tp) or tp[i].lower() not in want): + return False + return True + + +# ---------------------------------------------------------------- what a live pass reads +def selections(conn) -> list[dict]: + """The index step's and the price scan's reads (index.step, prices.fetch_pool_logs) as HyperSync selections.""" + from . import index, prices + sels = [H.selection([config.PONS_FACTORY, config.PONS_ROUTER, config.PONS_HOOK], index.PONS_TOPICS), + H.selection([config.V4_POOL_MANAGER], [index.V4_TOPICS0]), + H.selection([config.PONS_HOOK], [[index.FEE_TOPIC]])] + hops: dict = {} + for r in conn.execute("SELECT DISTINCT a.asset, a.ref_path::text AS ref_path FROM rh_assets a JOIN rh_pools p ON p.quote_asset = a.asset " + "WHERE p.is_pons AND a.class <> 'eth' AND a.ref_path IS NOT NULL").fetchall(): + for h in json.loads(r["ref_path"]): + hops.setdefault(prices._hop_key(h), h) + plain = sorted({h["emitter"].lower() for h in hops.values() if not prices._is_id(h["pool"])}) + if plain: + sels.append(H.selection(plain, [[prices.SWAP_V3]])) + managers: dict[str, set] = {} + for h in hops.values(): + if prices._is_id(h["pool"]): + managers.setdefault(h["emitter"].lower(), set()).add(h["pool"].lower()) + for em, ids in sorted(managers.items()): + sels.append(H.selection([em], [None, sorted(ids)])) + return sels + + +class Prefetch: + """The stream's rpc for one pass, answering the fetched range's log reads from memory (module docstring).""" + + def __init__(self, rpc, lo: int, hi: int, logs: list[dict], times: dict[int, int], sels: list[dict], source: str): + self._rpc, self.lo, self.hi, self.logs, self.times, self.sels, self.source = rpc, lo, hi, logs, times, sels, source + self.served = 0; self.passed = 0; self.served_last = False + + def __getattr__(self, name): + return getattr(self._rpc, name) + + def block(self, tag, full: bool = False): + if isinstance(tag, int) and not full and tag in self.times: + return {"number": hex(tag), "timestamp": hex(self.times[tag])} + return self._rpc.block(tag, full) + + def get_logs_multi(self, addresses, from_block: int, to_block: int, topics=None) -> list[dict]: + if self.lo <= from_block and to_block <= self.hi and covered(addresses, topics, self.sels): + want_a = {a.lower() for a in addresses}; want_t = _sets(topics) + self.served += 1; self.served_last = True + return [g for g in self.logs if from_block <= int(g["blockNumber"], 16) <= to_block and matches(g, want_a, want_t)] + self.passed += 1; self.served_last = False + if self.source == "rpc": # HyperSync is down: the RPC answers what memory cannot + return RhRpc.get_logs_multi(self._rpc, addresses, from_block, to_block, topics) + return self._rpc.get_logs_multi(addresses, from_block, to_block, topics) + + +def _too_big(e: Exception) -> bool: + msg = str(e).lower() + return "exceeds limit" in msg or "too many" in msg or "timed out" in msg or "timeout" in msg or "response size" in msg + + +def fetch(rpc, lo: int, hi: int) -> Prefetch: + """The pass's logs over lo..hi: one HyperSync query (to its archive height) unless HyperSync is failing, else one + RPC read of at most ``rpc_span`` blocks (halved while the RPC refuses it). The returned ``hi`` is what was covered.""" + with transaction() as conn: + sels = selections(conn) + hs = getattr(rpc, "hs", None) + if hs is not None and not hypersync_down(): + try: + logs, through, times = hs.get_logs_sel(sels, lo, hi, upto=True, tries=LIVE_TRIES, all_blocks=True) + _recovered() + return Prefetch(rpc, lo, through, logs, times, sels, "hypersync") + except Exception as e: # whatever HyperSync's failure, the pass goes on without it + _failed(e) + addrs = sorted({a for s in sels for a in (s.get("address") or [])}) + while True: + top = min(hi, lo + _STATE["rpc_span"] - 1) + try: + logs = RhRpc.get_logs_multi(rpc, addrs, lo, top, None) + except Exception as e: + if _too_big(e) and _STATE["rpc_span"] > RPC_SPAN_MIN: + _STATE["rpc_span"] = max(RPC_SPAN_MIN, _STATE["rpc_span"] // 2); continue + raise + _STATE["rpc_span"] = min(RPC_SPAN, _STATE["rpc_span"] * 2) + logs.sort(key=lambda g: (int(g["blockNumber"], 16), int(g["logIndex"], 16))) + return Prefetch(rpc, lo, top, logs, {}, [{"address": addrs, "topics": []}], "rpc") diff --git a/fly_trader/rh/prices.py b/fly_trader/rh/prices.py new file mode 100644 index 0000000000000000000000000000000000000000..1e2d9f7d078df0c8df33770ed568154077bfe100 --- /dev/null +++ b/fly_trader/rh/prices.py @@ -0,0 +1,229 @@ +"""Each Pons quote asset's value in ETH per minute (``rh_base_prices``): the base leg's mark, and the conversion that puts +every RH row in ETH. + +Reference path: the AMM pools KyberSwap routes 1 unit of the asset into WETH through, restricted to venues whose swap +logs carry the pool price (discovered once, stored as ``rh_assets.ref_path``): on 2026-09-28 USDG → WETH and a stock → +USDG → WETH resolve to Uniswap v3 pools; the unrestricted best route may be an order book, which leaves no log price. Every one of those venues puts the pool's sqrtPriceX96 +third in its swap event's data, so a hop's price at a minute is the last swap's sqrt price in (or before) that minute; +the asset's ETH value is the product over its hops. A minute with no swap on some hop carries the last price forward and +is marked ``stale`` once that price is older than ``STALE_S``. +""" +from __future__ import annotations + +import json +import logging + +from .. import config +from ..db.connection import transaction +from . import abi, kyber as K, scan, v4 +from .rpc import logs_rpc + +log = logging.getLogger(__name__) + +STALE_S = 600 +ETH_ASSET = "0x" + "00" * 20 +# plain v3-style pools first (address-based, the standard Swap event), then the singleton CL venues (bytes32 pool ids) +REF_SOURCES = ("uniswapv3,slipstream", "uniswapv3,uniswap-v4,slipstream,pancake-infinity-cl,orvex-cl-feemanager") + + +def _is_id(pool: str) -> bool: + return len(pool) == 66 + + +def discover(conn, rpc, asset: str, decimals: int, kyber: K.Kyber | None = None) -> list[dict]: + """The hops from ``asset`` to WETH (``[{pool, emitter, token0, token1, dec0, dec1, token_in}]``), stored on rh_assets.""" + ky = kyber or K.Kyber() + paths = [] + for sources in REF_SOURCES: # AMM venues only: an order book leaves no price in the logs + try: + paths = ky.get_route(asset, config.RH_WETH, 10 ** decimals, included_sources=sources)["routeSummary"].get("route") or [] + except K.KyberError: + paths = [] + if paths and paths[0]: + break + if not paths or not paths[0]: + raise K.KyberError(f"no AMM route for {asset}") + hops = [] + for h in paths[0]: # the first split carries the path; splits share hops + tin, tout, pool = h["tokenIn"].lower(), h["tokenOut"].lower(), h["pool"].lower() + t0, t1 = sorted([tin, tout], key=lambda a: int(a, 16)) + if _is_id(pool): # a singleton manager's pool id: find the manager from its logs + head = rpc.block_number(); emitter = None; lo = head - 200_000 + while emitter is None and lo > head - 2_000_000: + got = rpc.get_logs_multi(None, lo, lo + 20_000, [None, pool]); scan.pause() + emitter = next((g["address"].lower() for g in got if len(g["data"]) >= 2 + 64 * 3), None); lo -= 20_000 + if emitter is None: + raise K.KyberError(f"no swaps seen for reference pool {pool}") + else: + emitter = pool + dec = {} + for t in (t0, t1): + dec[t] = 18 if int(t, 16) == 0 else abi.decode(["uint8"], bytes.fromhex(rpc.eth_call(t, "0x" + abi.selector("decimals()").hex())[2:]))[0] + hops.append({"pool": pool, "emitter": emitter, "token0": t0, "token1": t1, "dec0": dec[t0], "dec1": dec[t1], "token_in": tin}) + conn.execute("UPDATE rh_assets SET ref_path = %s, ref_pool = %s, ref_kind = %s, updated_at = now() WHERE asset = %s", + (json.dumps(hops), hops[0]["pool"], paths[0][0].get("poolType"), asset.lower())) + return hops + + +def hop_price(h: dict, sqrt_price_x96: int) -> float: + """Whole token_out per whole token_in on this hop.""" + p10 = v4.price1_per_0(sqrt_price_x96, h["dec0"], h["dec1"]) # token1 per token0 + return p10 if h["token_in"] == h["token0"] else (1.0 / p10 if p10 > 0 else float("nan")) + + +SWAP_V3 = abi.event_topic("Swap(address,address,int256,int256,uint160,uint128,int24)") # Uniswap v3 and Slipstream pools + + +def _logs(rpc, addresses, lo: int, hi: int, topics) -> list[dict]: + from .index import _get_logs + return _get_logs(rpc, addresses, lo, hi, topics) + + +def _swap_logs(rpc, h: dict, lo: int, hi: int) -> list[dict]: + if _is_id(h["pool"]): + return _logs(rpc, [h["emitter"]], lo, hi, [None, h["pool"]]) + return _logs(rpc, [h["emitter"]], lo, hi, [SWAP_V3]) + + +def fetch_pool_logs(rpc, pools: dict, lo: int, hi: int) -> dict: + """Swap logs of every reference pool in ``pools`` (``_hop_key`` → hop) over lo..hi, in as few queries as the venues + allow: one for all the standalone pool contracts (one address list, one Swap topic) and one per singleton manager + (its pool ids as the topic-1 list). 2026-10-07: 69 pools queried one by one made a live pass take 1–3.5 min.""" + out = {k: [] for k in pools} + plain = sorted({h["emitter"].lower() for h in pools.values() if not _is_id(h["pool"])}) + if plain: + for g in _logs(rpc, plain, lo, hi, [SWAP_V3]): + k = (g["address"].lower(), g["address"].lower()) + if k in out: + out[k].append(g) + managers: dict[str, list[str]] = {} + for h in pools.values(): + if _is_id(h["pool"]): + managers.setdefault(h["emitter"].lower(), []).append(h["pool"].lower()) + for em, ids in sorted(managers.items()): + for g in _logs(rpc, [em], lo, hi, [None, sorted(set(ids))]): + tp = g.get("topics") or [] + k = (em, tp[1].lower()) if len(tp) > 1 else None + if k in out: + out[k].append(g) + return out + + +def _hop_key(h: dict) -> tuple[str, str]: + return h["emitter"].lower(), h["pool"].lower() + + +def scan_asset(conn, rpc, asset: str, hops: list[dict], lo: int, hi: int, state: dict, times, logs_by_hop: dict | None = None) -> int: + """Minute closes of ``asset`` over blocks lo..hi → rh_base_prices. ``state`` carries each hop's last (t, sqrtP) across + ranges; ``times(blocks) -> {block: ts}`` times the swaps. ``logs_by_hop``: swap logs already fetched per hop + (``_hop_key``), shared by every asset whose path uses that pool. Returns minutes written.""" + per_hop = [] + for i, h in enumerate(hops): + got = logs_by_hop[_hop_key(h)] if logs_by_hop is not None else _swap_logs(rpc, h, lo, hi) + logs = [g for g in got if len(g["data"]) >= 2 + 64 * 3 and lo <= int(g["blockNumber"], 16) <= hi] + ts = times(sorted({int(g["blockNumber"], 16) for g in logs})) if logs else {} + seq = sorted((ts[int(g["blockNumber"], 16)], int(g["logIndex"], 16), v4.sqrt_price_word(g)) for g in logs) + per_hop.append(seq) + minutes = sorted({int(t // 60) * 60 for seq in per_hop for t, _, _ in seq}) + rows = [] + idx = [0] * len(hops) + for m in minutes: + end = m + 60; stale = False; price = 1.0; n_obs = 0 + for i, h in enumerate(hops): + seq = per_hop[i] + while idx[i] < len(seq) and seq[idx[i]][0] < end: + t, _, sp = seq[idx[i]]; state[str(i)] = [t, sp]; idx[i] += 1; n_obs += 1 + last = state.get(str(i)) + if last is None: + price = None; break + if end - last[0] > STALE_S: + stale = True + price *= hop_price(h, int(last[1])) + if price is not None and price > 0: + rows.append((asset, m, price, n_obs, stale)) + if rows: + conn.cursor().executemany("INSERT INTO rh_base_prices (asset, ts, price_eth, n_obs, stale) VALUES (%s, to_timestamp(%s), %s, %s, %s) " + "ON CONFLICT (asset, ts) DO UPDATE SET price_eth = EXCLUDED.price_eth, n_obs = EXCLUDED.n_obs, stale = EXCLUDED.stale", rows) + return len(rows) + + +def base_eth(conn, asset: str | None, ts) -> tuple[float | None, bool]: + """(ETH per whole unit of ``asset`` at minute ``ts``, stale). ETH itself is 1. The last mark at or before the minute; + stale when older than ``STALE_S`` or flagged stale.""" + if asset is None or int(asset, 16) == 0: + return 1.0, False + r = conn.execute("SELECT ts, price_eth, stale FROM rh_base_prices WHERE asset = %s AND ts <= %s ORDER BY ts DESC LIMIT 1", (asset.lower(), ts)).fetchone() + if r is None: + return None, True + age = (ts - r["ts"]).total_seconds() if hasattr(ts, "tzinfo") else 0.0 + return float(r["price_eth"]), bool(r["stale"]) or age > STALE_S + + +def run_once(max_ranges: int = 1, rpc=None) -> dict: + """Advance every traded quote asset's price cursor (``price:``) toward the index cursor, all in lockstep: each + round takes the range after the laggard's cursor, fetches every distinct reference pool's swaps in it once (the + stock paths share USDG → WETH), times them once, and moves every asset whose cursor is inside the range.""" + from . import blocktime + rpc = rpc or logs_rpc(); out: dict = {} # the stream's (one connection, a live pass's memory), not a new one per pass + with transaction() as conn: + top = conn.execute("SELECT block, detail FROM rh_scan WHERE name = 'rh'").fetchone() + rows = conn.execute("SELECT DISTINCT a.asset, a.decimals, a.ref_path FROM rh_assets a JOIN rh_pools p ON p.quote_asset = a.asset " + "WHERE p.is_pons AND a.class <> 'eth'").fetchall() + if not top: + return out + through = int(top["block"]) + start = int(((top["detail"] or {}).get("start_block")) or config.RH_START_BLOCK or through) # where the index began + with transaction() as conn: # every price cursor in one read: a pass every few seconds cannot afford ~4 queries per asset + curs = {r["name"]: r for r in conn.execute("SELECT name, block, range_blocks, detail FROM rh_scan WHERE name LIKE 'price:%%'").fetchall()} + assets = [] + for a in rows: + c = curs.get(f"price:{a['asset']}") + if a["ref_path"] and c is not None: # the usual case: path known, cursor exists + hops = json.loads(a["ref_path"]) if isinstance(a["ref_path"], str) else a["ref_path"] + assets.append({"asset": a["asset"], "hops": hops, "at": int(c["block"]), "rng": int(c["range_blocks"]), "state": (c["detail"] or {}).get("state", {})}) + continue + try: + with transaction() as conn: + hops = a["ref_path"] if a["ref_path"] else discover(conn, rpc, a["asset"], int(a["decimals"])) + if isinstance(hops, str): + hops = json.loads(hops) + # marks are needed from the first graduation quoted in the asset (a day of history before it for the carry-forward) + fg = conn.execute("SELECT min(grad_block) AS b FROM rh_tokens WHERE quote_asset = %s AND grad_block IS NOT NULL", (a["asset"],)).fetchone()["b"] + first = max(start, int(fg) - 800_000) if fg is not None else through + at, rng = scan.cursor(conn, f"price:{a['asset']}", first - 1) + r = conn.execute("SELECT detail FROM rh_scan WHERE name = %s", (f"price:{a['asset']}",)).fetchone() + state = ((r["detail"] or {}) if r else {}).get("state", {}) + assets.append({"asset": a["asset"], "hops": hops, "at": at, "rng": rng, "state": state}) + except Exception as e: + log.warning("rh prices %s: %s", a["asset"], str(e)[:200]) + for _ in range(max_ranges): + active = [a for a in assets if a["at"] < through] + if not active: + break + lo = min(a["at"] for a in active) + 1 + rng = min(a["rng"] for a in active if a["at"] + 1 == lo) + hi = min(through, lo + rng - 1) + group = [a for a in active if a["at"] < hi] + try: + pools = {} + for a in group: + for h in a["hops"]: + pools.setdefault(_hop_key(h), h) + logs = fetch_pool_logs(rpc, pools, lo, hi) + ts = {int(g["blockNumber"], 16): int(g["blockTimestamp"], 16) for got in logs.values() for g in got + if int(g.get("blockTimestamp") or "0x0", 16)} # HyperSync logs carry their block's time + blocks = sorted({int(g["blockNumber"], 16) for got in logs.values() for g in got} - set(ts)) + if blocks: + ts.update(blocktime.times(rpc, lo, hi, blocks)) + with transaction() as conn: + for a in group: + n = scan_asset(conn, rpc, a["asset"], a["hops"], a["at"] + 1, hi, a["state"], lambda bs: {b: ts[b] for b in bs}, logs) + a["at"] = hi; a["rng"] = scan.grow(rng) + scan.set_cursor(conn, f"price:{a['asset']}", hi, a["rng"], {"state": a["state"]}) + out[a["asset"]] = out.get(a["asset"], 0) + n + except Exception as e: + log.warning("rh prices %s..%s: %s", lo, hi, str(e)[:200]) + for a in group: + a["rng"] = scan.shrink(a["rng"]) + break + return out diff --git a/fly_trader/rh/probe.py b/fly_trader/rh/probe.py new file mode 100644 index 0000000000000000000000000000000000000000..b4322b9ca0c400e07f192e0baf8a7487b57c499d --- /dev/null +++ b/fly_trader/rh/probe.py @@ -0,0 +1,68 @@ +"""``fly-trader rh-probe``: a read-only check of everything the RH trading path relies on, before any key is used. + +1. the RPC answers the expected chain id and every configured contract carries code; +2. for the most recent Pons graduations (per quote class): does KyberSwap route a buy and a sell, what gas does it + quote, and does the UniversalRouter v4 fallback quote and simulate a buy (eth_call from a contract holding ETH); +3. the gas units measured feed ``RH_GAS_RESERVE_ETH`` and the cost model's per-swap transaction fee. +""" +from __future__ import annotations + +import statistics +import time + +from .. import config +from . import abi, kyber as K, univ4 as U +from .rpc import RhRpc + +CONTRACTS = ("PONS_FACTORY", "PONS_ROUTER", "PONS_HOOK", "PONS_LOCKER", "PONS_GRAD_EXECUTOR", "V4_POOL_MANAGER", "V4_QUOTER", "V4_STATE_VIEW", + "UNIVERSAL_ROUTER", "PERMIT2", "V3_FACTORY", "V3_QUOTER_V2", "RH_WETH", "RH_USDG", "KYBER_ROUTER") +POOL_REGISTERED = "PoolRegistered(bytes32,address,address,address)" + + +def recent_graduations(rpc: RhRpc, blocks: int = 400_000, step: int = 50_000) -> list[dict]: + head = rpc.block_number(); t = abi.event_topic(POOL_REGISTERED); out = []; b = head - blocks + while b < head: + try: + logs = rpc.get_logs(config.PONS_HOOK, b, min(head, b + step), [t]); b += step + 1 + except Exception: + time.sleep(2.0); continue + for lg in logs: + meme, quote, creator = abi.decode(["address", "address", "address"], bytes.fromhex(lg["data"][2:])) + out.append({"pool_id": lg["topics"][1], "meme": meme, "quote": quote, "creator": creator, "block": int(lg["blockNumber"], 16)}) + return out + + +def run(sample: int = 12, amount_wei: int = 10 ** 14) -> dict: + rpc = RhRpc(); rep = {"chain_id": rpc.chain(), "expected": config.RH_EXPECTED_CHAIN_ID, "contracts": {}, "pools": []} + for name in CONTRACTS: + a = getattr(config, name); rep["contracts"][name] = {"address": a, "code_bytes": max(0, len(rpc.code(a)) // 2 - 1)} + grads = recent_graduations(RhRpc(config.RH_RPC_URL))[-sample:] # 50k-block log ranges: the public RPC, not the transaction endpoint + ky = K.Kyber(); gas_k = [] + for g in grads: + row = {"meme": g["meme"], "quote": "ETH" if int(g["quote"], 16) == 0 else g["quote"]} + try: + r = ky.get_route(K.NATIVE, g["meme"], amount_wei); row["kyber_buy_gas"] = int(r["routeSummary"].get("gas") or 0); gas_k.append(row["kyber_buy_gas"]) + out = int(r["routeSummary"]["amountOut"]) + s = ky.get_route(g["meme"], K.NATIVE, out); row["kyber_round_trip"] = int(s["routeSummary"]["amountOut"]) / amount_wei + except K.KyberError as e: + row["kyber"] = str(e)[:80] + if row["quote"] == "ETH": + try: + key = U.pool_key(U.ZERO, g["meme"]); q, gq = U.quote(rpc, key, U.ZERO, amount_wei) + data, value = U.swap_calldata(key, U.ZERO, amount_wei, q * 95 // 100) + rpc.call("eth_call", [{"from": config.RH_WETH, "to": config.UNIVERSAL_ROUTER, "data": "0x" + data.hex(), "value": hex(value)}, "latest"]) + row["ur_buy"] = "simulates"; row["ur_quote_gas"] = gq + except Exception as e: + row["ur_buy"] = str(e)[:80] + rep["pools"].append(row); time.sleep(0.3) + base = rpc.base_fee() + rep["base_fee_gwei"] = base / 1e9 + if gas_k: + med = statistics.median(gas_k) + rep["kyber_gas_median"] = med + rep["swap_fee_eth_at_base"] = med * base / 1e18 + rep["suggested_gas_reserve_eth"] = round(max(config.RH_GAS_RESERVE_ETH, 200 * 2 * med * base / 1e18), 6) + trips = [p["kyber_round_trip"] for p in rep["pools"] if "kyber_round_trip" in p] + if trips: + rep["round_trip_median"] = statistics.median(trips) + return rep diff --git a/fly_trader/rh/router.py b/fly_trader/rh/router.py new file mode 100644 index 0000000000000000000000000000000000000000..0a1ab6381f0a4b080bbb7a3c6175087ae5de4471 --- /dev/null +++ b/fly_trader/rh/router.py @@ -0,0 +1,78 @@ +"""One swap's transaction plan: KyberSwap first (atomic, multi-hop — a USDG- or stock-quoted Pons coin is bought as +ETH→base→meme in one transaction), the UniversalRouter v4 single-pool swap when Kyber has no route. + +A plan is what the executor (rh/exec.py) signs, in order: the approvals the route needs (exact amounts), then the swap. +Before the swap is signed the executor simulates it (eth_call as the wallet) — a plan is never trusted blind. +""" +from __future__ import annotations + +from dataclasses import dataclass, field + +from .. import config +from . import kyber as K, univ4 as U + + +class NoRoute(RuntimeError): + pass + + +@dataclass +class Call: + to: str + data: bytes + value: int = 0 + kind: str = "swap" # rh_txs.kind: 'approve' | 'swap' + + +@dataclass +class Plan: + route: str # 'atomic_kyber' | 'atomic_ur' + token_in: str + token_out: str + amount_in: int + expected_out: int + min_out: int + swap: Call + approvals: list = field(default_factory=list) # [(token, spender, exact amount)] made before the swap (skipped when covered) + pre_calls: list = field(default_factory=list) # other calls before the swap (Permit2 → router allowance) + gas_hint: int | None = None + quote: dict | None = None + build: dict | None = None + + +def _is_native(t: str) -> bool: + return t.lower() in (K.NATIVE, U.ZERO) + + +def plan_swap(account: str, token_in: str, token_out: str, amount_in: int, slippage_bps: float, *, kyber: K.Kyber | None = None, + rpc=None, pool: dict | None = None) -> Plan: + """``token_in``/``token_out``: an ERC-20 address, or native ETH (Kyber's 0xeeee… or the zero address). ``pool``: the + Pons pool (currency0, currency1, fee, tick_spacing, hooks) for the fallback.""" + kin = K.NATIVE if _is_native(token_in) else token_in.lower(); kout = K.NATIVE if _is_native(token_out) else token_out.lower() + kerr = None + try: + ky = kyber or K.Kyber() + route = ky.get_route(kin, kout, amount_in) + built = ky.build_route(route, account, slippage_bps) + exp = int(built["amountOut"]); mn = K.decode_desc(built["data"])["minReturnAmount"] + approvals = [] if kin == K.NATIVE else [(kin, config.KYBER_ROUTER.lower(), amount_in)] + return Plan("atomic_kyber", kin, kout, amount_in, exp, mn, Call(config.KYBER_ROUTER.lower(), bytes.fromhex(built["data"][2:]), int(built.get("transactionValue") or 0)), + approvals=approvals, gas_hint=int(route["routeSummary"].get("gas") or 0) or None, quote=route["routeSummary"], + build={k: built.get(k) for k in ("amountIn", "amountOut", "transactionValue", "routerAddress")}) + except K.KyberError as e: + kerr = str(e) + if pool is None or rpc is None: + raise NoRoute(f"KyberSwap: {kerr}; no v4 pool for the fallback") + key = U.pool_key(pool["currency0"], pool["currency1"], pool.get("fee", 0), pool.get("tick_spacing", 200), pool.get("hooks")) + tin = U.ZERO if _is_native(token_in) else token_in.lower(); tout = U.ZERO if _is_native(token_out) else token_out.lower() + if {tin, tout} != {key[0], key[1]}: + raise NoRoute(f"KyberSwap: {kerr}; the v4 fallback swaps within one pool only ({tin}→{tout} is not its pair)") + exp, gas = U.quote(rpc, key, tin, amount_in) + mn = K.min_received(exp, slippage_bps) + data, value = U.swap_calldata(key, tin, amount_in, mn) + approvals, pre = [], [] + if tin != U.ZERO: + approvals = [(tin, config.PERMIT2.lower(), amount_in)] + pre = [Call(config.PERMIT2.lower(), U.permit2_approve_calldata(tin, amount_in), 0, "approve")] + return Plan("atomic_ur", tin, tout, amount_in, exp, mn, Call(config.UNIVERSAL_ROUTER.lower(), data, value), approvals=approvals, pre_calls=pre, + gas_hint=gas, quote={"kyber_error": kerr, "v4_quote": str(exp)}) diff --git a/fly_trader/rh/rpc.py b/fly_trader/rh/rpc.py new file mode 100644 index 0000000000000000000000000000000000000000..e9aa7c17b6b037d408d1dcd69191ff7eb7623886 --- /dev/null +++ b/fly_trader/rh/rpc.py @@ -0,0 +1,129 @@ +"""JSON-RPC for the RH trading path: fly_trader/vault/evm.EvmRpc (reads, logged and scrubbed) plus what sending and +indexing need — raw transactions, receipts, nonces, balances, fee history, gas estimates, multi-address logs and +batched requests. vault/evm.py is left untouched (the vault is edited concurrently on master). +""" +from __future__ import annotations + +import time +from typing import Any + +from .. import config +from ..db.apilog import record_api_call +from ..vault.evm import EvmRpc, EvmRpcError + + +RATE_LIMIT_TRIES = 6 # 1, 2, 4, 8, 16 s: the public endpoint's 429s clear within seconds + + +def _rate_limited(e: Exception) -> bool: + s = str(e) + return "429" in s or "Too Many" in s or "rate limit" in s.lower() + + +class RhRpc(EvmRpc): + """Without an explicit ``url``: the transaction endpoint (``RH_RPC_URL_TX``, e.g. QuickNode), else the public RPC. Log + readers pass their endpoint (``logs_rpc``): the transaction endpoint may refuse log ranges.""" + def __init__(self, url: str | None = None, chain_id: int | None = None, service: str = "rh_rpc", timeout: float = 20.0): + super().__init__(url or config.RH_RPC_URL_TX or config.RH_RPC_URL, chain_id if chain_id is not None else config.RH_EXPECTED_CHAIN_ID, service, timeout) + + def call(self, method: str, params: list | None = None) -> Any: + """``EvmRpc.call`` that waits out a rate limit (429) with exponential backoff; any other error raises at once. + Never retried: eth_sendRawTransaction (the wallet decides what a failed broadcast means).""" + for i in range(RATE_LIMIT_TRIES): + try: + return super().call(method, params) + except EvmRpcError as e: + if method == "eth_sendRawTransaction" or not _rate_limited(e) or i == RATE_LIMIT_TRIES - 1: + raise + time.sleep(2 ** i) + + # ---- batching: many reads in one POST (tx lookups while indexing, balances while reconciling) + def batch(self, calls: list[tuple[str, list]]) -> list[Any]: + """Results in call order; an individual error comes back as an ``EvmRpcError`` instance (not raised). A rate-limited + batch waits and retries like ``call``.""" + for i in range(RATE_LIMIT_TRIES): + try: + return self._batch(calls) + except EvmRpcError as e: + if not _rate_limited(e) or i == RATE_LIMIT_TRIES - 1: + raise + time.sleep(2 ** i) + return [] + + def _batch(self, calls: list[tuple[str, list]]) -> list[Any]: + if not calls: + return [] + with self._lock: + base = self._id + 1; self._id += len(calls) + body = [{"jsonrpc": "2.0", "id": base + i, "method": m, "params": p} for i, (m, p) in enumerate(calls)] + t0 = time.monotonic(); status, ok, err = None, False, None + try: + r = self._client.post(self.url, json=body) + status = r.status_code; r.raise_for_status() + got = {x.get("id"): x for x in (r.json() or [])} + out = [] + for i, (m, _) in enumerate(calls): + x = got.get(base + i) + if x is None: + out.append(EvmRpcError(m, {"message": "missing from batch response"})) + elif x.get("error"): + out.append(EvmRpcError(m, x["error"])) + else: + out.append(x.get("result")) + ok = True + return out + except Exception as e: + err = type(e).__name__ + raise EvmRpcError("batch", {"message": err + (f" {status}" if status else "")}) from None + finally: + record_api_call(self.service, f"batch[{len(calls)}]", "POST", status, int((time.monotonic() - t0) * 1000), ok, err) + + # ---- sending + def send_raw(self, raw_hex: str) -> str: + return self.call("eth_sendRawTransaction", [raw_hex]) + + def receipt(self, tx_hash: str) -> dict | None: + return self.call("eth_getTransactionReceipt", [tx_hash]) + + def tx_by_hash(self, tx_hash: str) -> dict | None: + return self.call("eth_getTransactionByHash", [tx_hash]) + + def tx_count(self, address: str, tag: str = "pending") -> int: + return int(self.call("eth_getTransactionCount", [address, tag]), 16) + + def get_balance(self, address: str, tag: str = "latest") -> int: + return int(self.call("eth_getBalance", [address, tag]), 16) + + def estimate_gas(self, tx: dict) -> int: + return int(self.call("eth_estimateGas", [tx]), 16) + + def base_fee(self) -> int: + b = self.block("latest") + return int(b.get("baseFeePerGas") or "0x0", 16) + + def max_priority_fee(self) -> int: + try: + return int(self.call("eth_maxPriorityFeePerGas", []), 16) + except EvmRpcError: + return 0 # Arbitrum-style chains order by arrival; the tip is ignored + + def get_logs_multi(self, addresses: list[str] | None, from_block: int, to_block: int, topics: list | None = None) -> list[dict]: + """``eth_getLogs`` over several emitters (or any emitter when ``addresses`` is None).""" + flt: dict = {"fromBlock": hex(from_block), "toBlock": hex(to_block)} + if addresses: + flt["address"] = addresses if len(addresses) > 1 else addresses[0] + if topics: + flt["topics"] = topics + return self.call("eth_getLogs", [flt]) or [] + + +def logs_rpc() -> RhRpc: + """The endpoint for heavy reads (getLogs, tx lookups): ``RH_RPC_URL_LOGS`` when set, else the public RPC. Its ``blocks`` + attribute is the endpoint block headers are read from (``RH_RPC_URL_BLOCKS``).""" + if config.env_str("ENVIO_API_TOKEN"): # bulk history from HyperSync (rh/hypersync.py) + from .hypersync import HyperRpc, HyperSync + r = HyperRpc(HyperSync(config.env_str("ENVIO_API_TOKEN")), url=config.RH_RPC_URL_LOGS or config.RH_RPC_URL, service="rh_logs") + else: + r = RhRpc(config.RH_RPC_URL_LOGS or config.RH_RPC_URL, service="rh_logs") + r.blocks = RhRpc(config.RH_RPC_URL_BLOCKS, service="rh_blocks") if config.RH_RPC_URL_BLOCKS else r + return r diff --git a/fly_trader/rh/scan.py b/fly_trader/rh/scan.py new file mode 100644 index 0000000000000000000000000000000000000000..d736ef251d748545e6c1002ba9e195160efaa8fa --- /dev/null +++ b/fly_trader/rh/scan.py @@ -0,0 +1,67 @@ +"""A cursor over block ranges for the RH indexer: the vault indexer's adaptive-range pattern (vault/rh_index.run_once) +with the public RPC's limits measured on 2026-09-28 built in — a 20,000-block eth_getLogs range sustains ~12,000 blocks/s +at one request per second; wider ranges or faster requests return 429. + +``advance(conn, name, head, step_fn)`` runs ``step_fn(from_block, to_block)`` on the next range after the cursor, commits +the cursor with the work, grows the range after a success and halves it after a failure (never below ``MIN_RANGE``). +""" +from __future__ import annotations + +import logging +import time + +from .. import config + +log = logging.getLogger(__name__) + +MIN_RANGE, MAX_RANGE, START_RANGE = 250, 20_000, 5_000 +MAX_RANGE_FAST = 50_000 # HyperSync answers whole ranges in one paginated query; ~0.04 busy days of logs held at once +PAUSE_S = 0.8 # between RPC requests: the public endpoint answers 429 above ~1 request/s +BACKOFF_S = 3.0 + + +def fast() -> bool: + """A log source without the public endpoint's ~1 request/s limit (a dedicated RPC or HyperSync).""" + return bool(config.RH_RPC_URL_LOGS or config.env_str("ENVIO_API_TOKEN")) + + +def max_range() -> int: + return MAX_RANGE_FAST if config.env_str("ENVIO_API_TOKEN") else MAX_RANGE + + +def pause() -> None: + time.sleep(PAUSE_S if not fast() else 0.05) + + +def cursor(conn, name: str, default: int) -> tuple[int, int]: + r = conn.execute("SELECT block, range_blocks FROM rh_scan WHERE name = %s", (name,)).fetchone() + if r is None: + conn.execute("INSERT INTO rh_scan (name, block, range_blocks) VALUES (%s, %s, %s) ON CONFLICT (name) DO NOTHING", (name, default, START_RANGE)) + return default, START_RANGE + return int(r["block"]), int(r["range_blocks"]) + + +def set_cursor(conn, name: str, block: int, rng: int, detail: dict | None = None) -> None: + import json + conn.execute("UPDATE rh_scan SET block = %s, range_blocks = %s, detail = COALESCE(%s::jsonb, detail), updated_at = now() WHERE name = %s", + (block, rng, json.dumps(detail) if detail else None, name)) + + +def next_range(done_through: int, rng: int, head: int) -> tuple[int, int] | None: + lo = done_through + 1 + if lo > head: + return None + return lo, min(head, lo + rng - 1) + + +def grow(rng: int) -> int: + return min(max_range(), int(rng * 1.5) + 1) + + +def shrink(rng: int) -> int: + return max(MIN_RANGE, rng // 2) + + +def is_rate_limit(e: Exception) -> bool: + s = str(e) + return "429" in s or "rate" in s.lower() or "Too Many" in s diff --git a/fly_trader/rh/schema.py b/fly_trader/rh/schema.py new file mode 100644 index 0000000000000000000000000000000000000000..715c161ff01f9b5e3347e4a3c016414683de4478 --- /dev/null +++ b/fly_trader/rh/schema.py @@ -0,0 +1,141 @@ +"""Robinhood Chain tables: the Pons/Uniswap-v4 index, RH minutes and meta, and the live RH book's on-chain ledger. + +Idempotent DDL outside the version ladder in db/schema.py (applied after VAULT_DDL), like the vault's: no migration +number is taken, so master's concurrent Kalshi/vault migrations never collide. Only ever ADD here. + +Units are in the names: ``*_raw`` = integer token units (numeric(78,0): a uint256 fits), ``*_wei`` = native ETH wei, +``*_eth`` = ETH as a float, ``*_q`` = the pool's quote asset in whole units. Addresses are lowercase hex; pool ids are +the v4 PoolId (bytes32 hex). The existing ledger tables (positions, decisions, beats, wealth_marks) carry the RH books +too: their ``*_sol`` columns hold the book's own unit (ETH for paper_rh_* / live_rh, see fly_trader/markets.py). +""" +from __future__ import annotations + +RH_DDL: list[str] = [ + # ---- indexer cursors: one per log source, each with its adaptive getLogs range + """CREATE TABLE IF NOT EXISTS rh_scan ( + name text PRIMARY KEY, block bigint NOT NULL DEFAULT 0, range_blocks bigint NOT NULL DEFAULT 50000, + detail jsonb, updated_at timestamptz NOT NULL DEFAULT now())""", + # ---- quote assets a Pons launch may pair with (factory-approved): the base legs and their ETH marks + """CREATE TABLE IF NOT EXISTS rh_assets ( + asset text PRIMARY KEY, symbol text, class text NOT NULL CHECK (class IN ('eth', 'stable', 'btc', 'stock')), + decimals int NOT NULL, approved boolean NOT NULL DEFAULT true, graduation_threshold_raw numeric(78,0), + ref_pool text, ref_kind text, ref_quote text, ref_fee_bps int, trading_hours jsonb, note text, + updated_at timestamptz NOT NULL DEFAULT now())""", + # ---- Pons launches (the curve phase feeds features; trading starts at graduation) + """CREATE TABLE IF NOT EXISTS rh_tokens ( + token text PRIMARY KEY, launch_block bigint, launch_ts timestamptz, launch_tx text, creator text, curve text, + quote_asset text, quote_class text, decimals int NOT NULL DEFAULT 18, name text, symbol text, uri text, + supply_raw numeric(78,0), dev_quote_in_raw numeric(78,0), dev_tokens_raw numeric(78,0), creator_tax_bps int, curve_fee_bps int, + graduated_at timestamptz, grad_block bigint, pool_id text, grad_position_id numeric(78,0), grad_quote_raw numeric(78,0), + grad_token_raw numeric(78,0), status text NOT NULL DEFAULT 'curve' CHECK (status IN ('curve', 'graduated')), + updated_at timestamptz NOT NULL DEFAULT now())""", + "CREATE INDEX IF NOT EXISTS rh_tokens_creator_idx ON rh_tokens (creator, launch_ts)", + "CREATE INDEX IF NOT EXISTS rh_tokens_curve_idx ON rh_tokens (curve)", + """CREATE TABLE IF NOT EXISTS rh_pools ( + pool_id text PRIMARY KEY, token text, currency0 text NOT NULL, currency1 text NOT NULL, token_is_0 boolean, + fee int, tick_spacing int, hooks text, quote_asset text, init_block bigint, init_ts timestamptz, is_pons boolean NOT NULL DEFAULT false, + sqrt_price_x96 numeric(78,0), liquidity numeric(78,0), tick int, updated_block bigint)""", + "CREATE INDEX IF NOT EXISTS rh_pools_token_idx ON rh_pools (token)", + """CREATE TABLE IF NOT EXISTS rh_curve_trades ( + block bigint NOT NULL, log_index int NOT NULL, tx_hash text NOT NULL, ts timestamptz NOT NULL, token text NOT NULL, + side smallint NOT NULL, trader text, recipient text, quote_raw numeric(78,0), tokens_raw numeric(78,0), fee_raw numeric(78,0), + tax_raw numeric(78,0), snipe_tax_raw numeric(78,0), quote_eth double precision, PRIMARY KEY (block, log_index))""", + "CREATE INDEX IF NOT EXISTS rh_curve_trades_token_idx ON rh_curve_trades (token, ts)", + # ---- graduated-pool swaps (Uniswap v4 PoolManager Swap on Pons pools), priced in the quote and in ETH + """CREATE TABLE IF NOT EXISTS rh_swaps ( + block bigint NOT NULL, log_index int NOT NULL, tx_hash text NOT NULL, block_hash text, ts timestamptz NOT NULL, + pool_id text NOT NULL, token text NOT NULL, trader text, side smallint NOT NULL, token_raw numeric(78,0), quote_raw numeric(78,0), + hook_fee_raw numeric(78,0), hook_tax_raw numeric(78,0), sqrt_price_x96 numeric(78,0), liquidity numeric(78,0), tick int, + price_q double precision, base_eth double precision, price_eth double precision, quote_eth double precision, + resq_q double precision, resq_eth double precision, PRIMARY KEY (block, log_index))""", + "CREATE INDEX IF NOT EXISTS rh_swaps_token_ts_idx ON rh_swaps (token, ts)", + "CREATE INDEX IF NOT EXISTS rh_swaps_ts_idx ON rh_swaps (ts)", + """CREATE TABLE IF NOT EXISTS rh_liquidity ( + block bigint NOT NULL, log_index int NOT NULL, tx_hash text, ts timestamptz, pool_id text NOT NULL, sender text, + tick_lower int, tick_upper int, liquidity_delta numeric(78,0), salt text, PRIMARY KEY (block, log_index))""", + # ---- each quote asset's ETH (and USD) value per minute: the base leg's mark + """CREATE TABLE IF NOT EXISTS rh_base_prices ( + asset text NOT NULL, ts timestamptz NOT NULL, price_eth double precision, price_usd double precision, n_obs int NOT NULL DEFAULT 0, + stale boolean NOT NULL DEFAULT false, PRIMARY KEY (asset, ts))""", + # ---- RH minutes: pump_minutes' twin, amounts in ETH (non-ETH-quoted pools converted at the minute's base/ETH) + """CREATE TABLE IF NOT EXISTS rh_minutes ( + mint text NOT NULL, ts timestamptz NOT NULL, pool_id text, quote_asset text, quote_class text, + open double precision, high double precision, low double precision, close double precision, close_q double precision, base_eth double precision, + buy_eth double precision, sell_eth double precision, n_buys int, n_sells int, n_traders int, resq_eth double precision, resq_q double precision, + fee_rate double precision, n_buyers int, wash_eth double precision, wash_buy_eth double precision, top_sell_eth double precision, + insider_sell_eth double precision, skill_buy real[], revised boolean NOT NULL DEFAULT false, PRIMARY KEY (mint, ts))""", + "CREATE INDEX IF NOT EXISTS rh_minutes_ts_idx ON rh_minutes (ts)", + # ---- per-launch facts in corpus_meta's column names (values in ETH) so both chains feed the same model inputs + """CREATE TABLE IF NOT EXISTS rh_meta ( + mint text PRIMARY KEY, graduated_at timestamptz, create_ts timestamptz, creator text, quote_asset text, quote_class text, + supply double precision, ttg_min double precision, dev_sol double precision, dev_tokens double precision, dev_share double precision, + mayhem boolean NOT NULL DEFAULT false, rq0 double precision, pool_id text, + prior_launches int, prior_grads int, prior_known int, prior_rug_share real, prior_moon_share real, + own_dd60 real, own_max60 real, own_alive6h boolean, + bundle_share real, dev_hold_share real, dev_sold_frac real, grad_hhi real, curve_known real, + updated_at timestamptz NOT NULL DEFAULT now())""", + "CREATE INDEX IF NOT EXISTS rh_meta_creator_idx ON rh_meta (creator, graduated_at)", + """CREATE TABLE IF NOT EXISTS rh_insiders ( + mint text NOT NULL, wallet text NOT NULL, kind text NOT NULL, PRIMARY KEY (mint, wallet))""", + # ---- the live RH book's on-chain ledger (fly_trader/rh/exec.py, rh/accounting.py) + """CREATE TABLE IF NOT EXISTS rh_intents ( + id bigserial PRIMARY KEY, book text NOT NULL, decision_id bigint, position_id bigint, + kind text NOT NULL CHECK (kind IN ('buy', 'sell', 'base_out', 'sweep', 'liquidate', 'payout')), + token text, quote_asset text, route text CHECK (route IN ('atomic_kyber', 'atomic_ur', 'two_leg', 'transfer')), + amount_in_raw numeric(78,0), min_out_raw numeric(78,0), + state text NOT NULL DEFAULT 'new' CHECK (state IN ('new', 'leg1', 'leg2', 'done', 'failed', 'aborted')), + attempts int NOT NULL DEFAULT 0, committed_wei numeric(78,0) NOT NULL DEFAULT 0, plan jsonb, error text, + created_at timestamptz NOT NULL DEFAULT now(), updated_at timestamptz NOT NULL DEFAULT now())""", + "CREATE UNIQUE INDEX IF NOT EXISTS rh_intents_decision_uniq ON rh_intents (decision_id, kind) WHERE decision_id IS NOT NULL", + "CREATE INDEX IF NOT EXISTS rh_intents_open_idx ON rh_intents (state) WHERE state NOT IN ('done', 'failed', 'aborted')", + """CREATE TABLE IF NOT EXISTS rh_txs ( + id bigserial PRIMARY KEY, intent_id bigint, position_id bigint, + kind text NOT NULL CHECK (kind IN ('approve', 'swap', 'cancel', 'transfer', 'payout', 'wrap')), + from_addr text NOT NULL, nonce bigint NOT NULL, hash text NOT NULL UNIQUE, raw text NOT NULL, to_addr text, + value_wei numeric(78,0) NOT NULL DEFAULT 0, data_sha256 text, gas_limit bigint, max_fee_wei numeric(78,0), max_prio_wei numeric(78,0), + status text NOT NULL DEFAULT 'signed' CHECK (status IN ('signed', 'sent', 'mined_ok', 'reverted', 'dropped', 'replaced')), + block bigint, block_hash text, gas_used bigint, eff_gas_price_wei numeric(78,0), fee_wei numeric(78,0), + router text, quote jsonb, build jsonb, applied_at timestamptz, error text, + created_at timestamptz NOT NULL DEFAULT now(), updated_at timestamptz NOT NULL DEFAULT now())""", + "CREATE INDEX IF NOT EXISTS rh_txs_nonce_idx ON rh_txs (from_addr, nonce)", + "CREATE INDEX IF NOT EXISTS rh_txs_open_idx ON rh_txs (status) WHERE applied_at IS NULL", + """CREATE TABLE IF NOT EXISTS rh_legs ( + id bigserial PRIMARY KEY, intent_id bigint NOT NULL, tx_id bigint, position_id bigint, leg_no int NOT NULL, + asset_in text NOT NULL, asset_out text NOT NULL, amount_in_raw numeric(78,0) NOT NULL, amount_out_raw numeric(78,0) NOT NULL, + eth_in double precision, eth_out double precision, + verified_by text NOT NULL CHECK (verified_by IN ('balance_delta', 'receipt_logs', 'model')), ts timestamptz NOT NULL DEFAULT now())""", + "CREATE INDEX IF NOT EXISTS rh_legs_position_idx ON rh_legs (position_id)", + """CREATE TABLE IF NOT EXISTS rh_base_lots ( + id bigserial PRIMARY KEY, asset text NOT NULL, qty_raw numeric(78,0) NOT NULL, basis_eth double precision NOT NULL, + position_id bigint, intent_id bigint, source text NOT NULL CHECK (source IN ('entry_leg', 'exit_leg', 'stray')), + status text NOT NULL DEFAULT 'open' CHECK (status IN ('open', 'consumed', 'liquidated', 'dust')), + proceeds_eth double precision, fee_eth double precision, opened_at timestamptz NOT NULL DEFAULT now(), closed_at timestamptz)""", + "CREATE INDEX IF NOT EXISTS rh_base_lots_open_idx ON rh_base_lots (status) WHERE status = 'open'", + """CREATE TABLE IF NOT EXISTS rh_wallet_flows ( + id bigserial PRIMARY KEY, block bigint, ts timestamptz NOT NULL DEFAULT now(), direction text NOT NULL CHECK (direction IN ('in', 'out')), + kind text NOT NULL CHECK (kind IN ('deposit', 'withdrawal', 'payout', 'unexplained_in', 'unknown_outbound')), + wei numeric(78,0) NOT NULL, note text)""", + """CREATE TABLE IF NOT EXISTS rh_wallet_marks ( + ts timestamptz PRIMARY KEY, native_wei numeric(78,0), positions_eth double precision, lots_eth double precision, + exit_cost_eth double precision, reserved_wei numeric(78,0), wealth_eth double precision, consistent boolean NOT NULL DEFAULT true)""", + # ---- additive columns on shared tables (defaults keep every Solana row as it was) + # the hook's fee + tax as a fraction of the swap's gross amount (Pons takes it in whichever currency comes in) + "ALTER TABLE rh_swaps ADD COLUMN IF NOT EXISTS fee_frac double precision", + # when a receipt's fee was recorded: the reconciler counts gas by it (updated_at moves on later status changes) + "ALTER TABLE rh_txs ADD COLUMN IF NOT EXISTS fee_at timestamptz", + # a pool's hook fee + tax is fixed at launch (measured: 1,665 of 1,732 pools vary < 0.05 points): read from its first + # FEE_OBS swaps' HookFeeCollected logs, then applied to its later swaps without fetching their fee logs (rh/index.py) + "ALTER TABLE rh_pools ADD COLUMN IF NOT EXISTS hook_fee double precision", + "ALTER TABLE rh_pools ADD COLUMN IF NOT EXISTS fee_obs int NOT NULL DEFAULT 0", + "ALTER TABLE rh_tokens ADD COLUMN IF NOT EXISTS curve_indexed boolean NOT NULL DEFAULT false", + "ALTER TABLE rh_assets ADD COLUMN IF NOT EXISTS ref_path jsonb", + "ALTER TABLE positions ADD COLUMN IF NOT EXISTS chain text NOT NULL DEFAULT 'sol'", + "ALTER TABLE positions ADD COLUMN IF NOT EXISTS quote_asset text", + "ALTER TABLE positions ADD COLUMN IF NOT EXISTS gas_q double precision NOT NULL DEFAULT 0", + "ALTER TABLE fly_scored ADD COLUMN IF NOT EXISTS chain text NOT NULL DEFAULT 'sol'", + "ALTER TABLE notional_ledger ADD COLUMN IF NOT EXISTS market text NOT NULL DEFAULT 'sol'", + # rails circuit 3 (fly_trader/markets.RH_CIRCUIT); the id CHECK only allowed 1 before the Kalshi migration dropped it, + # and an install still on the distribution's older ladder may carry it + "ALTER TABLE circuit_state DROP CONSTRAINT IF EXISTS circuit_state_id_check", + "INSERT INTO circuit_state (id) VALUES (3) ON CONFLICT DO NOTHING", +] diff --git a/fly_trader/rh/smoke.py b/fly_trader/rh/smoke.py new file mode 100644 index 0000000000000000000000000000000000000000..3eee58affacc8e2f36dac2cdcc066975e02cb831 --- /dev/null +++ b/fly_trader/rh/smoke.py @@ -0,0 +1,100 @@ +"""The RH bot wallet's key, and the tiny mainnet smoke test (plan P11). + +``new_key``: a fresh secp256k1 key written to the project .env as RH_BOT_PRIVATE_KEY with its address pinned as +RH_BOT_ADDRESS (the guard refuses a key that does not derive to it). Refuses when a key is already there. + +``run``: three real round trips of ``size`` ETH through the live executor and accounting (book live_rh, strategy +'smoke'): KyberSwap on an ETH-quoted Pons pool, the Uniswap v4 fallback on the same pool, and a stock- or USDG-quoted +coin (ETH → base → coin in one transaction). Each is booked from measured balances; afterwards the reconciler must find +the chain and the books equal. Refuses unless signing is allowed (RH_LIVE_ENABLED=1, key, pinned address, chain 4663) +and the whole test fits under ``cap`` ETH of spend. +""" +from __future__ import annotations + +import os +import re +from pathlib import Path + +from .. import config +from ..db.connection import transaction +from .tx import address_of + +WEI = 10 ** 18 +SMOKE = "smoke" + + +def new_key(env_path: Path | None = None) -> str: + from coincurve import PrivateKey + path = Path(env_path or config.REPO_ROOT / ".env") + text = path.read_text() if path.exists() else "" + if re.search(r"^RH_BOT_PRIVATE_KEY=\S", text, re.M) or config.env_str("RH_BOT_PRIVATE_KEY"): + raise SystemExit("RH_BOT_PRIVATE_KEY is already set: refusing to replace a key that may hold funds") + while True: + key = os.urandom(32) + try: + PrivateKey(key); break + except ValueError: + continue + addr = address_of(key) + text = re.sub(r"^RH_BOT_ADDRESS=.*\n?", "", text, flags=re.M) + if text and not text.endswith("\n"): + text += "\n" + text += f"# the Robinhood Chain bot wallet (fly-trader rh-wallet new); fund it with ETH on Robinhood Chain\nRH_BOT_PRIVATE_KEY=0x{key.hex()}\nRH_BOT_ADDRESS={addr}\n" + path.write_text(text); os.chmod(path, 0o600) + return addr + + +def _pools(conn) -> dict: + """The most traded graduated ETH-quoted pool and non-ETH-quoted pool the indexer knows.""" + q = ("SELECT p.pool_id, p.token, p.currency0, p.currency1, p.fee, p.tick_spacing, p.hooks, p.quote_asset FROM rh_pools p JOIN rh_tokens t ON t.token = p.token " + "WHERE t.status = 'graduated' AND p.is_pons AND (p.quote_asset = %s) = %s ORDER BY (SELECT count(*) FROM rh_swaps s WHERE s.pool_id = p.pool_id) DESC LIMIT 1") + zero = "0x" + "00" * 20 + return {"eth": conn.execute(q, (zero, True)).fetchone(), "base": conn.execute(q, (zero, False)).fetchone()} + + +def run(size_eth: float = 0.002, cap_eth: float = 0.01) -> dict: + from . import accounting as A, kyber as K, router + from .exec import RhExecutor, RhRequest + from .rpc import RhRpc + from .wallet import RhWallet + if 3 * size_eth > cap_eth: + raise SystemExit(f"three round trips of {size_eth} ETH exceed the {cap_eth} ETH cap") + w = RhWallet(RhRpc()) # the guard: RH_LIVE_ENABLED, key, pinned address, chain id + native = w.balance() + if native < int((size_eth + config.RH_GAS_RESERVE_ETH) * WEI): + raise SystemExit(f"the RH wallet {w.address} holds {native / WEI:.6f} ETH: fund it with at least {size_eth + config.RH_GAS_RESERVE_ETH:.4f} ETH") + with transaction() as conn: + pools = {k: dict(v) for k, v in _pools(conn).items() if v} + A.reconcile(conn, w) + if "eth" not in pools: + raise SystemExit("no graduated ETH-quoted Pons pool indexed yet (run rh-backfill first)") + + class _NoKyber: + def get_route(self, *a, **k): + raise K.KyberError("smoke: the v4 fallback") + + fallback = lambda account, tin, tout, amount, slip, rpc=None, pool=None: router.plan_swap(account, tin, tout, amount, slip, kyber=_NoKyber(), rpc=rpc, pool=pool) # noqa: E731 + trips = [("kyber, ETH pool", pools["eth"], None), ("v4 fallback, ETH pool", pools["eth"], fallback)] + if "base" in pools: + trips.append(("kyber, non-ETH quote", pools["base"], None)) + out = [] + for name, pool, planner in trips: + ex = RhExecutor(w, planner=planner, start=False) + b = ex.execute(RhRequest(kind="buy", token=pool["token"], quote_asset=pool["quote_asset"], amount_in=int(size_eth * WEI), slippage_bps=300, + max_slippage_bps=800, pool=pool, hold_s=60.0, strategy=SMOKE)) + trip = {"trip": name, "token": pool["token"], "buy": {"ok": b.ok, "route": b.route, "error": b.error}} + if b.ok: + with transaction() as conn: + qty = int(conn.execute("SELECT qty FROM positions WHERE id = %s", (b.position_id,)).fetchone()["qty"]) + s = ex.execute(RhRequest(kind="sell", token=pool["token"], quote_asset=pool["quote_asset"], amount_in=qty, slippage_bps=300, max_slippage_bps=800, + pool=pool, position_id=b.position_id, strategy=SMOKE)) + with transaction() as conn: + p = conn.execute("SELECT cost_sol, realized_sol, gas_q, status FROM positions WHERE id = %s", (b.position_id,)).fetchone() + txs = [r["hash"] for r in conn.execute("SELECT t.hash FROM rh_txs t JOIN rh_intents i ON i.id = t.intent_id WHERE i.position_id = %s ORDER BY t.id", + (b.position_id,)).fetchall()] + trip.update(sell={"ok": s.ok, "route": s.route, "error": s.error}, position=dict(p), txs=txs) + out.append(trip) + with transaction() as conn: + rec = A.reconcile(conn, w) + return {"wallet": w.address, "start_eth": native / WEI, "end_eth": w.balance() / WEI, "trips": out, "reconcile": rec, + "passed": all(t["buy"]["ok"] and t.get("sell", {}).get("ok") for t in out) and rec["ok"]} diff --git a/fly_trader/rh/stream.py b/fly_trader/rh/stream.py new file mode 100644 index 0000000000000000000000000000000000000000..125ba2775fefa518c5d5fd620c8eec03f5cb03e4 --- /dev/null +++ b/fly_trader/rh/stream.py @@ -0,0 +1,144 @@ +"""The ``rh_stream`` worker: Robinhood Chain from the Pons V2 deployment to the head, then live. + +Each pass: the index cursor advances (rh/index.py; many ranges while backfilling, one near the head), the quote assets' +marks follow it (rh/prices.py), the complete minutes are written (rh/minutes.py), and the graduated launches' meta +(rh/meta.py) is refreshed. ``rh_stream_status.flushed_through`` gates the engine's RH minutes, as pumpstream_status gates +Solana's. Near the head a pass reads its logs in one query (rh/prefetch.py: HyperSync, or the RPC while HyperSync fails) +and the next pass starts as soon as the next minute can be written (``_wait_s``), at most ``RH_STREAM_POLL_S`` later. +""" +from __future__ import annotations + +import json +import logging +import threading +import time +from datetime import datetime, timezone + +from .. import config +from ..db.connection import transaction +from ..logging_setup import setup +from . import index, minutes, prefetch, prices, scan +from .rpc import logs_rpc + +log = logging.getLogger(__name__) +BACKFILL_RANGES = 20 # ranges per pass while far behind (then a status write) +LIVE_LAG_BLOCKS = 5_000 # closer than this to the head counts as live +RPC_CATCHUP_S = 3.0 # between catch-up passes while HyperSync is down (each reads ~2-3 times from the public RPC) +SLOW_EVERY_S = 60.0 # meta and corpus refresh with each written minute, else at least this often +_SLOW = {"at": 0.0} + + +def _status(**kv) -> None: + with transaction() as conn: + r = conn.execute("SELECT value FROM ui_settings WHERE key = 'rh_stream_status'").fetchone() + cur = (r["value"] if r and isinstance(r["value"], dict) else json.loads(r["value"])) if r else {} + cur.update(kv); cur["updated_at"] = datetime.now(timezone.utc).isoformat() + conn.execute("INSERT INTO ui_settings (key, value) VALUES ('rh_stream_status', %s) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()", + (json.dumps(cur, default=str),)) + + +def _head(rpc) -> int: + """The chain head: the header endpoint's (RH_RPC_URL_BLOCKS: the freshest head measured, 2026-10-09, and no ~20 + requests/min limit like the official RPC's), else the RPC's, else HyperSync's archive height — a pass every few + seconds must not stop because one of them is down.""" + err = None + for src in (getattr(rpc, "blocks", None), rpc): + if src is None: + continue + try: + return src.block_number() + except Exception as e: + err = e + hs = getattr(rpc, "hs", None) + if hs is not None and hs.height: + return int(hs.height) + raise err + + +def one_pass(rpc, skill=None) -> dict: + tm: dict[str, float] = {}; t = time.monotonic() + + def lap(step: str) -> None: # seconds per step, in the status: where a slow pass spends its time + nonlocal t + now = time.monotonic(); tm[step] = round(now - t, 2); t = now + head = _head(rpc) + with transaction() as conn: + r = conn.execute("SELECT block, range_blocks FROM rh_scan WHERE name = %s", (index.CURSOR,)).fetchone() + at = int(r["block"]) if r else -1 + behind = head - (at if r else 0) + lap("head") + src = rpc; pre = None + if r is not None and at >= 0 and (behind <= LIVE_LAG_BLOCKS or prefetch.hypersync_down()): + rng = scan.next_range(at, int(r["range_blocks"] or scan.START_RANGE), head - config.RH_CONFIRMATIONS) + if rng is not None: # one query for everything the pass reads (rh/prefetch.py) + pre = src = prefetch.fetch(rpc, *rng); lap("logs") + ix = index.run_once(src, max_ranges=1, head=pre.hi) + else: + ix = {"through": at, "head": head, "ranges": 0, "counts": {}} + lap("index") + px = prices.run_once(max_ranges=2, rpc=src); lap("prices") + else: + ix = index.run_once(rpc, max_ranges=BACKFILL_RANGES if behind > LIVE_LAG_BLOCKS else 1); lap("index") + px = prices.run_once(max_ranges=BACKFILL_RANGES if behind > LIVE_LAG_BLOCKS else 2, rpc=rpc); lap("prices") + from . import wallet_skill + wallet_skill.daily(); lap("skill_tables") # today's and tomorrow's tables, before their minutes are written + mn = minutes.run_once(src, wallet_skill.load_today() if skill is None else skill); lap("minutes") + from . import corpus, meta + mt, cp = {}, {} + if mn.get("minutes") or time.monotonic() - _SLOW["at"] >= SLOW_EVERY_S: # after a written minute: passes in between stay short + _SLOW["at"] = time.monotonic() + mt = meta.run_once(); lap("meta") + cp = corpus.run_once(max_days=1) if behind <= LIVE_LAG_BLOCKS * 20 else {} # the training corpus follows once near the head + lap("corpus") + out = {"head": head, "index_through": ix.get("through"), "behind_blocks": head - (ix.get("through") or 0), "index": ix.get("counts"), "prices": px, + "minutes": mn, "meta": mt, "corpus": cp, "mode": "backfill" if behind > LIVE_LAG_BLOCKS else "live", "timings_s": tm, + "logs_from": pre.source if pre is not None else ("hypersync" if hasattr(rpc, "hs") else "rpc"), + "memory_reads": {"served": pre.served, "passed": pre.passed} if pre is not None else None, "hypersync": prefetch.status() or None} + _status(**{k: v for k, v in out.items() if k != "minutes"}, minutes_through=mn.get("through")) + return out + + +def main(stop_event: threading.Event | None = None) -> None: + setup("rh_stream") + if not config.RH_ENABLED: + log.info("RH_ENABLED is off: rh_stream idles"); return + from ..db import schema + schema.apply_schema() + rpc = logs_rpc(); stop = stop_event or threading.Event() + log.info("rh_stream: logs from %s; the head from %s", "HyperSync" if config.env_str("ENVIO_API_TOKEN") else "the RPC", + "the dedicated RPC (RH_RPC_URL_LOGS)" if config.RH_RPC_URL_LOGS else "HyperSync / the public RPC") + while not stop.is_set(): + try: + out = one_pass(rpc) # the pass writes and loads the day's skill table itself + if out["mode"] == "backfill": + log.info("rh backfill: through %s, %s blocks behind, %s", out["index_through"], out["behind_blocks"], out["index"]) + if out.get("logs_from") == "rpc" and prefetch.hypersync_down(): + stop.wait(RPC_CATCHUP_S) # catching up on the public RPC: under its rate limit + continue + except Exception as e: + log.exception("rh_stream pass failed: %s", str(e)[:200]); _status(error=str(e)[:300]) + stop.wait(10.0); continue + stop.wait(_wait_s((out.get("minutes") or {}).get("through"))) + + +def _wait_s(written_through: int | None, now: float | None = None) -> float: + """Until the next pass: the moment the last ended minute can be written (its end + ``minutes.GRACE_S``) if it is not + yet, else the next minute's, but never longer than ``RH_STREAM_POLL_S`` (``written_through``: the end of the last + minute written, epoch s).""" + now = time.time() if now is None else now + end = int(now // 60) * 60 + due = (end if (written_through or 0) < end else end + 60) + minutes.GRACE_S + 0.5 + return max(0.05, min(config.RH_STREAM_POLL_S, due - now)) + + +def status() -> dict: + with transaction() as conn: + r = conn.execute("SELECT value FROM ui_settings WHERE key = 'rh_stream_status'").fetchone() + scans = [dict(x) for x in conn.execute("SELECT name, block, range_blocks, updated_at FROM rh_scan ORDER BY name").fetchall()] + counts = {t: conn.execute(f"SELECT count(*) AS n FROM {t}").fetchone()["n"] for t in ("rh_tokens", "rh_pools", "rh_swaps", "rh_curve_trades", "rh_minutes", "rh_base_prices")} + return {"status": (r["value"] if r else None), "cursors": scans, "rows": counts} + + +if __name__ == "__main__": + main() + time.sleep(0) diff --git a/fly_trader/rh/tx.py b/fly_trader/rh/tx.py new file mode 100644 index 0000000000000000000000000000000000000000..3b9df5ecf1767617e6aa1a4321e6e8edf9306eb0 --- /dev/null +++ b/fly_trader/rh/tx.py @@ -0,0 +1,156 @@ +"""EIP-1559 (type-2) transactions for Robinhood Chain: RLP, the signing hash, secp256k1 signatures and sender recovery. + +Signed with ``coincurve`` (libsecp256k1: deterministic RFC 6979 nonces, low-s) over keccak(0x02 ‖ rlp(fields)); the +wire form is 0x02 ‖ rlp(fields + [yParity, r, s]) and the hash is keccak of it. Checked against viem's serializer +(tests/vectors/evm_tx_v1.json) — the repo carries no web3/eth_account next to a hot key. +""" +from __future__ import annotations + +from dataclasses import dataclass, field + +import coincurve + +from ..vault.evm import keccak256 + +SECP256K1_N = 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEBAAEDCE6AF48A03BBFD25E8CD0364141 + + +# ---------------------------------------------------------------- RLP +def _int_bytes(n: int) -> bytes: + if n < 0: + raise ValueError("RLP cannot encode a negative integer") + return b"" if n == 0 else n.to_bytes((n.bit_length() + 7) // 8, "big") + + +def rlp_encode(item) -> bytes: + """RLP of bytes / int / nested lists (ints as minimal big-endian, 0 as the empty string).""" + if isinstance(item, int): + item = _int_bytes(item) + if isinstance(item, (bytes, bytearray)): + b = bytes(item) + if len(b) == 1 and b[0] < 0x80: + return b + return _len_prefix(len(b), 0x80) + b + if isinstance(item, (list, tuple)): + body = b"".join(rlp_encode(x) for x in item) + return _len_prefix(len(body), 0xC0) + body + raise TypeError(f"cannot RLP-encode {type(item).__name__}") + + +def _len_prefix(n: int, offset: int) -> bytes: + if n < 56: + return bytes([offset + n]) + ln = _int_bytes(n) + return bytes([offset + 55 + len(ln)]) + ln + + +def rlp_decode(data: bytes): + item, rest = _rlp_item(data) + if rest: + raise ValueError("trailing RLP bytes") + return item + + +def _rlp_item(d: bytes): + if not d: + raise ValueError("empty RLP") + p = d[0] + if p < 0x80: + return d[:1], d[1:] + if p < 0xB8: + n = p - 0x80; return d[1:1 + n], d[1 + n:] + if p < 0xC0: + ll = p - 0xB7; n = int.from_bytes(d[1:1 + ll], "big"); return d[1 + ll:1 + ll + n], d[1 + ll + n:] + if p < 0xF8: + n = p - 0xC0; body, rest = d[1:1 + n], d[1 + n:] + else: + ll = p - 0xF7; n = int.from_bytes(d[1:1 + ll], "big"); body, rest = d[1 + ll:1 + ll + n], d[1 + ll + n:] + out = [] + while body: + x, body = _rlp_item(body); out.append(x) + return out, rest + + +# ---------------------------------------------------------------- EIP-1559 +def _addr_bytes(a: str | None) -> bytes: + if a is None or a == "": + return b"" # contract creation (never used here) + h = a[2:] if a.startswith("0x") else a + if len(h) != 40: + raise ValueError(f"not an address: {a!r}") + return bytes.fromhex(h) + + +@dataclass(frozen=True) +class Eip1559Tx: + chain_id: int + nonce: int + max_priority_fee_per_gas: int + max_fee_per_gas: int + gas: int + to: str + value: int = 0 + data: bytes = b"" + access_list: list = field(default_factory=list) + + def fields(self) -> list: + al = [[_addr_bytes(a), [bytes.fromhex(k[2:]) for k in keys]] for a, keys in self.access_list] + return [self.chain_id, self.nonce, self.max_priority_fee_per_gas, self.max_fee_per_gas, self.gas, _addr_bytes(self.to), + self.value, bytes(self.data), al] + + def signing_hash(self) -> bytes: + return keccak256(b"\x02" + rlp_encode(self.fields())) + + +@dataclass(frozen=True) +class SignedTx: + tx: Eip1559Tx + y_parity: int + r: int + s: int + raw: bytes + + @property + def hash(self) -> str: + return "0x" + keccak256(self.raw).hex() + + @property + def raw_hex(self) -> str: + return "0x" + self.raw.hex() + + +def address_of(private_key: bytes) -> str: + """The lowercase 0x address of a 32-byte secp256k1 private key.""" + pub = coincurve.PrivateKey(private_key).public_key.format(compressed=False)[1:] + return "0x" + keccak256(pub)[-20:].hex() + + +def sign(tx: Eip1559Tx, private_key: bytes) -> SignedTx: + if tx.max_priority_fee_per_gas > tx.max_fee_per_gas: + raise ValueError("priority fee above the max fee") + sig = coincurve.PrivateKey(private_key).sign_recoverable(tx.signing_hash(), hasher=None) + r, s, v = int.from_bytes(sig[:32], "big"), int.from_bytes(sig[32:64], "big"), sig[64] + if s > SECP256K1_N // 2: # libsecp256k1 already normalises; refuse rather than send a malleable one + raise ValueError("high-s signature") + raw = b"\x02" + rlp_encode(tx.fields() + [v, r, s]) + return SignedTx(tx, v, r, s, raw) + + +def decode_raw(raw: bytes) -> SignedTx: + """Parse a signed type-2 transaction (the inverse of ``sign``).""" + if not raw or raw[0] != 0x02: + raise ValueError("not a type-2 transaction") + f = rlp_decode(raw[1:]) + if len(f) != 12: + raise ValueError("a type-2 transaction has 12 fields") + i = lambda b: int.from_bytes(b, "big") + al = [("0x" + a.hex(), ["0x" + k.hex() for k in keys]) for a, keys in f[8]] + tx = Eip1559Tx(chain_id=i(f[0]), nonce=i(f[1]), max_priority_fee_per_gas=i(f[2]), max_fee_per_gas=i(f[3]), gas=i(f[4]), + to="0x" + f[5].hex(), value=i(f[6]), data=bytes(f[7]), access_list=al) + return SignedTx(tx, i(f[9]), i(f[10]), i(f[11]), raw) + + +def recover_sender(stx: SignedTx) -> str: + sig = stx.r.to_bytes(32, "big") + stx.s.to_bytes(32, "big") + bytes([stx.y_parity]) + pub = coincurve.PublicKey.from_signature_and_message(sig, stx.tx.signing_hash(), hasher=None).format(compressed=False)[1:] + return "0x" + keccak256(pub)[-20:].hex() diff --git a/fly_trader/rh/univ4.py b/fly_trader/rh/univ4.py new file mode 100644 index 0000000000000000000000000000000000000000..4f6973a3cf14bcbed39c8011748133970d6e6d26 --- /dev/null +++ b/fly_trader/rh/univ4.py @@ -0,0 +1,88 @@ +"""Direct Uniswap v4 swaps through the UniversalRouter — the fallback when KyberSwap has no route for a Pons pool. + +One pool, exact input: ``execute(commands=[V4_SWAP], inputs=[abi.encode(actions, params)], deadline)`` with the v4 +actions SWAP_EXACT_IN_SINGLE → SETTLE_ALL → TAKE_ALL. Native ETH input is paid as the call's value; an ERC-20 input is +pulled through Permit2 (the token approves Permit2 exactly, Permit2 approves the router for exactly the amount, expiring +within the hour). minOut comes from the V4Quoter (an eth_call) minus the slippage. A Pons pool's key is (currency0, +currency1, fee 0, tickSpacing 200, the Pons hook); its id is keccak(abi.encode(key)). +""" +from __future__ import annotations + +import time + +from .. import config +from ..vault.evm import keccak256 +from . import abi + +ZERO = "0x" + "00" * 20 +V4_SWAP = 0x10 +SWAP_EXACT_IN_SINGLE, SETTLE_ALL, TAKE_ALL = 0x06, 0x0C, 0x0F +POOL_KEY = "(address,address,uint24,int24,address)" +EXACT_IN_SINGLE = f"({POOL_KEY},bool,uint128,uint128,bytes)" +QUOTE_SINGLE = f"({POOL_KEY},bool,uint128,bytes)" +EXECUTE_SIG = "execute(bytes,bytes[],uint256)" +PERMIT2_APPROVE_SIG = "approve(address,address,uint160,uint48)" +DEADLINE_S = 20 * 60 +PERMIT_EXPIRY_S = 3600 + + +def pool_key(currency0: str, currency1: str, fee: int = 0, tick_spacing: int = 200, hooks: str | None = None) -> tuple: + c0, c1 = currency0.lower(), currency1.lower() + if int(c0, 16) >= int(c1, 16): + raise ValueError("currency0 must sort below currency1") + return (c0, c1, int(fee), int(tick_spacing), (hooks or config.PONS_HOOK).lower()) + + +def pool_id(key: tuple) -> str: + return "0x" + keccak256(abi.encode([POOL_KEY], [key])).hex() + + +def swap_calldata(key: tuple, token_in: str, amount_in: int, min_out: int, deadline: int | None = None, hook_data: bytes = b"") -> tuple[bytes, int]: + """(UniversalRouter calldata, native value) for an exact-input swap of ``amount_in`` of ``token_in`` in pool ``key``.""" + tin = token_in.lower() + if tin not in (key[0], key[1]): + raise ValueError("token_in is not in the pool") + zero_for_one = tin == key[0] + tout = key[1] if zero_for_one else key[0] + if not (0 < amount_in < 2 ** 128 and 0 <= min_out < 2 ** 128): + raise ValueError("amounts out of uint128 range") + actions = bytes([SWAP_EXACT_IN_SINGLE, SETTLE_ALL, TAKE_ALL]) + params = [abi.encode([EXACT_IN_SINGLE], [(key, zero_for_one, amount_in, min_out, hook_data)]), + abi.encode(["address", "uint256"], [tin, amount_in]), + abi.encode(["address", "uint256"], [tout, min_out])] + inp = abi.encode(["bytes", "bytes[]"], [actions, params]) + dl = deadline if deadline is not None else int(time.time()) + DEADLINE_S + data = abi.encode_call(EXECUTE_SIG, bytes([V4_SWAP]), [inp], dl) + return data, (amount_in if tin == ZERO else 0) + + +def decode_swap(data: bytes) -> dict: + """Parse our own UniversalRouter calldata back (tests, and a last check before signing).""" + commands, inputs, deadline = abi.decode_call(EXECUTE_SIG, data) + if commands != bytes([V4_SWAP]) or len(inputs) != 1: + raise ValueError("not a single V4_SWAP") + actions, params = abi.decode(["bytes", "bytes[]"], inputs[0]) + if actions != bytes([SWAP_EXACT_IN_SINGLE, SETTLE_ALL, TAKE_ALL]): + raise ValueError("unexpected v4 actions") + (key, zfo, amt, mn, hook_data), = abi.decode([EXACT_IN_SINGLE], params[0]) + settle = abi.decode(["address", "uint256"], params[1]); take = abi.decode(["address", "uint256"], params[2]) + return {"key": key, "zero_for_one": zfo, "amount_in": amt, "min_out": mn, "settle": settle, "take": take, "deadline": deadline} + + +def quote_calldata(key: tuple, token_in: str, amount_in: int, hook_data: bytes = b"") -> bytes: + zfo = token_in.lower() == key[0] + return abi.encode_call(f"quoteExactInputSingle({QUOTE_SINGLE})", (key, zfo, amount_in, hook_data)) + + +def quote(rpc, key: tuple, token_in: str, amount_in: int) -> tuple[int, int]: + """(amountOut, gasEstimate) from the V4Quoter.""" + out = rpc.eth_call(config.V4_QUOTER, "0x" + quote_calldata(key, token_in, amount_in).hex()) + a, g = abi.decode(["uint256", "uint256"], bytes.fromhex(out[2:])) + return a, g + + +def permit2_approve_calldata(token: str, amount: int, now_s: int | None = None) -> bytes: + if not 0 < amount < 2 ** 160: + raise ValueError("Permit2 amounts are uint160 and must be exact") + exp = (int(time.time()) if now_s is None else now_s) + PERMIT_EXPIRY_S + return abi.encode_call(PERMIT2_APPROVE_SIG, token.lower(), config.UNIVERSAL_ROUTER.lower(), amount, exp) diff --git a/fly_trader/rh/v4.py b/fly_trader/rh/v4.py new file mode 100644 index 0000000000000000000000000000000000000000..6f6f2118c4dad693e9993d50fc98c129138c63bc --- /dev/null +++ b/fly_trader/rh/v4.py @@ -0,0 +1,81 @@ +"""Uniswap v4 PoolManager events and concentrated-liquidity math for the Pons pools. + +v4 emits ``Swap(id, sender, amount0, amount1, sqrtPriceX96, liquidity, tick, fee)`` with the amounts as the swapper's +balance delta (negative = paid into the pool). A Pons pool's locked graduation position is full range, so between two +swaps the pool behaves as constant product with virtual reserves x = L / √P (currency0) and y = L · √P (currency1), √P = +sqrtPriceX96 / 2^96 in raw units; P = y / x = currency1 per currency0. + +The same sqrt-price word sits third in the data of Uniswap v3, v4 and Pancake-Infinity-CL swaps, which is how the base +asset marks (rh/prices.py) read any reference pool. +""" +from __future__ import annotations + +from . import abi + +Q96 = 2 ** 96 +SIGS = { + "Initialize": "Initialize(bytes32,address,address,uint24,int24,address,uint160,int24)", # id, currency0, currency1 (indexed) + "Swap": "Swap(bytes32,address,int128,int128,uint160,uint128,int24,uint24)", # id, sender (indexed) + "ModifyLiquidity": "ModifyLiquidity(bytes32,address,int24,int24,int256,bytes32)", # id, sender (indexed) +} +TOPIC = {k: abi.event_topic(v) for k, v in SIGS.items()} +NAME = {v: k for k, v in TOPIC.items()} +MIN_TICK, MAX_TICK = -887272, 887272 + + +def decode(log: dict) -> dict | None: + name = NAME.get(log["topics"][0]) + if name is None: + return None + t = log["topics"]; d = bytes.fromhex(log["data"][2:]) + base = {"event": name, "pool_id": t[1].lower(), "block": int(log["blockNumber"], 16), "log_index": int(log["logIndex"], 16), + "tx_hash": log["transactionHash"], "block_hash": log.get("blockHash"), "ts": int(log.get("blockTimestamp") or "0x0", 16) or None} + if name == "Initialize": + fee, ts_, hooks, sp, tick = abi.decode(["uint24", "int24", "address", "uint160", "int24"], d) + return {**base, "currency0": "0x" + t[2][-40:].lower(), "currency1": "0x" + t[3][-40:].lower(), "fee": fee, "tick_spacing": ts_, "hooks": hooks, + "sqrt_price_x96": sp, "tick": tick} + if name == "Swap": + a0, a1, sp, liq, tick, fee = abi.decode(["int128", "int128", "uint160", "uint128", "int24", "uint24"], d) + return {**base, "sender": "0x" + t[2][-40:].lower(), "amount0": a0, "amount1": a1, "sqrt_price_x96": sp, "liquidity": liq, "tick": tick, "fee": fee} + if name == "ModifyLiquidity": + lo, hi, delta, salt = abi.decode(["int24", "int24", "int256", "bytes32"], d) + return {**base, "sender": "0x" + t[2][-40:].lower(), "tick_lower": lo, "tick_upper": hi, "liquidity_delta": delta, "salt": "0x" + salt.hex()} + return None + + +def sqrt_price_word(log: dict) -> int: + """The sqrtPriceX96 of any v3 / v4 / Pancake-Infinity-CL swap log (the third data word).""" + d = log["data"][2:] + return int(d[128:192], 16) + + +def price1_per_0(sqrt_price_x96: int, dec0: int, dec1: int) -> float: + """Whole currency1 per whole currency0.""" + p = (sqrt_price_x96 / Q96) ** 2 + return p * 10 ** (dec0 - dec1) + + +def token_price_in_quote(sqrt_price_x96: int, token_is_0: bool, dec_token: int, dec_quote: int) -> float: + """Whole quote per whole memecoin.""" + if token_is_0: + return price1_per_0(sqrt_price_x96, dec_token, dec_quote) + p = price1_per_0(sqrt_price_x96, dec_quote, dec_token) + return 1.0 / p if p > 0 else float("nan") + + +def virtual_reserves(sqrt_price_x96: int, liquidity: int) -> tuple[float, float]: + """(x raw currency0, y raw currency1) of the constant-product equivalent at this price.""" + s = sqrt_price_x96 / Q96 + return (liquidity / s if s > 0 else float("inf"), liquidity * s) + + +def quote_reserve(sqrt_price_x96: int, liquidity: int, token_is_0: bool, dec_quote: int) -> float: + """The quote side's virtual reserve in whole quote units.""" + x, y = virtual_reserves(sqrt_price_x96, liquidity) + return (y if token_is_0 else x) / 10 ** dec_quote + + +def is_full_range(tick_lower: int, tick_upper: int, tick_spacing: int) -> bool: + lo = (MIN_TICK // tick_spacing) * tick_spacing + (tick_spacing if MIN_TICK % tick_spacing else 0) + hi = (MAX_TICK // tick_spacing) * tick_spacing + return tick_lower <= lo and tick_upper >= hi diff --git a/fly_trader/rh/wallet.py b/fly_trader/rh/wallet.py new file mode 100644 index 0000000000000000000000000000000000000000..5e4a7b660030947fa6f5137bdfb65e7969ac9a70 --- /dev/null +++ b/fly_trader/rh/wallet.py @@ -0,0 +1,250 @@ +"""The RH bot wallet: nonces, fees, signing, persist-before-broadcast, receipts, exact ERC-20 approvals. + +Rules (docs: plan "Accounting / Idempotency"): +- one transaction in flight per wallet (a lock around send→receipt), so a native-ETH balance delta belongs to exactly + one transaction; +- the signed transaction (nonce, raw bytes, hash) is written to ``rh_txs`` and committed BEFORE it is broadcast, so a + crash after broadcast can always be recovered (rebroadcast the same bytes, or cancel at that nonce); +- nonce = max(the chain's pending count, the highest nonce we signed that is not dropped/replaced + 1); +- fee = 2 × base fee + tip, never above ``RH_MAX_FEE_GWEI`` per gas (a spike refuses rather than overpays); +- approvals are exact (never 2**256-1), skipped when the allowance already covers the amount, reset to 0 first only for a + token that refuses to change a non-zero allowance. +""" +from __future__ import annotations + +import hashlib +import json +import logging +import threading +import time + +from .. import config +from ..db.connection import transaction +from ..vault.evm import EvmRpcError +from . import abi, guard +from .tx import Eip1559Tx, sign + +log = logging.getLogger(__name__) + +GAS_MARGIN_NUM, GAS_MARGIN_DEN, GAS_MARGIN_ADD = 13, 10, 10_000 # estimate × 1.3 + 10k: Orbit estimates include the L1 part +RECEIPT_POLL_S = 1.0 +MAX_UINT = 2 ** 256 - 1 + + +class FeeTooHigh(RuntimeError): + pass + + +class WalletLock: + """One RH address, one transaction in flight — across threads (trading intents, vault ETH payouts) and processes + (CLI tools): a re-entrant thread lock plus a Postgres advisory lock held while the outermost holder has it. The + executor's balance deltas are only that intent's because nothing else can send from the address meanwhile.""" + + def __init__(self, address: str): + self.key = int.from_bytes(hashlib.sha256(("rh-wallet:" + address.lower()).encode()).digest()[:8], "big", signed=True) + self._t = threading.RLock(); self._depth = 0; self._conn = None + + def acquire(self, blocking: bool = True, timeout: float = -1) -> bool: + if not self._t.acquire(blocking, timeout): + return False + if self._depth == 0: + from ..db.connection import connect + conn = connect(autocommit=True) + deadline = None if timeout is None or timeout < 0 else time.monotonic() + timeout + while not conn.execute("SELECT pg_try_advisory_lock(%s) AS ok", (self.key,)).fetchone()["ok"]: + if not blocking or (deadline is not None and time.monotonic() > deadline): + conn.close(); self._t.release() + return False + time.sleep(0.2) + self._conn = conn + self._depth += 1 + return True + + def release(self) -> None: + self._depth -= 1 + if self._depth == 0 and self._conn is not None: + try: + self._conn.execute("SELECT pg_advisory_unlock(%s)", (self.key,)) + finally: + self._conn.close(); self._conn = None + self._t.release() + + def __enter__(self): + self.acquire() + return self + + def __exit__(self, *exc) -> None: + self.release() + + +_LOCKS: dict[str, WalletLock] = {} +_LOCKS_MU = threading.Lock() + + +def wallet_lock(address: str) -> WalletLock: + with _LOCKS_MU: + return _LOCKS.setdefault(address.lower(), WalletLock(address)) + + +class SendFailed(RuntimeError): + pass + + +def _calldata(d) -> bytes: + if isinstance(d, (bytes, bytearray)): + return bytes(d) + return bytes.fromhex(d[2:] if d.startswith("0x") else d) + + +class RhWallet: + def __init__(self, rpc, key: bytes | None = None, address: str | None = None): + self.rpc = rpc + if key is None: + key, address = guard.check(rpc) + self._key = key + self.address = (address or "").lower() + self.lock = wallet_lock(self.address) + + # ---- chain reads + def balance(self) -> int: + return self.rpc.get_balance(self.address) + + def balance_of(self, token: str, owner: str | None = None, block: str = "latest") -> int: + return abi.decode(["uint256"], bytes.fromhex(self.rpc.eth_call(token, "0x" + abi.encode_call("balanceOf(address)", owner or self.address).hex(), block)[2:]))[0] + + def allowance(self, token: str, spender: str, owner: str | None = None) -> int: + data = abi.encode_call("allowance(address,address)", owner or self.address, spender) + return abi.decode(["uint256"], bytes.fromhex(self.rpc.eth_call(token, "0x" + data.hex())[2:]))[0] + + def decimals(self, token: str) -> int: + return abi.decode(["uint8"], bytes.fromhex(self.rpc.eth_call(token, "0x" + abi.selector("decimals()").hex())[2:]))[0] + + # ---- fees, gas, nonces + def fees(self) -> tuple[int, int]: + """(max fee, tip) per gas in wei.""" + cap = int(config.RH_MAX_FEE_GWEI * 1e9) + base = self.rpc.base_fee(); tip = self.rpc.max_priority_fee() + if base + tip > cap: + raise FeeTooHigh(f"base fee {base / 1e9:.3f} gwei + tip over the {config.RH_MAX_FEE_GWEI} gwei cap") + mx = min(2 * base + tip, cap) + return mx, min(tip, mx) + + def estimate(self, to: str, data: bytes, value: int = 0) -> int: + g = self.rpc.estimate_gas({"from": self.address, "to": to, "data": "0x" + data.hex(), "value": hex(value)}) + return g * GAS_MARGIN_NUM // GAS_MARGIN_DEN + GAS_MARGIN_ADD + + def simulate(self, to: str, data: bytes, value: int = 0, block: str = "latest") -> str: + """eth_call as this wallet; raises EvmRpcError on a revert (checked before anything is signed).""" + return self.rpc.call("eth_call", [{"from": self.address, "to": to, "data": "0x" + data.hex(), "value": hex(value)}, block]) + + def next_nonce(self, conn) -> int: + chain = self.rpc.tx_count(self.address, "pending") + r = conn.execute("SELECT max(nonce) AS n FROM rh_txs WHERE from_addr = %s AND status NOT IN ('dropped', 'replaced')", (self.address,)).fetchone() + return max(chain, (int(r["n"]) + 1) if r and r["n"] is not None else 0) + + # ---- send + def send(self, *, to: str, data=b"", value: int = 0, kind: str, intent_id: int | None = None, position_id: int | None = None, + gas: int | None = None, nonce: int | None = None, fees: tuple[int, int] | None = None, router: str | None = None, + quote: dict | None = None, build: dict | None = None, on_persist=None) -> dict: + """Sign, persist, broadcast. Returns {id, hash, nonce, gas, max_fee}. Raises SendFailed (the row is marked dropped, + so its nonce is reused) or FeeTooHigh (nothing signed). ``on_persist(conn, tx_id)`` runs inside the transaction + that persists the signed row, so a caller's own link to it (a claim's payout) is committed before broadcast too.""" + data = _calldata(data) + with self.lock: + gas = gas or self.estimate(to, data, value) + mx, tip = fees or self.fees() + with transaction() as conn: + n = self.next_nonce(conn) if nonce is None else nonce + stx = sign(Eip1559Tx(chain_id=self.rpc.chain_id, nonce=n, max_priority_fee_per_gas=tip, max_fee_per_gas=mx, gas=gas, + to=to.lower(), value=value, data=data), self._key) + row = conn.execute( + "INSERT INTO rh_txs (intent_id, position_id, kind, from_addr, nonce, hash, raw, to_addr, value_wei, data_sha256, gas_limit, " + "max_fee_wei, max_prio_wei, router, quote, build) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) " + # the same fields re-signed after a failed broadcast give the same hash: revive that row, never a second one + "ON CONFLICT (hash) DO UPDATE SET status = CASE WHEN rh_txs.status = 'dropped' THEN 'signed' ELSE rh_txs.status END, " + "error = NULL, updated_at = now() RETURNING id", + (intent_id, position_id, kind, self.address, n, stx.hash, stx.raw_hex, to.lower(), value, hashlib.sha256(data).hexdigest(), gas, + mx, tip, router, json.dumps(quote) if quote else None, json.dumps(build) if build else None)).fetchone() + if on_persist is not None: + on_persist(conn, int(row["id"])) + tid = int(row["id"]) # committed: a crash from here on is recoverable + try: + self.rpc.send_raw(stx.raw_hex) + except EvmRpcError as e: + msg = (e.message or "").lower() + if "already known" in msg or "known transaction" in msg: + pass # the node has it: same hash, carry on + else: + with transaction() as conn: + conn.execute("UPDATE rh_txs SET status = 'dropped', error = %s, updated_at = now() WHERE id = %s", (str(e)[:300], tid)) + raise SendFailed(str(e)) from None + with transaction() as conn: + conn.execute("UPDATE rh_txs SET status = 'sent', updated_at = now() WHERE id = %s AND status = 'signed'", (tid,)) + return {"id": tid, "hash": stx.hash, "nonce": n, "gas": gas, "max_fee": mx} + + def wait(self, tx_hash: str, timeout_s: float = 120.0, confirmations: int | None = None) -> dict | None: + """The receipt once it has ``confirmations`` blocks on top (``RH_CONFIRMATIONS``), or None at the timeout.""" + conf = config.RH_CONFIRMATIONS if confirmations is None else confirmations + deadline = time.monotonic() + timeout_s + while time.monotonic() < deadline: + rc = self.rpc.receipt(tx_hash) + if rc is not None and rc.get("blockNumber"): + if self.rpc.block_number("latest") - int(rc["blockNumber"], 16) >= conf: + return rc + time.sleep(RECEIPT_POLL_S) + return None + + @staticmethod + def record_receipt(conn, tx_id: int, rc: dict) -> dict: + """Store a receipt's outcome on its rh_txs row. Returns {ok, fee_wei, gas_used, block}. A withdrawal (kind + 'transfer') is booked with its receipt (rh/withdraw.book), once, whichever path records it.""" + gas_used = int(rc["gasUsed"], 16); price = int(rc.get("effectiveGasPrice") or "0x0", 16); fee = gas_used * price + ok = int(rc.get("status", "0x0"), 16) == 1 + row = conn.execute("UPDATE rh_txs SET status = %s, block = %s, block_hash = %s, gas_used = %s, eff_gas_price_wei = %s, fee_wei = %s, " + "fee_at = COALESCE(fee_at, clock_timestamp()), updated_at = now() " + "WHERE id = %s RETURNING kind", ("mined_ok" if ok else "reverted", int(rc["blockNumber"], 16), rc.get("blockHash"), gas_used, price, fee, tx_id)).fetchone() + if row is not None and row["kind"] == "transfer": + from .withdraw import book + book(conn, tx_id, ok) + return {"ok": ok, "fee_wei": fee, "gas_used": gas_used, "block": int(rc["blockNumber"], 16)} + + def send_and_wait(self, timeout_s: float = 120.0, **kw) -> dict: + """``send`` then ``wait`` then ``record_receipt``. Returns the send dict plus {ok, fee_wei, gas_used, block, receipt}; + ``receipt`` None means no receipt yet (the row stays 'sent' for recovery).""" + with self.lock: + s = self.send(**kw) + rc = self.wait(s["hash"], timeout_s) + if rc is None: + return {**s, "ok": None, "receipt": None} + with transaction() as conn: + out = self.record_receipt(conn, s["id"], rc) + return {**s, **out, "receipt": rc} + + def cancel(self, nonce: int, prev_max_fee: int) -> dict: + """Replace whatever sits at ``nonce`` with a 0-value self-transfer at ≥ 1.25 × its fee (the node's replacement rule).""" + mx, tip = self.fees() + mx = max(mx, prev_max_fee * 5 // 4 + 1); tip = min(max(tip, 1), mx) + return self.send(to=self.address, data=b"", value=0, kind="cancel", gas=21_000, nonce=nonce, fees=(mx, tip)) + + # ---- ERC-20 approvals: exact, never unlimited + def approve_exact(self, token: str, spender: str, amount: int, *, intent_id: int | None = None, position_id: int | None = None) -> dict | None: + """Make ``spender``'s allowance at least ``amount`` with an approval of exactly ``amount``. None if it already is.""" + if amount <= 0 or amount >= MAX_UINT // 2: + raise ValueError(f"refusing an approval of {amount}: exact amounts only, never unlimited") + with self.lock: + cur = self.allowance(token, spender) + if cur >= amount: + return None + data = abi.encode_call("approve(address,uint256)", spender, amount) + if cur > 0: + try: + self.simulate(token, data) + except EvmRpcError: # a token that refuses non-zero → non-zero changes + z = self.send_and_wait(to=token, data=abi.encode_call("approve(address,uint256)", spender, 0), kind="approve", + intent_id=intent_id, position_id=position_id) + if not z.get("ok"): + raise SendFailed(f"resetting the allowance of {token} failed") + r = self.send_and_wait(to=token, data=data, kind="approve", intent_id=intent_id, position_id=position_id) + if not r.get("ok"): + raise SendFailed(f"approve {token} → {spender} {'reverted' if r.get('ok') is False else 'has no receipt'}") + return r diff --git a/fly_trader/rh/wallet_skill.py b/fly_trader/rh/wallet_skill.py new file mode 100644 index 0000000000000000000000000000000000000000..8f61251a32aad8ab5aa5d140f8188b686e5590dd --- /dev/null +++ b/fly_trader/rh/wallet_skill.py @@ -0,0 +1,158 @@ +"""Wallet skill on Robinhood Chain: train/wallet_skill's definitions over RH wallets only (a Solana wallet and an RH wallet +are different people; the tables never mix), under the same frozen (h, L) — so ``skill_top*_15m`` means the same thing on +both chains' rows. + +``build_wallet_day(D)``: each Pons swap leg of day D (trader, ETH amount, ETH price) marked out over every hold like the +Solana legs (the close of the last minute at or before t + h, after the one-side exit cost at both ends, clipped ±100 %) +→ data/corpus/rh/wallet_day/.parquet. ``write_table(D)`` → data/corpus/rh/wallet_skill/.parquet from the wallet days +ending GAP_DAYS before D (train/wallet_skill.skill_table with the RH directories). +""" +from __future__ import annotations + +import logging +from datetime import date, datetime, timedelta, timezone +from pathlib import Path + +import duckdb +import pandas as pd +import pyarrow as pa +import pyarrow.parquet as pq + +from .. import config +from ..db.connection import transaction +from ..train import wallet_skill as ws + +log = logging.getLogger(__name__) +ROOT = config.CORPUS_DIR / config.RH_DIR_NAME +DAY_DIR = ROOT / "wallet_day" +SKILL_DIR = ROOT / "wallet_skill" +MATURE_DIR = ROOT / "mature" +FEAT_DIR = ROOT / "features_mature" + + +def skill_version() -> str: + return "rh-" + ws.skill_version() + + +def legs(conn, d: date) -> pd.DataFrame: + t0 = datetime(d.year, d.month, d.day, tzinfo=timezone.utc); t1 = t0 + timedelta(days=1) + rows = conn.execute( + "SELECT s.token AS mint, s.trader, s.ts, s.side, s.quote_raw, a.decimals AS qdec, s.price_q, s.pool_id, p.quote_asset " + "FROM rh_swaps s JOIN rh_pools p ON p.pool_id = s.pool_id JOIN rh_assets a ON a.asset = p.quote_asset " + "WHERE s.ts >= %s AND s.ts < %s AND s.trader IS NOT NULL AND s.price_q > 0", (t0, t1)).fetchall() + df = pd.DataFrame(rows) + if df.empty: + return df + from . import prices + base: dict[tuple, float | None] = {} + def b(asset, ts): + k = (asset, int(ts.timestamp() // 60)) + if k not in base: + base[k] = prices.base_eth(conn, asset, ts.replace(second=59, microsecond=0))[0] + return base[k] + df["base"] = [b(a, t) for a, t in zip(df["quote_asset"], df["ts"])] + df = df[df["base"].notna()].copy() + df["sol"] = df["quote_raw"].astype(float) / (10.0 ** df["qdec"].astype(float)) * df["base"] + df["price"] = df["price_q"].astype(float) * df["base"] + return df[["mint", "trader", "ts", "side", "sol", "price"]] + + +def build_wallet_day(d: date) -> int: + nxt = d + timedelta(days=1) + aggs = [MATURE_DIR / f"{x.isoformat()}.parquet" for x in (d, nxt)]; parts = [FEAT_DIR / x.isoformat() / "part.parquet" for x in (d, nxt)] + if not all(p.exists() for p in aggs + parts): + return 0 + with transaction() as conn: + lg = legs(conn, d) + if lg.empty: + return 0 + con = duckdb.connect(); con.register("legs_df", lg) + con.execute("CREATE TEMP TABLE legs AS SELECT * FROM legs_df") + con.execute("CREATE TEMP TABLE closes AS SELECT mint, ts AS mts, close FROM read_parquet(?) ORDER BY mint, mts", [[str(p) for p in aggs]]) + con.execute("CREATE TEMP TABLE ec AS SELECT mint, ts AS mts, exit_cost_0p1 AS ec FROM read_parquet(?)", [[str(p) for p in parts]]) + out = [] + for h in ws.HOLDS_MIN: + out.append(con.execute(f""" + SELECT l.trader AS wallet, {h} AS h, count(*) AS n, sum(l.sol) AS sol, + sum(l.sol * greatest(-1.0, least(1.0, CASE WHEN l.side = 1 + THEN c.close / l.price * (1 - coalesce(e0.ec, 0)) * (1 - coalesce(e1.ec, 0)) - 1 + ELSE -(c.close / l.price - 1) END))) AS sol_m + FROM legs l ASOF JOIN closes c ON c.mint = l.mint AND c.mts <= l.ts + INTERVAL {h} MINUTE + LEFT JOIN ec e0 ON e0.mint = l.mint AND e0.mts = time_bucket(INTERVAL 1 MINUTE, l.ts) + LEFT JOIN ec e1 ON e1.mint = c.mint AND e1.mts = c.mts + GROUP BY l.trader""").fetch_arrow_table()) + con.close() + tab = pa.concat_tables(out); DAY_DIR.mkdir(parents=True, exist_ok=True) + tmp = DAY_DIR / f"{d.isoformat()}.parquet.tmp"; pq.write_table(tab, tmp, compression="zstd"); tmp.replace(DAY_DIR / f"{d.isoformat()}.parquet") + return tab.num_rows + + +def write_table(d: date) -> bool: + c = ws.skill_config() + if not c: + return False + return ws.write_table(d, int(c["h"]), int(c["L"]), out_dir=SKILL_DIR, day_dir=DAY_DIR) + + +def table_for(d: date) -> dict | None: + f = SKILL_DIR / f"{d.isoformat()}.parquet" + if not f.exists(): + return None + t = pq.read_table(f, columns=["wallet", "bucket"]) + return dict(zip(t.column("wallet").to_pylist(), t.column("bucket").to_pylist())) + + +_TODAY: dict = {"key": None, "table": None} + + +def load_today() -> dict | None: + """Live: the RH table in force today (wallet → decile), read once per file. Like the Solana feed + (ingest/pumpstream.load_skill), a missing table for today falls back to the newest one on disk: without one every + minute's skill inputs read as zero and the selector fails closed on every RH row (2026-10-05..07: no RH trade).""" + today = datetime.now(timezone.utc).date() + f = SKILL_DIR / f"{today.isoformat()}.parquet" + if not f.exists(): + have = sorted(SKILL_DIR.glob("????-??-??.parquet")) if SKILL_DIR.exists() else [] + if not have: + return None + f = have[-1] + log.warning("no RH wallet skill table for %s: using the newest on disk, %s", today, f.stem) + key = (str(f), f.stat().st_mtime) + if _TODAY["key"] != key: + t = pq.read_table(f, columns=["wallet", "bucket"]) + _TODAY["table"] = dict(zip(t.column("wallet").to_pylist(), t.column("bucket").to_pylist())); _TODAY["key"] = key + log.info("RH wallet skill table %s loaded: %d wallets", f.stem, len(_TODAY["table"])) + return _TODAY["table"] + + +def window_ready(d: date, L: int) -> bool: + """Every RH wallet day of D's window exists (days before the first RH wallet day do not count).""" + have = sorted(DAY_DIR.glob("????-??-??.parquet")) if DAY_DIR.exists() else [] + if not have: + return False + first = date.fromisoformat(have[0].stem) + need = [x for x in ws.window_days(d, L) if x >= first] + return bool(need) and all((DAY_DIR / f"{x.isoformat()}.parquet").exists() for x in need) + + +def daily(through: date | None = None) -> int: + """The table of every day from the first RH wallet day through tomorrow (UTC) once its window exists — written before + the day begins, as Solana's (train/wallet_skill.daily), so the live minutes and the corpus read the same file. The + window ends GAP_DAYS before the day, so writing early and writing at corpus time give the same table.""" + c = ws.skill_config() + have = sorted(DAY_DIR.glob("????-??-??.parquet")) if DAY_DIR.exists() else [] + if not c or not have: + return 0 + v = f"h{int(c['h'])}-L{int(c['L'])}"; n = 0 + d = date.fromisoformat(have[0].stem) + timedelta(days=1) + last = through or (datetime.now(timezone.utc).date() + timedelta(days=1)) + while d <= last: + f = SKILL_DIR / f"{d.isoformat()}.parquet" + if (not f.exists() or ws.table_version(f) != v) and window_ready(d, int(c["L"])) and write_table(d): + n += 1 + d += timedelta(days=1) + return n + + +def path_of(d: date) -> Path: + return SKILL_DIR / f"{d.isoformat()}.parquet" diff --git a/fly_trader/rh/withdraw.py b/fly_trader/rh/withdraw.py new file mode 100644 index 0000000000000000000000000000000000000000..09e7aacf1790742d283c02e2c8b2498127bedf84 --- /dev/null +++ b/fly_trader/rh/withdraw.py @@ -0,0 +1,123 @@ +"""Withdraw ETH from the RH bot wallet to one of the operator's own addresses (System → Safety & wallet in the console). + +The Solana withdrawal's rules (chain/withdraw.py), in ETH: +- only to an address listed in RH_FUNDING_ADDRESSES, never on the hosted vault fly, and every key and chain check of + rh/guard.py but RH_LIVE_ENABLED (guard.check_withdrawal); +- what is left covers, while live positions or base lots are open, the gas reserve (their exits pay gas from it); +- the wallet's lock (rh/wallet.py: one transaction in flight per address) is held from the balance read to the booking, + so the executor and the reconciler never see a withdrawal half done; +- the transfer is an ``rh_txs`` row of kind 'transfer', committed before broadcast. Whichever path records its receipt + (this one, ``settle_pending``, or the executor's recovery) books it once (``book``): the ETH sent becomes a + 'withdrawal' wallet flow, so rh/accounting.reconcile still finds the chain equal to the books, and circuit 3's + kill-switch peak re-bases at it; +- a withdrawal still without a receipt is settled before the next one (rh/exec.settle_tx): mined, superseded at its + nonce, rebroadcast or cancelled. +""" +from __future__ import annotations + +from decimal import Decimal + +from .. import config +from ..agent import rails +from ..db.apilog import record_event +from ..db.connection import transaction +from ..markets import RH, RH_CIRCUIT +from ..vault.evm import is_address +from . import guard + +WEI = 10 ** 18 +RECEIPT_WAIT_S = 120.0 + + +class WithdrawRefused(RuntimeError): + """Nothing was signed; the message says why.""" + + +def destinations() -> list[str]: + """The addresses ETH may go to: the valid EVM addresses in RH_FUNDING_ADDRESSES, lowercase.""" + return [a.lower() for a in config.RH_FUNDING_ADDRESSES if is_address(a)] + + +def wallet(to: str, rpc=None): + """The RH bot wallet, for a withdrawal to ``to`` (guard.check_withdrawal).""" + from .rpc import RhRpc + from .wallet import RhWallet + rpc = rpc or RhRpc() + key, addr = guard.check_withdrawal(rpc, to) + return RhWallet(rpc, key, addr) + + +def limits(w, to: str) -> dict: + """Fresh reads: the wallet's wei, the transfer's gas and fee cap, what must stay (the gas reserve while live positions or + base lots are open), and the most that can go out with the fee counted at its cap.""" + balance = w.balance(); mx, tip = w.fees(); gas = w.estimate(to, b"", 0) + with transaction() as conn: + n_open = int(conn.execute("SELECT (SELECT count(*) FROM positions WHERE book = %s AND status = 'open') + " + "(SELECT count(*) FROM rh_base_lots WHERE status = 'open') AS n", (RH.live_book,)).fetchone()["n"]) + keep = int(config.RH_GAS_RESERVE_ETH * WEI) if n_open else 0 + return {"balance": balance, "gas": gas, "max_fee": mx, "tip": tip, "fee": gas * mx, "keep": keep, "n_open": n_open, + "max": max(0, balance - gas * mx - keep)} + + +def plan(lim: dict, eth: float | None) -> tuple[int, str | None]: + """(wei to send, why not or None) for ``eth`` ETH, or everything withdrawable when ``eth`` is None.""" + wei = lim["max"] if eth is None else int(Decimal(str(eth)) * WEI) + if wei <= 0: + return 0, "nothing to withdraw" if eth is None else "enter an amount" + if wei > lim["max"]: + return wei, (f"at most {lim['max'] / WEI:.6f} ETH can go out" + + (f": {lim['keep'] / WEI:g} ETH stays for the gas of {lim['n_open']} open live position(s)" if lim["keep"] else " after the gas")) + return wei, None + + +def book(conn, tx_id: int, ok: bool) -> None: + """A withdrawal's receipt, in the transaction that records it (rh/wallet.RhWallet.record_receipt): the row is marked + applied and, if it went through, its ETH is booked as a 'withdrawal' wallet flow and circuit 3's peak re-based. Once.""" + tx = conn.execute("UPDATE rh_txs SET applied_at = now() WHERE id = %s AND applied_at IS NULL RETURNING value_wei, to_addr, hash, block", + (tx_id,)).fetchone() + if tx is None or not ok: + return + conn.execute("INSERT INTO rh_wallet_flows (block, direction, kind, wei, note) VALUES (%s, 'out', 'withdrawal', %s, %s)", + (tx["block"], int(tx["value_wei"]), f"to {tx['to_addr']} ({tx['hash']})")) + rails.record_withdrawal(conn, {"eth": int(tx["value_wei"]) / WEI, "to": tx["to_addr"], "hash": tx["hash"]}, RH_CIRCUIT) + + +def settle_pending(w) -> list[str]: + """Settle the wallet's withdrawals that have no receipt yet (a receipt books them). Returns the hashes still in flight. + Under the wallet's lock.""" + from .exec import settle_tx + q = ("SELECT * FROM rh_txs WHERE from_addr = %s AND kind = 'transfer' AND applied_at IS NULL AND status IN ('signed', 'sent') " + "ORDER BY nonce, id") + with transaction() as conn: + rows = [dict(r) for r in conn.execute(q, (w.address,)).fetchall()] + for tx in rows: + settle_tx(w, tx) + with transaction() as conn: + return [r["hash"] for r in conn.execute(q, (w.address,)).fetchall()] + + +def withdraw(eth: float | None, to: str, rpc=None, w=None) -> dict: + """Send ``eth`` ETH (None: everything withdrawable) from the RH bot wallet to ``to``. Returns {status, hash, wei, to, + block}; status 'mined_ok', 'reverted' (only the gas moved) or 'pending' (no receipt yet: it is booked when one is + recorded). Raises WithdrawRefused, guard.SigningRefused, wallet.FeeTooHigh or wallet.SendFailed when nothing went out.""" + to = to.lower() + if w is None: + w = wallet(to, rpc) + else: + guard.check_withdrawal(w.rpc, to) # the same checks for a wallet built elsewhere + with w.lock: + if left := settle_pending(w): + raise WithdrawRefused(f"an earlier withdrawal ({left[0]}) has not landed yet: try again in a few minutes") + lim = limits(w, to) + wei, why = plan(lim, eth) + if why: + raise WithdrawRefused(why) + s = w.send(to=to, data=b"", value=wei, kind="transfer", gas=lim["gas"], fees=(lim["max_fee"], lim["tip"])) + rc = w.wait(s["hash"], RECEIPT_WAIT_S) + if rc is None: + return {"status": "pending", "hash": s["hash"], "wei": wei, "to": to, "block": None} + with transaction() as conn: + res = w.record_receipt(conn, s["id"], rc) # books it + status = "mined_ok" if res["ok"] else "reverted" + record_event("info" if res["ok"] else "warning", "rh_wallet", f"withdrawal of {wei / WEI:.6f} ETH to {to}: {status}", {"hash": s["hash"]}) + return {"status": status, "hash": s["hash"], "wei": wei, "to": to, "block": res["block"]} diff --git a/fly_trader/signer/__init__.py b/fly_trader/signer/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d4180ec5a732a60f945bcfd1a0e186fbf3b43569 --- /dev/null +++ b/fly_trader/signer/__init__.py @@ -0,0 +1 @@ +"""The key-holding signer (see core.py). Imports nothing from the rest of fly_trader at module level.""" diff --git a/fly_trader/signer/__main__.py b/fly_trader/signer/__main__.py new file mode 100644 index 0000000000000000000000000000000000000000..350cbe3ac4a123c47c657273fae95dfd16480604 --- /dev/null +++ b/fly_trader/signer/__main__.py @@ -0,0 +1,56 @@ +"""The signer container's entry point (``python -m fly_trader.signer``), standard library + solders + httpx only: it +imports nothing else of fly_trader, so its image carries none of the brain's dependencies. + + python -m fly_trader.signer serve # the service (SIGNER_SOCKET, SIGNER_DB, key files, SIGNER_RPC_URL) + python -m fly_trader.signer status + python -m fly_trader.signer ping # the container's health check: the socket answers + python -m fly_trader.signer clear-panic # only from a shell on the server: the brain cannot call this +""" +from __future__ import annotations + +import json +import logging +import os +import sys + +from .core import Signer, SignerConfig +from .ledger import Ledger +from .rpc import Rpc + + +def _keypair(path: str): + import base58 + from solders.keypair import Keypair + raw = open(path).read().strip() + b = bytes(json.loads(raw)) if raw.startswith("[") else base58.b58decode(raw) + return Keypair.from_seed(b) if len(b) == 32 else Keypair.from_bytes(b) + + +def build() -> Signer: + env = os.environ + payout = env.get("PAYOUT_PRIVATE_KEY_FILE") + return Signer(_keypair(env["BOT_PRIVATE_KEY_FILE"]), _keypair(payout) if payout else None, Rpc(env.get("SIGNER_RPC_URL", "")), + SignerConfig.from_env(env), Ledger(env.get("SIGNER_DB", "/srv/signer/ledger.sqlite"))) + + +def main(argv: list[str]) -> None: + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s") + cmd = argv[1] if len(argv) > 1 else "serve" + if cmd == "ping": # no keys needed: ask the running server over its socket + from .client import SocketSigner + SocketSigner(os.environ.get("SIGNER_SOCKET", "/run/fly-signer/sock"), timeout=5).call("pubkeys") + return + s = build() + if cmd == "serve": + from .server import serve + serve(s, os.environ.get("SIGNER_SOCKET", "/run/fly-signer/sock")) + elif cmd == "status": + print(json.dumps(s.status(), indent=1)) + elif cmd == "clear-panic": + print(json.dumps(s.clear_panic())) + else: + raise SystemExit(__doc__) + + +if __name__ == "__main__": + main(sys.argv) diff --git a/fly_trader/signer/client.py b/fly_trader/signer/client.py new file mode 100644 index 0000000000000000000000000000000000000000..0162dcaa6cd71cf7e07d5fd14ee3ec668d75c34e --- /dev/null +++ b/fly_trader/signer/client.py @@ -0,0 +1,99 @@ +"""The brain's side of the signer. On the vault server it talks to the signer container over its Unix socket +(``SIGNER_SOCKET``); anywhere else (the Mac, tests) the same Signer runs in-process from the usual key settings, so +there is one code path and one policy.""" +from __future__ import annotations + +import json +import socket +import threading + +from .policy import PolicyError + + +class SignerUnavailable(RuntimeError): + pass + + +class SocketSigner: + def __init__(self, path: str, timeout: float = 90.0): + self.path, self.timeout = path, timeout + + def call(self, method: str, **params): + s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + s.settimeout(self.timeout) + try: + s.connect(self.path) + s.sendall(json.dumps({"method": method, "params": params}).encode() + b"\n") + buf = b"" + while not buf.endswith(b"\n"): + chunk = s.recv(65536) + if not chunk: + break + buf += chunk + except OSError as e: + raise SignerUnavailable(f"signer socket {self.path}: {e}") from e + finally: + s.close() + if not buf: + raise SignerUnavailable("the signer closed the connection without a reply") + r = json.loads(buf) + if r.get("ok"): + return r["result"] + if r.get("code") == "error": + raise SignerUnavailable(r.get("error")) + raise PolicyError(r.get("error") or "refused", code=r.get("code") or "policy") + + +class LocalSigner: + """The same Signer in this process (no socket); one call at a time, like the server.""" + + def __init__(self, signer): + self.signer, self.lock = signer, threading.Lock() + + def call(self, method: str, **params): + from .core import SERVED + if method not in SERVED: + raise PolicyError(f"unknown method {method!r}") + with self.lock: + return getattr(self.signer, method)(**params) + + +_client = None +_lock = threading.Lock() + + +def get(): + """The process-wide signer client: the socket when SIGNER_SOCKET is set, else an in-process signer built from the + key settings (BOT_PRIVATE_KEY / _FILE, PAYOUT_PRIVATE_KEY_FILE).""" + global _client + with _lock: + if _client is None: + from .. import config + if config.SIGNER_SOCKET: + _client = SocketSigner(config.SIGNER_SOCKET) + else: + _client = LocalSigner(local_signer()) + return _client + + +def local_signer(): + import os + from .. import config + from ..chain.keys import load_keypair + from .core import Signer, SignerConfig + from .ledger import Ledger + from .rpc import Rpc + payout = None + if config.PAYOUT_PRIVATE_KEY_FILE: + from ..chain.keys import _keypair_from + payout = _keypair_from(open(config.PAYOUT_PRIVATE_KEY_FILE).read()) + url = config.SIGNER_RPC_URL or config.vault_solana_rpc_url() + return Signer(load_keypair(), payout, Rpc(url), SignerConfig.from_env({**os.environ, **config.signer_env()}), + Ledger(os.environ.get("SIGNER_DB") or config.DATA_DIR / "signer" / "ledger.sqlite")) + + +def reset() -> None: + """Tests: forget the cached client.""" + global _client + with _lock: + _client = None diff --git a/fly_trader/signer/core.py b/fly_trader/signer/core.py new file mode 100644 index 0000000000000000000000000000000000000000..786d81b5641a899be6cb5fce73929d6f0db85345 --- /dev/null +++ b/fly_trader/signer/core.py @@ -0,0 +1,274 @@ +"""The signer: the only code on the server that holds the two hot keys. The brain asks it for signed transactions and +broadcasts them itself (it persists the signature first, as before); the signer builds every transaction except the +Jupiter swap itself, checks each against policy (policy.py) and the treasury's on-chain spending limits (squads.py), +and keeps its own ledger (ledger.py). It never trusts the brain's database. + +What each method will sign: + sign_swap a Jupiter (Metis) swap between SOL and one token that the static rules and a simulation accept; buys + capped per trade, per minute and per day; none while panicking + topup treasury -> trading under L1 (destination locked to the trading wallet by the owner) + return_float trading -> treasury (a fixed destination) + pay_claim treasury -> a holder under L2, once per claim id (re-signed only after the earlier one provably failed + or expired), within what L2 has left this week + close_atas empty token accounts of the trading wallet, rent back to it + backup_keys both keys age-encrypted to the operator's public key (the plaintext never leaves this process) + panic from now on only sells, return_float and closes (``clear-panic`` is a CLI command inside the container) +""" +from __future__ import annotations + +import base64 +import os +import subprocess +import tempfile +import time +from dataclasses import dataclass, field + +from solders.address_lookup_table_account import AddressLookupTable +from solders.hash import Hash +from solders.instruction import AccountMeta, Instruction +from solders.keypair import Keypair +from solders.message import MessageV0, to_bytes_versioned +from solders.pubkey import Pubkey +from solders.system_program import TransferParams, transfer +from solders.transaction import VersionedTransaction + +from . import policy, squads +from .ledger import Ledger +from .policy import NATIVE_MINT, TOKEN, TOKEN_2022, PolicyError +from .rpc import Rpc + +RENT_EXEMPT_MIN = 890_880 # a 0-byte system account: the treasury PDA must keep this much +LAMPORTS = 1_000_000_000 +SIM_MAX_ACCOUNTS = 20 # the RPC caps accounts per simulation (30, or fewer: it has said 27) + + +@dataclass +class SignerConfig: + multisig: str | None = None # the Squads multisig (the treasury is its vault 0) + limit_trading: str | None = None # L1 spending-limit account + limit_payout: str | None = None # L2 spending-limit account + max_buy_lamports: int = int(1.0 * LAMPORTS) + daily_buy_lamports: int = int(10 * LAMPORTS) + swaps_per_min: int = 6 + max_priority_lamports: int = 5_000_000 + min_value_ratio: float = 0.6 + topup_max_lamports: int = int(2 * LAMPORTS) + backup_recipient: str | None = None + extra: dict = field(default_factory=dict) + + @classmethod + def from_env(cls, env=os.environ) -> "SignerConfig": + f = lambda k, d: float(env.get(k) or d) # noqa: E731 + return cls(multisig=env.get("VAULT_MULTISIG") or None, limit_trading=env.get("VAULT_LIMIT_TRADING") or None, + limit_payout=env.get("VAULT_LIMIT_PAYOUT") or None, + max_buy_lamports=int(f("SIGNER_MAX_BUY_SOL", 1.0) * LAMPORTS), # a safety bound, not sizing + daily_buy_lamports=int(f("SIGNER_DAILY_BUY_SOL", 10) * LAMPORTS), swaps_per_min=int(f("SIGNER_SWAPS_PER_MIN", 6)), + max_priority_lamports=int(f("SIGNER_MAX_PRIORITY_SOL", 0.005) * LAMPORTS), min_value_ratio=f("SIGNER_MIN_VALUE_RATIO", 0.6), + topup_max_lamports=int(f("TREASURY_FLOAT_SOL", 2.0) * LAMPORTS), backup_recipient=env.get("VAULT_BACKUP_RECIPIENT") or None) + + +def _b64(tx: VersionedTransaction) -> str: + return base64.b64encode(bytes(tx)).decode() + + +class Signer: + def __init__(self, trading: Keypair, payout: Keypair | None, rpc: Rpc, cfg: SignerConfig, ledger: Ledger | None = None): + if payout is not None and payout.pubkey() == trading.pubkey(): + raise ValueError("the payout key must differ from the trading key") + self.trading, self.payout, self.rpc, self.cfg = trading, payout, rpc, cfg + self.ledger = ledger or Ledger() + self.multisig = Pubkey.from_string(cfg.multisig) if cfg.multisig else None + self.treasury = squads.vault_pda(self.multisig, 0) if self.multisig else None + + # ------------------------------------------------------------------ helpers + def _built(self, payer: Keypair, ixs: list[Instruction], signers: list[Keypair]) -> dict: + bh, lvbh = self.rpc.blockhash() + msg = MessageV0.try_compile(payer.pubkey(), ixs, [], Hash.from_string(bh)) + tx = VersionedTransaction(msg, signers) + return {"tx": _b64(tx), "signature": str(tx.signatures[0]), "last_valid_block_height": lvbh, "blockhash": bh} + + def _limit(self, which: str, member: Pubkey) -> tuple[Pubkey, dict]: + addr = self.cfg.limit_trading if which == "L1" else self.cfg.limit_payout + if not (self.multisig and addr): + raise PolicyError(f"{which} is not configured (VAULT_MULTISIG / VAULT_LIMIT_*)") + key = Pubkey.from_string(addr) + acc = self.rpc.account(key) + if acc is None or acc["owner"] != squads.PROGRAM_ID: + raise PolicyError(f"{which} ({addr}) is not a Squads account: revoked?", code="cap") + lim = squads.decode_spending_limit(acc["data"]) + if lim["multisig"] != self.multisig or lim["vault_index"] != 0 or lim["mint"] != squads.SYSTEM_PROGRAM: + raise PolicyError(f"{which} is not a SOL limit on this treasury") + if member not in lim["members"]: + raise PolicyError(f"{which} does not list this key as a member: revoked?", code="cap") + return key, lim + + def _treasury_spendable(self) -> int: + return max(0, self.rpc.balance(self.treasury) - RENT_EXEMPT_MIN) + + def _resolve_alt(self, table: Pubkey) -> list[Pubkey]: + acc = self.rpc.account(table) + if acc is None: + raise PolicyError(f"lookup table {table} not found") + return list(AddressLookupTable.deserialize(acc["data"]).addresses) + + def _snapshot(self, accounts: list[dict | None], addrs: list[Pubkey]) -> policy.Snapshot: + tokens = {} + for a, acc in zip(addrs[1:], accounts[1:]): + if acc and acc["owner"] in (TOKEN, TOKEN_2022) and len(acc["data"]) >= 72: + tokens[a] = (policy.token_mint(acc["data"]), policy.token_amount(acc["data"]), acc["lamports"]) + return policy.Snapshot(lamports=accounts[0]["lamports"] if accounts[0] else 0, tokens=tokens) + + # ------------------------------------------------------------------ methods + def pubkeys(self) -> dict: + return {"trading": str(self.trading.pubkey()), "payout": str(self.payout.pubkey()) if self.payout else None, + "treasury": str(self.treasury) if self.treasury else None, "multisig": self.cfg.multisig} + + def sign_swap(self, tx: str, in_mint: str, out_mint: str, in_amount: int, min_out: int) -> dict: + me = self.trading.pubkey() + in_m, out_m = Pubkey.from_string(in_mint), Pubkey.from_string(out_mint) + buy = in_m == NATIVE_MINT + if buy and self.ledger.flag("panic"): + raise PolicyError("panic: no buys until cleared on the server") + now = time.time() + if len(self.ledger.swaps_since(now - 60)) >= self.cfg.swaps_per_min: + raise PolicyError("swap rate limit", code="cap") + if buy: + if in_amount > self.cfg.max_buy_lamports: + raise PolicyError(f"buy of {in_amount} lamports is above the per-trade cap {self.cfg.max_buy_lamports}") + spent = sum(r["in_amount"] for r in self.ledger.swaps_since(now - 86400) if r["direction"] == "buy") + if spent + in_amount > self.cfg.daily_buy_lamports: + raise PolicyError("daily buy volume cap reached", code="cap") + vt = VersionedTransaction.from_bytes(base64.b64decode(tx)) + chk = policy.check_swap_static(vt, me, self._resolve_alt, self.cfg.max_priority_lamports) + # every token account the trading wallet owns, plus the two this swap may open + mint_acc = self.rpc.accounts([in_m, out_m]) + progs = {m: (a["owner"] if a else TOKEN) for m, a in zip((in_m, out_m), mint_acc)} + addrs = [me] + self.rpc.token_accounts(me, TOKEN) + self.rpc.token_accounts(me, TOKEN_2022) + for m in (in_m, out_m): + a = policy.ata(me, m, progs[m]) + if a not in addrs: + addrs.append(a) + pre = self._snapshot(self.rpc.accounts(addrs), addrs) + post_accs: list[dict | None] = [] + for i in range(0, len(addrs), SIM_MAX_ACCOUNTS): # in chunks: the RPC caps accounts per simulation + sim = self.rpc.simulate(tx, addrs[i:i + SIM_MAX_ACCOUNTS]) + if sim.get("err") is not None: + raise PolicyError(f"the swap fails in simulation: {sim['err']} {(sim.get('logs') or [])[-3:]}", code="in_flight") + post_accs += [None if a is None else {"lamports": a["lamports"], "owner": Pubkey.from_string(a["owner"]), "data": base64.b64decode(a["data"][0])} + for a in sim["accounts"]] + post = self._snapshot(post_accs, addrs) + slack = policy.BASE_FEE_LAMPORTS * vt.message.header.num_required_signatures + chk.priority_lamports + 10_000 + eff = policy.check_swap_effects(pre, post, in_mint=in_m, out_mint=out_m, in_amount=int(in_amount), min_out=int(min_out), slack_lamports=slack) + tok = out_m if buy else in_m + tok_acc = mint_acc[1] if buy else mint_acc[0] + prices = self.rpc.usd_prices([str(tok), str(NATIVE_MINT)]) + ratio = policy.check_value("buy" if buy else "sell", eff["sol"], abs(eff["token"]), tok_acc["data"][44] if tok_acc else 0, + prices.get(str(tok)), prices.get(str(NATIVE_MINT)), self.cfg.min_value_ratio) + sigs = list(vt.signatures) + sigs[chk.signer_index] = self.trading.sign_message(to_bytes_versioned(vt.message)) + signed = VersionedTransaction.populate(vt.message, sigs) + sig = str(sigs[chk.signer_index]) + self.ledger.record_swap("buy" if buy else "sell", in_mint, out_mint, int(in_amount), -eff["sol"], sig) + return {"tx": _b64(signed), "signature": sig, "simulated": eff, "value_ratio": ratio} + + def topup(self, lamports: int) -> dict: + if self.ledger.flag("panic"): + raise PolicyError("panic: no top-ups") + lamports = int(lamports) + if not 0 < lamports <= self.cfg.topup_max_lamports: + raise PolicyError(f"top-up of {lamports} is outside (0, {self.cfg.topup_max_lamports}]") + me = self.trading.pubkey() + key, lim = self._limit("L1", me) + if lim["destinations"] != [me]: + raise PolicyError("L1 must be locked to the trading wallet as its only destination (owner: recreate it)") + if squads.remaining_now(lim, int(time.time())) < lamports: + raise PolicyError("L1 has too little left today", code="cap") + if self._treasury_spendable() < lamports: + raise PolicyError("the treasury holds too little", code="liquidity") + ix = squads.spending_limit_use_sol(self.multisig, me, key, self.treasury, me, lamports, "fly-topup") + return self._built(self.trading, [ix], [self.trading]) + + def return_float(self, lamports: int) -> dict: + lamports = int(lamports) + if lamports <= 0 or self.treasury is None: + raise PolicyError("nothing to return, or no treasury configured") + if self.rpc.balance(self.trading.pubkey()) < lamports + 10_000: + raise PolicyError("the trading wallet holds less than that") + ix = transfer(TransferParams(from_pubkey=self.trading.pubkey(), to_pubkey=self.treasury, lamports=lamports)) + return self._built(self.trading, [ix], [self.trading]) + + def pay_claim(self, claim_id: str, dest: str, lamports: int) -> dict: + if self.ledger.flag("panic"): + raise PolicyError("panic: claims are halted", code="cap") + if self.payout is None: + raise PolicyError("no payout key") + claim_id, lamports, to = str(claim_id), int(lamports), Pubkey.from_string(dest) + if lamports <= 0: + raise PolicyError("a claim must be positive") + prev = self.ledger.claim(claim_id) + if prev is not None: + if prev["dest"] != dest or prev["lamports"] != lamports: + raise PolicyError(f"claim {claim_id} was already signed for {prev['lamports']} lamports to {prev['dest']}") + st = self.rpc.signature_status(prev["signature"]) + if st is not None and st.get("err") is None and st.get("confirmationStatus") in ("confirmed", "finalized"): + raise PolicyError(f"claim {claim_id} is already paid ({prev['signature']})", code="paid") + if st is None and self.rpc.block_height() <= prev["last_valid_block_height"]: + raise PolicyError(f"claim {claim_id}'s earlier payment may still land", code="in_flight") + key, lim = self._limit("L2", self.payout.pubkey()) + if lim["destinations"] and to not in lim["destinations"]: + raise PolicyError("L2 does not allow this destination") + if squads.remaining_now(lim, int(time.time())) < lamports: + raise PolicyError("L2 has too little left this week: raise it from the owner wallet", code="cap") + if self._treasury_spendable() < lamports: + raise PolicyError("the treasury holds too little", code="liquidity") + ix = squads.spending_limit_use_sol(self.multisig, self.payout.pubkey(), key, self.treasury, to, lamports, f"fly-vault-claim:{claim_id}") + out = self._built(self.trading, [ix], [self.trading, self.payout]) # trading pays the fee; payout uses L2 + self.ledger.record_claim(claim_id, dest, lamports, out["signature"], out["last_valid_block_height"]) + return out + + def close_atas(self, accounts: list[str]) -> list[dict]: + me = self.trading.pubkey() + keys = [Pubkey.from_string(a) for a in accounts] + ixs = [] + for k, acc in zip(keys, self.rpc.accounts(keys)): + if not acc or acc["owner"] not in (TOKEN, TOKEN_2022) or len(acc["data"]) < 72: + continue + if Pubkey.from_bytes(acc["data"][32:64]) != me or policy.token_amount(acc["data"]) != 0: + raise PolicyError(f"{k} is not an empty token account of the trading wallet") + ixs.append(Instruction(acc["owner"], bytes([9]), [AccountMeta(k, False, True), AccountMeta(me, False, True), AccountMeta(me, True, False)])) + return [self._built(self.trading, ixs[i:i + 10], [self.trading]) for i in range(0, len(ixs), 10)] + + def backup_keys(self) -> list[dict]: + rcpt = self.cfg.backup_recipient + if not rcpt: + raise PolicyError("no backup recipient (VAULT_BACKUP_RECIPIENT)") + import base58 + out = [] + with tempfile.NamedTemporaryFile("w", delete=False) as f: + f.write(rcpt.strip() + "\n"); rfile = f.name + try: + for name, kp in (("trading", self.trading), ("payout", self.payout)): + if kp is None: + continue + enc = subprocess.run(["age", "-R", rfile], input=base58.b58encode(bytes(kp)), check=True, capture_output=True).stdout + out.append({"name": name, "pubkey": str(kp.pubkey()), "age_b64": base64.b64encode(enc).decode()}) + finally: + os.unlink(rfile) + return out + + def panic(self, reason: str = "") -> dict: + self.ledger.set_flag("panic", reason or "panic") + return {"panic": True} + + def clear_panic(self) -> dict: # CLI only (not served on the socket) + self.ledger.set_flag("panic", None) + return {"panic": False} + + def status(self) -> dict: + now = time.time() + return {"panic": self.ledger.flag("panic"), "swaps_last_min": len(self.ledger.swaps_since(now - 60)), + "buys_24h_lamports": sum(r["in_amount"] for r in self.ledger.swaps_since(now - 86400) if r["direction"] == "buy"), + **self.pubkeys()} + + +SERVED = ("pubkeys", "sign_swap", "topup", "return_float", "pay_claim", "close_atas", "backup_keys", "panic", "status") diff --git a/fly_trader/signer/ledger.py b/fly_trader/signer/ledger.py new file mode 100644 index 0000000000000000000000000000000000000000..3b8ca3ef9be3684abb9492cf66766b14f7fee464 --- /dev/null +++ b/fly_trader/signer/ledger.py @@ -0,0 +1,55 @@ +"""The signer's own records (SQLite in its own volume, which the brain container cannot reach): swaps it signed (for +the rate and daily caps), claims it signed (one payment per claim id, whatever the brain's database says), and flags +(panic). Nothing here is trusted from the brain.""" +from __future__ import annotations + +import sqlite3 +import time +from pathlib import Path + +DDL = """ +CREATE TABLE IF NOT EXISTS swaps (ts REAL NOT NULL, direction TEXT NOT NULL, in_mint TEXT, out_mint TEXT, in_amount INTEGER, + sol_spent INTEGER, signature TEXT); +CREATE INDEX IF NOT EXISTS swaps_ts ON swaps(ts); +CREATE TABLE IF NOT EXISTS claims (claim_id TEXT PRIMARY KEY, dest TEXT NOT NULL, lamports INTEGER NOT NULL, signature TEXT NOT NULL, + last_valid_block_height INTEGER NOT NULL, ts REAL NOT NULL, attempts INTEGER NOT NULL DEFAULT 1); +CREATE TABLE IF NOT EXISTS flags (key TEXT PRIMARY KEY, value TEXT, ts REAL); +""" + + +class Ledger: + def __init__(self, path: str | Path = ":memory:"): + if str(path) != ":memory:": + Path(path).parent.mkdir(parents=True, exist_ok=True) + self.db = sqlite3.connect(str(path), isolation_level=None, check_same_thread=False) + self.db.row_factory = sqlite3.Row + self.db.execute("PRAGMA journal_mode=WAL") + self.db.executescript(DDL) + + # swaps + def swaps_since(self, since: float) -> list[sqlite3.Row]: + return self.db.execute("SELECT * FROM swaps WHERE ts >= ?", (since,)).fetchall() + + def record_swap(self, direction: str, in_mint: str, out_mint: str, in_amount: int, sol_spent: int, signature: str) -> None: + self.db.execute("INSERT INTO swaps VALUES (?,?,?,?,?,?,?)", (time.time(), direction, in_mint, out_mint, in_amount, sol_spent, signature)) + + # claims + def claim(self, claim_id: str) -> sqlite3.Row | None: + return self.db.execute("SELECT * FROM claims WHERE claim_id = ?", (claim_id,)).fetchone() + + def record_claim(self, claim_id: str, dest: str, lamports: int, signature: str, lvbh: int) -> None: + self.db.execute("INSERT INTO claims (claim_id, dest, lamports, signature, last_valid_block_height, ts) VALUES (?,?,?,?,?,?) " + "ON CONFLICT(claim_id) DO UPDATE SET signature = excluded.signature, last_valid_block_height = excluded.last_valid_block_height, " + "ts = excluded.ts, attempts = attempts + 1", (claim_id, dest, lamports, signature, lvbh, time.time())) + + # flags + def flag(self, key: str) -> str | None: + r = self.db.execute("SELECT value FROM flags WHERE key = ?", (key,)).fetchone() + return r["value"] if r else None + + def set_flag(self, key: str, value: str | None) -> None: + if value is None: + self.db.execute("DELETE FROM flags WHERE key = ?", (key,)) + else: + self.db.execute("INSERT INTO flags VALUES (?,?,?) ON CONFLICT(key) DO UPDATE SET value = excluded.value, ts = excluded.ts", + (key, value, time.time())) diff --git a/fly_trader/signer/policy.py b/fly_trader/signer/policy.py new file mode 100644 index 0000000000000000000000000000000000000000..239196f066e02c025a39fd7513018ec824eb74b2 --- /dev/null +++ b/fly_trader/signer/policy.py @@ -0,0 +1,192 @@ +"""What the signer will sign for a swap. Two layers, both required: + +1. Static: every top-level instruction of the Jupiter transaction is one we expect from a Metis route + (tools: ComputeBudget, System wrap, SPL Token sync/close, ATA create, Jupiter v6), each with the trading wallet in + the only roles it may have; the priority fee is capped. Program ids are always static keys in a v0 message, so a + lookup table can hide an account but never a program. +2. Simulated: the signer simulates the transaction itself (its own RPC) and compares the trading wallet's SOL and every + token account it owns before and after. A buy may spend at most the ordered SOL (+ rent and fees) and must deliver + the bought token to the trading wallet; a sell may spend at most the ordered tokens and must return SOL to it; no + other holding may shrink. So a route that pays someone else, drains another holding or overspends is refused + whatever the brain claimed. + +Pure functions only: the RPC calls live in core.py, which passes their results in here. +""" +from __future__ import annotations + +import struct +from dataclasses import dataclass + +from solders.message import MessageV0 +from solders.pubkey import Pubkey +from solders.transaction import VersionedTransaction + +SYSTEM = Pubkey.from_string("11111111111111111111111111111111") +COMPUTE_BUDGET = Pubkey.from_string("ComputeBudget111111111111111111111111111111") +TOKEN = Pubkey.from_string("TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA") +TOKEN_2022 = Pubkey.from_string("TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb") +ATA = Pubkey.from_string("ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL") +JUPITER_V6 = Pubkey.from_string("JUP6LkbZbjS1jKKwapdHNy74zcZ3tLUZoi5QNyVTaV4") +NATIVE_MINT = Pubkey.from_string("So11111111111111111111111111111111111111112") +MEMO = Pubkey.from_string("MemoSq4gqABAXKb96qnH8TysNcWxMyWCqXgDLGmfcHr") +SWAP_PROGRAMS = frozenset({SYSTEM, COMPUTE_BUDGET, TOKEN, TOKEN_2022, ATA, JUPITER_V6}) +# the Jupiter routers the broker asks for (Metis only): every other router brings programs we have not pinned +EXCLUDE_ROUTERS = "jupiterz,dflow,okx" +ATA_RENT_LAMPORTS = 2_039_280 +BASE_FEE_LAMPORTS = 5_000 + + +class PolicyError(RuntimeError): + """``code``: 'policy' (refused: never retry as is), 'cap' (a limit's period is used up: wait), 'liquidity' (the + treasury is short: wait), 'in_flight' (an earlier signature may still land: wait), 'paid' (already landed).""" + + def __init__(self, msg: str, code: str = "policy"): + super().__init__(msg) + self.code = code + + +def ata(owner: Pubkey, mint: Pubkey, token_program: Pubkey = TOKEN) -> Pubkey: + return Pubkey.find_program_address([bytes(owner), bytes(token_program), bytes(mint)], ATA)[0] + + +def account_keys(msg: MessageV0, resolve_alt) -> list[Pubkey]: + """Static keys, then every lookup table's writable, then readonly addresses (the v0 message order). + ``resolve_alt(table) -> list[Pubkey]`` reads a lookup table (the signer's own RPC).""" + keys = list(msg.account_keys) + lookups = list(msg.address_table_lookups) + tables = {lk.account_key: resolve_alt(lk.account_key) for lk in lookups} + try: + keys += [tables[lk.account_key][i] for lk in lookups for i in bytes(lk.writable_indexes)] + keys += [tables[lk.account_key][i] for lk in lookups for i in bytes(lk.readonly_indexes)] + except IndexError as e: + raise PolicyError("a lookup table index is out of range") from e + return keys + + +@dataclass +class SwapCheck: + signer_index: int + fee_payer: Pubkey + priority_lamports: int + creates_atas: int + + +def check_swap_static(tx: VersionedTransaction, trading: Pubkey, resolve_alt, max_priority_lamports: int) -> SwapCheck: + msg = tx.message + if not isinstance(msg, MessageV0) and not hasattr(msg, "account_keys"): + raise PolicyError("not a transaction message") + static = list(msg.account_keys) + n_sign = msg.header.num_required_signatures + try: + idx = static.index(trading) + except ValueError: + raise PolicyError("the trading wallet does not sign this transaction") from None + if idx >= n_sign: + raise PolicyError("the trading wallet is not a required signer") + keys = account_keys(msg, resolve_alt) if hasattr(msg, "address_table_lookups") else static + wsol = ata(trading, NATIVE_MINT) + cu_limit, cu_price, creates = 200_000, 0, 0 + for ix in msg.instructions: + prog = static[ix.program_id_index] + if prog not in SWAP_PROGRAMS: + raise PolicyError(f"program {prog} is not one a swap may call") + data = bytes(ix.data); acc = [keys[i] for i in bytes(ix.accounts)] + tag = data[0] if data else None + if prog == COMPUTE_BUDGET: + if tag == 2 and len(data) >= 5: + cu_limit = struct.unpack("= 9: + cu_price = struct.unpack("= 3: # CloseAccount(account, destination, owner) + if acc[1] != trading or acc[2] != trading: + raise PolicyError("a token account close that does not return its rent to the trading wallet") + continue + raise PolicyError(f"token instruction {tag} is not part of a swap (transfer, approve, authority changes are refused)") + elif prog == ATA: + if tag not in (None, 0, 1) or len(acc) < 3 or acc[2] != trading: # Create / CreateIdempotent for our wallet + raise PolicyError("an associated-token-account instruction for another wallet") + creates += 1 + # JUPITER_V6: the route itself; what it does to our balances is checked by simulation + prio = cu_limit * cu_price // 1_000_000 + if prio > max_priority_lamports: + raise PolicyError(f"priority fee {prio} lamports is above the cap {max_priority_lamports}") + return SwapCheck(signer_index=idx, fee_payer=static[0], priority_lamports=prio, creates_atas=creates) + + +def token_amount(data: bytes | None) -> int: + """SPL token account amount (Token and Token-2022 share the base layout); a closed account holds 0.""" + if not data or len(data) < 72: + return 0 + return struct.unpack(" Pubkey: + return Pubkey.from_bytes(bytes(data[0:32])) + + +@dataclass +class Snapshot: + """The trading wallet's lamports and each of its token accounts: address -> (mint, amount, lamports).""" + lamports: int + tokens: dict + + +def check_swap_effects(pre: Snapshot, post: Snapshot, *, in_mint: Pubkey, out_mint: Pubkey, in_amount: int, min_out: int, + slack_lamports: int) -> dict: + """The balance changes a simulation produced, against what was ordered. Returns the deltas; raises PolicyError.""" + if (in_mint == NATIVE_MINT) == (out_mint == NATIVE_MINT): + raise PolicyError("only SOL <-> token swaps are signed") + if in_amount <= 0 or min_out <= 0: + raise PolicyError("an order needs a positive amount and a positive minimum out") + sol = post.lamports - pre.lamports + by_mint: dict[Pubkey, int] = {} + for addr in set(pre.tokens) | set(post.tokens): + mint = (pre.tokens.get(addr) or post.tokens.get(addr))[0] + a0 = (pre.tokens.get(addr) or (mint, 0, 0)) + a1 = (post.tokens.get(addr) or (mint, 0, 0)) + if mint == NATIVE_MINT: # wrapped SOL counts as SOL: by lamports + sol += a1[2] - a0[2] + else: + by_mint[mint] = by_mint.get(mint, 0) + (a1[1] - a0[1]) + sol += a1[2] - a0[2] # rent of token accounts opened or closed + for mint, d in by_mint.items(): + if d < 0 and mint != in_mint: + raise PolicyError(f"the swap would reduce another holding ({mint})") + if in_mint == NATIVE_MINT: # buy + got = by_mint.get(out_mint, 0) + if -sol > in_amount + slack_lamports: + raise PolicyError(f"the swap spends {-sol} lamports, more than the {in_amount} ordered (+{slack_lamports} fees and rent)") + if got < min_out: + raise PolicyError(f"the trading wallet would receive {got} of the token, below the minimum {min_out}") + return {"sol": sol, "token": got} + spent = -by_mint.get(in_mint, 0) # sell + if spent > in_amount: + raise PolicyError(f"the swap sells {spent} tokens, more than the {in_amount} ordered") + if sol < min_out - slack_lamports: + raise PolicyError(f"the trading wallet would receive {sol} lamports, below the minimum {min_out}") + return {"sol": sol, "token": -spent} + + +def check_value(direction: str, sol_moved: int, token_moved: int, token_decimals: int, token_usd: float | None, sol_usd: float | None, + min_ratio: float) -> float | None: + """Value out / value in at independent prices (the signer's own Jupiter price query); None if a price is missing. + Refuses a swap that gives away more than ``1 - min_ratio`` of its value (a self-dealing route into a pool the + attacker priced).""" + if not token_usd or not sol_usd or token_moved <= 0 or sol_moved == 0: + return None + tok_value = token_moved / 10 ** token_decimals * token_usd + sol_value = abs(sol_moved) / 1e9 * sol_usd + ratio = tok_value / sol_value if direction == "buy" else sol_value / tok_value + if ratio < min_ratio: + raise PolicyError(f"the swap returns {ratio:.0%} of the value it gives (floor {min_ratio:.0%})") + return ratio diff --git a/fly_trader/signer/rpc.py b/fly_trader/signer/rpc.py new file mode 100644 index 0000000000000000000000000000000000000000..8e35aad30607ce2dee516b66a0f2afe4ea231887 --- /dev/null +++ b/fly_trader/signer/rpc.py @@ -0,0 +1,71 @@ +"""The signer's own Solana JSON-RPC client (its own provider: SIGNER_RPC_URL, not the brain's), and Jupiter's public +price endpoint for the value check. httpx only.""" +from __future__ import annotations + +import base64 + +import httpx +from solders.pubkey import Pubkey + +PRICE_URL = "https://lite-api.jup.ag/price/v3" + + +class RpcError(RuntimeError): + pass + + +class Rpc: + def __init__(self, url: str, timeout: float = 20.0): + if not url: + raise RpcError("the signer needs its own RPC URL (SIGNER_RPC_URL)") + self.url, self.http = url, httpx.Client(timeout=timeout) + + def call(self, method: str, params: list): + r = self.http.post(self.url, json={"jsonrpc": "2.0", "id": 1, "method": method, "params": params}) + r.raise_for_status() + d = r.json() + if d.get("error"): + raise RpcError(f"{method}: {d['error']}") + return d.get("result") + + def accounts(self, keys: list[Pubkey], commitment: str = "confirmed") -> list[dict | None]: + out: list[dict | None] = [] + for i in range(0, len(keys), 100): + v = self.call("getMultipleAccounts", [[str(k) for k in keys[i:i + 100]], {"encoding": "base64", "commitment": commitment}])["value"] + out += [None if a is None else {"lamports": a["lamports"], "owner": Pubkey.from_string(a["owner"]), "data": base64.b64decode(a["data"][0])} + for a in v] + return out + + def account(self, key: Pubkey, commitment: str = "confirmed") -> dict | None: + return self.accounts([key], commitment)[0] + + def balance(self, key: Pubkey, commitment: str = "confirmed") -> int: + return int(self.call("getBalance", [str(key), {"commitment": commitment}])["value"]) + + def token_accounts(self, owner: Pubkey, program: Pubkey) -> list[Pubkey]: + v = self.call("getTokenAccountsByOwner", [str(owner), {"programId": str(program)}, {"encoding": "base64", "commitment": "confirmed", + "dataSlice": {"offset": 0, "length": 0}}])["value"] + return [Pubkey.from_string(a["pubkey"]) for a in v] + + def blockhash(self) -> tuple[str, int]: + v = self.call("getLatestBlockhash", [{"commitment": "confirmed"}])["value"] + return v["blockhash"], int(v["lastValidBlockHeight"]) + + def block_height(self) -> int: + return int(self.call("getBlockHeight", [{"commitment": "confirmed"}])) + + def signature_status(self, sig: str) -> dict | None: + return self.call("getSignatureStatuses", [[sig], {"searchTransactionHistory": True}])["value"][0] + + def simulate(self, tx_b64: str, addresses: list[Pubkey]) -> dict: + return self.call("simulateTransaction", [tx_b64, {"sigVerify": False, "replaceRecentBlockhash": True, "encoding": "base64", + "commitment": "processed", + "accounts": {"encoding": "base64", "addresses": [str(a) for a in addresses]}}])["value"] + + def usd_prices(self, mints: list[str]) -> dict[str, float]: + try: + r = self.http.get(PRICE_URL, params={"ids": ",".join(mints)}, timeout=5) + r.raise_for_status() + return {m: float(v["usdPrice"]) for m, v in (r.json() or {}).items() if v and v.get("usdPrice")} + except Exception: + return {} diff --git a/fly_trader/signer/server.py b/fly_trader/signer/server.py new file mode 100644 index 0000000000000000000000000000000000000000..59682880ef75f84b83a83cfd0ce5506e5f501e87 --- /dev/null +++ b/fly_trader/signer/server.py @@ -0,0 +1,60 @@ +"""The signer's Unix socket: one JSON request per line, one JSON reply per line, one request at a time (so the caps +and the claim ledger see every signature in order). The socket file is group-writable for the brain's group only; the +directory it lives in is the only thing the two containers share.""" +from __future__ import annotations + +import json +import logging +import os +import socketserver +from pathlib import Path + +from .core import SERVED, Signer +from .policy import PolicyError + +log = logging.getLogger("fly.signer") +MAX_REQUEST = 256 * 1024 + + +def handle(signer: Signer, line: bytes) -> dict: + try: + req = json.loads(line) + method, params = req.get("method"), req.get("params") or {} + if method not in SERVED or not isinstance(params, dict): + return {"ok": False, "code": "policy", "error": f"unknown method {method!r}"} + out = getattr(signer, method)(**params) + log.info("signed %s", method) + return {"ok": True, "result": out} + except PolicyError as e: + log.warning("refused %s: %s", locals().get("method"), e) + return {"ok": False, "code": e.code, "error": str(e)} + except TypeError as e: + return {"ok": False, "code": "policy", "error": f"bad parameters: {e}"} + except Exception as e: # an RPC failure etc.: the caller retries later + log.exception("error in %s", locals().get("method")) + return {"ok": False, "code": "error", "error": f"{type(e).__name__}: {e}"} + + +def serve(signer: Signer, path: str, group_mode: int = 0o660) -> None: + p = Path(path) + p.parent.mkdir(parents=True, exist_ok=True) + if p.exists(): + p.unlink() + + class H(socketserver.StreamRequestHandler): + timeout = 60 + + def handle(self): + line = self.rfile.readline(MAX_REQUEST + 1) + if not line or len(line) > MAX_REQUEST: + return + self.wfile.write(json.dumps(handle(signer, line)).encode() + b"\n") + + old = os.umask(0o117) + try: + srv = socketserver.UnixStreamServer(str(p), H) + finally: + os.umask(old) + os.chmod(p, group_mode) + log.info("signer listening on %s for %s", p, signer.pubkeys()) + srv.serve_forever() diff --git a/fly_trader/signer/squads.py b/fly_trader/signer/squads.py new file mode 100644 index 0000000000000000000000000000000000000000..d5a551a36fbf6ccca53f7bfbeb4fdfcc273977a9 --- /dev/null +++ b/fly_trader/signer/squads.py @@ -0,0 +1,137 @@ +"""Squads v4 multisig: the treasury's spending limits, built by hand from the program's Anchor IDL (v2.1.0, +Squads-Protocol/v4 ``sdk/multisig/idl/squads_multisig_program.json``); stdlib + solders only, so the signer image needs +nothing else. + +The treasury is vault 0 of a multisig whose only member is Bryce's wallet (Solflare now, the Ledger later). Two +spending limits let the server's hot keys move SOL out of it without a vote, and nothing more: + + L1 member = trading key, destinations = [trading wallet], period Day -> the trading float refills itself + L2 member = payout key, destinations = [] (any holder), period Week -> claims + +``spendingLimitUse`` checks on chain that the signer is a member of the limit, that the destination is allowed and that +the amount fits what is left of the period; the owner removes a limit (or the whole server's access) from his phone. +""" +from __future__ import annotations + +import hashlib +import struct + +from solders.instruction import AccountMeta, Instruction +from solders.pubkey import Pubkey + +PROGRAM_ID = Pubkey.from_string("SQDS4ep65T869zMMBKyuUq6aD6EgTu8psMjkvj52pCf") +SYSTEM_PROGRAM = Pubkey.from_string("11111111111111111111111111111111") +SEED_PREFIX, SEED_MULTISIG, SEED_VAULT, SEED_SPENDING_LIMIT = b"multisig", b"multisig", b"vault", b"spending_limit" +PERIODS = ("OneTime", "Day", "Week", "Month") +PERIOD_S = {"OneTime": None, "Day": 86400, "Week": 7 * 86400, "Month": 30 * 86400} + + +def _disc(kind: str, name: str) -> bytes: + return hashlib.sha256(f"{kind}:{name}".encode()).digest()[:8] + + +IX_SPENDING_LIMIT_USE = _disc("global", "spending_limit_use") +ACC_SPENDING_LIMIT = _disc("account", "SpendingLimit") +ACC_MULTISIG = _disc("account", "Multisig") + + +def multisig_pda(create_key: Pubkey) -> Pubkey: + return Pubkey.find_program_address([SEED_PREFIX, SEED_MULTISIG, bytes(create_key)], PROGRAM_ID)[0] + + +def vault_pda(multisig: Pubkey, index: int = 0) -> Pubkey: + return Pubkey.find_program_address([SEED_PREFIX, bytes(multisig), SEED_VAULT, bytes([index])], PROGRAM_ID)[0] + + +def spending_limit_pda(multisig: Pubkey, create_key: Pubkey) -> Pubkey: + return Pubkey.find_program_address([SEED_PREFIX, bytes(multisig), SEED_SPENDING_LIMIT, bytes(create_key)], PROGRAM_ID)[0] + + +def spending_limit_use_sol(multisig: Pubkey, member: Pubkey, spending_limit: Pubkey, vault: Pubkey, destination: Pubkey, + lamports: int, memo: str | None = None) -> Instruction: + """Move ``lamports`` SOL from ``vault`` to ``destination`` under ``spending_limit`` (``member`` signs).""" + if lamports <= 0: + raise ValueError("amount must be positive") + data = IX_SPENDING_LIMIT_USE + struct.pack(" bytes: + if self.i + n > len(self.b): + raise ValueError("account data too short") + out = self.b[self.i:self.i + n]; self.i += n + return out + + def u8(self) -> int: + return self.take(1)[0] + + def u16(self) -> int: + return struct.unpack(" int: + return struct.unpack(" int: + return struct.unpack(" int: + return struct.unpack(" Pubkey: + return Pubkey.from_bytes(self.take(32)) + + def keys(self) -> list[Pubkey]: + return [self.key() for _ in range(self.u32())] + + +def decode_spending_limit(data: bytes) -> dict: + if bytes(data[:8]) != ACC_SPENDING_LIMIT: + raise ValueError("not a Squads SpendingLimit account") + r = _Reader(bytes(data)); r.take(8) + d = {"multisig": r.key(), "create_key": r.key(), "vault_index": r.u8(), "mint": r.key(), "amount": r.u64()} + p = r.u8() + if p >= len(PERIODS): + raise ValueError("unknown period") + d.update({"period": PERIODS[p], "remaining": r.u64(), "last_reset": r.i64(), "bump": r.u8(), "members": r.keys(), "destinations": r.keys()}) + return d + + +def decode_multisig(data: bytes) -> dict: + if bytes(data[:8]) != ACC_MULTISIG: + raise ValueError("not a Squads Multisig account") + r = _Reader(bytes(data)); r.take(8) + d = {"create_key": r.key(), "config_authority": r.key(), "threshold": r.u16(), "time_lock": r.u32(), "transaction_index": r.u64(), + "stale_transaction_index": r.u64()} + d["rent_collector"] = r.key() if r.u8() else None + d["bump"] = r.u8() + d["members"] = [(r.key(), r.u8()) for _ in range(r.u32())] # (key, permission mask: 1 initiate, 2 vote, 4 execute) + return d + + +def remaining_now(limit: dict, now: int) -> int: + """What ``spendingLimitUse`` would allow right now (the program resets whole periods lazily, on use).""" + secs = PERIOD_S[limit["period"]] + if secs is not None and now - limit["last_reset"] > secs: + return int(limit["amount"]) + return int(limit["remaining"]) + + +def encode_spending_limit(d: dict) -> bytes: + """Account bytes for tests and the local demo (the inverse of ``decode_spending_limit``).""" + b = ACC_SPENDING_LIMIT + bytes(d["multisig"]) + bytes(d["create_key"]) + bytes([d.get("vault_index", 0)]) + bytes(d.get("mint", SYSTEM_PROGRAM)) + b += struct.pack(" np.ndarray: + """The rows' days as proleptic ordinals (int32), computed once per set: comparing tens of millions of date objects + (``np.isin`` sorts them) cost minutes per walk-forward block on the Kalshi corpus.""" + o = getattr(ds, "_day_ord", None) + if o is None or len(o) != len(ds.day): + uniq = {} + o = np.fromiter((uniq.setdefault(d, d.toordinal()) for d in ds.day), dtype=np.int32, count=len(ds.day)) + ds._day_ord = o + return o + + @dataclass class DecisionSet: X: np.ndarray # [N, F] float32, raw (not standardized) @@ -68,16 +83,21 @@ class DecisionSet: cols: list[str] horizon_s: float fwd_h: dict = field(default_factory=dict) # hold (min) → [N] float32 net return (next-open fill) over that hold; NaN past the data + chain: np.ndarray | None = None # [N] 'sol' | 'rh' (None: every row Solana) + + def chains(self) -> np.ndarray: + return self.chain if self.chain is not None else np.full(len(self.y), "sol", dtype=object) @property def days(self) -> list[date]: - return sorted(set(self.day.tolist())) + return [date.fromordinal(int(o)) for o in np.unique(day_ords(self))] def subset(self, mask: np.ndarray) -> "DecisionSet": """The rows of ``mask`` (e.g. the days before a bootstrap), labels for every hold included.""" m = np.asarray(mask, dtype=bool) return DecisionSet(X=self.X[m], y=self.y[m], fwd=self.fwd[m], fwd_pess=self.fwd_pess[m], day=self.day[m], ts=self.ts[m], mint=self.mint[m], - cols=list(self.cols), horizon_s=self.horizon_s, fwd_h={h: v[m] for h, v in self.fwd_h.items()}) + cols=list(self.cols), horizon_s=self.horizon_s, fwd_h={h: v[m] for h, v in self.fwd_h.items()}, + chain=None if self.chain is None else self.chain[m]) def mask_days(self, lo: date | None = None, hi: date | None = None) -> np.ndarray: m = np.ones(len(self.y), bool) @@ -91,35 +111,70 @@ class DecisionSet: def _label_exit_cost(df) -> np.ndarray: """One side's real cost per row, at the size the book trades (market/exit_cost.cost_at_size). The live engine puts the same value in every bar it hands the fly (agent/minute_engine.py), so what the fly learns from and what it was - trained on are one cost model.""" + trained on are one cost model. Robinhood Chain rows are priced at their own market's sizes (markets.RH).""" from ..market.exit_cost import cost_at_size - return cost_at_size(df["exit_cost_0p1"].to_numpy(dtype=float), df["resq"].to_numpy(dtype=float)) + ec = cost_at_size(df["exit_cost_0p1"].to_numpy(dtype=float), df["resq"].to_numpy(dtype=float)) + if "_chain" in df and (df["_chain"] == "rh").any(): + from ..markets import RH + rh = (df["_chain"] == "rh").to_numpy() + ec = np.asarray(ec, dtype=float).copy() + ec[rh] = cost_at_size(df["exit_cost_0p1"].to_numpy(dtype=float)[rh], df["resq"].to_numpy(dtype=float)[rh], market=RH) + return ec + + +def _roots(feature_dir) -> list[tuple]: + """(feature part root, chain, part_current) per market: the given directory alone (Solana: tests, studies), else the + Solana corpus plus Robinhood Chain's when it is enabled and built (rh/corpus.py).""" + if feature_dir is not None: + return [(feature_dir, "sol", part_current)] + out = [(config.CORPUS_DIR / "features_mature", "sol", part_current)] + if config.RH_ENABLED: + from ..rh import corpus as rh_corpus + if any(rh_corpus.FEAT_DIR.glob("*/part.parquet")): + out.append((rh_corpus.FEAT_DIR, "rh", rh_corpus.part_current)) + return out def build(days: int | None = 45, horizon_min: int = HOLD_MIN, fee: float | None = None, label_thr: float = 0.03, feature_dir=None, holds=None) -> DecisionSet: """``holds``: extra holds (minutes) whose labels are computed in the same pass (``DecisionSet.fwd_h``).""" - root = feature_dir or (config.CORPUS_DIR / "features_mature") - files = sorted(glob.glob(str(root / "*" / "part.parquet"))) - if days: - files = files[-days - 1:] - if not files: - raise RuntimeError("no mature feature parts; run build-mature") - stale = [f for f in files if not part_current(f)] - if stale: - raise RuntimeError(f"{len(stale)} feature part(s) were built with another feature or aggregation version (e.g. {stale[0]}); run build-mature") need = ["mint", "ts", "open", "close", "resq", "age_h", "traders_15m", "traders_1h", "n_trades_1m"] + FEATURES + FLOW_COLS + SKILL_COLS + MARKET_COLS optional = set(FLOW_COLS + SKILL_COLS + MARKET_COLS) # parts that predate the wallet-flow columns (tests' synthetic parts) read as 0 - df = pd.concat([pq.read_table(f, columns=[c for c in need if c not in optional or c in pq.read_schema(f).names]).to_pandas() for f in files], - ignore_index=True).sort_values(["mint", "ts"]).reset_index(drop=True) + roots = _roots(feature_dir); per_root = [] + for root, chain, current in roots: + files = sorted(glob.glob(str(root / "*" / "part.parquet"))) + per_root.append((files, chain, current)) + if days: # the last ``days`` calendar days (+1 of warm-up) of the newest corpus + last = max((f.split("/")[-2] for files, _, _ in per_root for f in files), default=None) + if last is not None: + lo = (date.fromisoformat(last) - pd.Timedelta(days=days)).isoformat() + per_root = [([f for f in files if f.split("/")[-2] >= lo], c, cur) for files, c, cur in per_root] + if not any(files for files, _, _ in per_root): + raise RuntimeError("no mature feature parts; run build-mature") + frames = [] + for files, chain, current in per_root: + stale = [f for f in files if not current(f)] + if stale: + raise RuntimeError(f"{len(stale)} feature part(s) were built with another feature or aggregation version (e.g. {stale[0]}); run build-mature") + for f in files: + fr = pq.read_table(f, columns=[c for c in need if c not in optional or c in pq.read_schema(f).names]).to_pandas() + fr["_chain"] = chain; frames.append(fr) + df = pd.concat(frames, ignore_index=True).sort_values(["mint", "ts"]).reset_index(drop=True) + kk = np.where(df["_chain"].to_numpy() == "rh", config.RH_ETH_PER_SOL, 1.0) # ETH per SOL on RH rows: fixed-unit thresholds for c in optional: if c not in df: df[c] = 0.0 # a mint whose series breaks scale (secondary pool in another quote, impossible reserve) is ineligible FROM THAT MINUTE ON — # never before it, so no decision row is judged with the token's future (rows before a dump keep their labels) jump = (df["close"] / df.groupby("mint")["close"].shift(1)).fillna(1.0) - df["_off"] = ((jump > 50) | (jump < 1 / 50) | (df["resq"] > 1e5)).astype(np.int8) + df["_off"] = ((jump > 50) | (jump < 1 / 50) | (df["resq"] > 1e5 * kk)).astype(np.int8) broken = df.groupby("mint")["_off"].cummax().to_numpy().astype(bool) - df = df.merge(load_features(), on="mint", how="left") + meta = load_features() + if (df["_chain"] == "rh").any(): # each chain's launch facts, same columns (rh/meta.py) + from ..rh.meta import load_features as rh_features + rmeta = rh_features() + with_q = rmeta.merge(_rh_quote_classes(), on="mint", how="left") if len(rmeta) else rmeta + meta = pd.concat([meta, with_q], ignore_index=True) + df = df.merge(meta, on="mint", how="left") df["meta_known"] = df["ttg_min"].notna().astype(np.float32) if "ttg_min" in df else np.float32(0.0) H = horizon_min * 60.0 ts = _epoch_s(df["ts"]); cl = df["close"].to_numpy(); op = df["open"].to_numpy() @@ -154,15 +209,28 @@ def build(days: int | None = 45, horizon_min: int = HOLD_MIN, fee: float | None df["age_known"] = df["age_h"].notna().astype(np.float32) df["mayhem"] = df["mayhem"].astype(float) if "mayhem" in df else 0.0 last_ts = df["ts"].max() - elig = (np.isfinite(df["resq"]) & (df["resq"] >= MIN_RESQ_SOL) & (df["logvol_15m"] >= math.log1p(MIN_VOL_15M_SOL)) + df["chain_rh"] = (df["_chain"] == "rh").astype(np.float32) + qc = df["quote_class"] if "quote_class" in df else pd.Series(None, index=df.index) + for c, cls in (("qc_stable", "stable"), ("qc_btc", "btc"), ("qc_stock", "stock")): + df[c] = (qc == cls).astype(np.float32) + vol_line = np.where(kk == 1.0, math.log1p(MIN_VOL_15M_SOL), np.log1p(MIN_VOL_15M_SOL * kk)) # Solana: exactly today's threshold + elig = (np.isfinite(df["resq"]) & (df["resq"] >= MIN_RESQ_SOL * kk) & (df["logvol_15m"] >= vol_line) & np.isfinite(fwd) & (df["ts"] < last_ts - pd.Timedelta(minutes=horizon_min + 5))).to_numpy() & ~broken for hm in hold_list: # no label where that hold runs past the data fh[hm][(df["ts"] >= last_ts - pd.Timedelta(minutes=hm + 5)).to_numpy()] = np.nan + chain = df["_chain"].to_numpy()[elig] df = df[elig].reset_index(drop=True); fwd = fwd[elig]; fwdp = fwdp[elig]; ts = ts[elig] X = df[X_COLS].astype(np.float32).replace([np.inf, -np.inf], np.nan).fillna(0.0).to_numpy() return DecisionSet(X=X, y=(fwd > label_thr).astype(np.int8), fwd=fwd.astype(np.float32), fwd_pess=fwdp.astype(np.float32), day=df["ts"].dt.date.to_numpy(), ts=ts, mint=df["mint"].to_numpy(), cols=list(X_COLS), horizon_s=H, - fwd_h={hm: fh[hm][elig].astype(np.float32) for hm in hold_list}) + fwd_h={hm: fh[hm][elig].astype(np.float32) for hm in hold_list}, chain=None if (chain == "sol").all() else chain.astype(object)) + + +def _rh_quote_classes() -> pd.DataFrame: + from ..db.connection import transaction + with transaction() as conn: + rows = conn.execute("SELECT mint, quote_class FROM rh_meta").fetchall() + return pd.DataFrame(rows, columns=["mint", "quote_class"]) if rows else pd.DataFrame(columns=["mint", "quote_class"]) def taken_idx(ts: np.ndarray, mint: np.ndarray, horizon_s: float, idx: np.ndarray) -> np.ndarray: diff --git a/fly_trader/train/fly_calibrate.py b/fly_trader/train/fly_calibrate.py index 5f6e7cec72849cf071b7bd12b3e0cf12c3427943..78d02c5e6c96fff4413a635d070a723a4463e8fb 100644 --- a/fly_trader/train/fly_calibrate.py +++ b/fly_trader/train/fly_calibrate.py @@ -6,6 +6,16 @@ quantiles (a compressed score scale still gets a usable line); the line with the position per token, the hold) over at least ``MIN_LINE_TRADES`` trades wins — the selector's rule — and a line that lost money over the window never does; otherwise the previous line stays. Sizing: ``agent/sizing.build_table`` on the window's trades at that line (the previous table if too few). The fly never trades the selector's line: its scores live on their own scale. + +``rule`` picks the line (default ``total``; the memecoin fly passes ``RULE`` = ``config.FLY_CALIB_RULE``, the Kalshi +fly keeps the default): +- ``total``: the most total net profit over at least ``MIN_LINE_TRADES`` trades (the rule above). Volume wins: on fly + #133's week it chose 414 trades at +2.8 % over 115 at +7.3 %, where its teacher took 13 at +8.5 %. +- ``selector``: the selector's own objective (train/strategies.better): admissible lines first (≥ ``WIN_MIN`` winners, + profit factor ≥ ``PF_MIN``, ≥ ``MIN_LINE_TRADES`` trades), then those meeting the profit-factor bar, then total profit; + a window with no line meeting at least the profit-factor bar keeps the previous line, as the selector would refuse. +- ``match``: the teacher's selectivity: the lowest line at which the fly takes no more trades over the window than its + teacher would have (``target_trades``, one position per token over the hold); the ranking is the fly's own. """ from __future__ import annotations @@ -15,10 +25,14 @@ import numpy as np from ..agent import sizing from .decisions import taken_idx +from .. import config from .selector import LINE_CANDIDATES, MIN_EV, MIN_LINE_TRADES +from .strategies import PF_MIN, WIN_MIN WINDOW_DAYS = 7 QUANTILES = (0.95, 0.98, 0.99, 0.995, 0.998, 0.999) +RULE = config.FLY_CALIB_RULE +RULES = ("total", "selector", "match") @dataclass @@ -38,19 +52,62 @@ def candidates(scores: np.ndarray) -> list[float]: return sorted({float(c) for c in (*LINE_CANDIDATES, *q)}) +def _stats(r: np.ndarray) -> dict: + g, l = float(r[r > 0].sum()), float(-r[r <= 0].sum()) + return {"trades": int(len(r)), "mean": float(r.mean()) if len(r) else None, "total": float(r.sum()), + "win": float((r > 0).mean()) if len(r) else None, "pf": (g / l if l > 0 else (float("inf") if g > 0 else None))} + + +def match_line(ts, mint, horizon_s, scores, rows, target: int) -> float | None: + """The lowest score line at which the taken trades over ``rows`` number at most ``target`` (bisection over the + rows' own scores, highest first: the count only grows as the line falls, give or take a re-entry).""" + s = np.sort(np.unique(scores[rows]))[::-1] + if target <= 0 or not len(s): + return float(s[0]) + 1e-9 if len(s) else None + n = lambda k: len(taken_idx(ts, mint, horizon_s, rows[scores[rows] >= s[k]])) + lo, hi = 0, len(s) - 1 # n(lo) ≤ target is the invariant; find the largest such k + if n(hi) <= target: + return float(s[hi]) + while hi - lo > 1: + mid = (lo + hi) // 2 + if n(mid) <= target: + lo = mid + else: + hi = mid + return float(s[lo]) + + def calibrate(ts: np.ndarray, mint: np.ndarray, horizon_s: float, scores: np.ndarray, rets: np.ndarray, prev_line: float | None = None, - prev_sizing: list | None = None, min_trades: int = MIN_LINE_TRADES) -> Calibration: - """Line and sizing from one window of (ts, mint, score, realized net return) rows.""" + prev_sizing: list | None = None, min_trades: int = MIN_LINE_TRADES, rule: str | None = None, + target_trades: int | None = None) -> Calibration: + """Line and sizing from one window of (ts, mint, score, realized net return) rows. ``rule``: see the module + docstring (default ``total``); ``match`` needs ``target_trades`` (the teacher's trades over the same window).""" + rule = rule or "total" + if rule not in RULES: + raise ValueError(f"unknown calibration rule {rule!r}: one of {RULES}") scores = np.asarray(scores, dtype=np.float64); rets = np.asarray(rets, dtype=np.float64) rows = np.flatnonzero(np.isfinite(scores) & np.isfinite(rets)) table = [] for c in candidates(scores[rows]): - tr = taken_idx(ts, mint, horizon_s, rows[scores[rows] >= c]); r = rets[tr] - table.append({"line": c, "trades": int(len(r)), "mean": float(r.mean()) if len(r) else None, "total": float(r.sum())}) - # A line must have made money over the window to be chosen: with a day of resolved rows the only candidate with - # enough trades is the lowest line, and on 2026-09-23 it took the live fly from 4.1 % to 0.25 % on 119 losing trades. - ok = [t for t in table if t["trades"] >= min_trades and t["total"] > 0] - line = max(ok, key=lambda t: t["total"])["line"] if ok else (prev_line if prev_line is not None else MIN_EV) + tr = taken_idx(ts, mint, horizon_s, rows[scores[rows] >= c]) + table.append({"line": c, **_stats(rets[tr])}) + fallback = prev_line if prev_line is not None else MIN_EV + if rule == "match": + if target_trades is None: + raise ValueError("the match rule needs target_trades (the teacher's trades over the window)") + m = match_line(ts, mint, horizon_s, scores, rows, int(target_trades)) if len(rows) else None + line = m if m is not None else fallback + elif rule == "selector": + # train/strategies.better: (admissible, profit-factor bar met, total); nothing meeting the bar keeps the old line + key = lambda t: (t["trades"] >= min_trades and (t["pf"] or 0) >= PF_MIN and (t["win"] or 0) >= WIN_MIN, + t["trades"] >= min_trades and (t["pf"] or 0) >= PF_MIN, t["total"]) + ok = [t for t in table if key(t)[1] and t["total"] > 0] + line = max(ok, key=key)["line"] if ok else fallback + else: + # A line must have made money over the window to be chosen: with a day of resolved rows the only candidate with + # enough trades is the lowest line, and on 2026-09-23 it took the live fly from 4.1 % to 0.25 % on 119 losing trades. + ok = [t for t in table if t["trades"] >= min_trades and t["total"] > 0] + line = max(ok, key=lambda t: t["total"])["line"] if ok else fallback tr = taken_idx(ts, mint, horizon_s, rows[scores[rows] >= line]); r = rets[tr] table_s = sizing.build_table(scores[tr] - line, r) or list(prev_sizing or []) return Calibration(line=float(line), sizing=table_s, trades=int(len(r)), mean=float(r.mean()) if len(r) else None, total=float(r.sum()), diff --git a/fly_trader/train/fly_replay.py b/fly_trader/train/fly_replay.py index 5319dc6c96057f257b4a1994a1121003bc0b50f0..12b5600fcedd70167fe95392b94aa92e55d8c666 100644 --- a/fly_trader/train/fly_replay.py +++ b/fly_trader/train/fly_replay.py @@ -27,6 +27,7 @@ from __future__ import annotations import json import logging import math +from dataclasses import dataclass import threading import time from datetime import date, datetime, timezone @@ -34,6 +35,7 @@ from datetime import date, datetime, timezone import numpy as np import torch +from .. import config from ..brain import device as brain_device, plastic from ..db.apilog import record_event from ..db.connection import transaction @@ -49,6 +51,45 @@ CHUNK = 2048 CONFIGS = [(0.0, math.inf)] + [(a, h) for a in (1e-4, 3e-4, 1e-3, 3e-3) for h in (1.0, 3.0, 7.0, 30.0)] LABEL_LAG_S = 60.0 # a label is known once the exit minute has closed HOUR_S, DAY_S = 3600.0, 86400.0 +TRADES_DIR = config.BRAIN_DIR / "replay" +# The fly's gate asks whether its edge is real, not whether it traded a lot: selector.deploy_decision's 100-trade floor +# failed the teacher-matched fly (92 trades at +10.1 %, 58 % winners, PF 1.92) and passed the hungry one (1,607 at +2.0 %). +MIN_TRADES = 30 # enough trades for a mean to be judged at all (and for a configuration to be chosen) +EDGE_P = 0.05 # at most this chance, resampling whole days, that the mean per trade is zero or less +EDGE_RESAMPLES = 2000 +RH_REPLAY_TRAIN_DAYS = 28 # days of its own a second chain's rows give the teacher before the bootstrap: the teacher fits only on days +# before S − CALIB_DAYS − PURGE_DAYS (2026-10-06: 21 days from RH's first day left it 12 thin early days, a tenth of +# September's volume; its RH picks dried up as the market grew and the matched fly line followed) + + +@dataclass +class Slots: + """Per-chain lines (branch rh-memecoins): one line per (strategy, chain) slot, slot = strategy + NS × chain index. + ``cidx_o``: each ordered row's chain index. Absent (a Solana-only replay), every slot is its strategy.""" + NS: int + chains: list + cidx_o: np.ndarray + + @property + def n(self) -> int: + return self.NS * len(self.chains) + + def strategy(self, slot: int) -> int: + return slot % self.NS + + def chain(self, slot: int) -> str: + return self.chains[slot // self.NS] + + def name(self, names: list[str], slot: int) -> str: + c = self.chain(slot); n = names[self.strategy(slot)] + return n if c == "sol" else f"{n}@{c}" + + +def _slots_of(slots: "Slots | None", NS: int) -> list[tuple[int, int, np.ndarray | None]]: + """(slot, strategy, chain index or None) for every slot.""" + if slots is None: + return [(s_, s_, None) for s_ in range(NS)] + return [(sl, slots.strategy(sl), sl // NS) for sl in range(slots.n)] def _day_start(t: float) -> float: @@ -60,8 +101,10 @@ def _labels(ds: DecisionSet, hold_min: int) -> np.ndarray: def run(days: int | None = None, start_day: int = START_DAY, configs: list | None = None, stop: threading.Event | None = None, device=None, - ds: DecisionSet | None = None, fly=None, boot: dict | None = None, graph=None, save_verdict: bool = True) -> dict: - """The replay; ``ds``/``fly``/``boot`` may be given (tests, or a bootstrap already run).""" + ds: DecisionSet | None = None, fly=None, boot: dict | None = None, graph=None, save_verdict: bool = True, dump_trades: bool | None = None, + only_chains: tuple | None = None) -> dict: + """The replay; ``ds``/``fly``/``boot`` may be given (tests, or a bootstrap already run). ``dump_trades`` (default: as + ``save_verdict``) writes the chosen book's trades to ``TRADES_DIR``; a study passes it without saving a verdict.""" from .strategies import HOLDS_MIN configs = [(float(a), float(h)) for a, h in (configs or CONFIGS)] if configs[0][0] != 0.0: @@ -71,6 +114,10 @@ def run(days: int | None = None, start_day: int = START_DAY, configs: list | Non prog.update("fly replay: building decision points", 0, 1, force=True) ds = build(days=days, holds=HOLDS_MIN) dl = ds.days + ch_all = ds.chains() + if (ch_all != "sol").any(): # the other chain's evaluation needs training days of its own before it (RH_REPLAY_TRAIN_DAYS) + first_other = min(ds.day[ch_all != "sol"]) + start_day = max(start_day, dl.index(first_other) + other_chain_lead() if first_other in dl else start_day) if len(dl) <= start_day + 2: raise RuntimeError(f"the corpus has {len(dl)} days; the replay needs more than {start_day + 2}") S = dl[start_day] @@ -93,8 +140,29 @@ def run(days: int | None = None, start_day: int = START_DAY, configs: list | Non log.info("fly replay on %s: %d rows per batch", brain_device.describe(bank.dev), chunk) C = bank.C scores = np.full((C, NS, n), np.nan, np.float32) - lines = np.array([[fly.lines[k] for k in names]] * C, dtype=np.float64); sizings = [[list(fly.sizings[k]) for k in names] for _ in range(C)] - line_log: dict = {} + ch_o = ds.chains()[order]; chains = ["sol"] + sorted({c for c in ch_o.tolist() if c != "sol"}) + slots = Slots(NS, chains, np.array([chains.index(c) for c in ch_o], dtype=int)) if len(chains) > 1 else None + NSL = slots.n if slots else NS + slot_line = lambda sl: fly.line_for(names[sl % NS], chains[sl // NS]) if slots else fly.lines[names[sl]] + slot_size = lambda sl: list(fly.sizing_for(names[sl % NS], chains[sl // NS]) if slots else fly.sizings[names[sl]]) + slot_name = lambda sl: slots.name(names, sl) if slots else names[sl] + lines = np.array([[slot_line(sl) for sl in range(NSL)]] * C, dtype=np.float64); sizings = [[slot_size(sl) for sl in range(NSL)] for _ in range(C)] + line_log: dict = {}; size_log: dict = {slot_name(sl): [(-math.inf, slot_size(sl))] for sl in range(NSL)} + allowed = None + if fly_calibrate.RULE == "match": # the teacher's own decisions over every row, for the window's trade budget + if getattr(fly, "teacher", None) is None: + raise RuntimeError("the match calibration rule needs the bootstrap's teacher (fly.teacher)") + prog.update("fly replay: the teacher's decisions for the match rule", 0, 1, force=True) + allowed = fly_selector.teacher_allowed(fly.teacher, ds.X[order], ds.ts[order], names) + gate = None + if config.FLY_TEACHER_GATE and allowed is not None: # the fly buys only what its teacher buys, as the live fly (agent/fly_session._gated) + gate = np.zeros((len(ds.y), NS), bool); gate[order] = allowed + tsc = None + if config.FLY_SCORE == "teacher": # it scores as its teacher plus its learned correction (agent/fly_session._score) + if not hasattr(getattr(fly, "teacher", None), "raw_scores"): + raise RuntimeError("FLY_SCORE=teacher needs the bootstrap's teacher stack (fly.teacher)") + prog.update("fly replay: the teacher's scores, the fly's starting point", 0, 1, force=True) + tsc = fly.teacher.raw_scores(ds.X[order], ds.ts[order], names) pending = plastic.PendingTags(fly.net.n_kc, fly.net.k_active) labels = [torch.tensor(_labels(ds, h), dtype=torch.float32) for h in holds_min] starts = np.flatnonzero(np.r_[True, ts_o[1:] != ts_o[:-1]]); ends = np.r_[starts[1:], n] @@ -123,30 +191,41 @@ def run(days: int | None = None, start_day: int = START_DAY, configs: list | Non _learn_add(learn, now, bank.update(tags, r, now)); n_updates += len(tags) if day_done is not None and _day_start(now) > day_done: day_done = _day_start(now) - _recalibrate(ds, order, ts_o, scores, lines, sizings, now, lags, holds_min, line_log) + _recalibrate(ds, order, ts_o, scores, lines, sizings, now, lags, holds_min, line_log, size_log=size_log, names=names, allowed=allowed, + slots=slots) if next_gov is not None and now >= next_gov: next_gov = now + HOUR_S - _governance(ds, order, ts_o, scores, lines, line_log, bank, now, lags, holds_min, gov_counts) + _governance(ds, order, ts_o, scores, lines, line_log, bank, now, lags, holds_min, gov_counts, slots=slots) for s_ in range(NS): loc = np.flatnonzero(trig[lo:hi, s_]) + if tsc is not None: + loc = loc[np.isfinite(tsc[s0 + loc, s_])] # a candidate the teacher scores (its own trigger fired) if not len(loc): continue li = torch.as_tensor(lo + loc, device=Y.device) + y = (bank.head_for(torch.as_tensor(tsc[s0 + loc, s_], dtype=torch.float32, device=bank.dev), u0[li], s=np.full(len(loc), s_)) + if tsc is not None else Y[li, s_]) # frozen = the teacher's score exactly; learning adds to it with torch.no_grad(): - sc, _ = bank.predict(Y[li, s_], u0[li], k[li], s=np.full(len(loc), s_)) + sc, _ = bank.predict(y, u0[li], k[li], s=np.full(len(loc), s_)) scores[:, s_, s0 + loc] = sc.float().cpu().numpy() if s0 >= live_from: - w = plastic.row_weights(sc, torch.tensor(lines[:, s_], dtype=sc.dtype, device=sc.device)) + if gate is not None: # gated, as the live fly: it learns from its teacher's trades of this strategy (agent/fly_session._score) + w = torch.as_tensor(np.broadcast_to(allowed[s0 + loc, s_].astype(np.float32)[None], (sc.shape[0], len(loc))).copy(), + dtype=sc.dtype, device=sc.device) + else: + ln = lines[:, s_] if slots is None else lines[:, s_ + NS * slots.cidx_o[s0 + loc]] # each row against its chain's line + w = plastic.row_weights(sc, torch.tensor(ln, dtype=sc.dtype, device=sc.device)) keep = np.flatnonzero((w > 0).any(0).cpu().numpy()) # a row no configuration would trade teaches none of them if len(keep): - kl = loc[keep]; ki = torch.as_tensor(lo + kl, device=Y.device) + kl = loc[keep]; ki = torch.as_tensor(lo + kl, device=Y.device); kk = torch.as_tensor(keep, device=y.device) pending.push([(int(order[s0 + j]), s_) for j in kl], ts_o[s0 + kl], now + lags[s_], - Y[ki, s_], u0[ki], k[ki], w[:, torch.as_tensor(keep, device=w.device)], s=s_) + y[kk], u0[ki], k[ki], w[:, torch.as_tensor(keep, device=w.device)], s=s_) if g == 0 or (g1 // 50) != (g // 50): prog.update("fly replay: learning from the market", int(b), n, day=str(datetime.fromtimestamp(float(ts_o[b - 1]), timezone.utc).date()), updates=n_updates, pending=len(pending), drift=[round(float(x), 4) for x in bank.drift()], eta_s=(n - b) * (time.time() - t0) / max(b, 1)) g = g1 - return _verdict(ds, order, ts_o, live_from, scores, line_log, configs, bank, gov_counts, boot, S, save_verdict, time.time() - t0, fly, names, holds_min, lines, learn) + return _verdict(ds, order, ts_o, live_from, scores, line_log, configs, bank, gov_counts, boot, S, save_verdict, time.time() - t0, fly, names, holds_min, lines, learn, + size_log=size_log if (save_verdict if dump_trades is None else dump_trades) else None, slots=slots, only_chains=only_chains, teacher_gate=gate) def _learn_add(learn: dict, now: float, st: dict) -> None: @@ -193,39 +272,58 @@ def _line_at(line_log: dict, key, ts: np.ndarray, default: float) -> np.ndarray: return np.where(i >= 0, vals[np.clip(i, 0, None)], default) -def _recalibrate(ds, order, ts_o, scores, lines, sizings, now, lags, holds_min, line_log) -> None: +def _recalibrate(ds, order, ts_o, scores, lines, sizings, now, lags, holds_min, line_log, size_log: dict | None = None, names=None, + allowed: np.ndarray | None = None, slots: Slots | None = None) -> None: """One line and sizing per strategy, calibrated on the frozen fly's scores and shared by every configuration. Until 2026-09-22 each configuration calibrated its own. With the plastic and frozen weights no more than 5 % apart, the frozen arm still took 4,216 selection-half trades to the chosen arm's 2,609: the arms traded different slices because of their lines, so "plastic beats frozen" measured line calibration, not plasticity. A shared line leaves the weights as the only difference between arms, which is the question the replay exists to answer.""" - for s_, (lag, H) in enumerate(zip(lags, holds_min)): + for sl, s_, ci in _slots_of(slots, len(lags)): + lag, H = lags[s_], holds_min[s_] a, b = _window(ts_o, now, lag, fly_calibrate.WINDOW_DAYS * DAY_S) rows = order[a:b]; y = _labels(ds, H)[rows] sc = scores[0, s_, a:b]; has = np.isfinite(sc) - cal = fly_calibrate.calibrate(ds.ts[rows[has]], ds.mint[rows[has]], H * 60.0, sc[has], y[has], prev_line=float(lines[0, s_]), prev_sizing=sizings[0][s_]) + if ci is not None: + has = has & (slots.cidx_o[a:b] == ci) # this chain's rows only: its own line and budget + target = None + if allowed is not None: # the teacher's trades over the same window: the match rule's budget + al = allowed[a:b, s_][has] & np.isfinite(y[has]) + target = len(taken_idx(ds.ts[rows[has]], ds.mint[rows[has]], H * 60.0, np.flatnonzero(al))) + cal = fly_calibrate.calibrate(ds.ts[rows[has]], ds.mint[rows[has]], H * 60.0, sc[has], y[has], prev_line=float(lines[0, sl]), prev_sizing=sizings[0][sl], + rule=fly_calibrate.RULE, target_trades=target) for c in range(lines.shape[0]): - key = (c, s_) + key = (c, sl) if key not in line_log: - line_log[key] = [(-math.inf, float(lines[c, s_]))] - lines[c, s_] = cal.line; sizings[c][s_] = cal.sizing + line_log[key] = [(-math.inf, float(lines[c, sl]))] + lines[c, sl] = cal.line; sizings[c][sl] = cal.sizing line_log[key].append((now, float(cal.line))) + if size_log is not None and names is not None: # shared by every configuration, like the line + size_log.setdefault(slots.name(names, sl) if slots else names[s_], []).append((now, list(cal.sizing))) -def _governance(ds, order, ts_o, scores, lines, line_log, bank, now, lags, holds_min, counts) -> None: - for s_, (lag, H) in enumerate(zip(lags, holds_min)): +def _governance(ds, order, ts_o, scores, lines, line_log, bank, now, lags, holds_min, counts, slots: Slots | None = None) -> None: + for sl, s_, ci in _slots_of(slots, len(lags)): + lag, H = lags[s_], holds_min[s_] a3, b3 = _window(ts_o, now, lag, gov.SHADOW_DAYS * DAY_S); a1, b1 = _window(ts_o, now, lag, gov.IC_HOURS * HOUR_S) r3 = order[a3:b3]; r1 = order[a1:b1]; y = _labels(ds, H) - lf = _line_at(line_log, (0, s_), ts_o[a3:b3], float(lines[0, s_])) + lf = _line_at(line_log, (0, sl), ts_o[a3:b3], float(lines[0, sl])) sf = scores[0, s_, a3:b3] + if ci is not None: # this chain's rows: the others are judged against their own lines + other3 = slots.cidx_o[a3:b3] != ci; sf = np.where(other3, np.nan, sf) rf = gov.picks_returns(ds.ts[r3], ds.mint[r3], H * 60.0, np.nan_to_num(sf, nan=-np.inf), lf, y[r3]) drift = bank.drift(s_).tolist() if bank.learn.shape[0] > 1 else bank.drift().tolist() for c in range(1, lines.shape[0]): - lc = _line_at(line_log, (c, s_), ts_o[a3:b3], float(lines[c, s_])) - sc3 = np.nan_to_num(scores[c, s_, a3:b3], nan=-np.inf) + lc = _line_at(line_log, (c, sl), ts_o[a3:b3], float(lines[c, sl])) + sc3 = scores[c, s_, a3:b3] + if ci is not None: + sc3 = np.where(other3, np.nan, sc3) + sc3 = np.nan_to_num(sc3, nan=-np.inf) rp = gov.picks_returns(ds.ts[r3], ds.mint[r3], H * 60.0, sc3, lc, y[r3]) s1 = scores[c, s_, a1:b1]; has = np.isfinite(s1) + if ci is not None: + has = has & (slots.cidx_o[a1:b1] == ci) out = gov.check((rp, rf), (s1[has], y[r1][has]), drift[c]) counts[(c, s_)]["checks"] += 1 for t in out["triggers"]: @@ -233,70 +331,188 @@ def _governance(ds, order, ts_o, scores, lines, line_log, bank, now, lags, holds counts[(c, s_)]["any"] += bool(out["triggers"]) -def _arm(ds, order, ts_o, scores_cs, line_log, key, default_line, mask_pos) -> tuple[np.ndarray, np.ndarray]: - """Full-length (N) scores and per-row lines of (configuration, strategy) ``key`` on ``mask_pos`` (NaN elsewhere).""" +def _arm(ds, order, ts_o, scores_cs, line_log, key, default_line, mask_pos, slots: Slots | None = None, slot_defaults=None) -> tuple[np.ndarray, np.ndarray]: + """Full-length (N) scores and per-row lines of (configuration, strategy) ``key`` on ``mask_pos`` (NaN elsewhere); with + ``slots`` each row gets its own chain's line (``slot_defaults``: the bootstrap line per chain).""" S_full = np.full(len(ds.y), np.nan); L_full = np.full(len(ds.y), np.inf) pos = np.flatnonzero(mask_pos & np.isfinite(scores_cs)) - S_full[order[pos]] = scores_cs[pos]; L_full[order[pos]] = _line_at(line_log, key, ts_o[pos], default_line) + S_full[order[pos]] = scores_cs[pos] + if slots is None: + L_full[order[pos]] = _line_at(line_log, key, ts_o[pos], default_line) + else: + c, s_ = key + for ci in range(len(slots.chains)): + pc = pos[slots.cidx_o[pos] == ci] + L_full[order[pc]] = _line_at(line_log, (c, s_ + slots.NS * ci), ts_o[pc], slot_defaults[ci]) return S_full, L_full -def _book(ds, order, ts_o, scores, line_log, choice: list[int], boot_lines: list[float], holds_min, mask_pos): +def _gated_ok(ds, order, ts_o, scores, line_log, c: int, s_: int, Sf, Lf, mask_pos, gate, slots: Slots | None, slot_defaults, boot_line) -> np.ndarray: + """Rows configuration ``c`` trades for strategy ``s_``: its line without a gate; with one, the teacher's own decisions + (``gate``) minus the learned vetoes (its score more than FLY_LEARNED_VETO below the frozen fly's, configuration 0).""" + if gate is None: + return Sf >= Lf + in_mask = np.zeros(len(ds.y), bool); in_mask[order[mask_pos]] = True + S0, _ = _arm(ds, order, ts_o, scores[0, s_], line_log, (0, s_), boot_line, mask_pos, slots, slot_defaults) + veto = np.isfinite(Sf) & np.isfinite(S0) & (Sf < S0 - config.FLY_LEARNED_VETO) + return gate[:, s_] & in_mask & ~veto + + +def _book(ds, order, ts_o, scores, line_log, choice: list[int], boot_lines: list[float], holds_min, mask_pos, detail: bool = False, + slots: Slots | None = None, boot_slot_lines=None, gate: np.ndarray | None = None): """The combined book of the per-strategy choices: per row, among the strategies at/above their line in force, the one - with the largest margin; returns (pick, per-row hold s, per-row return).""" - N = len(ds.y); key = np.full(N, -np.inf); pick = np.zeros(N, bool); hold = np.zeros(N); ret = np.full(N, np.nan, np.float32) + with the largest margin; returns (pick, per-row hold s, per-row return), and with ``detail`` also (per-row strategy index + or -1, per-row margin over its line).""" + N = len(ds.y); key = np.full(N, -np.inf); pick = np.zeros(N, bool); hold = np.zeros(N); ret = np.full(N, np.nan, np.float32); who = np.full(N, -1) for s_, c in enumerate(choice): - Sf, Lf = _arm(ds, order, ts_o, scores[c, s_], line_log, (c, s_), boot_lines[s_], mask_pos) + Sf, Lf = _arm(ds, order, ts_o, scores[c, s_], line_log, (c, s_), boot_lines[s_], mask_pos, slots, + [boot_slot_lines[s_ + slots.NS * ci] for ci in range(len(slots.chains))] if slots else None) lab = _labels(ds, holds_min[s_]) - ok = (Sf >= Lf) & np.isfinite(lab) # a row whose hold runs past the corpus has no label: not a trade - m = np.where(ok, Sf - Lf, -np.inf); take = ok & (m > key) - key = np.where(take, m, key); pick |= ok + sd = [boot_slot_lines[s_ + slots.NS * ci] for ci in range(len(slots.chains))] if slots else None + ok = _gated_ok(ds, order, ts_o, scores, line_log, c, s_, Sf, Lf, mask_pos, gate, slots, sd, boot_lines[s_]) & np.isfinite(lab) # no label: not a trade + m = np.where(ok, np.where(np.isfinite(Sf - Lf), Sf - Lf, 0.0), -np.inf); take = ok & (m > key) + key = np.where(take, m, key); pick |= ok; who = np.where(take, s_, who) hold = np.where(take, holds_min[s_] * 60.0, hold); ret = np.where(take, lab, ret) - return pick, hold, ret - - -def _verdict(ds, order, ts_o, live_from, scores, line_log, configs, bank, gov_counts, boot, S, save_verdict, secs, fly, names, holds_min, lines, learn=None) -> dict: + return (pick, hold, ret, who, key) if detail else (pick, hold, ret) + + +def _table_at(log_s: list, t: float) -> list: + """The sizing table in force at ``t`` (the last recalibration at or before it).""" + cur = log_s[0][1] if log_s else [] + for t0, tab in log_s: + if t0 <= t: + cur = tab + return cur + + +def _dump_trades(ds, order, ts_o, scores, line_log, size_log, choice, boot_lines, holds_min, names, mask_pos, day_o, sel_days, slots: Slots | None = None, + boot_slot_lines=None, gate: np.ndarray | None = None) -> str | None: + """The chosen book's trades over the replay's live days (one position per token, as ``evaluate`` counts them): entry + time, token, strategy, margin over the line in force, hold, net return (at the label size), the pool's quote reserve, + the half, and the sizing table in force — what ``agent/sizing.simulate_bankroll`` needs to replay a sizing rule on the + fly's own out-of-sample trades. Written next to the verdict as ``data/brain/replay/trades_.json``.""" + pick, hold, ret, who, margin = _book(ds, order, ts_o, scores, line_log, choice, boot_lines, holds_min, mask_pos, detail=True, slots=slots, + boot_slot_lines=boot_slot_lines, gate=gate) + row_chain = ds.chains() + rows = np.flatnonzero(who >= 0) + if not len(rows): + return None + tr = np.sort(taken_idx(ds.ts, ds.mint, np.where(hold > 0, hold, 60.0), rows)) + tr = tr[np.argsort(ds.ts[tr], kind="stable")] + resq = np.expm1(ds.X[tr, ds.cols.index("log_liquidity_sol")].astype(np.float64)) / 2.0 if "log_liquidity_sol" in ds.cols else np.full(len(tr), np.nan) + tables: list = []; tix: dict = {} + def tab_id(tab): + k = json.dumps(tab, sort_keys=True, default=str) + if k not in tix: + tix[k] = len(tables); tables.append(tab) + return tix[k] + sel = set(sel_days) + key = lambda i: names[int(who[i])] if row_chain[i] == "sol" else f"{names[int(who[i])]}@{row_chain[i]}" + trades = [{"ts": float(ds.ts[i]), "mint": str(ds.mint[i]), "strategy": names[int(who[i])], "chain": str(row_chain[i]), "margin": float(margin[i]), + "hold_s": float(hold[i]), "ret": float(ret[i]), "resq": float(resq[k]), "half": "selection" if ds.day[i] in sel else "evaluation", + "table": tab_id(_table_at(size_log.get(key(i), []), float(ds.ts[i])))} for k, i in enumerate(tr)] + TRADES_DIR.mkdir(parents=True, exist_ok=True) + path = TRADES_DIR / f"trades_{datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')}.json" + tmp = path.with_suffix(".tmp"); tmp.write_text(json.dumps(prog._finite({"strategies": names, "tables": tables, "trades": trades}), default=str)); tmp.replace(path) + return str(path) + + +def other_chain_lead() -> int: + """Days from a second chain's first day to the bootstrap: its own training days, then the calibration week and purge.""" + return RH_REPLAY_TRAIN_DAYS + fly_selector.CALIB_DAYS + fly_selector.PURGE_DAYS + + +def merge_chains(row, out: dict, only_chains: tuple) -> dict: + """The stored verdict with ``out``'s verdicts for ``only_chains`` put in; the other chains keep theirs (each with the + replay start it was judged from). A stored verdict on other definitions is replaced whole.""" + old = (row["value"] if isinstance(row["value"], dict) else json.loads(row["value"])) if row else None + if not old or old.get("data") != out.get("data") or "chains" not in old: + return out + chains = dict(old["chains"]) + for c in only_chains: + if c in (out.get("chains") or {}): + chains[c] = {**out["chains"][c], "S": str(out.get("S"))} + merged = {**old, "chains": chains, "passed": any(v.get("passed") for v in chains.values()), + "reason": "; ".join(f"{c}: {'passed' if v.get('passed') else 'failed'} — {v.get('reason')}" for c, v in chains.items())} + merged.setdefault("rejudged", []).append({"chains": list(only_chains), "S": str(out.get("S")), "at": out.get("finished_at")}) + return merged + + +def _verdict(ds, order, ts_o, live_from, scores, line_log, configs, bank, gov_counts, boot, S, save_verdict, secs, fly, names, holds_min, lines, learn=None, + size_log: dict | None = None, slots: Slots | None = None, only_chains: tuple | None = None, teacher_gate: np.ndarray | None = None) -> dict: days_r = sorted(set(ds.day[order[live_from:]].tolist())) sel_days, ev_days = days_r[: len(days_r) // 2], days_r[len(days_r) // 2:] day_o = ds.day[order] live_pos = np.arange(len(order)) >= live_from sel_pos = live_pos & np.isin(day_o, sel_days); ev_pos = live_pos & np.isin(day_o, ev_days) boot_lines = [fly.lines[k] for k in names] + NS = len(names) + boot_slot = [fly.line_for(names[sl % NS], slots.chains[sl // NS]) for sl in range(slots.n)] if slots else None + slot_def = lambda s_: [boot_slot[s_ + NS * ci] for ci in range(len(slots.chains))] if slots else None per_strategy = {}; choice = [] for s_, name in enumerate(names): y = _labels(ds, holds_min[s_]); table = [] for c, (alpha, hl) in enumerate(configs): - Sf, Lf = _arm(ds, order, ts_o, scores[c, s_], line_log, (c, s_), boot_lines[s_], sel_pos) - tr = taken_idx(ds.ts, ds.mint, holds_min[s_] * 60.0, np.flatnonzero(Sf >= Lf)); r = y[tr] + Sf, Lf = _arm(ds, order, ts_o, scores[c, s_], line_log, (c, s_), boot_lines[s_], sel_pos, slots, slot_def(s_)) + okc = _gated_ok(ds, order, ts_o, scores, line_log, c, s_, Sf, Lf, sel_pos, teacher_gate, slots, slot_def(s_), boot_lines[s_]) + tr = taken_idx(ds.ts, ds.mint, holds_min[s_] * 60.0, np.flatnonzero(okc)); r = y[tr] table.append({"config": c, "alpha": alpha, "half_life_days": hl if math.isfinite(hl) else None, "trades": int(len(r)), "total": float(np.nansum(r)), "mean": float(np.nanmean(r)) if len(r) else None, "governance": gov_counts.get((c, s_))}) - ok = [t for t in table if t["alpha"] > 0 and t["trades"] >= selector.MIN_LINE_TRADES] + ok = [t for t in table if t["alpha"] > 0 and t["trades"] >= MIN_TRADES] best = max(ok, key=lambda t: t["total"])["config"] if ok else 0 choice.append(best) per_strategy[name] = {"configs": table, "chosen": table[best] if ok else None, "alpha": table[best]["alpha"], "half_life_days": table[best]["half_life_days"], "hold_min": holds_min[s_], "plastic": bool(ok)} - def judge(ch): - pick, hold, ret = _book(ds, order, ts_o, scores, line_log, ch, boot_lines, holds_min, ev_pos) + row_chain = ds.chains() + + def judge(ch, chain: str | None = None): + """The evaluation half's book of configuration choice ``ch``: every chain together, or only ``chain``'s rows.""" + pick, hold, ret = _book(ds, order, ts_o, scores, line_log, ch, boot_lines, holds_min, ev_pos, slots=slots, boot_slot_lines=boot_slot, gate=teacher_gate) test = np.zeros(len(ds.y), bool); test[order[ev_pos]] = True + if chain is not None: + test &= row_chain == chain ev = evaluate(ds, np.where(pick, 1.0, np.nan), test & pick, 0.5, "fly", hold_s=hold, returns=ret) rnd = summarize(random_trades(ds, test, max(1, int(ev["pooled"]["n"] or 1)))) + tr = taken_idx(ds.ts, ds.mint, np.where(hold > 0, hold, 60.0), np.flatnonzero(test & pick & np.isfinite(ret))) + ev["edge_p"] = edge_p(ret[tr], ds.day[tr]) return ev, rnd out = {"S": str(S), "selection_days": [str(sel_days[0]), str(sel_days[-1])] if sel_days else None, "evaluation_days": [str(ev_days[0]), str(ev_days[-1])] if ev_days else None, "strategies": names, "per_strategy": per_strategy, "configs": per_strategy[names[0]]["configs"], - "bootstrap": {k: boot.get(k) for k in ("lines", "calibration", "diagnostics", "gates_ok", "gate_failures")}, "data": fly_selector.FLY_VERSION, + "bootstrap": {k: boot.get(k) for k in ("lines", "calibration", "diagnostics", "gates_ok", "gate_failures")}, "data": fly_selector.verdict_version(), "secs": secs, "finished_at": datetime.now(timezone.utc).isoformat(), "learning": _learning_series(learn or {})} ev0, rnd0 = judge([0] * len(names)) out["frozen"] = {**ev0["pooled"], "random": rnd0} if not any(v["plastic"] for v in per_strategy.values()): - out.update(passed=False, reason="no strategy's plastic configuration made 100 trades on the selection half") + out.update(passed=False, reason=f"no strategy's plastic configuration made {MIN_TRADES} trades on the selection half") else: ev, rnd = judge(choice) - passed, why = selector.deploy_decision(ev["pooled"], rnd) + passed, why = gate(ev["pooled"], rnd, ev["edge_p"]) out.update(passed=bool(passed), reason=why, chosen=per_strategy[names[0]]["chosen"], evaluation=ev["pooled"], random=rnd, per_day=ev["per_day"], alpha=per_strategy[names[0]]["alpha"], half_life_days=per_strategy[names[0]]["half_life_days"], plastic_beats_frozen=bool((ev["pooled"]["mean"] or -1) > (ev0["pooled"]["mean"] or -1))) + + if slots is not None: + # each chain is judged on its own trades (its random baseline, its days): a chain's book trades only if it passes. + # Without a plastic configuration every chain fails, and its frozen fly's result is reported. + plastic_ok = any(v["plastic"] for v in per_strategy.values()) + out["chains"] = {} + for chain in slots.chains: + e0, r0 = judge([0] * len(names), chain) + if plastic_ok: + evc, rndc = judge(choice, chain); pc, wc = gate(evc["pooled"], rndc, evc["edge_p"]) + else: + evc, rndc, pc, wc = e0, r0, False, out["reason"] + out["chains"][chain] = {"passed": bool(pc), "reason": wc, "evaluation": evc["pooled"], "random": rndc, "edge_p": evc.get("edge_p"), "frozen": e0["pooled"]} + out["passed"] = any(v["passed"] for v in out["chains"].values()) + out["reason"] = "; ".join(f"{c}: {'passed' if v['passed'] else 'failed'} — {v['reason']}" for c, v in out["chains"].items()) + if size_log is not None and any(v["plastic"] for v in per_strategy.values()): + try: + out["trades_file"] = _dump_trades(ds, order, ts_o, scores, line_log, size_log, choice, boot_lines, holds_min, names, live_pos, day_o, sel_days, + slots=slots, boot_slot_lines=boot_slot, gate=teacher_gate) + except Exception: + log.exception("could not write the replay's trades (the verdict is unaffected)") log.info("fly replay from %s: %s — %s | frozen %s", S, "PASSED" if out.get("passed") else "FAILED", out.get("reason"), out["frozen"]) log.info("fly replay verdict: %s", {k: out.get(k) for k in ("passed", "reason", "evaluation", "frozen", "random")}) if save_verdict: @@ -305,8 +521,11 @@ def _verdict(ds, order, ts_o, live_from, scores, line_log, configs, bank, gov_co {"per_strategy": {k: {x: v[x] for x in ("alpha", "half_life_days", "hold_min", "plastic")} for k, v in per_strategy.items()}}) try: # NaN is valid Python and invalid JSON: never let the store discard a finished replay with transaction() as conn: + stored = out + if only_chains: # judge only these chains: the others keep their stored verdicts + stored = merge_chains(conn.execute("SELECT value FROM ui_settings WHERE key = %s", (KEY,)).fetchone(), out, only_chains) conn.execute("INSERT INTO ui_settings (key, value) VALUES (%s, %s) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()", - (KEY, json.dumps(prog._finite(out), default=str))) + (KEY, json.dumps(prog._finite(stored), default=str))) except Exception: log.exception("could not store the replay verdict; it is in the log above and the fly cannot be armed until it is stored") try: # the chosen configuration's learning, where the console already looks for it @@ -325,18 +544,47 @@ def _verdict(ds, order, ts_o, live_from, scores, line_log, configs, bank, gov_co return out +def edge_p(r: np.ndarray, days: np.ndarray, reps: int = EDGE_RESAMPLES, seed: int = 0) -> float | None: + """The chance that the mean per trade is zero or less, resampling whole days with replacement (a day's trades move + together, and one +400 % token must not carry a verdict by itself). None without two days of trades.""" + r = np.asarray(r, dtype=np.float64); ok = np.isfinite(r); r, days = r[ok], np.asarray(days)[ok] + keys = sorted(set(days.tolist())) + if len(keys) < 2: + return None + sums = np.array([r[days == k].sum() for k in keys]); cnts = np.array([(days == k).sum() for k in keys], dtype=np.float64) + pick = np.random.default_rng(seed).integers(0, len(keys), size=(reps, len(keys))) + means = sums[pick].sum(1) / np.maximum(cnts[pick].sum(1), 1.0) + return float((means <= 0).mean()) + + +def gate(p: dict, rb: dict, p_edge: float | None) -> tuple[bool, str]: + """The fly's verdict: at least ``MIN_TRADES``, a positive mean that beats random picks, and a day-resampled chance of + no edge of at most ``EDGE_P``.""" + n = int(p.get("n") or 0) + if n < MIN_TRADES: + return False, f"too few backtest trades to judge ({n} < {MIN_TRADES})" + if p.get("mean") is None or p["mean"] <= 0: + return False, f"its backtest lost money ({(p.get('mean') or 0) * 100:+.2f}% per trade after costs)" + if rb.get("mean") is not None and p["mean"] <= rb["mean"]: + return False, "it did not beat random picks" + if p_edge is None or p_edge > EDGE_P: + return False, f"its edge is not distinguishable from none ({'-' if p_edge is None else f'{p_edge:.1%}'} of day resamples at or below zero)" + return True, (f"its backtest made {p['mean'] * 100:+.2f}% per trade after costs over {n} trades (random {(rb.get('mean') or 0) * 100:+.2f}%; " + f"{p_edge:.1%} of day resamples at or below zero)") + + def verdict() -> dict | None: """The stored replay verdict for the current fly definitions, or None.""" with transaction() as conn: r = conn.execute("SELECT value FROM ui_settings WHERE key = %s", (KEY,)).fetchone() v = (r["value"] if isinstance(r["value"], dict) else json.loads(r["value"] or "{}")) if r else None - return v if v and v.get("data") == fly_selector.FLY_VERSION else None + return v if v and v.get("data") == fly_selector.verdict_version() else None -def main(days: int | None = None, start_day: int = START_DAY, stop_event: threading.Event | None = None) -> dict: +def main(days: int | None = None, start_day: int = START_DAY, stop_event: threading.Event | None = None, only_chains: tuple | None = None) -> dict: """The replay; when it passes and no fly may trade yet, the live fly is bootstrapped at once (taught on the whole corpus).""" prog.set_stop_event(stop_event); prog.clear() - out = run(days=days, start_day=start_day, stop=stop_event) + out = run(days=days, start_day=start_day, stop=stop_event, only_chains=only_chains) if out.get("passed") and not (stop_event is not None and stop_event.is_set()): with transaction() as conn: have = fly_selector.latest_deployable(conn) diff --git a/fly_trader/train/fly_selector.py b/fly_trader/train/fly_selector.py index 64a6278c64c552450c260e01bff2fb90552e587f..aa981d19eea5f2aa03c4aacafae41397210e9fcb 100644 --- a/fly_trader/train/fly_selector.py +++ b/fly_trader/train/fly_selector.py @@ -17,8 +17,9 @@ weights at that last step (``D`` = None is the frozen fly). The descending neuro the previous step, so ``D`` reaches the output only through the MBON readout. Bootstrap (``bootstrap``, the one time the selector teaches): the teacher is the selector the live book trades -(``deployed_teacher``); only when none is deployed is a stack refit on the days before ``S − 8`` (one purge day before -the calibration week), which is also what the replay uses, since there the refit is the honest out-of-sample teacher. +(``deployed_teacher``) as refit without its holdout days, so the calibration week is one it never saw; only when none is +deployed, or it has no such fit, is a stack refit on the days before ``S − 8`` (one purge day before the calibration +week), which is also what the replay uses, since there the refit is the honest out-of-sample teacher. The fly imitates it on the tradable minutes of those days (the teacher's top ``DISTIL_TOP`` plus a random ``DISTIL_SAMPLE``; squared error weighted ``TOP_WEIGHT`` × on the teacher's top 1 %; passes stop when a held-out slice stops improving by more than its own standard error, so the epoch count comes from the data), both @@ -33,6 +34,7 @@ from __future__ import annotations import hashlib import json import logging +import math import threading import time from datetime import date, datetime, timedelta, timezone @@ -50,7 +52,7 @@ from ..brain.connectome import AFFERENT_POPS, EFFERENT_POP, Connectome, SubConne from ..brain.plastic import assign_channels from ..db.apilog import record_event from ..db.connection import transaction -from . import fly_calibrate +from . import fly_calibrate, model_io from . import progress as prog from . import selector from .decisions import HOLD_MIN, DecisionSet, build, rank_corr, taken_idx @@ -75,7 +77,9 @@ DIAG_ROWS = 50_000 GATES = {"mbon_share_min": 0.10, "mbon_saturated_max": 0.20, "slope_min": 0.6, "slope_max": 1.4} TEACHER_NOTE = "a stack refit on the fly's training days (no deployed selector to copy)" # the definitions a fly was trained on (the selector's, as its teacher, + the fly's network and plasticity design) -FLY_VERSION = {**selector.DATA_VERSION, "fly": "flynet-3-ch", "plastic": "mb4-ch", "teacher": "deployed-1"} +# "calib": how its buy line is picked (config.FLY_CALIB_RULE): a verdict or fly calibrated under another rule is not this one +FLY_VERSION = {**selector.DATA_VERSION, "fly": "flynet-3-ch", "plastic": "mb4-ch", "teacher": "deployed-1", "calib": config.FLY_CALIB_RULE, + "lines": "per-chain-1"} # its own line per (strategy, chain), judged per chain (train/fly_replay.Slots) def _inv_softplus(x: torch.Tensor) -> torch.Tensor: @@ -87,17 +91,20 @@ class FlyNet(nn.Module): normalised descending neurons → decoder, + signed readout of the normalised MBONs → predicted net return (× ``SCALE``).""" def __init__(self, graph, obs_dim: int, k_steps: int = K_STEPS, leak: float = LEAK, hidden: int = HIDDEN, kc_active: float = KC_ACTIVE, device=None, - n_strategies: int = 1): + n_strategies: int = 1, aff_rows=None, scale: float = SCALE): + """``aff_rows``: the neurons the features enter (default: the memecoin fly's ``AFFERENT_POPS``; the Kalshi fly passes + the photoreceptors). ``scale``: the unit of the head's output (net return × 20 for memecoins; probability × 1 for Kalshi).""" super().__init__() dev = torch.device(device) if device else graph.device self.N, self.k_steps, self.leak, self.obs_dim, self.hidden, self.kc_active = graph.N, k_steps, leak, obs_dim, hidden, kc_active + self.scale = float(scale) self.register_buffer("pre", graph.indices[1].to(dev)); self.register_buffer("post", graph.indices[0].to(dev)) vals = graph.values_raw.float(); self.register_buffer("sign", torch.sign(vals).to(dev)) s = float(getattr(graph, "s", 0.0) or 0.0) or 0.99 / float(getattr(graph, "spectral_radius_raw", None) or 2788.0) self.theta = nn.Parameter(_inv_softplus(vals.abs() * s).to(dev)) # softplus^-1(|w|) self.gain = nn.Parameter(torch.ones(self.N, device=dev)); self.bias = nn.Parameter(torch.zeros(self.N, device=dev)) - aff = [torch.arange(*graph.pop_ranges[p]) for p in AFFERENT_POPS if p in graph.pop_ranges] - self.register_buffer("aff_rows", torch.cat(aff).to(dev)); self.register_buffer("eff_rows", torch.arange(*graph.pop_ranges[EFFERENT_POP]).to(dev)) + aff = torch.as_tensor(aff_rows, dtype=torch.long) if aff_rows is not None else torch.cat([torch.arange(*graph.pop_ranges[p]) for p in AFFERENT_POPS if p in graph.pop_ranges]) + self.register_buffer("aff_rows", aff.to(dev)); self.register_buffer("eff_rows", torch.arange(*graph.pop_ranges[EFFERENT_POP]).to(dev)) # the mushroom body: KC→MBON pairs of the connectome (MBON columns: approach, avoid, other) (self.kc0, self.kc1), self.mb0, self.mb1 = graph.pop_ranges["KC"], graph.pop_ranges["MBON_APP"][0], graph.pop_ranges["MBON_OTHER"][1] self.n_kc, self.n_mbon = self.kc1 - self.kc0, self.mb1 - self.mb0 @@ -212,7 +219,8 @@ class FlyModel: single-strategy interface of ``selector.SelectorModel`` (score, threshold, sizing, universe) is the first strategy's.""" def __init__(self, net: FlyNet, scaler: RobustScaler, cols: list[str], horizon_min: int, threshold: float | None = None, sizing: list | None = None, - rules: dict | None = None, lines: dict | None = None, sizings: dict | None = None, combine: str = "score"): + rules: dict | None = None, lines: dict | None = None, sizings: dict | None = None, combine: str = "score", + chain_lines: dict | None = None, chain_sizings: dict | None = None): self.net, self.scaler, self.cols, self.horizon_min = net, scaler, list(cols), horizon_min self.rules = dict(rules or {"ev": {"thr": {}, "high": None, "hold_min": int(horizon_min)}}) self.strategies = list(self.rules) @@ -220,6 +228,26 @@ class FlyModel: self.lines = {k: float((lines or {}).get(k, base)) for k in self.strategies} self.sizings = {k: list((sizings or {}).get(k, sizing or [])) for k in self.strategies} self.combine = combine + # the other chains' own lines and sizing per strategy (fly_trader/markets: 'rh'); ``lines`` / ``sizings`` are Solana's. + # One network scores both chains; each chain's line is calibrated on that chain's rows (its teacher budget too). + self.chain_lines = {ch: {k: float(v) for k, v in d.items()} for ch, d in (chain_lines or {}).items()} + self.chain_sizings = {ch: {k: list(v) for k, v in d.items()} for ch, d in (chain_sizings or {}).items()} + + def line_for(self, name: str, chain: str | None = "sol") -> float: + if chain in (None, "sol"): + return self.lines[name] + return self.chain_lines.get(chain, {}).get(name, self.lines[name]) + + def sizing_for(self, name: str, chain: str | None = "sol") -> list: + if chain in (None, "sol"): + return self.sizings[name] + return self.chain_sizings.get(chain, {}).get(name, self.sizings[name]) + + def set_line(self, name: str, chain: str, line: float, sizing_: list) -> None: + if chain in (None, "sol"): + self.lines[name], self.sizings[name] = float(line), list(sizing_) + else: + self.chain_lines.setdefault(chain, {})[name] = float(line); self.chain_sizings.setdefault(chain, {})[name] = list(sizing_) @property def threshold(self) -> float: @@ -249,7 +277,7 @@ class FlyModel: device's memory affords (``FlyNet.batch_rows``).""" self.net.eval(); batch = batch or self.net.batch_rows(); out = np.empty((len(X), len(self.strategies)), np.float64) for i in range(0, len(X), batch): - out[i:i + batch] = (self.net.forward_all(self._x(X[i:i + batch])) / SCALE).float().cpu().numpy() + out[i:i + batch] = (self.net.forward_all(self._x(X[i:i + batch])) / self.net.scale).float().cpu().numpy() return out def score(self, X: np.ndarray, batch: int = 2048) -> np.ndarray: @@ -332,6 +360,11 @@ class StackTeacher: self.threshold = self.lines[self.strategies[0]] if self.strategies else selector.MIN_EV self.combine = models.get("combine", "score"); self.eps: dict = {} + def raw_scores(self, X: np.ndarray, ts, names: list[str]) -> np.ndarray: + """[B, S] its own score of each row per strategy (NaN off the trigger): the starting score of a fly that scores as + its teacher plus a learned correction (config.FLY_SCORE = teacher).""" + return teacher_scores(self.models, X, ts, names, self.cols) + def targets(self, X: np.ndarray, ts: np.ndarray) -> tuple[np.ndarray, np.ndarray]: """(targets [B, S] with NaN off each strategy's candidates, allowed [B, S]).""" from . import strategies @@ -345,14 +378,29 @@ class StackTeacher: return T, A -def _teacher_targets(teacher, ds: DecisionSet, idx: np.ndarray) -> tuple[np.ndarray, np.ndarray]: +def teacher_allowed(teacher, X: np.ndarray, ts: np.ndarray, names: list[str], chunk: int = 200_000) -> np.ndarray: + """[B, S]: where the teacher itself would trade each of the fly's strategies (its final decision: trigger, line and + every filter) — the selectivity ``fly_calibrate``'s ``match`` rule copies.""" + A = np.zeros((len(X), len(names)), bool) + for i in range(0, len(X), chunk): + if isinstance(teacher, StackTeacher): + _, a = teacher.targets(X[i:i + chunk], ts[i:i + chunk]) + for j, k in enumerate(names): + if k in teacher.strategies: + A[i:i + chunk, j] = a[:, teacher.strategies.index(k)] + else: + A[i:i + chunk, :] = (teacher.score(X[i:i + chunk]) >= teacher.threshold)[:, None] + return A + + +def _teacher_targets(teacher, ds: DecisionSet, idx: np.ndarray, clip: tuple = (-1.0, 1.0)) -> tuple[np.ndarray, np.ndarray]: """[n, S] targets (NaN: not a candidate of that strategy) and loss weights for the rows ``idx``.""" if isinstance(teacher, StackTeacher): T = np.full((len(idx), len(teacher.strategies)), np.nan, np.float32); W = np.ones_like(T) for i in range(0, len(idx), 200_000): t, a = teacher.targets(ds.X[idx[i:i + 200_000]], ds.ts[idx[i:i + 200_000]]) T[i:i + 200_000] = t; W[i:i + 200_000] = np.where(a, TOP_WEIGHT, 1.0) - return np.clip(T, -1.0, 1.0), W + return np.clip(T, clip[0], clip[1]), W tgt = np.empty(len(idx), np.float32) for i in range(0, len(idx), 500_000): tgt[i:i + 500_000] = teacher.score(ds.X[idx[i:i + 500_000]]) @@ -371,7 +419,7 @@ def _val_mse(net: FlyNet, scaler: RobustScaler, ds: DecisionSet, idx: np.ndarray for i in range(0, len(val), batch): sl = val[i:i + batch] x = torch.tensor(scaler.transform(ds.X[idx[sl]]), device=dev) - t = torch.tensor(np.nan_to_num(T[sl]) * SCALE, device=dev); m = torch.tensor(np.isfinite(T[sl]), device=dev).float() + t = torch.tensor(np.nan_to_num(T[sl]) * net.scale, device=dev); m = torch.tensor(np.isfinite(T[sl]), device=dev).float() w = torch.tensor(W[sl], device=dev) * m parts.append(((w * (net.forward_all(x) - t) ** 2).sum(1) / w.sum(1).clamp(min=1e-9)).cpu()) net.train() @@ -380,13 +428,17 @@ def _val_mse(net: FlyNet, scaler: RobustScaler, ds: DecisionSet, idx: np.ndarray def train_fly(ds: DecisionSet, rows: np.ndarray, teacher, epochs: int = EPOCHS, batch: int = BATCH, stop: threading.Event | None = None, - seed: int = 0, graph=None, device=None) -> tuple[FlyModel | None, dict]: + seed: int = 0, graph=None, device=None, net_kwargs: dict | None = None, model_cls=None, target_clip: tuple = (-1.0, 1.0), + horizon_min: int | None = None) -> tuple[FlyModel | None, dict]: """Distil ``teacher`` (a StackTeacher, or anything with ``score`` and ``scaler``) into a FlyNet over the training rows - ``rows`` (mask): one head per strategy, each learning its strategy's targets on that strategy's candidates.""" + ``rows`` (mask): one head per strategy, each learning its strategy's targets on that strategy's candidates. + ``net_kwargs`` (``aff_rows``, ``scale``), ``model_cls`` and ``target_clip`` let another trading type (kalshi/fly.py) run + its own fly through this loop; the memecoin defaults are unchanged.""" brain_device.seed_all(seed); rng = np.random.default_rng(seed); dev = torch.device(device) if device else _device() n_s = len(teacher.strategies) if isinstance(teacher, StackTeacher) else 1 - net = FlyNet(graph if graph is not None else _graph(), obs_dim=len(ds.cols), device=dev, n_strategies=n_s) - idx = np.flatnonzero(rows); T, W = _teacher_targets(teacher, ds, idx) + net = FlyNet(graph if graph is not None else _graph(), obs_dim=len(ds.cols), device=dev, n_strategies=n_s, **(net_kwargs or {})) + model_cls = model_cls or FlyModel + idx = np.flatnonzero(rows); T, W = _teacher_targets(teacher, ds, idx, target_clip) keep = np.isfinite(T).any(1); idx, T, W = idx[keep], T[keep], W[keep] if not len(idx): return None, {"stopped": False, "reason": "no teacher candidates"} @@ -396,7 +448,7 @@ def train_fly(ds: DecisionSet, rows: np.ndarray, teacher, epochs: int = EPOCHS, n_v = min(VAL_ROWS, len(use) // 10); p0 = rng.permutation(len(use)); val, use = np.sort(use[p0[:n_v]]), np.sort(use[p0[n_v:]]) with torch.no_grad(): for j, hd in enumerate(net.heads): - col = T[:, j]; hd.bias.fill_(float(np.nanmean(col)) * SCALE if np.isfinite(col).any() else 0.0); hd.weight.mul_(0.1) + col = T[:, j]; hd.bias.fill_(float(np.nanmean(col)) * net.scale if np.isfinite(col).any() else 0.0); hd.weight.mul_(0.1) batch = min(batch, net.batch_rows(ceiling=batch, training=True)) # a training step keeps every propagation step: ~4x a forward pass opt = torch.optim.Adam(net.param_groups(LR_GRAPH, LR_HEAD)); n_b = max(1, len(use) // batch); total = max(1, epochs * n_b); hist = []; t0 = time.time() best = best_v = prev_v = early = None @@ -407,7 +459,7 @@ def train_fly(ds: DecisionSet, rows: np.ndarray, teacher, epochs: int = EPOCHS, return None, {"epochs": hist, "stopped": True} sel = perm[b * batch:(b + 1) * batch] x = torch.tensor(teacher.scaler.transform(ds.X[idx[sel]]), device=dev) - t = torch.tensor(np.nan_to_num(T[sel]) * SCALE, device=dev); m = torch.tensor(np.isfinite(T[sel]), device=dev).float() + t = torch.tensor(np.nan_to_num(T[sel]) * net.scale, device=dev); m = torch.tensor(np.isfinite(T[sel]), device=dev).float() w = torch.tensor(W[sel], device=dev) * m loss = (w * (net.forward_all(x) - t) ** 2).sum() / w.sum().clamp(min=1e-9) opt.zero_grad(); loss.backward(); torch.nn.utils.clip_grad_norm_(net.parameters(), 5.0); opt.step(); tot += loss.item() @@ -437,9 +489,10 @@ def train_fly(ds: DecisionSet, rows: np.ndarray, teacher, epochs: int = EPOCHS, rec = idx[rng.choice(len(idx), min(RECAL_ROWS, len(idx)), replace=False)] recalibrate(net, ds.X[np.sort(rec)], teacher.scaler, batch) if isinstance(teacher, StackTeacher): - fly = FlyModel(net, teacher.scaler, ds.cols, int(teacher.rules[teacher.strategies[0]]["hold_min"]), rules=teacher.rules, combine=teacher.combine) + h0 = teacher.rules[teacher.strategies[0]]["hold_min"] + fly = model_cls(net, teacher.scaler, ds.cols, horizon_min if horizon_min is not None else int(h0), rules=teacher.rules, combine=teacher.combine) else: - fly = FlyModel(net, teacher.scaler, ds.cols, int(ds.horizon_s // 60)) + fly = model_cls(net, teacher.scaler, ds.cols, horizon_min if horizon_min is not None else int(ds.horizon_s // 60)) return fly, {"epochs": hist, "stopped": False, "rows": int(len(use)), "training_rows": int(len(idx)), "top_weight": TOP_WEIGHT, "strategies": fly.strategies, "epochs_run": len(hist), "epoch_cap": int(epochs), "val_rows": int(len(val)), "val_mse": best_v, "early_stop": early} @@ -458,7 +511,7 @@ def diagnose(fly: FlyModel, X: np.ndarray, teacher_scores: np.ndarray | None = N out = {"mbon_share": float(mb.var() / tot.var()) if tot.var() > 0 else 0.0, "mbon_saturated": float((sat.median(0).values > 2.0).float().mean()), "kc_active": float(act.mean()), "rows": int(len(X))} if teacher_scores is not None: - s = (tot / SCALE).numpy() + s = (tot / net.scale).numpy() out["slope"] = float(np.polyfit(np.asarray(teacher_scores, dtype=np.float64), s, 1)[0]) if np.std(teacher_scores) > 0 else None out["rank_corr"] = rank_corr(s, teacher_scores) return out @@ -486,7 +539,7 @@ def _stack_teacher(ds: DecisionSet, train: np.ndarray, stop=None): return StackTeacher(strategies.final_models(sub, stack), RobustScaler.fit(sub.X, seed=7), ds.cols), stack -def deployed_teacher(ds: DecisionSet, train: np.ndarray) -> tuple[StackTeacher | None, str, int | None]: +def deployed_teacher(ds: DecisionSet, train: np.ndarray, calib_start: date | None = None) -> tuple[StackTeacher | None, str, int | None]: """The selector the live book actually trades, wrapped as the fly's teacher, with the reason and the snapshot id. Until 2026-09-21 the bootstrap always refit its own stack on the days before ``S − 8`` and distilled that instead. @@ -495,8 +548,12 @@ def deployed_teacher(ds: DecisionSet, train: np.ndarray) -> tuple[StackTeacher | traded ``capitulation`` at 0.010 over 240 — five of eight components flipped by one extra week of corpus. The fly was never a copy of the selector it was being judged against. None (with the reason) falls back to that refit. - The deployed selector is fit on the whole corpus, the fly's calibration week included, so that week is not - out-of-sample for the teacher; the fly's own line is still calibrated on realised labels, never on the teacher's.""" + ``calib_start``: the first day of the fly's calibration week. The deployed selector is fit on the whole corpus, that + week included, so distilling it let the teacher's in-sample fit flatter the fly's line and sizing bands there (fly + #107: 474 calibration trades at +2.79 %, then its biggest bets lost live). Given ``calib_start``, the teacher is the + same stack refit without its holdout days (``SelectorModel.blind``: same strategies, settings and lines), used only + when that fit ends at least ``PURGE_DAYS`` before the week; a selector whose own fit already ends there is used as + is; otherwise None with the reason, and the bootstrap refits a stack on its training days.""" from ..agent.selector_session import pinned_snapshot sid = pinned_snapshot() if sid is None: @@ -511,27 +568,51 @@ def deployed_teacher(ds: DecisionSet, train: np.ndarray) -> tuple[StackTeacher | st = getattr(m, "stack", None) or {} if not st.get("strategies"): return None, f"selector #{sid} has no strategy stack; {TEACHER_NOTE}", None + seen = "" + if calib_start is not None: + last_ok = calib_start - timedelta(days=PURGE_DAYS + 1) # the last day a fit may reach, labels spilling over included + blind = getattr(m, "blind", None) or {} + through = str(getattr(m, "trained_through", "") or "") + if blind.get("models", {}).get("strategies") and date.fromisoformat(blind["fit_through"]) <= last_ok: + st = blind["models"]; seen = f", refit through {blind['fit_through']} (blind to the calibration week from {calib_start})" + elif through and date.fromisoformat(through) <= last_ok: + seen = f", fit through {through} (before the calibration week from {calib_start})" + else: + fit = blind.get("fit_through") or through or "an unknown day" + return None, (f"selector #{sid} saw the calibration week from {calib_start} (its fit reaches {fit}" + f"{'' if blind else ', and it has no blind models'}); {TEACHER_NOTE}"), None missing = sorted({c for v in st["strategies"].values() for c in v["cols"]} - set(ds.cols)) if missing: return None, f"selector #{sid} wants columns the corpus lacks ({', '.join(missing[:4])}); {TEACHER_NOTE}", None names = ", ".join(f"{k} (line {v['line']:.4f}, {v['hold_min']} min)" for k, v in st["strategies"].items()) - return StackTeacher(st, RobustScaler.fit(ds.X[np.flatnonzero(train)], seed=7), list(ds.cols)), f"deployed selector #{sid}: {names}", sid + return StackTeacher(st, RobustScaler.fit(ds.X[np.flatnonzero(train)], seed=7), list(ds.cols)), f"deployed selector #{sid}: {names}{seen}", sid + + +def row_chains(X: np.ndarray, cols: list[str]) -> np.ndarray: + """Each row's chain from its ``chain_rh`` input ('sol' where the column is absent or 0).""" + X = np.atleast_2d(X) + if "chain_rh" not in cols: + return np.full(len(X), "sol", dtype=object) + return np.where(X[:, cols.index("chain_rh")] > 0.5, "rh", "sol").astype(object) def fly_decide(fly: FlyModel, X: np.ndarray, cols: list[str], values: np.ndarray | None = None) -> dict: - """The fly's own per-row decision: among the strategies whose trigger fires and whose value clears the fly's line, - the one with the larger sizing-band Kelly fraction or margin (the teacher's combination rule).""" + """The fly's own per-row decision: among the strategies whose trigger fires and whose value clears the fly's line + for that row's chain, the one with the larger sizing-band Kelly fraction or margin (the teacher's combination rule).""" V = fly.score_all(X) if values is None else values; trig = fly.triggers(X, cols); n = len(X) strat = np.full(n, None, dtype=object); score = np.full(n, -1.0); hold = np.zeros(n); thr = np.full(n, np.inf); tables = [[] for _ in range(n)] - key = np.full(n, -np.inf) + key = np.full(n, -np.inf); ch = row_chains(X, cols) for j, name in enumerate(fly.strategies): - v = V[:, j]; line = fly.lines[name]; ok = trig[:, j] & (v >= line) - kv = (np.array([((sizing.band_for(fly.sizings[name], m) or {}).get("kelly") or 0.0) for m in v - line]) + 1e-9 * v - if fly.combine == "kelly" and fly.sizings[name] else v - line) + v = V[:, j] + line = np.array([fly.line_for(name, c) for c in ch]) if n else np.zeros(0) + tabs = [fly.sizing_for(name, c) for c in ch] + ok = trig[:, j] & (v >= line) + kv = (np.array([((sizing.band_for(t, m) or {}).get("kelly") or 0.0) for t, m in zip(tabs, v - line)]) + 1e-9 * v + if fly.combine == "kelly" and any(tabs) else v - line) take = ok & (kv > key) key = np.where(take, kv, key) for i in np.flatnonzero(take): - strat[i] = name; score[i] = v[i]; hold[i] = fly.rules[name]["hold_min"] * 60.0; thr[i] = line; tables[i] = fly.sizings[name] + strat[i] = name; score[i] = v[i]; hold[i] = fly.rules[name]["hold_min"] * 60.0; thr[i] = line[i]; tables[i] = tabs[i] return {"strategy": strat, "score": score, "hold_s": hold, "threshold": thr, "tables": tables} @@ -564,19 +645,24 @@ def bootstrap(ds: DecisionSet, S: date, epochs: int = EPOCHS, stop: threading.Ev week = (ds.day >= S - timedelta(days=CALIB_DAYS)) & (ds.day < S) & uni prog.update("fly: calibrating its own buy lines", 0, 1, force=True) wi = np.flatnonzero(week); V = fly.score_all(ds.X[wi]) if len(wi) else np.zeros((0, len(fly.strategies))); trig = fly.triggers(ds.X[wi], ds.cols) - per = {} - for j, name in enumerate(fly.strategies): - H = fly.rules[name]["hold_min"]; y = ds.fwd_h.get(H, ds.fwd_pess) - known = trig[:, j] & (ds.ts[wi] + H * 60.0 + 60.0 <= s_epoch) # labels known at S - ci = wi[known] - cal = fly_calibrate.calibrate(ds.ts[ci], ds.mint[ci], H * 60.0, V[known, j], y[ci]) - fly.lines[name], fly.sizings[name] = cal.line, cal.sizing - per[name] = {"line": cal.line, "trades": cal.trades, "mean": cal.mean, "total": cal.total, "hold_min": H} + A = teacher_allowed(teacher, ds.X[wi], ds.ts[wi], fly.strategies) if fly_calibrate.RULE == "match" else None + per = {}; wch = ds.chains()[wi] + for chain in sorted(set(wch.tolist()), key=lambda c: c != "sol"): # Solana first: another chain without rows keeps its lines + for j, name in enumerate(fly.strategies): + H = fly.rules[name]["hold_min"]; y = ds.fwd_h.get(H, ds.fwd_pess) + known = trig[:, j] & (ds.ts[wi] + H * 60.0 + 60.0 <= s_epoch) & (wch == chain) # labels known at S, this chain's rows + ci = wi[known] + target = (len(taken_idx(ds.ts[ci], ds.mint[ci], H * 60.0, np.flatnonzero(A[known, j] & np.isfinite(y[ci])))) if A is not None else None) + cal = fly_calibrate.calibrate(ds.ts[ci], ds.mint[ci], H * 60.0, V[known, j], y[ci], rule=fly_calibrate.RULE, target_trades=target, + prev_line=fly.lines.get(name) if chain != "sol" else None) + fly.set_line(name, chain, cal.line, cal.sizing) + per[name if chain == "sol" else f"{name}@{chain}"] = {"line": cal.line, "trades": cal.trades, "mean": cal.mean, "total": cal.total, "hold_min": H, + "rule": fly_calibrate.RULE, "teacher_trades": target, "chain": chain} d = fly_decide(fly, ds.X[wi], ds.cols, V) if len(wi) else {"strategy": np.array([], dtype=object), "hold_s": np.array([])} picked = np.array([x is not None for x in d["strategy"]], bool) ret = np.array([ds.fwd_h.get(fly.rules[x]["hold_min"], ds.fwd_pess)[i] if x is not None else np.nan for i, x in zip(wi, d["strategy"])]) known = picked & (ds.ts[wi] + d["hold_s"] + 60.0 <= s_epoch) - tr = taken_idx(ds.ts[wi], ds.mint[wi], d["hold_s"], np.flatnonzero(known)); r = ret[tr] + tr = taken_idx(ds.ts[wi], ds.mint[wi], d["hold_s"], np.flatnonzero(known & np.isfinite(ret))); r = ret[tr] # trades with a known outcome comb = {"trades": int(len(r)), "mean": float(np.mean(r)) if len(r) else None, "total": float(np.sum(r)) if len(r) else 0.0, "per_strategy": per} ci = np.flatnonzero(week & (ds.ts + ds.horizon_s + 60.0 <= s_epoch)) di = ci if len(ci) <= DIAG_ROWS else np.sort(np.random.default_rng(seed).choice(ci, DIAG_ROWS, replace=False)) @@ -585,8 +671,9 @@ def bootstrap(ds: DecisionSet, S: date, epochs: int = EPOCHS, stop: threading.Ev ok_t = np.isfinite(t_sc); di, t_sc = di[ok_t], t_sc[ok_t] diag = diagnose(fly, ds.X[di], t_sc) if len(di) else {"mbon_share": 0.0, "mbon_saturated": 1.0, "kc_active": 0.0, "rows": 0} ok, why = gate_check(diag) + fly.teacher = teacher # not saved: the replay's match rule reads the teacher's own decisions over its rows info = {"S": str(S), "train_through": str(train_end - timedelta(days=1)), "calibration_days": [str(S - timedelta(days=CALIB_DAYS)), str(S - timedelta(days=1))], - "line": fly.threshold, "sizing": fly.sizing, "lines": fly.lines, "calibration": comb, "strategies": fly.strategies, "rules": fly.rules, + "line": fly.threshold, "sizing": fly.sizing, "lines": fly.lines, "chain_lines": fly.chain_lines, "calibration": comb, "strategies": fly.strategies, "rules": fly.rules, "diagnostics": diag, "gates_ok": ok, "gate_failures": why, "fit": fit_info, "teacher": teacher_note or TEACHER_NOTE, "teacher_line": getattr(teacher, "threshold", None), "teacher_stack": stack_info} log.info("fly bootstrap for %s: strategies %s, lines %s | calibration %d trades, mean %s | MBON share %.1f%% saturated %.0f%% | gates %s", @@ -595,71 +682,96 @@ def bootstrap(ds: DecisionSet, S: date, epochs: int = EPOCHS, stop: threading.Ev return fly, info -def is_current(meta: dict | None) -> bool: - return bool(meta) and meta.get("data") == FLY_VERSION +def verdict_version() -> dict: + """What a replay verdict holds for: the bootstrap's definitions (``FLY_VERSION``) and how the fly scores a row + (config.FLY_SCORE; "own" is what FLY_VERSION alone meant until 2026-10-11, so its verdicts stay valid for it).""" + return FLY_VERSION if config.FLY_SCORE == "own" else {**FLY_VERSION, "score": config.FLY_SCORE} + + +def teacher_scores(models: dict, X: np.ndarray, ts, names: list[str], cols: list[str], chunk: int = 200_000) -> np.ndarray: + """[B, S]: the teacher stack's own score of each row for each of the fly's strategies (its predicted net return; NaN + where that strategy's trigger does not fire) -- the starting score of the fly when config.FLY_SCORE = teacher.""" + from . import strategies + X = np.atleast_2d(X); out = np.full((len(X), len(names)), np.nan) + tsa = np.broadcast_to(np.asarray(ts, dtype=np.float64), (len(X),)) + for i in range(0, len(X), chunk): + per = strategies.decide_all(models, X[i:i + chunk], cols, tsa[i:i + chunk]) + for j, k in enumerate(names): + if k in per: + sc = per[k]["score"] + out[i:i + chunk, j] = np.where(per[k]["trig"] & np.isfinite(sc), np.clip(sc, -1.0, 1.0), np.nan) + return out + + +def is_current(meta: dict | None, version: dict | None = None) -> bool: + return bool(meta) and meta.get("data") == (version or FLY_VERSION) -def save(fly: FlyModel, metrics: dict, run_id: str | None = None) -> tuple[Path, int]: - root = config.BRAIN_DIR / "policies"; root.mkdir(parents=True, exist_ok=True) - path = root / f"fly_{time.strftime('%Y%m%dT%H%M%SZ', time.gmtime())}.pt" +def save(fly: FlyModel, metrics: dict, run_id: str | None = None, kind: str = "fly_selector", subdir: str = "policies", version: dict | None = None, + prefix: str = "fly") -> tuple[Path, int]: + """``kind`` / ``subdir`` / ``version`` / ``prefix``: another trading type's fly keeps its own snapshots apart (kalshi/fly.py).""" + root = config.BRAIN_DIR / subdir; root.mkdir(parents=True, exist_ok=True) + path = root / f"{prefix}_{time.strftime('%Y%m%dT%H%M%SZ', time.gmtime())}.pt" n = fly.net - torch.save({"state_dict": n.state_dict(), "scaler": fly.scaler.state(), "cols": fly.cols, "threshold": fly.threshold, "sizing": fly.sizing, + model_io.torch_save({"state_dict": n.state_dict(), "scaler": fly.scaler.state(), "cols": fly.cols, "threshold": fly.threshold, "sizing": fly.sizing, "horizon_min": fly.horizon_min, "rules": fly.rules, "lines": fly.lines, "sizings": fly.sizings, "combine": fly.combine, - "config": {"k_steps": n.k_steps, "leak": n.leak, "hidden": n.hidden, "obs_dim": n.obs_dim, "kc_active": n.kc_active, "scale": SCALE, - "n_strategies": n.n_strategies}, "metrics": metrics}, path) + "chain_lines": fly.chain_lines, "chain_sizings": fly.chain_sizings, + "config": {"k_steps": n.k_steps, "leak": n.leak, "hidden": n.hidden, "obs_dim": n.obs_dim, "kc_active": n.kc_active, "scale": n.scale, + "n_strategies": n.n_strategies, "aff_rows": n.aff_rows.cpu().tolist() if kind != "fly_selector" else None}, "metrics": metrics}, path) sha = hashlib.sha256(path.read_bytes()).hexdigest() with transaction() as conn: - row = conn.execute("INSERT INTO brain_snapshots (run_id, path, sha256, kind, note) VALUES (%s,%s,%s,'fly_selector',%s) RETURNING id", - (run_id, str(path), sha, json.dumps({**metrics, "data": FLY_VERSION}, default=str))).fetchone() + row = conn.execute("INSERT INTO brain_snapshots (run_id, path, sha256, kind, note) VALUES (%s,%s,%s,%s,%s) RETURNING id", + (run_id, str(path), sha, kind, json.dumps({**metrics, "data": version or FLY_VERSION}, default=str))).fetchone() return path, int(row["id"]) -def load(path: str | Path, graph=None, device=None) -> FlyModel: - d = torch.load(path, map_location="cpu", weights_only=False); c = d["config"] +def load(path: str | Path, graph=None, device=None, model_cls=None) -> FlyModel: + d = model_io.torch_load(path); c = d["config"] net = FlyNet(graph if graph is not None else _graph(), obs_dim=c["obs_dim"], k_steps=c["k_steps"], leak=c["leak"], hidden=c["hidden"], - kc_active=c["kc_active"], device=device or _device(), n_strategies=c.get("n_strategies", 1)) + kc_active=c["kc_active"], device=device or _device(), n_strategies=c.get("n_strategies", 1), aff_rows=c.get("aff_rows"), scale=c.get("scale", SCALE)) net.load_state_dict(d["state_dict"]); net.eval() for bn in (net.eff_norm, net.mb_norm): bn.momentum = None - return FlyModel(net, RobustScaler.from_state(d["scaler"]), d["cols"], d["horizon_min"], d["threshold"], d.get("sizing"), rules=d.get("rules"), - lines=d.get("lines"), sizings=d.get("sizings"), combine=d.get("combine", "score")) + return (model_cls or FlyModel)(net, RobustScaler.from_state(d["scaler"]), d["cols"], d["horizon_min"], d["threshold"], d.get("sizing"), rules=d.get("rules"), + lines=d.get("lines"), sizings=d.get("sizings"), combine=d.get("combine", "score"), + chain_lines=d.get("chain_lines"), chain_sizings=d.get("chain_sizings")) -def latest_current(conn) -> dict | None: +def latest_current(conn, kind: str = "fly_selector", version: dict | None = None) -> dict | None: """The newest fly bootstrap trained on the current definitions (``FLY_VERSION``) whose file exists, or None.""" - for r in conn.execute("SELECT id, path, note FROM brain_snapshots WHERE kind = 'fly_selector' ORDER BY id DESC").fetchall(): + for r in conn.execute("SELECT id, path, note FROM brain_snapshots WHERE kind = %s ORDER BY id DESC", (kind,)).fetchall(): try: meta = json.loads(r["note"] or "{}") except ValueError: continue - if is_current(meta) and Path(r["path"]).exists(): + if is_current(meta, version) and Path(r["path"]).exists(): return {**dict(r), "meta": meta} return None -def deployable(meta: dict | None) -> tuple[bool, str]: +def deployable(meta: dict | None, version: dict | None = None) -> tuple[bool, str]: """A bootstrap may trade: current definitions, healthy network (``GATES``) and its own line made money over at least 100 trades on its calibration week.""" - if not is_current(meta): + if not is_current(meta, version): return False, "trained on other definitions" if not meta.get("gates_ok"): return False, "; ".join(meta.get("gate_failures") or ["network gates failed"]) cal = meta.get("calibration") or {} if (cal.get("trades") or 0) < selector.MIN_LINE_TRADES: return False, f"too few calibration trades ({cal.get('trades') or 0})" - if (cal.get("mean") or 0) <= 0: + if not (isinstance(cal.get("mean"), (int, float)) and math.isfinite(cal["mean"]) and cal["mean"] > 0): # NaN never passes return False, f"its calibration week lost money ({(cal.get('mean') or 0) * 100:+.2f}% per trade)" return True, f"its line made {cal['mean'] * 100:+.2f}% per trade over {cal['trades']} calibration trades" -def latest_deployable(conn) -> dict | None: +def latest_deployable(conn, kind: str = "fly_selector", version: dict | None = None) -> dict | None: """The newest bootstrap that may trade (``deployable``), or None.""" - for r in conn.execute("SELECT id, path, note FROM brain_snapshots WHERE kind = 'fly_selector' ORDER BY id DESC").fetchall(): + for r in conn.execute("SELECT id, path, note FROM brain_snapshots WHERE kind = %s ORDER BY id DESC", (kind,)).fetchall(): try: meta = json.loads(r["note"] or "{}") except ValueError: continue - if deployable(meta)[0] and Path(r["path"]).exists(): + if deployable(meta, version)[0] and Path(r["path"]).exists(): return {**dict(r), "meta": meta} return None @@ -673,7 +785,7 @@ def main(days: int | None = None, epochs: int | None = None, stop_event: threadi prog.update("fly: building decision points", 0, 1, force=True) from .strategies import HOLDS_MIN ds = build(days=days, horizon_min=HOLD_MIN, holds=HOLDS_MIN); S = ds.days[-1] + timedelta(days=1) - teacher, note, taught_by = deployed_teacher(ds, ds.day < S - timedelta(days=CALIB_DAYS + PURGE_DAYS)) + teacher, note, taught_by = deployed_teacher(ds, ds.day < S - timedelta(days=CALIB_DAYS + PURGE_DAYS), calib_start=S - timedelta(days=CALIB_DAYS)) log.info("fly bootstrap teacher: %s", note) fly, info = bootstrap(ds, S, epochs=EPOCHS if epochs is None else epochs, stop=stop_event, teacher=teacher, teacher_note=note) if fly is None: diff --git a/fly_trader/train/mature.py b/fly_trader/train/mature.py index 50824b6683f60171980ad24ee63a350750a5186e..0a8b8afa1ad9a47067598f421c2d2bb009563256 100644 --- a/fly_trader/train/mature.py +++ b/fly_trader/train/mature.py @@ -300,13 +300,17 @@ def _supplies() -> dict: return {r["mint"]: float(r["supply"]) for r in conn.execute("SELECT mint, supply FROM corpus_meta WHERE supply > 0").fetchall()} -def build_day(d: date, lookback_days: int = 1, grads: dict | None = None, supplies: dict | None = None) -> int: +def build_day(d: date, lookback_days: int = 1, grads: dict | None = None, supplies: dict | None = None, *, src_dir: Path | None = None, + out_dir: Path | None = None, program_label: str = "Pump.fun Amm", meta_extra: dict | None = None, extra_md: dict | None = None) -> int: """Feature rows for every minute of day D, warmed up on the previous day's candles. ``grads``: mint → graduated_at - (default: read from ``corpus_meta``); the part records how many of its mints had one (metadata ``fly_known``).""" + (default: read from ``corpus_meta``); the part records how many of its mints had one (metadata ``fly_known``). + The keyword arguments build another market's parts from its own aggregates (rh/corpus.py: Robinhood Chain, whose + ``*_sol`` columns hold ETH): ``meta_extra`` = mint → extra TokenMeta fields (a Pons pool's fee, the exit-cost size).""" + src_dir = src_dir or MATURE_DIR; out_dir = out_dir or MATURE_FEAT_DIR t0 = time.time() frames = [] for k in range(lookback_days, -1, -1): - f = MATURE_DIR / f"{(d - timedelta(days=k)).isoformat()}.parquet" + f = src_dir / f"{(d - timedelta(days=k)).isoformat()}.parquet" if f.exists(): frames.append(pq.read_table(f).to_pandas()) if not frames: @@ -324,7 +328,8 @@ def build_day(d: date, lookback_days: int = 1, grads: dict | None = None, suppli n_mints += 1 g = grads[mint].timestamp() if grads.get(mint) else None n_known += g is not None - st = TokenState(mint); tm = TokenMeta(mint=mint, program_label="Pump.fun Amm", graduated_at=g, supply=supplies.get(mint) or PUMP_SUPPLY) + st = TokenState(mint); tm = TokenMeta(mint=mint, program_label=program_label, graduated_at=g, supply=supplies.get(mint) or PUMP_SUPPLY, + **((meta_extra or {}).get(mint) or {})) pre = {k: float("nan") for k in PRE_COLS} o, h, l, c = x["open"].to_numpy(), x["high"].to_numpy(), x["low"].to_numpy(), x["close"].to_numpy() bs, ss, nb, ns, nt, rq = x["buy_sol"].to_numpy(), x["sell_sol"].to_numpy(), x["n_buys"].to_numpy(), x["n_sells"].to_numpy(), x["n_traders"].to_numpy(), x["resq_sol"].to_numpy() @@ -358,7 +363,8 @@ def build_day(d: date, lookback_days: int = 1, grads: dict | None = None, suppli out.append(row) if not out: return 0 - write_part(pa.Table.from_pylist(out, schema=SCHEMA), MATURE_FEAT_DIR / d.isoformat() / "part.parquet", FEATURE_VERSION, {"fly_known": n_known, "fly_agg": AGG_VERSION}) + write_part(pa.Table.from_pylist(out, schema=SCHEMA), out_dir / d.isoformat() / "part.parquet", FEATURE_VERSION, + {"fly_known": n_known, "fly_agg": AGG_VERSION, **(extra_md or {})}) log.info("mature features %s: %d mints (%d with graduation), %d rows in %.0fs", d, n_mints, n_known, len(out), time.time() - t0) return len(out) diff --git a/fly_trader/train/model_io.py b/fly_trader/train/model_io.py new file mode 100644 index 0000000000000000000000000000000000000000..a0196f9208f83e1704db416ee068e6e5cbcab29e --- /dev/null +++ b/fly_trader/train/model_io.py @@ -0,0 +1,99 @@ +"""Model files that cannot run code when they load. + +A pickle (``joblib.load``, ``torch.load(weights_only=False)``) executes whatever its author put in it, and model +releases install on the hosted fly without a human step (ops/release.py). So every model the engine loads goes through +here: + +* fly checkpoints and plastic banks: ``torch.load(weights_only=True)`` -- tensors, numbers, strings, lists and dicts + only; anything else refuses to load. ``torch_save`` re-reads what it wrote the same way, so a checkpoint that would + not load safely is caught where it is made, not on the server. +* selectors: skops (``.skops``), whose loader builds only the types listed in ``TRUSTED_SELECTOR_TYPES`` (plus the + scikit-learn / numpy types skops itself trusts). Older ``.joblib`` selectors are refused; ``fly-trader models + convert`` rewrites this machine's own ones once. +""" +from __future__ import annotations + +import hashlib +from pathlib import Path + +TRUSTED_SELECTOR_TYPES = [ + "fly_trader.train.scaling.RobustScaler", + "fly_trader.train.selector.SelectorModel", + "sklearn.ensemble._hist_gradient_boosting.predictor.TreePredictor", +] +# what a release's models/ may hold (connectome .npz loads with allow_pickle=False, wallet skill is parquet, geometry +# is raw floats + JSON); TABLE and current.txt are plain text +SAFE_SUFFIXES = {".pt", ".skops", ".npz", ".json", ".bin", ".parquet", ".txt", ".sha256", ""} + + +class UnsafeModel(RuntimeError): + pass + + +def torch_load(path: str | Path): + import torch + return torch.load(path, map_location="cpu", weights_only=True) + + +def torch_save(obj, path: str | Path) -> None: + import torch + torch.save(obj, path) + try: + torch_load(path) + except Exception as e: + Path(path).unlink(missing_ok=True) + raise UnsafeModel(f"{Path(path).name} would not load with weights_only=True (a non-tensor, non-primitive value?): {e}") from e + + +def save_selector(m, path: str | Path) -> None: + import skops.io as sio + sio.dump(m, path) + untrusted = set(sio.get_untrusted_types(file=path)) - set(TRUSTED_SELECTOR_TYPES) + if untrusted: + Path(path).unlink(missing_ok=True) + raise UnsafeModel(f"selector holds types outside TRUSTED_SELECTOR_TYPES: {sorted(untrusted)}") + + +def load_selector(path: str | Path): + import skops.io as sio + p = Path(path) + if p.suffix != ".skops": + raise UnsafeModel(f"{p.name}: only .skops selectors load (a pickle can run code); run `fly-trader models convert`") + return sio.load(p, trusted=TRUSTED_SELECTOR_TYPES) + + +def check_file(path: str | Path) -> None: + """Raise UnsafeModel unless ``path`` is a model file that loads without running code (ops/release.apply_release).""" + p = Path(path) + if p.suffix not in SAFE_SUFFIXES: + raise UnsafeModel(f"{p.name}: {p.suffix} files are not accepted in a release (pickle formats can run code)") + if p.suffix == ".pt": + try: + torch_load(p) + except Exception as e: + raise UnsafeModel(f"{p.name} does not load with weights_only=True: {e}") from e + elif p.suffix == ".skops": + import skops.io as sio + untrusted = set(sio.get_untrusted_types(file=p)) - set(TRUSTED_SELECTOR_TYPES) + if untrusted: + raise UnsafeModel(f"{p.name} holds untrusted types {sorted(untrusted)}") + + +def convert_legacy() -> list[dict]: + """Rewrite this machine's own ``.joblib`` selector snapshots as ``.skops`` (path and sha256 updated in place, ids + kept so pins still hold). Only for files this machine trained: it unpickles them once to convert.""" + import joblib + from ..db.connection import transaction + done = [] + with transaction() as conn: + rows = conn.execute("SELECT id, path FROM brain_snapshots WHERE kind = 'selector' AND path LIKE '%%.joblib'").fetchall() + for r in rows: + src = Path(r["path"]) + if not src.exists(): + continue + dst = src.with_suffix(".skops") + save_selector(joblib.load(src), dst) + sha = hashlib.sha256(dst.read_bytes()).hexdigest() + conn.execute("UPDATE brain_snapshots SET path = %s, sha256 = %s WHERE id = %s", (str(dst), sha, r["id"])) + done.append({"id": int(r["id"]), "path": str(dst), "sha256": sha}) + return done diff --git a/fly_trader/train/selector.py b/fly_trader/train/selector.py index 6eda96237cdcfb155a50e9071f3d4d9ae25ac8b4..ab2c79506e8948588b6d10430290a90432f51a09 100644 --- a/fly_trader/train/selector.py +++ b/fly_trader/train/selector.py @@ -15,7 +15,7 @@ Walk-forward protocol (the strategy gate): every day after a 21-day warm-up is a every eligible minute of the days at least two days earlier (refit every 7 days). A model goes to work only if the evaluation half made money after costs, beat random picks from the same universe and has at least 100 trades. The deployed model is then fit on every day and saved with its line, scaler and sizing table -(``data/brain/selectors/selector_.joblib`` + ``brain_snapshots`` kind 'selector'). Runs inside the training +(``data/brain/selectors/selector_.skops`` + ``brain_snapshots`` kind 'selector'). Runs inside the training pipeline (``train/pipeline.py``) and as ``fly-trader train-selector``. """ from __future__ import annotations @@ -29,7 +29,6 @@ from dataclasses import dataclass, field from datetime import timedelta from pathlib import Path -import joblib import numpy as np from sklearn.ensemble import HistGradientBoostingRegressor @@ -38,7 +37,7 @@ from ..agent import sizing from ..db.apilog import record_event from ..db.connection import transaction from ..market.features import FEATURE_VERSION -from . import progress as prog +from . import model_io, progress as prog from .decisions import HOLD_MIN, MIN_RESQ_SOL, MIN_VOL_15M_SOL, X_COLS, DecisionSet, build, evaluate, random_trades, rank_corr, summarize, taken_rows from .mature import AGG_VERSION from .scaling import RobustScaler @@ -53,8 +52,20 @@ def _skill_version() -> str: return skill_version() +def _rh_version() -> dict: + """The Robinhood Chain half of the combined dataset: its aggregation, skill space and cost constants (a model trained + before RH rows existed, or on other RH definitions, is not current). Not RH_ENABLED: whether an install indexes and + trades RH is its own choice, and a combined model trades Solana alone (until 2026-10-10 an install with RH off + computed "chains": "sol", refused the shipped combined models and crash-looped at boot).""" + from .. import config + from ..rh.corpus import RH_AGG_VERSION + from ..rh.wallet_skill import skill_version as rh_skill + return {"chains": "sol+rh-1", "rh_agg": RH_AGG_VERSION, "rh_skill": rh_skill(), + "rh_costs": f"pons-kyber-1:{config.RH_ETH_PER_SOL:g}:{config.RH_LABEL_SIZE_ETH:g}:{config.RH_TX_FEE_ETH:g}:{config.RH_HOOK_FEE:g}"} + + DATA_VERSION = {"agg": AGG_VERSION, "features": FEATURE_VERSION, "costs": "real-fees-sized-1", "selector": "multi-1", - "cols": hashlib.sha1(",".join(X_COLS).encode()).hexdigest()[:8], "skill": _skill_version(), "meta": "rug-1"} + "cols": hashlib.sha1(",".join(X_COLS).encode()).hexdigest()[:8], "skill": _skill_version(), "meta": "rug-1", **_rh_version()} WARMUP_DAYS, BLOCK_DAYS = 21, 7 # walk-forward: the first 21 days only train; every later day is tested, refit every 7 days MIN_AGE_H = 6.0 # trade only tokens at least this long past graduation (unknown age = graduated before the archive); fitted, see decisions.HOLD_MIN MIN_EV = 0.01 # the buy line of a model before its walk-forward chose one @@ -103,6 +114,10 @@ class SelectorModel: metrics: dict = field(default_factory=dict) sizing: list = field(default_factory=list) # agent/sizing.py table from the backtest's out-of-sample trades stack: dict = field(default_factory=dict) # train/strategies.final_models: strategies, veto, combination rule (empty: single ev model) + # The same stack refit without its holdout days (the models the holdout is scored with): {"models", "fit_through", + # "holdout_days"}. The fly's bootstrap distils these, so the week it calibrates its line and sizing on is one its + # teacher never saw (train/fly_selector.deployed_teacher). Empty: no holdout, or saved before 2026-09-25. + blind: dict = field(default_factory=dict) def score(self, X: np.ndarray) -> np.ndarray: """Predicted net return over the hold.""" @@ -226,8 +241,8 @@ def walk_forward(ds: DecisionSet, warmup_days: int = WARMUP_DAYS, block_days: in def save(m: SelectorModel, run_id: str | None = None) -> tuple[Path, int]: SELECTOR_DIR.mkdir(parents=True, exist_ok=True) - path = SELECTOR_DIR / f"selector_{time.strftime('%Y%m%dT%H%M%SZ', time.gmtime())}.joblib" - joblib.dump(m, path); sha = hashlib.sha256(path.read_bytes()).hexdigest() + path = SELECTOR_DIR / f"selector_{time.strftime('%Y%m%dT%H%M%SZ', time.gmtime())}.skops" + model_io.save_selector(m, path); sha = hashlib.sha256(path.read_bytes()).hexdigest() with transaction() as conn: row = conn.execute("INSERT INTO brain_snapshots (run_id, path, sha256, kind, note) VALUES (%s,%s,%s,'selector',%s) RETURNING id", (run_id, str(path), sha, json.dumps({"threshold": m.threshold, "horizon_min": m.horizon_min, "trained_through": m.trained_through, @@ -251,13 +266,13 @@ def load_snapshot(snapshot_id: int) -> SelectorModel | None: """One named selector snapshot, whatever has been deployed since (agent/selector_session.py's pin).""" with transaction() as conn: row = conn.execute("SELECT path FROM brain_snapshots WHERE id = %s AND kind = 'selector'", (snapshot_id,)).fetchone() - return joblib.load(row["path"]) if row and Path(row["path"]).exists() else None + return model_io.load_selector(row["path"]) if row and Path(row["path"]).exists() else None def load_latest() -> SelectorModel | None: with transaction() as conn: r = latest_current(conn) - return joblib.load(r["path"]) if r else None + return model_io.load_selector(r["path"]) if r else None def main(days: int | None = None, horizon_min: int = HOLD_MIN, stop_event: threading.Event | None = None) -> dict: @@ -278,7 +293,9 @@ def main(days: int | None = None, horizon_min: int = HOLD_MIN, stop_event: threa prog.update("selector: fitting the deployable models on every day", 0, 1, force=True) models = strategies.final_models(ds, stack) if stack.fits else {"strategies": {}} # scored with models refit without the holdout: the deployed ones above have seen those days and would score themselves - holdout = strategies.score_holdout(ds, strategies.final_models(ds, stack, exclude_days=stack.holdout_days), stack.holdout_days) if stack.holdout_days else {} + blind = strategies.final_models(ds, stack, exclude_days=stack.holdout_days) if stack.holdout_days else None + holdout = strategies.score_holdout(ds, blind, stack.holdout_days) if blind is not None else {} + by_chain = strategies.score_holdout_by_chain(ds, blind, stack.holdout_days) if blind is not None and ds.chain is not None else {} if holdout.get("n"): log.info("holdout %s..%s (no fit ever saw these days): %d trades, %s winners, PF %s, %s per trade vs random %s", holdout["days"][0], holdout["days"][1], holdout["n"], @@ -295,11 +312,14 @@ def main(days: int | None = None, horizon_min: int = HOLD_MIN, stop_event: threa final = SelectorModel(gbm=ev["gbm"], scaler=ev["scaler"], cols=ev["cols"], threshold=ev["line"], horizon_min=ev["hold_min"], trained_through=str(ds.days[-1]), sizing=ev["sizing"]) final.stack = models + if blind is not None and blind["strategies"]: # kept, not thrown away: the fly's honest teacher for its calibration week + hd = sorted(stack.holdout_days) + final.blind = {"models": blind, "fit_through": str(hd[0] - timedelta(days=1)), "holdout_days": [str(hd[0]), str(hd[-1])]} final.metrics = {"walk_forward": {**stack.evaluation, "days": None}, "selection": stack.selection, "random_baseline": {"mean": stack.evaluation.get("random_mean")}, "components": stack.components, "strategies": {k: {x: v[x] for x in ("line", "hold_min", "thr", "high", "hours", "regimes", "sizing", "selection", "evaluation")} for k, v in models["strategies"].items()}, "groups": stack.groups, "combine": stack.combine, "veto": {k: v for k, v in (stack.veto or {}).items() if k not in ("p",)} or None, - "fallback": stack.fallback, "holdout": holdout, "line": final.threshold, "costs": "real fees (pool fee by market cap, Jupiter 10 bps, network fee) + impact", + "fallback": stack.fallback, "holdout": holdout, "chains": by_chain, "line": final.threshold, "costs": "real fees (pool fee by market cap, Jupiter 10 bps, network fee) + impact", "data": DATA_VERSION, "deployable": stack.deployable, "deploy_reason": stack.reason, "sizing": final.sizing, "model": {"kind": "stack", "min_age_h": MIN_AGE_H, "min_resq_sol": MIN_RESQ_SOL, "min_vol_15m_sol": MIN_VOL_15M_SOL}, "rows": int(len(ds.y)), "days": len(ds.days), "first_day": str(ds.days[0]), "last_day": str(ds.days[-1])} diff --git a/fly_trader/train/strategies.py b/fly_trader/train/strategies.py index 410859acf55f840bdbf3663d320af4badbeea916..a2367a816642a308c85f53102890d3005f8b23ae 100644 --- a/fly_trader/train/strategies.py +++ b/fly_trader/train/strategies.py @@ -31,7 +31,7 @@ import logging import threading from dataclasses import dataclass, field from pathlib import Path -from datetime import timedelta +from datetime import datetime, timedelta, timezone import numpy as np from scipy.stats import binom @@ -40,11 +40,30 @@ from sklearn.ensemble import HistGradientBoostingClassifier, HistGradientBoostin from .. import config from ..agent import sizing from . import progress as prog -from .decisions import GROUPS, LEGACY_COLS, DecisionSet, taken_idx +from .decisions import GROUPS, LEGACY_COLS, DecisionSet, day_ords, taken_idx from .scaling import RobustScaler log = logging.getLogger(__name__) HOLDS_MIN = (10, 20, 30, 45, 60, 90, 120, 180, 240) + + +@dataclass +class StackSpec: + """What a trading type supplies to the stack machinery: its candidate sets, inputs, universe, labels and holds. The + memecoin selector is ``MEMECOIN`` (the default of every function here); the Kalshi selector supplies its own + (fly_trader/kalshi/strategies.py), where a "hold" is the arm (taker | maker) and every position is held to settlement.""" + name: str + strategies: dict # name -> {"holds", "vars", "highs"} + base_mask: object # (name, X, cols, high) -> mask + trigger_value: object # (name, var, X, cols) -> values + groups: dict # input groups the ev step tries, on top of legacy_cols + legacy_cols: list + ev_hold: object # the hold the input-group step fits at + in_universe: object # (X, cols) -> mask + label: object # (ds, H) -> net return per row for hold H, or None when the set has no label for it + hold_s: object # (ds, H) -> seconds: a scalar, or one per row + oos: object # (ds, rows, y, col_idx, H, stop, label) -> out-of-sample score per row (NaN elsewhere) + random_hold: object = None # (ds) -> hold_s for the random baseline (None: ds.horizon_s) EV_HOLD = 120 WIN_MIN, PF_MIN, SIGN_ALPHA = 0.65, 1.3, 0.05 # The bars a setting must clear, as a key: results fitted under another objective (train/wallet_skill.py's saved candidates) @@ -52,6 +71,11 @@ WIN_MIN, PF_MIN, SIGN_ALPHA = 0.65, 1.3, 0.05 # each walk-forward half it demanded 8 winning weeks (89 %), a stricter bar than the 65 % winners it was meant to protect, # and nothing in the corpus passed it (operator decision, 2026-09-16). OBJECTIVE = f"win{WIN_MIN}-pf{PF_MIN}-trades-noweekly" +# A decision set may carry ``gate = "significance"`` (the Kalshi set does: kalshi/decisions.build): a setting then needs a +# positive mean and the weekly sign test (p ≤ SIGN_ALPHA) instead of PF ≥ PF_MIN and WIN_MIN winners. A contract held to +# settlement has its win rate fixed by its price and PF by price and edge together (bought at 95c, PF 1.3 takes a 1.3 % +# edge), and the Kalshi halves are long enough (~39 weeks each) for the sign test to be passable (operator decision, +# 2026-10-09). QUANTILES = (0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.95, 0.975, 0.99, 0.995, 0.999) LOSS_QUANTILES = (0.05, 0.10, 0.20, 0.30) RANDOM_SEEDS = 20 @@ -91,25 +115,35 @@ class Score: weeks: int = 0 weeks_pos: int = 0 sign_ok: bool = False + gate: str = "memecoin" # "significance": the bars are a positive mean and the weekly sign test (see OBJECTIVE) @property def base_ok(self) -> bool: + if self.gate == "significance": + return self.n >= _min_trades() and self.mean is not None and self.mean > 0 and self.sign_ok return self.n >= _min_trades() and self.pf is not None and self.pf >= PF_MIN @property def admissible(self) -> bool: + if self.gate == "significance": + return self.base_ok return self.base_ok and self.win is not None and self.win >= WIN_MIN def dict(self) -> dict: return {"n": self.n, "win": self.win, "pf": self.pf, "total": self.total, "mean": self.mean, "weeks": self.weeks, "weeks_pos": self.weeks_pos, - "sign_ok": self.sign_ok, "admissible": self.admissible, "base_ok": self.base_ok} + "sign_ok": self.sign_ok, "admissible": self.admissible, "base_ok": self.base_ok, "gate": self.gate} + + +def bars(gate: str) -> str: + """The bars a setting must clear, in words.""" + return f"a positive mean with a weekly sign test p ≤ {SIGN_ALPHA:g}" if gate == "significance" else f"≥{WIN_MIN:.0%} winners and PF ≥{PF_MIN:g}" def score_trades(ds: DecisionSet, tr: np.ndarray, returns: np.ndarray, day0) -> Score: - r = np.asarray(returns[tr], dtype=np.float64) + r = np.asarray(returns[tr], dtype=np.float64); gate = getattr(ds, "gate", "memecoin") ok = np.isfinite(r); r = r[ok]; tr = np.asarray(tr)[ok] if len(r) == 0: - return Score() + return Score(gate=gate) g, l = r[r > 0].sum(), -r[r <= 0].sum() wk = np.array([(d - day0).days // 7 for d in ds.day[tr]]) tot = np.bincount(wk - wk.min(), weights=r) if len(wk) else np.array([]) @@ -117,7 +151,7 @@ def score_trades(ds: DecisionSet, tr: np.ndarray, returns: np.ndarray, day0) -> weeks = int(has.sum()); pos = int(((tot > 0) & has).sum()) p = float(binom.sf(pos - 1, weeks, 0.5)) if weeks else 1.0 # P(X ≥ pos) under no edge return Score(n=len(r), win=float((r > 0).mean()), pf=float(g / l) if l > 0 else float("inf"), total=float(r.sum()), mean=float(r.mean()), - weeks=weeks, weeks_pos=pos, sign_ok=p <= SIGN_ALPHA) + weeks=weeks, weeks_pos=pos, sign_ok=p <= SIGN_ALPHA, gate=gate) def score_pick(ds: DecisionSet, pick: np.ndarray, hold_s, returns: np.ndarray, day0) -> Score: @@ -156,7 +190,10 @@ def beats_random_removal(ds: DecisionSet, pick_before: np.ndarray, pick_after: n def _cache_path(ds: DecisionSet, cols: list[str], tag: str) -> "Path": from .selector import DATA_VERSION k = repr((str(ds.days[0]), str(ds.days[-1]), int(len(ds.y)), float(ds.horizon_s), list(cols), tag, - DATA_VERSION.get("agg"), DATA_VERSION.get("features"), DATA_VERSION.get("costs"), OBJECTIVE)) + DATA_VERSION.get("agg"), DATA_VERSION.get("features"), DATA_VERSION.get("costs"), OBJECTIVE) + + (("label-known-purge",) if getattr(ds, "label_ts", None) is not None else ()) + + ((("data", getattr(ds, "data_version")),) if getattr(ds, "data_version", None) is not None else ()) # e.g. the Kalshi feature version + + ((("train_max_rows", int(getattr(ds, "train_max_rows"))),) if getattr(ds, "train_max_rows", None) else ())) return CACHE_DIR / f"{hashlib.sha256(k.encode()).hexdigest()[:20]}.npy" @@ -183,6 +220,29 @@ def _cached(ds: DecisionSet, cols: list[str], tag: str, stop, compute): +def _day_start(d) -> float: + return datetime(d.year, d.month, d.day, tzinfo=timezone.utc).timestamp() + + +def known_before(ds: DecisionSet, t: float) -> np.ndarray: + """Rows whose label was known before epoch ``t``. A set with per-row label times (``ds.label_ts``: Kalshi, where a + position is held to its settlement up to 10 days later) may train a model for a period only on rows that had resolved + when the period began; the day-before purge alone let markets that settle inside the test block — the very markets + being tested — into training. Sets without ``label_ts`` (memecoins, 2 h holds) are unaffected.""" + lt = getattr(ds, "label_ts", None) + return np.ones(len(ds.y), bool) if lt is None else (np.asarray(lt, dtype=np.float64) < float(t)) + + +def cap_rows(ds: DecisionSet, idx: np.ndarray, seed: int) -> np.ndarray: + """At most ``ds.train_max_rows`` of the training rows ``idx``, a uniform random subset fixed by ``seed`` (blind to labels). + The Kalshi corpus repeats each market every 5 minutes, so a block's 35 M training rows carry little more than 8 M do, + while fitting on all of them took the selector past 70 GB. Sets without the attribute (memecoins) are unchanged.""" + cap = getattr(ds, "train_max_rows", None) + if not cap or len(idx) <= cap: + return idx + return np.sort(np.random.default_rng(seed).choice(idx, int(cap), replace=False)) + + def wf_blocks(days: list) -> list[list]: from .selector import BLOCK_DAYS, WARMUP_DAYS return [days[k:k + BLOCK_DAYS] for k in range(WARMUP_DAYS, len(days), BLOCK_DAYS)] @@ -206,7 +266,8 @@ def wf_regress(ds: DecisionSet, rows: np.ndarray, y: np.ndarray, cols: np.ndarra for k, blk in enumerate(blocks): if stop is not None and stop.is_set(): break - tr = np.flatnonzero(fin & (ds.day < (min(blk) - timedelta(days=1)))); te = np.flatnonzero(rows & np.isin(ds.day, blk)) + od = day_ords(ds); b0 = min(blk).toordinal() + tr = cap_rows(ds, np.flatnonzero(fin & (od < b0 - 1) & known_before(ds, _day_start(min(blk)))), 1000 + k); te = np.flatnonzero(rows & np.isin(od, [d.toordinal() for d in blk])) if len(tr) < 10 * _gbm_params()["min_samples_leaf"] or len(te) == 0: continue Xtr = ds.X[np.ix_(tr, cols)] # this block's own copy: scaled in place, freed before the next @@ -226,7 +287,8 @@ def wf_classify(ds: DecisionSet, rows: np.ndarray, feats, y: np.ndarray, n_class for k, blk in enumerate(blocks): if stop is not None and stop.is_set(): break - tr = np.flatnonzero(rows & (ds.day < (min(blk) - timedelta(days=1)))); te = np.flatnonzero(rows & np.isin(ds.day, blk)) + od = day_ords(ds); b0 = min(blk).toordinal() + tr = cap_rows(ds, np.flatnonzero(rows & (od < b0 - 1) & known_before(ds, _day_start(min(blk)))), 1000 + k); te = np.flatnonzero(rows & np.isin(od, [d.toordinal() for d in blk])) if len(tr) < 10 * _gbm_params()["min_samples_leaf"] or len(te) == 0 or len(np.unique(y[tr])) < 2: continue Xtr = np.asarray(feats(tr), dtype=np.float32) # as above: one block-sized array at a time @@ -270,15 +332,29 @@ def base_mask(name: str, X: np.ndarray, cols: list[str], high: str | None = None raise KeyError(name) +# Trigger variables carried in the row's own currency (log1p of an amount): on Robinhood Chain rows (ETH) they are put in +# SOL terms before a threshold applies, log1p(expm1(v) / K), so one fitted threshold means the same on both chains. +UNIT_LOG_VARS = frozenset(["log_org_vol_15m"]) + + def trigger_value(name: str, var: str, X: np.ndarray, cols: list[str]) -> np.ndarray: - return -_col(cols, X, "ret_1m") if var == "neg_ret_1m" else _col(cols, X, var) + if var == "neg_ret_1m": + return -_col(cols, X, "ret_1m") + v = _col(cols, X, var) + if var in UNIT_LOG_VARS and "chain_rh" in cols: + rh = X[:, cols.index("chain_rh")] > 0.5 + if rh.any(): + from .. import config + v = np.array(v, dtype=np.float64, copy=True); v[rh] = np.log1p(np.expm1(v[rh]) / config.RH_ETH_PER_SOL) + return v -def trigger_mask(name: str, thr: dict, X: np.ndarray, cols: list[str]) -> np.ndarray: +def trigger_mask(name: str, thr: dict, X: np.ndarray, cols: list[str], spec: "StackSpec | None" = None) -> np.ndarray: + tv = (spec or memecoin_spec()).trigger_value m = np.ones(len(X), bool) for var, t in thr.items(): if t is not None and np.isfinite(t): - m &= trigger_value(name, var, X, cols) >= t + m &= tv(name, var, X, cols) >= t return m @@ -301,10 +377,16 @@ class StrategyFit: regimes: list = field(default_factory=list) # blocked mkt_vol_1h decile bins [(lo, hi)] passed: bool = False reason: str = "" + spec: object = None # the StackSpec it was fitted under (None: MEMECOIN) + + @property + def hold(self): + """Seconds each position is held, for ``ds``-independent scalars; per-row holds come from ``spec.hold_s(ds, H)``.""" + return self.hold_min def pick(self, ds: DecisionSet, rows: np.ndarray, uni: np.ndarray) -> np.ndarray: - X, c = ds.X, ds.cols - p = rows & uni & base_mask(self.name, X, c, self.high) & trigger_mask(self.name, self.thr, X, c) & (self.oos >= self.line) + sp = self.spec or memecoin_spec(); X, c = ds.X, ds.cols + p = rows & uni & sp.base_mask(self.name, X, c, self.high) & trigger_mask(self.name, self.thr, X, c, sp) & (self.oos >= self.line) if self.meta_cut is not None and self.meta_p is not None: p &= self.meta_p >= self.meta_cut if self.hours: @@ -315,36 +397,36 @@ class StrategyFit: def fit_strategy(ds: DecisionSet, name: str, cols: list[str], uni: np.ndarray, sel: np.ndarray, day0, stop=None, holds=None, - cache_tag: str = "") -> StrategyFit | None: + cache_tag: str = "", spec: "StackSpec | None" = None) -> StrategyFit | None: """Coordinate ascent over (hold, line, trigger thresholds, high window) on the selection half; each hold's OOS scores come from its own walk-forward (models fit on the strategy's candidates, label = net return over that hold). ``holds``: fit these holds instead of the strategy's whole grid — one walk-forward per hold is the expensive part, so the input-group comparison runs at a single hold and only the strategies themselves scan the grid. ``cache_tag`` distinguishes fits whose inputs share their column names but hold different values (the wallet-skill candidates), which would otherwise collide in the walk-forward cache.""" - spec = STRATEGIES[name]; X, c = ds.X, ds.cols; ci = np.asarray([c.index(x) for x in cols]) + sp = spec or memecoin_spec(); st = sp.strategies[name]; X, c = ds.X, ds.cols; ci = np.asarray([c.index(x) for x in cols]) oos = {} loose = uni & np.zeros(len(ds.y), bool) - for high in spec["highs"]: - loose |= uni & base_mask(name, X, c, high) - for H in (holds or spec["holds"]): - y = ds.fwd_h[H] if H in ds.fwd_h else (ds.fwd_pess if H * 60 == ds.horizon_s else None) + for high in st["highs"]: + loose |= uni & sp.base_mask(name, X, c, high) + for H in (holds or st["holds"]): + y = sp.label(ds, H) if y is None: continue - oos[H] = _cached(ds, cols, f"{name}|hold{H}{cache_tag}", stop, lambda H=H, y=y: wf_regress(ds, loose, y, ci, stop, label=f"{name} {H} min")) + oos[H] = _cached(ds, cols, f"{sp.name}|{name}|hold{H}{cache_tag}", stop, lambda H=H, y=y: sp.oos(ds, loose, y, ci, H, stop, f"{name} {H}")) if not oos: return None - ret = lambda H: ds.fwd_h[H] if H in ds.fwd_h else ds.fwd_pess + ret = lambda H: sp.label(ds, H) best = None; best_p = None; trials = 0; seen = None - thr = {v: None for v in spec["vars"]}; high = spec["highs"][0]; H = next(iter(oos)) + thr = {v: None for v in st["vars"]}; high = st["highs"][0]; H = next(iter(oos)) g0 = quantile_grid(oos[H][sel & loose]); line = g0[0] if g0 else float("inf") # the loosest line until a trial passes - grids = {v: quantile_grid(trigger_value(name, v, X[sel & loose], c)) for v in spec["vars"]} + grids = {v: quantile_grid(sp.trigger_value(name, v, X[sel & loose], c)) for v in st["vars"]} def trial(H_, line_, thr_, high_): nonlocal best, best_p, trials, seen trials += 1 - pk = sel & uni & base_mask(name, X, c, high_) & trigger_mask(name, thr_, X, c) & (oos[H_] >= line_) - s = score_pick(ds, pk, H_ * 60.0, ret(H_), day0) + pk = sel & uni & sp.base_mask(name, X, c, high_) & trigger_mask(name, thr_, X, c, sp) & (oos[H_] >= line_) + s = score_pick(ds, pk, sp.hold_s(ds, H_), ret(H_), day0) if s.n > 0 and (seen is None or s.total > seen.total): seen = s # the most profitable setting that actually traded (an empty pick scores 0 and must not win this) if better(s, best): @@ -355,29 +437,31 @@ def fit_strategy(ds: DecisionSet, name: str, cols: list[str], uni: np.ndarray, s trial(H_, line_, thr, high) if best_p: H, line, thr, high = best_p - for v in spec["vars"]: + for v in st["vars"]: for t in [None] + grids[v]: trial(H, line, {**thr, v: t}, high) if best_p: H, line, thr, high = best_p - for high_ in spec["highs"]: + for high_ in st["highs"]: trial(H, line, thr, high_) if best_p: H, line, thr, high = best_p if best_p is None: return StrategyFit(name, H, float("inf"), thr, high, cols, oos=oos[H], trials=trials, best_seen=seen.dict() if seen else {}, - reason="no setting met PF and the trade count on the selection half") + reason=("no setting met the sign test, a positive mean and the trade count on the selection half" if getattr(ds, "gate", "") == "significance" + else "no setting met PF and the trade count on the selection half"), spec=sp) H, line, thr, high = best_p - return StrategyFit(name, H, float(line), thr, high, cols, oos=oos[H], selection=best.dict(), trials=trials, best_seen=seen.dict() if seen else {}) + return StrategyFit(name, H, float(line), thr, high, cols, oos=oos[H], selection=best.dict(), trials=trials, best_seen=seen.dict() if seen else {}, spec=sp) # ---------------------------------------------------------------- meta-label, veto, gates def fit_meta(ds: DecisionSet, s: StrategyFit, uni: np.ndarray, sel: np.ndarray, day0, stop=None) -> tuple[np.ndarray, float | None, dict]: + sp = s.spec or memecoin_spec() """Nested: a classifier of P(net return > 0 over the strategy's hold) on its OOS candidates (primary score ≥ 0 and the fitted trigger), walk-forward, the primary score among its inputs; the cut-off (deciles among the selection half's candidates) jointly with the line.""" - X, c = ds.X, ds.cols; ci = np.asarray([c.index(x) for x in s.cols]); y_r = ds.fwd_h.get(s.hold_min, ds.fwd_pess) - rows = uni & base_mask(s.name, X, c, s.high) & trigger_mask(s.name, s.thr, X, c) & np.isfinite(s.oos) & (s.oos >= 0) & np.isfinite(y_r) + X, c = ds.X, ds.cols; ci = np.asarray([c.index(x) for x in s.cols]); y_r = sp.label(ds, s.hold_min) + rows = uni & sp.base_mask(s.name, X, c, s.high) & trigger_mask(s.name, s.thr, X, c, sp) & np.isfinite(s.oos) & (s.oos >= 0) & np.isfinite(y_r) log.info("meta-label %s: %d candidate rows of %d, %d inputs, hold %d min", s.name, int(rows.sum()), len(ds.y), len(ci) + 1, s.hold_min) feats = lambda idx: np.c_[X[np.ix_(idx, ci)], s.oos[idx]] p = wf_classify(ds, rows, feats, (y_r > 0).astype(np.int8), 2, stop)[:, 1] @@ -391,7 +475,7 @@ def fit_meta(ds: DecisionSet, s: StrategyFit, uni: np.ndarray, sel: np.ndarray, if trials % 25 == 0: log.info("meta-label %s: trial %d/%d, %.1f GB resident", s.name, trials, len(lines) * len(cuts), _rss_gb()) pk = sel & rows & (s.oos >= line) & (p >= cut) - sc = score_pick(ds, pk, s.hold_min * 60.0, y_r, day0) + sc = score_pick(ds, pk, sp.hold_s(ds, s.hold_min), y_r, day0) if sc.n > 0 and (seen is None or sc.total > seen.total): seen = sc # the most profitable cut-off that actually traded if better(sc, best): @@ -405,6 +489,7 @@ def system_pick(ds: DecisionSet, fits: list[StrategyFit], rows: np.ndarray, uni: N = len(ds.y); pick = np.zeros(N, bool); hold = np.full(N, np.nan); ret = np.full(N, np.nan, np.float32); who = np.full(N, -1) key = np.full(N, -np.inf) for k, f in enumerate(fits): + sp = f.spec or memecoin_spec() pk = f.pick(ds, rows, uni) if combine == "kelly" and tables and tables.get(f.name): v = np.array([((sizing.band_for(tables[f.name], m) or {}).get("kelly") or 0.0) for m in (f.oos[pk] - f.line)]) @@ -413,7 +498,7 @@ def system_pick(ds: DecisionSet, fits: list[StrategyFit], rows: np.ndarray, uni: kv = np.where(pk, f.oos, -np.inf) take = pk & (kv > key) key = np.where(take, kv, key); who = np.where(take, k, who); pick |= pk - hold = np.where(take, f.hold_min * 60.0, hold); ret = np.where(take, ds.fwd_h.get(f.hold_min, ds.fwd_pess), ret) + hold = np.where(take, sp.hold_s(ds, f.hold_min), hold); ret = np.where(take, sp.label(ds, f.hold_min), ret) return pick, np.where(np.isnan(hold), 0.0, hold), ret, who @@ -474,13 +559,14 @@ def _comp(name, passed, reason, sel=None, ev=None, params=None, trials=0, best=N return c -def fit_stack(ds: DecisionSet, stop: threading.Event | None = None, holdout_days_n: int | None = None) -> Stack: +def fit_stack(ds: DecisionSet, stop: threading.Event | None = None, holdout_days_n: int | None = None, spec: "StackSpec | None" = None) -> Stack: """Every component in order, each on top of the accepted ones; see the module docstring. ``holdout_days_n``: how many of the last days to withhold from every fit (default ``HOLDOUT_DAYS``). A caller whose ``ds`` is already restricted to days before some later date -- the fly's bootstrap teacher (train/fly_selector.py) -- passes 0: its out-of-sample proof is the replay that follows, and withholding again would only cost it its most recent three weeks.""" - from .selector import deploy_decision, in_universe - uni = in_universe(ds.X, ds.cols) + from .selector import deploy_decision + sp = spec or memecoin_spec() + uni = sp.in_universe(ds.X, ds.cols) n_hold = HOLDOUT_DAYS if holdout_days_n is None else int(holdout_days_n) holdout_days = list(ds.days[-n_hold:]) if n_hold and len(ds.days) > n_hold + 4 else [] fit_days = [d for d in ds.days if d not in set(holdout_days)] @@ -488,14 +574,14 @@ def fit_stack(ds: DecisionSet, stop: threading.Event | None = None, holdout_days if len(tdays) < 4: raise RuntimeError("too few walk-forward days for the strategy stack") half = tdays[len(tdays) // 2]; day0 = tdays[0] - tested = np.isin(ds.day, tdays); sel = tested & (ds.day < half); ev = tested & (ds.day >= half) + od = day_ords(ds); tested = np.isin(od, [d.toordinal() for d in tdays]); sel = tested & (od < half.toordinal()); ev = tested & (od >= half.toordinal()) log.info("fitting on %d days (%s..%s); holdout of %d days (%s..%s) is never fitted on", len(tdays), tdays[0], tdays[-1], len(holdout_days), holdout_days[0] if holdout_days else "-", holdout_days[-1] if holdout_days else "-") comps = [] # 1. input groups, judged on the ev strategy: all groups at once; if that is rejected, each group on its own, accumulating - best = fit_strategy(ds, "ev", LEGACY_COLS, uni, sel, day0, stop, holds=(EV_HOLD,)) - groups, cols = [], list(LEGACY_COLS) - trial_sets = [list(GROUPS)] + [[g] for g in GROUPS] + best = fit_strategy(ds, "ev", sp.legacy_cols, uni, sel, day0, stop, holds=(sp.ev_hold,), spec=sp) + groups, cols = [], list(sp.legacy_cols) + trial_sets = ([list(sp.groups)] if sp.groups else []) + [[g] for g in sp.groups] def _score_of(f): return Score(**{k: v for k, v in f.selection.items() if k in Score.__dataclass_fields__}) if f is not None and f.selection else None @@ -503,23 +589,23 @@ def fit_stack(ds: DecisionSet, stop: threading.Event | None = None, holdout_days for gs in trial_sets: if stop is not None and stop.is_set(): break - if gs != list(GROUPS) and set(groups) == set(GROUPS): + if gs != list(sp.groups) and set(groups) == set(sp.groups): break add = [g for g in gs if g not in groups] if not add: continue - gi = [ds.cols.index(x) for x in (x for g in add for x in GROUPS[g]) if x in ds.cols] + gi = [ds.cols.index(x) for x in (x for g in add for x in sp.groups[g]) if x in ds.cols] sample = np.flatnonzero(tested)[:200_000] if gi and len(sample) and not (np.ptp(ds.X[np.ix_(sample, gi)], axis=0) > 0).any(): comps.append(_comp(f"inputs: {'+'.join(add)}", False, "these inputs are empty in this corpus (every value identical), so there is nothing to learn from", params={"groups": add})); continue - cand_cols = cols + [x for g in add for x in GROUPS[g]] - f = fit_strategy(ds, "ev", cand_cols, uni, sel, day0, stop, holds=(EV_HOLD,)) + cand_cols = cols + [x for g in add for x in sp.groups[g]] + f = fit_strategy(ds, "ev", cand_cols, uni, sel, day0, stop, holds=(sp.ev_hold,), spec=sp) if f is None: continue sel_s = _score_of(f) or Score() if better(sel_s, _score_of(best)): - evs = score_pick(ds, f.pick(ds, ev, uni), f.hold_min * 60.0, ds.fwd_h.get(f.hold_min, ds.fwd_pess), day0) + evs = score_pick(ds, f.pick(ds, ev, uni), sp.hold_s(ds, f.hold_min), sp.label(ds, f.hold_min), day0) ok = passes(evs, fallback=not sel_s.admissible) comps.append(_comp(f"inputs: {'+'.join(add)}", ok, "improves the ev strategy on the selection half" + ("" if ok else ", fails on the evaluation half"), f.selection, evs.dict(), {"groups": add}, f.trials, best=f.best_seen)) @@ -530,17 +616,17 @@ def fit_stack(ds: DecisionSet, stop: threading.Event | None = None, holdout_days best=f.best_seen)) # 2. strategies fits = [] - for name in STRATEGIES: - log.info("fitting strategy %s over holds %s", name, STRATEGIES[name]["holds"]) - f = fit_strategy(ds, name, cols, uni, sel, day0, stop) # the full hold grid: the group step above fitted one hold only + for name in sp.strategies: + log.info("fitting strategy %s over holds %s", name, sp.strategies[name]["holds"]) + f = fit_strategy(ds, name, cols, uni, sel, day0, stop, spec=sp) # the full hold grid: the group step above fitted one hold only if f is None or not f.selection: comps.append(_comp(f"strategy: {name}", False, (f.reason if f else "no candidates") or "no admissible setting", trials=f.trials if f else 0, best=f.best_seen if f else None)); continue - y_r = ds.fwd_h.get(f.hold_min, ds.fwd_pess) - evs = score_pick(ds, f.pick(ds, ev, uni), f.hold_min * 60.0, y_r, day0) + y_r = sp.label(ds, f.hold_min) + evs = score_pick(ds, f.pick(ds, ev, uni), sp.hold_s(ds, f.hold_min), y_r, day0) sel_adm = f.selection.get("admissible") f.evaluation = evs.dict(); f.passed = passes(evs, fallback=not sel_adm) - f.reason = ("meets ≥65 % winners and PF ≥1.3 on both halves" if sel_adm and evs.admissible else + f.reason = (f"meets {bars(evs.gate)} on both halves" if sel_adm and evs.admissible else "profit fallback: below 65 % winners, PF ≥1.3 on both halves" if f.passed else "fails on the evaluation half") comps.append(_comp(f"strategy: {name}", f.passed, f.reason, f.selection, f.evaluation, {"hold_min": f.hold_min, "line": f.line, "thr": f.thr, "high": f.high}, f.trials, best=f.best_seen)) @@ -551,7 +637,7 @@ def fit_stack(ds: DecisionSet, stop: threading.Event | None = None, holdout_days # sizing tables from each strategy's out-of-sample trades (selection half: used to choose the combination rule) tables = {} for f in fits: - y_r = ds.fwd_h.get(f.hold_min, ds.fwd_pess); t = taken_idx(ds.ts, ds.mint, f.hold_min * 60.0, np.flatnonzero(f.pick(ds, sel, uni))) + y_r = sp.label(ds, f.hold_min); t = taken_idx(ds.ts, ds.mint, sp.hold_s(ds, f.hold_min), np.flatnonzero(f.pick(ds, sel, uni))) tables[f.name] = sizing.build_table(f.oos[t] - f.line, y_r[t]) # 3. combination rule best_c = None; combine = "score" @@ -611,12 +697,13 @@ def fit_stack(ds: DecisionSet, stop: threading.Event | None = None, holdout_days s_sel = score_pick(ds, pk & sel, hd, rt, day0); s_ev = score_pick(ds, pk & ev, hd, rt, day0) tr_ev = taken_idx(ds.ts, ds.mint, hd, np.flatnonzero(pk & ev)) from .decisions import random_trades, summarize - rnd = summarize(random_trades(ds, ev & uni, max(1, len(tr_ev)))) + rnd = summarize(random_trades(ds, ev & uni, max(1, len(tr_ev)), hold_s=sp.random_hold(ds) if sp.random_hold else None, + returns=sp.label(ds, sp.ev_hold) if sp.random_hold else None)) dep, why = deploy_decision({"n": s_ev.n, "mean": s_ev.mean}, rnd) strict = s_sel.admissible and s_ev.admissible fb = (not strict) and s_sel.base_ok and s_ev.base_ok deployable = dep and (strict or fb) - reason = (why + ("; meets ≥65 % winners and PF ≥1.3 on both halves" if strict else "; profit fallback (below 65 % winners)" if fb else "; fails the stack's bars")) + reason = (why + (f"; meets {bars(s_ev.gate)} on both halves" if strict else "; profit fallback (below 65 % winners)" if fb else f"; fails the stack's bars ({bars(s_ev.gate)})")) return Stack(groups, cols, fits, combine, veto, comps, s_sel.dict(), {**s_ev.dict(), "random_mean": rnd.get("mean")}, deployable, fb and deployable, reason, tables, holdout_days=holdout_days) @@ -627,54 +714,68 @@ def _without_meta(f: StrategyFit, old) -> StrategyFit: return g -def score_holdout(ds: DecisionSet, models: dict, days: list) -> dict: +def score_holdout_by_chain(ds: DecisionSet, models: dict, days: list, spec: "StackSpec | None" = None) -> dict: + """``score_holdout`` on each chain's rows alone (Solana, Robinhood Chain), with each chain's own random baseline, and + whether that chain's book may trade the stack: at least 30 holdout trades, a positive mean that beats random picks + (the combined stack may carry an edge on one chain and none on the other).""" + ch = ds.chains(); out = {} + for c in sorted(set(ch.tolist())): + h = score_holdout(ds.subset(ch == c), models, [d for d in days if d in set(ds.day[ch == c].tolist())], spec) + n = int(h.get("n") or 0); m = h.get("mean"); rm = h.get("random_mean") + out[c] = {**h, "deployable": bool(n >= 30 and m is not None and m > 0 and (rm is None or m > rm))} + return out + + +def score_holdout(ds: DecisionSet, models: dict, days: list, spec: "StackSpec | None" = None) -> dict: """The deployed decision (``decide``: strategies, win filters, dump veto and gates as they would trade) scored on the days ``fit_stack`` withheld from every fit. It decides nothing — recorded and reported only. Feeding it back into any choice would make it a second evaluation half rather than a holdout.""" from .decisions import random_trades, summarize - from .selector import in_universe + sp = spec or memecoin_spec() if not days or not (models.get("strategies") or {}): return {} - eligible = np.isin(ds.day, days) & in_universe(ds.X, ds.cols) + eligible = np.isin(day_ords(ds), [d.toordinal() for d in days]) & sp.in_universe(ds.X, ds.cols) rows = np.flatnonzero(eligible) out = {"days": [str(days[0]), str(days[-1])], "rows": int(len(rows))} if not len(rows): return out - d = decide(models, ds.X[rows], ds.cols, ds.ts[rows]) + d = decide(models, ds.X[rows], ds.cols, ds.ts[rows], spec=sp) take = d["allow"] if not take.any(): return {**out, "n": 0} - idx = rows[take]; holds = d["hold_s"][take] - hold_full = np.zeros(len(ds.y)); hold_full[idx] = holds - ret_full = np.full(len(ds.y), np.nan, np.float32) - for hm in {int(h // 60) for h in holds}: - r = idx[(holds // 60).astype(int) == hm] - ret_full[r] = ds.fwd_h.get(hm, ds.fwd_pess)[r] + idx = rows[take] + hold_full = np.zeros(len(ds.y)); ret_full = np.full(len(ds.y), np.nan, np.float32) + for name, m in models["strategies"].items(): + r = idx[d["strategy"][take] == name] + if len(r): + hs = sp.hold_s(ds, m["hold_min"]); hold_full[r] = hs[r] if np.ndim(hs) else hs + ret_full[r] = sp.label(ds, m["hold_min"])[r] tr = taken_idx(ds.ts, ds.mint, hold_full, idx) s = score_trades(ds, tr, ret_full, days[0]) - rnd = summarize(random_trades(ds, eligible, max(1, len(tr)))) + rnd = summarize(random_trades(ds, eligible, max(1, len(tr)), hold_s=sp.random_hold(ds) if sp.random_hold else None, + returns=sp.label(ds, sp.ev_hold) if sp.random_hold else None)) return {**out, **s.dict(), "random_mean": rnd.get("mean")} # ---------------------------------------------------------------- deployment: final models and live decisions -def final_models(ds: DecisionSet, stack: Stack, exclude_days: list | None = None) -> dict: +def final_models(ds: DecisionSet, stack: Stack, exclude_days: list | None = None, spec: "StackSpec | None" = None) -> dict: """Every accepted component refit on all days (the walk-forward only chose their settings): per strategy its GBM over its candidates and hold, its win classifier on all its out-of-sample candidates; the veto on all candidate trades. ``exclude_days``: leave those days out of every fit, so the result can be scored on days it has never seen. Deployment refits on everything; only the holdout measurement excludes, or it would be scoring itself in sample.""" - from .selector import in_universe - uni = in_universe(ds.X, ds.cols); out = {"strategies": {}, "veto": None, "combine": stack.combine} - keep = ~np.isin(ds.day, exclude_days) if exclude_days else np.ones(len(ds.y), bool) + sp = spec or memecoin_spec() + uni = sp.in_universe(ds.X, ds.cols); out = {"strategies": {}, "veto": None, "combine": stack.combine, "spec": sp.name} + keep = (~np.isin(day_ords(ds), [d.toordinal() for d in exclude_days]) & known_before(ds, _day_start(min(exclude_days)))) if exclude_days else np.ones(len(ds.y), bool) for f in stack.fits: - ci = np.asarray([ds.cols.index(x) for x in f.cols]); y = ds.fwd_h.get(f.hold_min, ds.fwd_pess) + ci = np.asarray([ds.cols.index(x) for x in f.cols]); y = sp.label(ds, f.hold_min) loose = np.zeros(len(ds.y), bool) - for high in STRATEGIES[f.name]["highs"]: - loose |= uni & base_mask(f.name, ds.X, ds.cols, high) + for high in sp.strategies[f.name]["highs"]: + loose |= uni & sp.base_mask(f.name, ds.X, ds.cols, high) rows = np.flatnonzero(loose & np.isfinite(y) & keep) - gbm, scaler = fit_regressor(ds.X[np.ix_(rows, ci)], y[rows], seed=99) + gbm, scaler = sp.fit_final(ds, rows, ci, f.hold_min) if getattr(sp, "fit_final", None) else fit_regressor(ds.X[np.ix_(rows, ci)], y[rows], seed=99) meta = None if f.meta_cut is not None: - mr = np.flatnonzero(uni & base_mask(f.name, ds.X, ds.cols, f.high) & trigger_mask(f.name, f.thr, ds.X, ds.cols) & np.isfinite(f.oos) & (f.oos >= 0) & np.isfinite(y) & keep) + mr = np.flatnonzero(uni & sp.base_mask(f.name, ds.X, ds.cols, f.high) & trigger_mask(f.name, f.thr, ds.X, ds.cols, sp) & np.isfinite(f.oos) & (f.oos >= 0) & np.isfinite(y) & keep) mc, ms = fit_classifier(np.c_[ds.X[np.ix_(mr, ci)], f.oos[mr]], (y[mr] > 0).astype(np.int8), seed=99) meta = {"model": mc, "scaler": ms, "cut": f.meta_cut} out["strategies"][f.name] = {"gbm": gbm, "scaler": scaler, "cols": list(f.cols), "line": f.line, "hold_min": f.hold_min, "thr": dict(f.thr), "high": f.high, @@ -689,20 +790,22 @@ def final_models(ds: DecisionSet, stack: Stack, exclude_days: list | None = None return out -def decide_all(models: dict, X: np.ndarray, cols: list[str], t_start) -> dict: +def decide_all(models: dict, X: np.ndarray, cols: list[str], t_start, spec: "StackSpec | None" = None) -> dict: """Every accepted strategy's own view of each row: ``trig`` (its fitted trigger fires), ``score`` (its predicted net return over its hold, −inf off the trigger), ``allow`` (at/above its line and not blocked by its win filter, its gates or the dump veto), ``why`` (the blocking filter), plus its line, hold and sizing table. ``t_start``: the minute start (a scalar, or one per row) — the hour gates use it.""" + sp = spec or memecoin_spec() X = np.atleast_2d(X); n = len(X) hour = ((np.broadcast_to(np.asarray(t_start, dtype=np.float64), (n,)) // 3600) % 24).astype(int) out = {} for name, m in (models.get("strategies") or {}).items(): ci = np.asarray([cols.index(x) for x in m["cols"]]) - trig = base_mask(name, X, cols, m["high"]) & trigger_mask(name, m["thr"], X, cols) + trig = sp.base_mask(name, X, cols, m["high"]) & trigger_mask(name, m["thr"], X, cols, sp) sc = np.full(n, -np.inf); why = np.full(n, "", dtype=object) if trig.any(): - ti = np.flatnonzero(trig); sc[ti] = m["gbm"].predict(m["scaler"].transform(X[np.ix_(ti, ci)])) + ti = np.flatnonzero(trig) + sc[ti] = sp.score_final(m, X[np.ix_(ti, ci)], X[ti], cols, m["hold_min"]) if getattr(sp, "score_final", None) else m["gbm"].predict(m["scaler"].transform(X[np.ix_(ti, ci)])) ok = trig & (sc >= m["line"]) if m.get("meta") is not None and ok.any(): idx = np.flatnonzero(ok) @@ -731,10 +834,11 @@ def decide_all(models: dict, X: np.ndarray, cols: list[str], t_start) -> dict: return out -def decide(models: dict, X: np.ndarray, cols: list[str], t_start) -> dict: +def decide(models: dict, X: np.ndarray, cols: list[str], t_start, spec: "StackSpec | None" = None) -> dict: """The selector's decision for each row of one minute (``t_start`` = the minute's start): which strategy trades it, its predicted net return, hold, line and sizing table, and whether a filter blocks it (with the reason).""" - X = np.atleast_2d(X); n = len(X); per = decide_all(models, X, cols, t_start) + sp = spec or memecoin_spec() + X = np.atleast_2d(X); n = len(X); per = decide_all(models, X, cols, t_start, sp) best_key = np.full(n, -np.inf); strat = np.full(n, None, dtype=object); score = np.full(n, -1.0); hold = np.zeros(n) thr = np.full(n, np.inf); tables = [[] for _ in range(n)]; allow = np.zeros(n, bool); reason = np.full(n, "no strategy fires", dtype=object) for name, d in per.items(): @@ -744,6 +848,28 @@ def decide(models: dict, X: np.ndarray, cols: list[str], t_start) -> dict: take = d["trig"] & (sc >= d["line"]) & (key > best_key) best_key = np.where(take, key, best_key) for i in np.flatnonzero(take): - strat[i] = name; score[i] = sc[i]; hold[i] = d["hold_min"] * 60.0; thr[i] = d["line"]; tables[i] = d["sizing"] + strat[i] = name; score[i] = sc[i]; hold[i] = sp.hold_seconds(d["hold_min"]) if getattr(sp, "hold_seconds", None) else d["hold_min"] * 60.0; thr[i] = d["line"]; tables[i] = d["sizing"] allow[i] = bool(d["allow"][i]); reason[i] = d["why"][i] or "trade" return {"strategy": strat, "score": score, "hold_s": hold, "threshold": thr, "tables": tables, "allow": allow, "reason": reason} + + +# ---------------------------------------------------------------- the memecoin stack's spec (every default above) +def _memecoin_label(ds: DecisionSet, H): + if H in ds.fwd_h: + return ds.fwd_h[H] + return ds.fwd_pess if H * 60 == ds.horizon_s else None + + +def _memecoin_in_universe(X, cols): + from .selector import in_universe + return in_universe(X, cols) + + +def memecoin_spec() -> StackSpec: + """The memecoin spec, read from this module's globals at call time (tests monkeypatch STRATEGIES, GROUPS, LEGACY_COLS).""" + return StackSpec(name="memecoin", strategies=STRATEGIES, base_mask=base_mask, trigger_value=trigger_value, groups=GROUPS, legacy_cols=list(LEGACY_COLS), + ev_hold=EV_HOLD, in_universe=_memecoin_in_universe, label=_memecoin_label, hold_s=lambda ds, H: H * 60.0, + oos=lambda ds, rows, y, ci, H, stop, label: wf_regress(ds, rows, y, ci, stop, label=label + " min")) + + +MEMECOIN = memecoin_spec() diff --git a/fly_trader/ui/app.py b/fly_trader/ui/app.py index c02b786f853ed43f8f1b1b995cdd31df76b41812..c30e80d28943a41bb6f49a46bc39d851d661cf61 100644 --- a/fly_trader/ui/app.py +++ b/fly_trader/ui/app.py @@ -15,6 +15,7 @@ import streamlit as st st.set_page_config(page_title="fly-trader", page_icon="🪰", layout="wide") +from fly_trader import config from fly_trader.ops.supervisor import get_supervisor from fly_trader.ui.common import status_strip @@ -30,15 +31,28 @@ def _autostart_once() -> None: _autostart_once() -pages = [ - st.Page("app_pages/overview.py", title="Overview", icon=":material/dashboard:", default=True), - st.Page("app_pages/trades.py", title="Trades", icon=":material/swap_horiz:"), - st.Page("app_pages/model.py", title="Model & training", icon=":material/psychology:"), - st.Page("app_pages/data.py", title="Data pipelines", icon=":material/database:"), - st.Page("app_pages/processes.py", title="Processes", icon=":material/settings_applications:"), - st.Page("app_pages/safety.py", title="Safety & wallet", icon=":material/shield:"), - st.Page("app_pages/vault.py", title="$FLY vault", icon=":material/savings:", url_path="vault"), -] +pages = { + "Memecoins": [ + st.Page("app_pages/overview.py", title="Overview", icon=":material/dashboard:", default=True), + st.Page("app_pages/trades.py", title="Trades", icon=":material/swap_horiz:"), + st.Page("app_pages/model.py", title="Model & training", icon=":material/psychology:"), + st.Page("app_pages/data.py", title="Data pipelines", icon=":material/database:"), + st.Page("app_pages/rh.py", title="Robinhood Chain", icon=":material/link:", url_path="rh"), + ], + "Prediction markets": [ + st.Page("app_pages/kalshi_overview.py", title="Kalshi overview", icon=":material/visibility:", url_path="kalshi"), + st.Page("app_pages/kalshi_trades.py", title="Kalshi trades", icon=":material/receipt_long:", url_path="kalshi-trades"), + st.Page("app_pages/kalshi_model.py", title="Kalshi model & training", icon=":material/neurology:", url_path="kalshi-model"), + st.Page("app_pages/kalshi_data.py", title="Kalshi data", icon=":material/database:", url_path="kalshi-data"), + ], + "System": [ + st.Page("app_pages/processes.py", title="Processes", icon=":material/settings_applications:"), + st.Page("app_pages/safety.py", title="Safety & wallet", icon=":material/shield:"), + st.Page("app_pages/vault.py", title="$FLY vault", icon=":material/savings:", url_path="vault"), + ], +} +if not config.KALSHI_API_KEY_ID: # the distribution ships Kalshi dormant: its pages appear once it is set up + pages.pop("Prediction markets") page = st.navigation(pages, position="top") status_strip() try: diff --git a/fly_trader/ui/app_pages/kalshi_data.py b/fly_trader/ui/app_pages/kalshi_data.py new file mode 100644 index 0000000000000000000000000000000000000000..bc2d5ed00c7e3da0574fcaea4630b82760590f1d --- /dev/null +++ b/fly_trader/ui/app_pages/kalshi_data.py @@ -0,0 +1,88 @@ +"""Kalshi data: the live feed, the history corpus (dataset seed, candles, feature builds) and the catalogue's coverage.""" +import glob +from datetime import datetime, timezone + +import pandas as pd +import streamlit as st + +from fly_trader import config +from fly_trader.db.queries import q, q1 +from fly_trader.ops.supervisor import get_supervisor +from fly_trader.ui.common import ago, setting + + +def _state_badge(alive: bool, ok: bool = True) -> None: + st.badge("running" if alive and ok else "stale" if alive else "stopped", color="green" if alive and ok else "orange" if alive else "gray") + + +@st.cache_data(ttl=60, show_spinner=False) +def feature_parts() -> dict: + from fly_trader.kalshi.mature import part_current + files = sorted(glob.glob(str(config.KALSHI_DIR / "features" / "*" / "part-*.parquet"))) + days = sorted({f.split("/")[-2] for f in files}) + return {"n": len(files), "days": len(days), "first": days[0] if days else None, "last": days[-1] if days else None, + "current": sum(1 for f in files if part_current(f))} + + +@st.cache_data(ttl=60, show_spinner=False) +def corpus_counts() -> dict: + c = {r["status"]: int(r["n"]) for r in q("SELECT status, count(*) AS n FROM kalshi_corpus GROUP BY status")} + m = q1("SELECT count(*) AS n, count(*) FILTER (WHERE result IN ('yes','no')) AS settled, count(*) FILTER (WHERE result IS NULL AND close_time > now()) AS open, min(close_time) AS first FROM kalshi_markets") or {} + seed = q1("SELECT value FROM ui_settings WHERE key = 'kalshi_dataset_seed'") + cats = q("""SELECT COALESCE(s.category, e.category, 'unknown') AS category, count(*) AS markets FROM kalshi_markets m LEFT JOIN kalshi_events e USING (event_ticker) + LEFT JOIN kalshi_series s ON s.ticker = e.series_ticker WHERE m.result IN ('yes','no') GROUP BY 1 ORDER BY 2 DESC LIMIT 15""") + return {"corpus": c, "markets": m, "seed": seed["value"] if seed else None, "categories": cats} + + +@st.fragment(run_every="10s") +def pipelines() -> None: + ws = get_supervisor().status(); now = datetime.now(timezone.utc) + ks, ks_at = setting("kalshi_stream_status"); hs, hs_at = setting("kalshi_history_status") + a, b = st.columns(2) + with a: + with st.container(border=True): + with st.container(horizontal=True, vertical_alignment="center"): + st.markdown(":material/sensors: **Kalshi feed** (live)") + ft = ks.get("flushed_through"); age = (now - datetime.fromisoformat(ft)).total_seconds() - 60 if ft else None + _state_badge(ws["kalshi_stream"]["alive"], age is not None and age < 180) + with st.container(horizontal=True): + st.metric("Messages / s", f"{float(ks.get('events_per_s') or 0):.0f}") + st.metric("Newest minute", f"{age:.0f} s old" if age is not None else "—") + st.metric("Markets quoted", f"{int(ks.get('tickers_quoted') or 0):,}", help="Distinct markets with a quote since the feed started (combos excluded).") + st.metric("Settlements seen", int(ks.get("settled") or 0)); st.metric("Reconnects", int(ks.get("reconnects") or 0)) + st.caption(f"ticker, trade and lifecycle channels → kalshi_minutes / kalshi_quotes · {'private channels on' if ks.get('private') else 'public channels only'} · " + f"late {int(ks.get('dropped_late') or 0):,} · updated {ago(ks_at)}") + with b: + with st.container(border=True): + with st.container(horizontal=True, vertical_alignment="center"): + st.markdown(":material/history: **Kalshi history** (training data)") + _state_badge(ws["kalshi_history"]["alive"]) + cc = corpus_counts(); c = cc["corpus"]; total = sum(c.values()) + with st.container(horizontal=True): + st.metric("Settled markets", f"{int((cc['markets'] or {}).get('settled') or 0):,}") + st.metric("Candles filled", f"{c.get('done', 0) + c.get('built', 0):,}/{total:,}") + st.metric("Built", f"{c.get('built', 0):,}"); st.metric("Errors", f"{c.get('error', 0):,}") + if total: + st.progress(min(1.0, (c.get("done", 0) + c.get("built", 0)) / total)) + st.caption(f"Since {config.KALSHI_HISTORY_START} · stage {hs.get('stage') or '—'}" + (f" · about {float(hs['eta_h']):.1f} h left" if hs.get("eta_h") else "") + + f" · dataset seed {'done' if cc['seed'] else 'not yet'} · updated {ago(hs_at)}") + a, b = st.columns(2) + with a: + with st.container(border=True): + st.markdown(":material/table_chart: **Kalshi features**") + fp = feature_parts() + with st.container(horizontal=True): + st.metric("Days built", fp["days"]); st.metric("Range", f"{fp['first'] or '—'} → {fp['last'] or '—'}") + st.metric("Current version", f"{fp['current']}/{fp['n']}", help="Parts built with the feature engine's current definitions; training refuses a mix.") + st.caption("One row per (market, minute, side) every 5 minutes inside the entry window, through the same feature engine the live engine runs.") + with b: + with st.container(border=True): + st.markdown(":material/category: **Coverage by category**") + cats = corpus_counts()["categories"] + if cats: + st.dataframe(pd.DataFrame(cats), hide_index=True, height=260) + else: + st.caption("No settled markets in the catalogue yet.") + + +pipelines() diff --git a/fly_trader/ui/app_pages/kalshi_model.py b/fly_trader/ui/app_pages/kalshi_model.py new file mode 100644 index 0000000000000000000000000000000000000000..0270f3815bfa66e95c00d06d0cb21995357d1df0 --- /dev/null +++ b/fly_trader/ui/app_pages/kalshi_model.py @@ -0,0 +1,179 @@ +"""Kalshi model & training: the strategy stack the visual fly was taught by, the automatic pipeline, the replay verdict, +the plastic visual fly's controls and health, and every saved Kalshi model.""" +import json +from datetime import datetime, timezone + +import pandas as pd +import streamlit as st + +from fly_trader import config +from fly_trader.db.queries import q +from fly_trader.kalshi import fly as KF, fly_session, pipeline, selector as kselector +from fly_trader.ops.reset import reset_kalshi_fly +from fly_trader.ops.supervisor import get_supervisor +from fly_trader.ui.common import ago, jv, kalshi_state, parse_note, pct, usd + + +def _until(iso: str | None) -> str: + if not iso: + return "—" + s = (datetime.fromisoformat(iso) - datetime.now(timezone.utc)).total_seconds() + return "due now" if s <= 0 else f"in {s / 3600:.0f} h" if s < 172800 else f"in {s / 86400:.1f} days" + + +def _half(x, k): + return None if not x or x.get(k) is None else float(x[k]) + + +@st.fragment(run_every="15s") +def stack() -> None: + k = kalshi_state(); sel = k["selector"] + with st.container(border=True): + st.markdown("**The teacher: the Kalshi strategy stack**") + if not sel: + st.info("No Kalshi selector has been fitted on the current definitions yet. The pipeline below fits one as soon as the corpus is built.", icon=":material/info:") + return + meta = sel["meta"]; wf = meta.get("walk_forward") or {}; rb = meta.get("random_baseline") or {} + st.caption(f"Selector #{sel['id']} · fitted walk-forward by settlement day on {meta.get('first_day', '—')} → {meta.get('last_day', '—')} ({meta.get('rows', '—'):,} rows) · saved {ago(sel['ts'])} · " + + ("put to work: " if meta.get("deployable") else "not put to work: ") + str(meta.get("deploy_reason") or "")) + with st.container(horizontal=True): + st.metric("Evaluation trades", wf.get("n", "—"), help="Positions on the evaluation half, each scored by a classifier that never saw that settlement day.") + st.metric("Average per trade", pct(wf.get("mean")), delta=(f"random {pct(rb['mean'])}" if rb.get("mean") is not None else None), delta_color="off", + help="Net return per dollar of payout at Kalshi's fees; 'random' is the same number of random rows bought at the ask.") + st.metric("Winning", pct(wf.get("win"), 0, False)); st.metric("Profit factor", f"{wf['pf']:.2f}" if wf.get("pf") else "—") + st.metric("Profitable days", f"{wf.get('days_positive', '—')} of {wf.get('days', '—')}") + comps = meta.get("components") or [] + if comps: + st.dataframe(pd.DataFrame([{"component": c["name"], "kept": c["passed"], "why": c["reason"], "winning (evaluation)": (_half(c.get("evaluation"), "win") or 0) * 100 if _half(c.get("evaluation"), "win") is not None else None, + "profit factor": _half(c.get("evaluation"), "pf"), "trades": _half(c.get("evaluation"), "n"), "settings tried": c.get("trials")} for c in comps]), hide_index=True, + column_config={"winning (evaluation)": st.column_config.NumberColumn(format="%.0f%%"), "profit factor": st.column_config.NumberColumn(format="%.2f")}) + strat = meta.get("strategies") or {} + if strat: + st.dataframe(pd.DataFrame([{"strategy": name, "arm": v.get("hold_min"), "edge line": v.get("line"), "trigger": json.dumps(v.get("thr") or {}, default=str), + "trades (evaluation)": _half(v.get("evaluation"), "n"), "winning": (_half(v.get("evaluation"), "win") or 0) * 100 if _half(v.get("evaluation"), "win") is not None else None, + "profit factor": _half(v.get("evaluation"), "pf")} for name, v in strat.items()]), hide_index=True, + column_config={"edge line": st.column_config.NumberColumn(format="%.3f", help="Minimum p̂ − fee-inclusive price to trade."), "winning": st.column_config.NumberColumn(format="%.0f%%"), + "profit factor": st.column_config.NumberColumn(format="%.2f")}) + + +@st.fragment(run_every="3s") +def retraining() -> None: + k = kalshi_state(); sup = get_supervisor(); ps = pipeline.state(); alive = sup.alive("kalshi_train"); pr = k["train_progress"] + running = alive and str(ps.get("stage", "")).startswith(("training", "bootstrapping")) + with st.container(border=True): + with st.container(horizontal=True, vertical_alignment="center"): + st.markdown(f":material/autorenew: **Kalshi training pipeline** · every {pipeline.INTERVAL_DAYS} days until the visual fly trades, then on request") + st.space("stretch") + if st.button("Retrain now", icon=":material/play_arrow:", type="primary", key="kretrain_now", disabled=running): + pipeline.request_run() + if not alive: + try: + sup.start("kalshi_train", started_by="console") + except RuntimeError as e: + st.error(str(e)) + st.rerun() + if running and st.button("Stop", icon=":material/stop:", key="ktrain_stop"): + sup.request_stop("kalshi_train"); st.rerun() + st.caption("1 · The Kalshi strategy stack (favorite, momentum, constraint, ev; taker and maker arms) is fitted walk-forward by settlement day at Kalshi's fees and judged against random picks. \n" + "2 · The visual fly is distilled from it once — when none exists for the current definitions, or on request — with its features entering the photoreceptors; " + "then it learns from every settlement. \n3 · The replay (`fly-trader kalshi-fly-replay`) must pass before it trades.") + if not alive: + st.warning("The Kalshi trainer is not running: start it on the Processes page or click Retrain now.", icon=":material/warning:") + with st.container(horizontal=True): + st.metric("Status", (ps.get("stage") or "idle").capitalize() if alive else "Off") + st.metric("Last run", ago(ps["last_run_at"]) if ps.get("last_run_at") else "never") + st.metric("Next run", _until(ps.get("next_run_at")) if ps.get("next_run_at") else ("as soon as the corpus is built" if alive else "—")) + st.metric("Latest selector", f"#{k['selector']['id']}" if k["selector"] else "—") + st.metric("Latest bootstrap", f"#{k['bootstrap']['id']}" if k["bootstrap"] else "—", delta=("may trade" if k["deployable"] else "may not trade") if k["bootstrap"] else None, delta_color="off") + if running and pr.get("total"): + st.progress(min(1.0, float(pr.get("step") or 0) / max(float(pr["total"]), 1.0)), text=f"{pr.get('stage', '')}" + (f" · about {float(pr['eta_s']) / 60:.0f} min left" if pr.get("eta_s") else "")) + if ps.get("stage") == "waiting for data" and ps.get("waiting"): + st.caption(f"Waiting for the corpus: {ps['waiting']}. Checking again {_until(ps.get('retry_after'))}.") + if ps.get("stage") == "failed" and ps.get("last_error"): + st.error(f"The last run failed: {ps['last_error']}. Retrying {_until(ps.get('retry_after'))}.") + + +@st.fragment(run_every="15s") +def visual_fly() -> None: + k = kalshi_state(); fly = k["fly"]; rp = k["replay"]; sup = get_supervisor() + running = bool(fly.get("stage")) and fly.get("stage") != "not trading"; frozen = bool(fly.get("learning_frozen")) + with st.container(border=True): + with st.container(horizontal=True, vertical_alignment="center"): + st.markdown(":material/visibility: **The visual fly** · the optic lobes taught once by the Kalshi stack, then learning from every settlement") + st.space("stretch") + if st.button("Resume learning" if frozen else "Pause learning", key="kfly_learn", disabled=not running): + fly_session.send_command("resume" if frozen else "pause"); st.toast("Sent: applied at the next minute") + if st.button("Roll back", key="kfly_rollback", disabled=not running, help="Restore the newest good hourly snapshot at least a day old (or the bootstrap)."): + fly_session.send_command("rollback"); st.toast("Sent: applied at the next minute") + if st.button("Re-bootstrap", key="kfly_reboot", help="The Kalshi stack refits and teaches a new visual fly."): + pipeline.request_run(fly=True, reason="console"); st.toast("Re-bootstrap requested") + with st.popover("Reset fly", disabled=sup.alive("kalshi_runner"), help="Stop the Kalshi engine first." if sup.alive("kalshi_runner") else None): + st.markdown("Archive and clear the visual fly's paper books, orders and everything it learned?") + if st.button("Reset", type="primary", key="kfly_reset_confirm"): + st.success(f"Archived to {reset_kalshi_fly(reason='console button')['archived_to']}") + if not rp: + st.info("The Kalshi replay has not run on the current definitions: `fly-trader kalshi-fly-replay` bootstraps a visual fly, lets it learn from months of settlements " + "without retraining, and judges it — the proof required before it trades.", icon=":material/info:") + else: + ev, rnd, fr = rp.get("evaluation") or {}, rp.get("random") or {}, rp.get("frozen") or {} + (st.success if rp.get("passed") else st.warning)(f"Replay {'passed' if rp.get('passed') else 'failed'}: {rp.get('reason')}", icon=":material/verified:" if rp.get("passed") else ":material/block:") + with st.container(horizontal=True): + st.metric("Replay trades", ev.get("n", "—"), border=True) + st.metric("Average per trade", pct(ev.get("mean")), delta=(f"random {pct(rnd.get('mean'))}" if rnd.get("mean") is not None else None), delta_color="off", border=True) + st.metric("Frozen fly", pct(fr.get("mean")), border=True, help="The same bootstrap without plasticity.") + st.metric("Profitable days", f"{ev.get('days_positive', '—')} of {ev.get('days', '—')}", border=True) + st.metric("Learning rate α", f"{rp.get('alpha') or 0:g}", border=True) + st.metric("Forgetting half-life", f"{rp['half_life_days']:g} days" if rp.get("half_life_days") else "none", border=True) + per = rp.get("per_strategy") or {} + if per: + st.caption("Per strategy: " + " · ".join(f"{s} ({v.get('arm')}): α {v.get('alpha')}, τ½ {v.get('half_life_days') or '∞'}" for s, v in per.items())) + if running: + ch = fly.get("checks") or {}; lines = fly.get("lines") or {}; fl = fly.get("frozen_lines") or {}; arms = fly.get("arms") or {} + st.dataframe(pd.DataFrame([{"strategy": s, "arm": arms.get(s), "edge line": float(lines[s]), "shadow line": float(fl.get(s, lines[s])), "drift": float((fly.get("drift_by_strategy") or {}).get(s) or 0) * 100, + "24 h IC": (ch.get(s) or {}).get("ic"), "checks": ", ".join((ch.get(s) or {}).get("triggers") or []) or "ok"} for s in lines]), hide_index=True, + column_config={"edge line": st.column_config.NumberColumn(format="%.3f"), "shadow line": st.column_config.NumberColumn(format="%.3f"), + "drift": st.column_config.NumberColumn(format="%.2f%%"), "24 h IC": st.column_config.NumberColumn(format="%.3f")}) + with st.container(horizontal=True): + st.metric("Settlements pending", fly.get("pending", "—"), help="Scored rows waiting for their market to resolve.") + st.metric("Learned this minute", (fly.get("learned") or {}).get("n", 0)) + st.metric("Learning", "frozen" if frozen else "on"); st.metric("Device", fly.get("device") or "—") + upd = q("SELECT hour, n, mean_abs_delta, drift, ic FROM kalshi_fly_updates ORDER BY hour DESC LIMIT 72") + if upd: + st.line_chart(pd.DataFrame(upd), x="hour", y=["drift", "ic"], x_label="", height=200) + rb = q("SELECT ts, reason, from_snapshot, to_snapshot FROM kalshi_fly_rollbacks ORDER BY id DESC LIMIT 20") + if rb: + with st.expander(f"Rollbacks ({len(rb)})", icon=":material/history:"): + st.dataframe(pd.DataFrame(rb), hide_index=True) + else: + st.caption(f"Not trading: {fly.get('detail') or k['trading_why']}") + b = k["bootstrap"] + if b: + m = b["meta"]; d = m.get("diagnostics") or {}; ok, why = KF.deployable(m) + st.caption(f"Bootstrap #{b['id']} ({ago(b['ts'])}): {m.get('photoreceptors', '?'):,} photoreceptors carry the inputs into {m.get('neurons', '?'):,} neurons; the mushroom body carries " + f"{pct(d.get('mbon_share'), 0, False)} of the score, {pct(d.get('mbon_saturated'), 0, False)} of MBONs saturated, slope {d.get('slope') or 0:.2f} against its teacher. " + + ("May trade: " if ok else "May not trade: ") + why + ".") + + +def history() -> None: + snaps = q("SELECT id, ts, kind, note FROM brain_snapshots WHERE kind IN ('kalshi_selector', 'kalshi_fly_selector') ORDER BY id DESC LIMIT 30") + with st.expander("Saved Kalshi models", icon=":material/folder_open:"): + rows = [] + for r in snaps: + m = parse_note(r["note"]); sel = r["kind"] == "kalshi_selector" + ok = kselector.is_current(m) if sel else KF.is_current(m) + wf = (m.get("walk_forward") or {}) if sel else {"n": (m.get("calibration") or {}).get("trades"), "mean": (m.get("calibration") or {}).get("mean")} + rows.append({"model": f"#{r['id']}", "type": "stack" if sel else "visual fly bootstrap", "saved": r["ts"], "data": "current" if ok else "outdated", + "trades": wf.get("n") if ok else None, "% / trade": (wf["mean"] * 100) if ok and wf.get("mean") is not None else None, + "put to work": (bool(m.get("deployable")) if sel else KF.deployable(m)[0]) if ok else None}) + if rows: + st.dataframe(pd.DataFrame(rows), hide_index=True, column_config={"saved": st.column_config.DatetimeColumn(format="MMM D HH:mm"), "% / trade": st.column_config.NumberColumn(format="%+.2f")}) + else: + st.caption("No saved Kalshi models.") + with st.expander("Kalshi trainer log", icon=":material/terminal:"): + st.code(get_supervisor().log_tail("kalshi_train", 40) or "(no log yet)", language="json") + + +stack() +retraining() +visual_fly() +history() diff --git a/fly_trader/ui/app_pages/kalshi_overview.py b/fly_trader/ui/app_pages/kalshi_overview.py new file mode 100644 index 0000000000000000000000000000000000000000..771b4084b3ad7bb1e8230d1c365ae983ed4dbc76 --- /dev/null +++ b/fly_trader/ui/app_pages/kalshi_overview.py @@ -0,0 +1,152 @@ +"""Kalshi overview: what the visual fly is doing in plain words, both arms' books, this minute's scoring, the shared +brain with both flies lit, and the open positions by category.""" +from datetime import datetime, timezone + +import pandas as pd +import streamlit as st + +from fly_trader import config +from fly_trader.db.queries import q, q1 +from fly_trader.ui import brain3d +from fly_trader.ui.brain3d import feed +from fly_trader.ui.common import KALSHI_BOOKS, ago, kalshi_state, pct, system_state, usd + + +def narrative(k: dict) -> str: + fly = k["fly"]; lines = [] + if fly.get("stage") and fly.get("stage") != "not trading": + arms = fly.get("arms") or {}; ln = fly.get("lines") or {} + parts = ", ".join(f"**{s}** ({arms.get(s, '?')}, edge ≥ {float(ln.get(s, 0)):.3f})" for s in ln) + money = ("Both arms also trade Kalshi subaccount " + f"{config.KALSHI_SUBACCOUNT} with real dollars (cap ${config.KALSHI_CAPITAL_USD:g})." if k["live_money"] + else "No real money is at risk" + (": live mirroring is off (KALSHI_LIVE_ENABLED=0)." if not config.KALSHI_LIVE_ENABLED else f" yet: {', '.join(k['prerequisites_missing'])}.")) + lines.append(f"**{k['trading_why']}** The visual fly (bootstrap **#{fly.get('bootstrap')}**) scores both sides of every quoted Kalshi market inside " + f"{config.KALSHI_MAX_DAYS_TO_CLOSE} days of its close, every minute: it predicts the probability the side pays and trades the **edge** — that probability " + f"minus the fee-inclusive price. Strategies: {parts}. The **taker arm** buys at the ask (an IOC at the book walk's worst level); the **maker arm** rests a " + f"bid one tick inside the ask and cancels or replaces it as the quote moves. Every position runs to settlement, which is also what teaches the fly. {money}") + else: + lines.append(f"**{k['trading_why']}**" + ("" if k["runner"] else " Start the Kalshi trading engine on the Processes page.")) + if k["training"]: + tr = k["train"]; pr = k["train_progress"] + lines.append(f"Kalshi training is running: {tr.get('stage', '')}" + (f" · {pr.get('stage', '')} ({pr.get('step')}/{pr.get('total')})" if pr.get("total") else "") + ".") + return "\n\n".join(lines) + + +@st.fragment(run_every="5s") +def brain() -> None: + """The whole brain: the memecoin fly's central-brain neurons and the Kalshi fly's optic lobes, each lit by its own minute.""" + s = system_state(); k = kalshi_state(); mfly, kfly = s["fly"], k["fly"] + with st.container(border=True): + st.markdown(":material/neurology: **One brain, two flies** · the optic lobes belong to the Kalshi fly (its inputs enter the photoreceptors), the antennal lobe " + "and the other senses to the memecoin fly; the central brain is shared and shows the fly chosen below") + try: + meta = brain3d.geometry_full() + except FileNotFoundError as e: + st.caption(f"No brain to draw: {e}"); return + st.session_state.setdefault("kbrain_live", True) + with st.container(horizontal=True, vertical_alignment="bottom"): + st.toggle("Follow live", key="kbrain_live", help="Show each fly's newest minute as it arrives; moving the slider turns this off.") + owner = st.segmented_control("Central brain shows", ["kalshi", "memecoin"], key="kbrain_owner", default="kalshi") or "kalshi" + mode = st.segmented_control("Learned pathways", list(feed.MODES), key="kbrain_pmode", default="since bootstrap") or "since bootstrap" + flies = [] + for name, fly, d, sdir in (("memecoin", mfly, feed.activity_dir(), feed.fly_state_dir()), ("kalshi", kfly, feed.kalshi_activity_dir(), feed.kalshi_fly_state_dir())): + running = bool(fly.get("stage")) and fly.get("stage") != "not trading" + minutes = feed.minutes(feed.activity_mtime(d), str(d)); act = None + if minutes: + key = f"kbrain_minute_{name}" + if st.session_state["kbrain_live"] or st.session_state.get(key) not in minutes: + st.session_state[key] = minutes[-1] + stamp = st.select_slider(f"{name} minute (UTC)", options=minutes, key=key, format_func=feed.label, on_change=lambda: st.session_state.__setitem__("kbrain_live", False)) + act = feed.activity_at(stamp, str(d)) + pth = feed.pathways(mode, fly, meta, fly_name=name) if running else {"mode": feed.MODES[mode], "key": "none", "reference": None, "items": [], "note": None, "changed": 0} + flies.append(brain3d.fly_payload(name, meta, act, pth, list((fly.get("channels") or {}).keys()))) + msg = None if any(f["activity_b64"] for f in flies) else "Neither fly has scored a minute yet: this is the resting brain." + brain3d.brain_view(brain3d.payload_flies(meta, flies, owner, message=msg), key="kbrain3d", height=600) + st.caption(" · ".join(f"{f['name']}: " + (f"{f['minute'][11:16]} UTC, {f['candidates']} candidates of {f['rows']} rows" if f["minute"] else "no minute") for f in flies)) + + +@st.fragment(run_every="5s") +def summary() -> None: + k = kalshi_state() + with st.container(border=True): + st.markdown(narrative(k)) + cols = st.columns(2) + for col, (arm, book) in zip(cols, (("taker", "paper_kalshi_taker"), ("maker", "paper_kalshi_maker"))): + w = k["books"].get(book) or {} + c = q1("SELECT count(*) AS n, sum(realized_cents) AS pnl, count(*) FILTER (WHERE realized_cents > 0) AS wins FROM kalshi_positions WHERE book = %s AND status <> 'open'", (book,)) or {} + spark = [float(r["wealth"]) for r in q("SELECT wealth FROM wealth_marks WHERE book = %s ORDER BY ts DESC LIMIT 180", (book,))][::-1] + n = int(c.get("n") or 0) + with col: + with st.container(border=True): + st.markdown(f"**{KALSHI_BOOKS[book]}**") + with st.container(horizontal=True): + st.metric("Wealth", usd(w.get("wealth")), delta=(f"{float(w['wealth']) - config.KALSHI_CAPITAL_USD:+.2f} $ since start" if w.get("wealth") is not None else None), + chart_data=spark or None, help=f"Started at ${config.KALSHI_CAPITAL_USD:g}; open positions at the bid net of the exit fee, resting bids at their collateral.") + st.metric("Settled P&L", usd((c.get("pnl") or 0) / 100.0, signed=True), help="Settled and closed positions, after fees.") + st.metric("Settled", n, delta=(f"{int(c.get('wins') or 0) / n:.0%} winners" if n else None), delta_color="off") + st.metric("Open", int(w.get("n_open") or 0), delta=(f"${float(w.get('exposure') or 0):.2f} at cost" if w else None), delta_color="off") + st.metric("Drawdown", pct(w.get("drawdown"), signed=False)) + if k["live_money"] or config.KALSHI_LIVE_ENABLED: + lv = k["fly"].get("live") or {} + with st.container(border=True): + st.markdown("**Live mirror** · " + (lv.get("stage") or "not running")) + if lv.get("wealth") is not None: + with st.container(horizontal=True): + st.metric("Live wealth", usd(lv.get("wealth"))); st.metric("Cash", usd(lv.get("cash"))) + st.metric("Taker", usd(lv.get("wealth_taker"))); st.metric("Maker", usd(lv.get("wealth_maker"))) + g = lv.get("gap") or {} + st.metric("Gap · taker", pct((g.get("taker") or {}).get("live_mean")), delta=f"mirror {pct((g.get('taker') or {}).get('mirror_mean'))}", delta_color="off") + st.metric("Gap · maker", pct((g.get("maker") or {}).get("live_mean")), delta=f"mirror {pct((g.get('maker') or {}).get('mirror_mean'))}", delta_color="off") + if lv.get("error"): + st.error(lv["error"]) + fly = k["fly"] + with st.container(border=True): + st.markdown("**This minute**") + if not fly.get("minute"): + st.caption("No minute scored yet." if k["runner"] else "The Kalshi engine is stopped.") + else: + b = fly.get("books") or {} + with st.container(horizontal=True): + st.metric("Minute (UTC)", str(fly["minute"])[11:16]) + st.metric("Markets quoted", int(fly.get("markets_active") or 0), help="Markets with a message this minute inside the entry window.") + st.metric("Eligible rows", int(fly.get("eligible") or 0), help="Both sides of each market passing the spread, depth, volume and time gates.") + st.metric("Picks", int(fly.get("picks") or 0)) + st.metric("Taker bought", int((b.get("taker") or {}).get("entered") or 0)) + st.metric("Maker posted / resting", f"{int((b.get('maker') or {}).get('posted') or 0)} / {int((b.get('maker') or {}).get('resting') or 0)}") + st.metric("Settled", int((b.get("taker") or {}).get("settled") or 0) + int((b.get("maker") or {}).get("settled") or 0)) + st.caption(f"Scored {ago(k['fly_at'])} · {fly.get('stage')}" + (f" · {k['trading_why']}" if k["trading"] != "trading" else "")) + + +@st.fragment(run_every="15s") +def positions() -> None: + left, right = st.columns([3, 2]) + with left: + with st.container(border=True): + st.markdown("**Wealth by book**") + marks = q("SELECT ts, book, wealth FROM wealth_marks WHERE book = ANY(%s) AND ts > now() - interval '30 days' ORDER BY ts", (list(KALSHI_BOOKS),)) + if marks: + st.line_chart(pd.DataFrame(marks).pivot_table(index="ts", columns="book", values="wealth"), x_label="", y_label="USD", height=240) + else: + st.caption("No marks yet.") + with right: + with st.container(border=True): + st.markdown("**Open positions**") + rows = q("""SELECT p.book, p.ticker, p.side, p.contracts, p.cost_cents, p.fee_cents, p.avg_price_cents, p.last_mark_cents, p.strategy, p.opened_at, m.close_time, m.title, + COALESCE(s.category, e.category) AS category + FROM kalshi_positions p LEFT JOIN kalshi_markets m USING (ticker) LEFT JOIN kalshi_events e USING (event_ticker) LEFT JOIN kalshi_series s ON s.ticker = e.series_ticker + WHERE p.book = ANY(%s) AND p.status = 'open' ORDER BY p.opened_at DESC""", (list(KALSHI_BOOKS),)) + if not rows: + st.caption("None open.") + else: + now = datetime.now(timezone.utc) + df = pd.DataFrame([{"book": KALSHI_BOOKS.get(r["book"], r["book"]), "market": (r["title"] or r["ticker"])[:60], "side": r["side"], "strategy": r["strategy"] or "—", + "contracts": float(r["contracts"]), "paid (c)": float(r["avg_price_cents"] or 0), "now (c)": float(r["last_mark_cents"]) if r["last_mark_cents"] is not None else None, + "closes in (h)": (r["close_time"] - now).total_seconds() / 3600 if r["close_time"] else None, "category": r["category"] or "—"} for r in rows]) + st.dataframe(df, hide_index=True, column_config={"paid (c)": st.column_config.NumberColumn(format="%.1f"), "now (c)": st.column_config.NumberColumn(format="%.0f"), + "closes in (h)": st.column_config.NumberColumn(format="%.1f")}) + cat = df.groupby("category")["contracts"].count().sort_values(ascending=False) + st.caption("By category: " + ", ".join(f"{c} {n}" for c, n in cat.items())) + + +brain() +summary() +positions() diff --git a/fly_trader/ui/app_pages/kalshi_trades.py b/fly_trader/ui/app_pages/kalshi_trades.py new file mode 100644 index 0000000000000000000000000000000000000000..c94fe22cf2ac29d32b75c30cb79ec367aa834dc6 --- /dev/null +++ b/fly_trader/ui/app_pages/kalshi_trades.py @@ -0,0 +1,74 @@ +"""Kalshi trades: settled positions with their results, resting and recent maker orders, exchange fills and the decision log, per book.""" +import pandas as pd +import streamlit as st + +from fly_trader.db.queries import q, q1 +from fly_trader.ui.common import KALSHI_BOOKS, pct, usd + +with st.container(horizontal=True, vertical_alignment="bottom"): + view = st.segmented_control("View", ["Settled positions", "Orders", "Fills", "Decision log"], default="Settled positions", key="ktrades_view") + book = st.selectbox("Book", list(KALSHI_BOOKS), format_func=lambda b: KALSHI_BOOKS[b], key="ktrades_book", width=220, + help="Paper books simulate at the quoted book with real fees; live books are the subaccount's own orders and fills.") + +if view == "Settled positions": + c = q1("SELECT count(*) AS n, sum(realized_cents) AS pnl, avg(realized_cents / NULLIF(cost_cents + fee_cents, 0)) AS avg_ret, count(*) FILTER (WHERE realized_cents > 0) AS wins, " + "sum(fee_cents) AS fees FROM kalshi_positions WHERE book = %s AND status <> 'open'", (book,)) or {} + n = int(c.get("n") or 0) + with st.container(horizontal=True): + st.metric("Positions", n, border=True) + st.metric("Realized P&L", usd((c.get("pnl") or 0) / 100.0, signed=True), border=True) + st.metric("Average return", pct(c.get("avg_ret")), border=True, help="Per position, on the dollars put down (cost plus fee).") + st.metric("Winners", f"{int(c.get('wins') or 0) / n:.0%}" if n else "—", border=True) + st.metric("Fees paid", usd((c.get("fees") or 0) / 100.0), border=True) + rows = q("""SELECT p.ticker, p.side, p.contracts, p.avg_price_cents, p.fee_cents, p.result, p.payout_cents, p.realized_cents, p.cost_cents, p.strategy, p.opened_at, p.closed_at, p.status, m.title + FROM kalshi_positions p LEFT JOIN kalshi_markets m USING (ticker) WHERE p.book = %s AND p.status <> 'open' ORDER BY p.closed_at DESC LIMIT 300""", (book,)) + if rows: + df = pd.DataFrame([{"market": (r["title"] or r["ticker"])[:70], "side": r["side"], "strategy": r["strategy"] or "—", "contracts": float(r["contracts"]), + "paid (c)": float(r["avg_price_cents"] or 0), "fee ($)": float(r["fee_cents"] or 0) / 100, "result": r["result"] or r["status"], + "payout ($)": float(r["payout_cents"] or 0) / 100, "P&L ($)": float(r["realized_cents"] or 0) / 100, + "return": float(r["realized_cents"] or 0) / (float(r["cost_cents"] or 0) + float(r["fee_cents"] or 0)) * 100 if r["cost_cents"] else None, + "opened": r["opened_at"], "settled": r["closed_at"]} for r in rows]) + st.dataframe(df, hide_index=True, column_config={"opened": st.column_config.DatetimeColumn(format="MMM D HH:mm"), "settled": st.column_config.DatetimeColumn(format="MMM D HH:mm"), + "paid (c)": st.column_config.NumberColumn(format="%.1f"), "fee ($)": st.column_config.NumberColumn(format="%.2f"), + "payout ($)": st.column_config.NumberColumn(format="%.2f"), "P&L ($)": st.column_config.NumberColumn(format="%+.2f"), + "return": st.column_config.NumberColumn(format="%+.1f%%")}) + else: + st.caption("No settled positions in this book yet.") + +elif view == "Orders": + rows = q("""SELECT o.ts, o.status, o.ticker, o.side, o.price_cents, o.count, o.fill_count, o.tif, o.post_only, o.expiration_ts, o.strategy, o.order_id, o.error, m.title + FROM kalshi_orders o LEFT JOIN kalshi_markets m USING (ticker) WHERE o.book = %s ORDER BY o.id DESC LIMIT 300""", (book,)) + resting = sum(1 for r in rows if r["status"] == "resting") + st.caption(f"{resting} resting · the maker arm rests a post-only bid one tick inside the ask and cancels or replaces it when the ask moves; a taker order is an IOC at the walk's worst level.") + if rows: + df = pd.DataFrame([{"time": r["ts"], "status": r["status"], "market": (r["title"] or r["ticker"])[:60], "side": r["side"], "price (c)": int(r["price_cents"]), "count": float(r["count"]), + "filled": float(r["fill_count"] or 0), "type": ("post-only " if r["post_only"] else "") + r["tif"].replace("_", " "), "expires": r["expiration_ts"], + "strategy": r["strategy"] or "—", "exchange id": r["order_id"] or "", "error": r["error"] or ""} for r in rows]) + st.dataframe(df, hide_index=True, column_config={"time": st.column_config.DatetimeColumn(format="MMM D HH:mm:ss"), "expires": st.column_config.DatetimeColumn(format="MMM D HH:mm")}) + else: + st.caption("No orders in this book yet.") + +elif view == "Fills": + rows = q("SELECT ts, ticker, side, action, price_cents, count, fee_cents, is_taker, order_id, trade_id FROM kalshi_fills WHERE book = %s OR (%s LIKE 'live%%' AND book IS NULL) ORDER BY ts DESC LIMIT 300", + (book, book)) + if rows: + df = pd.DataFrame([{"time": r["ts"], "market": r["ticker"], "side": r["side"], "action": r["action"], "price (c)": float(r["price_cents"]), "count": float(r["count"]), + "fee ($)": float(r["fee_cents"] or 0) / 100, "taker": bool(r["is_taker"]), "order": r["order_id"], "trade": r["trade_id"]} for r in rows]) + st.dataframe(df, hide_index=True, column_config={"time": st.column_config.DatetimeColumn(format="MMM D HH:mm:ss")}) + else: + st.caption("No exchange fills for this book (paper books have none: their fills are the positions themselves).") + +else: + kinds = {"Taker bought": ("kalshi_taker_enter",), "Maker posted": ("kalshi_maker_post",), "Maker filled": ("kalshi_maker_fill",), "Maker cancelled": ("kalshi_maker_cancel",), + "Settled": ("kalshi_settle",), "Exits": ("kalshi_exit",), "Not taken": ("blocked",)} + pick = st.pills("Show", list(kinds), selection_mode="multi", default=["Taker bought", "Maker posted", "Maker filled", "Settled"], key="kdec_kinds") + want = [k for p in (pick or []) for k in kinds[p]] + rows = q("SELECT ts, kind, mint, pool, m_hat, size_sol, rail, reason, detail FROM decisions WHERE kind = ANY(%s) AND detail->>'book' = %s ORDER BY id DESC LIMIT 300", (want, book)) if want else [] + if rows: + action = {v[0]: k for k, v in kinds.items()} + df = pd.DataFrame([{"time": r["ts"], "action": action.get(r["kind"], r["kind"]), "market": r["mint"], "side": r["pool"], "edge": float(r["m_hat"]) if r["m_hat"] is not None else None, + "dollars": float(r["size_sol"] or 0), "why": (f"{r['rail']}: " if r["rail"] else "") + (r["reason"] or "")} for r in rows]) + st.dataframe(df, hide_index=True, column_config={"time": st.column_config.DatetimeColumn(format="MMM D HH:mm:ss"), "edge": st.column_config.NumberColumn(format="%.3f"), + "dollars": st.column_config.NumberColumn(format="%.2f")}) + else: + st.caption("No decisions recorded for this book yet.") diff --git a/fly_trader/ui/app_pages/overview.py b/fly_trader/ui/app_pages/overview.py index ddcb3b916e526055f3dbfc2b6ef9994e572346b6..d7cdc2d7fcd11391d0b0853074964622d1034802 100644 --- a/fly_trader/ui/app_pages/overview.py +++ b/fly_trader/ui/app_pages/overview.py @@ -9,19 +9,21 @@ from fly_trader.agent.selector_session import pinned_snapshot from fly_trader.db.queries import q, q1 from fly_trader.ui import brain3d from fly_trader.ui.brain3d import feed -from fly_trader.ui.common import BOOK, ago, backtest_line, labels, pct, sol, system_state +from fly_trader.ui.common import ago, amount, backtest_line, chain_picker, labels, pct, system_state def narrative(s: dict) -> str: - m = s["model"]; lines = [] + m = s["model"]; lines = []; mk = s["market"]; u = mk.unit; rh = mk.chain != "sol" if m: - run = m.get("run") or {}; meta = m["meta"] - money = "No real money is at risk." if not s["live_money"] else "Trades spend real SOL from the bot wallet." + run = m.get("run") or {}; meta = m["meta"]; mm = meta.get("model") or {} + money = "No real money is at risk." if not s["live_money"] else f"Trades spend real {u} from the bot wallet." + what = "graduated Pons pool on Robinhood Chain (prices in ETH, a non-ETH quote converted at the minute's base price)" if rh else "PumpSwap token" + resq = f"{mk.min_resq():.3g}" if rh else mm.get("min_resq_sol", "—"); vol = f"{mk.min_vol_15m():.3g}" if rh else mm.get("min_vol_15m_sol", "—") lines.append(f"**{s['trading_why']}** The selector is paper trading with model **#{m['id']}** (trained through {meta.get('trained_through', '—')}). " - f"Every minute it predicts the net {meta.get('horizon_min', '—')}-minute return of each PumpSwap token that traded, at least {(meta.get('model') or {}).get('min_age_h', '—')} hours " - f"past graduation, with a pool of at least {(meta.get('model') or {}).get('min_resq_sol', '—')} SOL and {(meta.get('model') or {}).get('min_vol_15m_sol', '—')} SOL traded in the last 15 minutes, and buys those predicted to make at least " + f"Every minute it predicts the net {meta.get('horizon_min', '—')}-minute return of each {what} that traded, at least {mm.get('min_age_h', '—')} hours " + f"past graduation, with a pool of at least {resq} {u} and {vol} {u} traded in the last 15 minutes, and buys those predicted to make at least " f"**{pct(float(run.get('threshold') or meta.get('threshold') or 0), 1)}** — sized by how certain the " - f"model is (up to {config.MAX_POSITION_FRACTION:.0%} of the bankroll, never the {config.GAS_RESERVE_SOL:g} SOL gas reserve) — " + f"model is (up to {config.MAX_POSITION_FRACTION:.0%} of the bankroll, never the {mk.gas_reserve():g} {u} gas reserve) — " f"and sells it **{run.get('horizon_min') or meta.get('horizon_min') or 30} minutes** later. {money}") lines.append(backtest_line(meta)) if pinned_snapshot() == m["id"]: @@ -68,8 +70,8 @@ def brain() -> None: @st.fragment(run_every="5s") -def summary() -> None: - s = system_state() +def summary(mk) -> None: + s = system_state(mk); BOOK = mk.selector_book; u = mk.unit; cap = mk.capital() with st.container(border=True): st.markdown(narrative(s)) w = q1("SELECT wealth, exposure, n_open, drawdown FROM wealth_marks WHERE book = %s ORDER BY ts DESC LIMIT 1", (BOOK,)) or {} @@ -77,11 +79,11 @@ def summary() -> None: spark = [float(r["wealth"]) for r in q("SELECT wealth FROM wealth_marks WHERE book = %s ORDER BY ts DESC LIMIT 180", (BOOK,))][::-1] n = int(c.get("n") or 0) with st.container(horizontal=True): - st.metric("Paper wealth", sol(w.get("wealth")), delta=(f"{float(w['wealth']) - config.CAPITAL_SOL:+.4f} SOL since start" if w.get("wealth") is not None else None), - border=True, chart_data=spark or None, help=f"Started at {config.CAPITAL_SOL:g} SOL; open positions valued net of exit costs.") - st.metric("Realized P&L", sol(c.get("pnl"), signed=True), border=True, help="Closed trades, after fees and price impact.") + st.metric("Paper wealth", amount(w.get("wealth"), u), delta=(f"{float(w['wealth']) - cap:+.4f} {u} since start" if w.get("wealth") is not None else None), + border=True, chart_data=spark or None, help=f"Started at {cap:g} {u}; open positions valued net of exit costs.") + st.metric("Realized P&L", amount(c.get("pnl"), u, signed=True), border=True, help="Closed trades, after fees and price impact.") st.metric("Closed trades", n, delta=(f"{int(c.get('wins') or 0) / n:.0%} winners" if n else None), delta_color="off", border=True) - st.metric("Open positions", int(w.get("n_open") or 0), delta=(f"{float(w.get('exposure') or 0):.2f} SOL at cost" if w else None), delta_color="off", border=True) + st.metric("Open positions", int(w.get("n_open") or 0), delta=(f"{float(w.get('exposure') or 0):.3g} {u} at cost" if w else None), delta_color="off", border=True) st.metric("Drawdown", pct(w.get("drawdown"), signed=False), border=True, help="Below the book's peak wealth.") sel = s["selector"] with st.container(border=True): @@ -93,7 +95,7 @@ def summary() -> None: with st.container(horizontal=True): st.metric("Minute (UTC)", str(sel["minute"])[11:16]) st.metric("Tokens traded", int(sel.get("mints_traded") or 0)) - st.metric("Eligible", int(sel.get("eligible") or 0), help="Pool ≥ 20 SOL and ≥ 5 SOL traded in the last 15 minutes.") + st.metric("Eligible", int(sel.get("eligible") or 0), help=f"Pool ≥ {mk.min_resq():.3g} {u} and ≥ {mk.min_vol_15m():.3g} {u} traded in the last 15 minutes.") st.metric("Best score", f"{float(sel.get('score_max') or 0):.3f}", delta=f"needs {thr:.3f}", delta_color="off") st.metric("Picks", int(sel.get("picks") or 0)) st.metric("Bought / sold", f"{int(sel.get('entered') or 0)} / {int(sel.get('exited') or 0)}") @@ -108,14 +110,15 @@ def summary() -> None: @st.fragment(run_every="15s") -def book() -> None: +def book(mk) -> None: + BOOK = mk.selector_book; u = mk.unit left, right = st.columns([3, 2]) with left: with st.container(border=True): st.markdown("**Paper wealth**") hist = q("SELECT ts, wealth FROM wealth_marks WHERE book = %s ORDER BY ts", (BOOK,)) if hist: - st.line_chart(pd.DataFrame(hist), x="ts", y="wealth", x_label="", y_label="SOL", height=240) + st.line_chart(pd.DataFrame(hist), x="ts", y="wealth", x_label="", y_label=u, height=240) else: st.caption("No marks yet.") with right: @@ -125,49 +128,50 @@ def book() -> None: if not rows: st.caption("None open.") else: - hold = int((system_state().get("model") or {}).get("run", {}).get("horizon_min") or 30) + hold = int((system_state(mk).get("model") or {}).get("run", {}).get("horizon_min") or 30) now = datetime.now(timezone.utc); names = labels([r["mint"] for r in rows]) df = pd.DataFrame([{"token": names.get(r["mint"], r["mint"][:6]), "strategy": r["strategy"] or "—", "held (min)": (now - r["opened_at"]).total_seconds() / 60, "exits in (min)": max(0.0, (float(r["hold_s"]) / 60 if r["hold_s"] else hold) - (now - r["opened_at"]).total_seconds() / 60), - "cost (SOL)": float(r["cost_sol"]), + f"cost ({u})": float(r["cost_sol"]), "return": (float(r["last_mark_price"]) / float(r["entry_price"]) - 1) * 100 if r["entry_price"] and r["last_mark_price"] else None} for r in rows]) st.dataframe(df, hide_index=True, column_config={"held (min)": st.column_config.NumberColumn(format="%.0f"), "exits in (min)": st.column_config.NumberColumn(format="%.0f"), - "cost (SOL)": st.column_config.NumberColumn(format="%.3f"), + f"cost ({u})": st.column_config.NumberColumn(format="%.4g"), "return": st.column_config.NumberColumn("return (gross)", format="%+.2f%%")}) @st.fragment(run_every="30s") -def race() -> None: - s = system_state(); ho = s["handover"]; fly = s["fly"] - marks = q("SELECT ts, book, wealth FROM wealth_marks WHERE book IN ('paper_selector', 'paper_fly', 'live') AND ts > now() - interval '30 days' ORDER BY ts") +def race(mk) -> None: + s = system_state(mk); ho = s["handover"]; fly = s["fly"]; u = mk.unit; books = list(mk.books) + marks = q("SELECT ts, book, wealth FROM wealth_marks WHERE book = ANY(%s) AND ts > now() - interval '30 days' ORDER BY ts", (books,)) if not marks and not (fly.get("stage") and fly.get("stage") != "not trading"): return since = datetime.now(timezone.utc) - timedelta(days=14) - pnl = {r["book"]: r for r in q("SELECT book, COALESCE(sum(realized_sol), 0) AS pnl, count(*) AS n FROM positions WHERE book IN ('paper_selector', 'paper_fly', 'live') " - "AND status = 'closed' AND closed_at >= %s GROUP BY book", (since,))} - start = (q1("SELECT min(ts) AS t FROM wealth_marks WHERE book = 'paper_fly'") or {}).get("t") + pnl = {r["book"]: r for r in q("SELECT book, COALESCE(sum(realized_sol), 0) AS pnl, count(*) AS n FROM positions WHERE book = ANY(%s) " + "AND status = 'closed' AND closed_at >= %s GROUP BY book", (books, since))} + start = (q1("SELECT min(ts) AS t FROM wealth_marks WHERE book = %s", (mk.fly_book,)) or {}).get("t") with st.container(border=True): st.markdown(":material/sports_score: **The race: selector vs plastic fly**") if ho: - st.success(f"The fly took the selector's seat {ago(ho.get('at'))}: {float(ho.get('fly_pnl_sol') or 0):+.3f} SOL vs {float(ho.get('selector_pnl_sol') or 0):+.3f} SOL " + st.success(f"The fly took the selector's seat {ago(ho.get('at'))}: {float(ho.get('fly_pnl_sol') or 0):+.4g} {u} vs {float(ho.get('selector_pnl_sol') or 0):+.4g} {u} " f"over {ho.get('days')} days. " + ("It trades the bot wallet; its paper book is the mirror." if s["live_money"] - else "Live trading waits for LIVE_ENABLED=1, the live prerequisites and a funded wallet."), icon=":material/swap_horiz:") + else f"Live trading waits for {'RH_LIVE_ENABLED' if mk.chain != 'sol' else 'LIVE_ENABLED'}=1, the live prerequisites and a funded wallet."), icon=":material/swap_horiz:") else: days = (datetime.now(timezone.utc) - start).total_seconds() / 86400 if start else 0.0 st.caption("The fly takes the seat when its realized P&L over the last 14 days is at least the selector's, with at least 30 trades" + (f" · racing for {days:.1f} days." if start else " · the race starts when the fly first trades.")) with st.container(horizontal=True): - for book, title in (("paper_selector", "Selector · 14 days"), ("paper_fly", "Fly · 14 days")) + ((("live", "Live · 14 days"),) if ho else ()): + for book, title in ((mk.selector_book, "Selector · 14 days"), (mk.fly_book, "Fly · 14 days")) + (((mk.live_book, "Live · 14 days"),) if ho else ()): r = pnl.get(book) or {} - st.metric(title, sol(r.get("pnl"), signed=True), delta=f"{int(r.get('n') or 0)} trades", delta_color="off", border=True, help="Realized P&L of closed trades.") + st.metric(title, amount(r.get("pnl"), u, signed=True), delta=f"{int(r.get('n') or 0)} trades", delta_color="off", border=True, help="Realized P&L of closed trades.") if marks: df = pd.DataFrame(marks).pivot_table(index="ts", columns="book", values="wealth") - st.line_chart(df, x_label="", y_label="SOL", height=240) + st.line_chart(df, x_label="", y_label=u, height=240) +mk = chain_picker() brain() -summary() -race() -book() +summary(mk) +race(mk) +book(mk) diff --git a/fly_trader/ui/app_pages/processes.py b/fly_trader/ui/app_pages/processes.py index 29c52b457f8731ef52c805e99599527513895a51..1714c817fa5013dcfcf1c292e5c074b8c9a5159a 100644 --- a/fly_trader/ui/app_pages/processes.py +++ b/fly_trader/ui/app_pages/processes.py @@ -9,7 +9,7 @@ from fly_trader.db.connection import transaction from fly_trader.db.queries import q, q1 from fly_trader.ops import procs from fly_trader.ops.supervisor import get_supervisor -from fly_trader.ui.common import ORDER, ROLE_LABEL, WORKER_INFO, ago, jv +from fly_trader.ui.common import ORDER, ROLE_LABEL, WORKER_INFO, WORKER_TYPE, ago, jv def _set_autostart(name: str) -> None: @@ -41,7 +41,10 @@ def workers() -> None: st.markdown(f"External **{name}** · pid {a['pid']} · started {ago(a['started_at'])}") if st.button("Stop external", key=f"ext_stop_{name}"): procs.stop(name); st.rerun() + shown = None for name in ORDER: + if WORKER_TYPE.get(name) != shown: + shown = WORKER_TYPE.get(name); st.markdown(f"#### {shown.capitalize()}") title, icon, desc, role = WORKER_INFO[name]; stt = status[name] state = "stopping" if stt["stopping"] else "running" if stt["alive"] else "crashed" if stt.get("error") else "stopped" with st.container(border=True): diff --git a/fly_trader/ui/app_pages/rh.py b/fly_trader/ui/app_pages/rh.py new file mode 100644 index 0000000000000000000000000000000000000000..e7b6f936b73fd2fac0f9d488c26be088b5405849 --- /dev/null +++ b/fly_trader/ui/app_pages/rh.py @@ -0,0 +1,128 @@ +"""Robinhood Chain: the Pons indexer, the quote assets' marks, the per-chain verdict and seat, and the ETH wallet's +intents, transactions, legs and base lots. Rails (circuit 3) and limits live on the Safety page.""" +from datetime import datetime, timezone + +import pandas as pd +import streamlit as st + +from fly_trader import config, markets +from fly_trader.db.queries import q, q1 +from fly_trader.ui.common import ago, amount, labels, pct, setting, system_state + +if not config.RH_ENABLED: + st.info("Robinhood Chain is off (RH_ENABLED=0).", icon=":material/info:") + st.stop() + +RH = markets.RH + + +@st.fragment(run_every="10s") +def indexer() -> None: + st_, st_at = setting("rh_stream_status") + with st.container(border=True): + st.markdown(":material/sensors: **Indexer** · Pons launches, graduations and Uniswap v4 swaps from the chain into 1-minute rows in ETH") + if not st_: + st.caption("The indexer has not run yet: start the Robinhood Chain feed on the Processes page."); return + with st.container(horizontal=True): + st.metric("Mode", str(st_.get("mode") or "—"), border=True) + st.metric("Blocks behind", f"{int(st_.get('behind_blocks') or 0):,}", border=True, help=f"head {st_.get('head')}, indexed through {st_.get('index_through')}") + mt = st_.get("minutes_through") + st.metric("Minutes through", datetime.fromtimestamp(int(mt), timezone.utc).strftime("%b %d %H:%M") if mt else "—", border=True, + help="The engine trades only minutes the indexer has written.") + st.metric("Updated", ago(st_at), border=True) + if st_.get("error"): + st.warning(f"Last error: {st_['error']}", icon=":material/warning:") + rows = {r["t"]: r["n"] for r in q("SELECT 'launches' AS t, count(*) AS n FROM rh_tokens UNION ALL SELECT 'graduated', count(*) FROM rh_tokens WHERE status = 'graduated' " + "UNION ALL SELECT 'swaps', count(*) FROM rh_swaps UNION ALL SELECT 'minutes', count(*) FROM rh_minutes")} + st.caption(" · ".join(f"{k}: {int(v):,}" for k, v in rows.items())) + + +@st.fragment(run_every="30s") +def assets() -> None: + rows = q("""SELECT a.symbol, a.class, a.decimals, a.asset, p.ts, p.price_eth, p.stale FROM rh_assets a + LEFT JOIN LATERAL (SELECT ts, price_eth, stale FROM rh_base_prices b WHERE b.asset = a.asset ORDER BY ts DESC LIMIT 1) p ON true + WHERE a.approved ORDER BY a.class, a.symbol""") + with st.container(border=True): + st.markdown(":material/currency_exchange: **Quote assets** · a coin quoted in one of these is bought ETH → base → coin in one transaction and sold back to ETH at once") + if not rows: + st.caption("No approved quote assets indexed yet."); return + now = datetime.now(timezone.utc) + df = pd.DataFrame([{"asset": r["symbol"] or r["asset"][:10], "class": r["class"], "ETH per unit": 1.0 if r["class"] == "eth" else r["price_eth"], + "marked": "native" if r["class"] == "eth" else (ago(r["ts"]) if r["ts"] else "never"), + "entries": "open" if r["class"] == "eth" or (r["ts"] and not r["stale"] and (now - r["ts"]).total_seconds() < 600) else "blocked: mark stale"} + for r in rows]) + st.dataframe(df, hide_index=True, column_config={"ETH per unit": st.column_config.NumberColumn(format="%.6g")}) + + +@st.fragment(run_every="30s") +def verdict_and_seat() -> None: + v, v_at = setting("fly_replay"); ho, _ = setting(RH.handover_key); s = system_state(RH) + with st.container(border=True): + st.markdown(":material/gavel: **Verdict and seat** · one combined replay judges each chain on its own trades") + chains = (v or {}).get("chains") or {} + if not chains: + st.caption("No combined replay verdict yet: the RH books wait for one (the Robinhood corpus must be built and the models trained on both chains).") + else: + with st.container(horizontal=True): + for c, r in chains.items(): + ev = r.get("evaluation") or {} + st.metric(markets.for_chain(c).name, "passed" if r.get("passed") else "failed", border=True, + delta=f"{int(ev.get('n') or 0)} trades · mean {pct(ev.get('mean'))}" if ev else None, delta_color="off", help=r.get("reason")) + st.caption(f"Replay {ago(v_at)}.") + if ho: + st.success(f"The RH fly holds the seat since {ago(ho.get('at'))}." + (" It trades the ETH wallet." if s["live_money"] else + " Live trading waits for RH_LIVE_ENABLED=1, the key, the pinned address and a funded wallet."), icon=":material/swap_horiz:") + else: + st.caption("The RH fly takes the seat when its realized P&L over 14 days is at least the RH selector's, with at least 30 trades.") + + +@st.fragment(run_every="15s") +def wallet() -> None: + wm = q1("SELECT ts, native_wei, consistent FROM rh_wallet_marks ORDER BY ts DESC LIMIT 1") or {} + w = q1("SELECT wealth, exposure, n_open, drawdown FROM wealth_marks WHERE book = %s ORDER BY ts DESC LIMIT 1", (RH.live_book,)) or {} + lots = q("SELECT asset, qty_raw, basis_eth, source, position_id, opened_at FROM rh_base_lots WHERE status = 'open' ORDER BY id") + with st.container(border=True): + st.markdown(":material/account_balance_wallet: **ETH wallet** · book live_rh") + st.code(config.RH_BOT_ADDRESS or "RH_BOT_ADDRESS is not set", language=None) + with st.container(horizontal=True): + st.metric("Native ETH", amount(int(wm["native_wei"]) / 1e18 if wm.get("native_wei") is not None else None, "ETH", 6), border=True, + help=f"at the last reconciliation, {ago(wm.get('ts'))}" if wm else "not reconciled yet") + st.metric("Live wealth", amount(w.get("wealth"), "ETH", 6), border=True) + st.metric("Open positions", int(w.get("n_open") or 0), border=True) + st.metric("Books match chain", "yes" if wm.get("consistent", True) else "NO", border=True, + help="Token balances vs open positions and lots; native ETH vs every booked flow since the last check.") + if lots: + st.markdown("Base lots (two-leg routes that stopped half way; sold back to ETH hourly)") + st.dataframe(pd.DataFrame(lots), hide_index=True) + + +@st.fragment(run_every="15s") +def activity() -> None: + with st.container(border=True): + st.markdown(":material/receipt_long: **Live activity**") + view = st.segmented_control("Show", ["Intents", "Transactions", "Legs"], default="Intents", key="rh_activity") or "Intents" + if view == "Intents": + rows = q("SELECT created_at, kind, token, route, state, attempts, error FROM rh_intents ORDER BY id DESC LIMIT 100") + if rows: + names = labels([r["token"] for r in rows if r["token"]]) + for r in rows: + r["token"] = names.get(r["token"], (r["token"] or "")[:10]) + elif view == "Transactions": + rows = q("SELECT created_at, kind, nonce, status, gas_used, fee_wei::float / 1e18 AS fee_eth, router, hash FROM rh_txs ORDER BY id DESC LIMIT 100") + else: + rows = q("SELECT ts, leg_no, asset_in, asset_out, amount_in_raw::text AS amount_in, amount_out_raw::text AS amount_out, eth_in, eth_out, verified_by " + "FROM rh_legs ORDER BY id DESC LIMIT 100") + if rows: + st.dataframe(pd.DataFrame(rows), hide_index=True) + else: + st.caption("Nothing yet: the live RH book has not traded.") + + +indexer() +left, right = st.columns([3, 2]) +with left: + verdict_and_seat() + assets() +with right: + wallet() +activity() diff --git a/fly_trader/ui/app_pages/safety.py b/fly_trader/ui/app_pages/safety.py index dc26b30a1c2f1af700cf5e0ad645f8f50d35143f..79757bebee0e2564e2d79ebf14ec4a4b7ac241bd 100644 --- a/fly_trader/ui/app_pages/safety.py +++ b/fly_trader/ui/app_pages/safety.py @@ -1,40 +1,191 @@ -"""Safety & wallet: the trading rails and their controls, the configured mode and limits, the bot wallet, maintenance.""" +"""Safety & wallet: both chains' bot wallets (balance in USD, withdrawals to your own wallets), the trading rails and their +controls, the configured mode and limits, maintenance.""" import pandas as pd import streamlit as st -from fly_trader import config +from fly_trader import config, markets from fly_trader.agent import rails from fly_trader.db.queries import q, q1 +from fly_trader.ops import wallets from fly_trader.ops.supervisor import get_supervisor -from fly_trader.ui.common import ago +from fly_trader.ui.common import ago, amount, usd + +BASE_UNITS = {"sol": config.LAMPORTS_PER_SOL, "rh": 10 ** 18} +PRICE_SOURCE = {"sol": "Jupiter", "rh": "KyberSwap ETH→USDG"} +NO_WALLET = {"sol": "No wallet yet: run `fly-trader wallet new`, then fund it with SOL.", + "rh": "No wallet yet: run `fly-trader rh-wallet new`, then fund it with ETH on Robinhood Chain."} + + +@st.cache_data(ttl="15s", show_spinner=False) +def wallet_summary(chain: str) -> dict: + return wallets.summary(chain) + + +@st.cache_data(ttl="10s", show_spinner=False) +def withdraw_limits(chain: str, to: str) -> dict: + return wallets.limits(chain, to) + + +def short(a: str | None) -> str: + return f"{a[:6]}…{a[-4:]}" if a else "—" + + +@st.fragment(run_every="30s") +def wallet_balances(chain: str) -> None: + spec, w = markets.MARKETS[chain], wallet_summary(chain) + with st.container(horizontal=True, vertical_alignment="center"): + st.markdown(f":material/account_balance_wallet: **{spec.name} bot wallet**") + st.space("stretch") + if w["address"]: + st.link_button("Explorer", wallets.explorer(chain, "address", w["address"]), icon=":material/open_in_new:", type="tertiary") + if not w["address"]: + st.caption(NO_WALLET[chain]) + return + st.code(w["address"], language=None) + with st.container(horizontal=True): + st.metric("Balance", usd(w["native_usd"]), delta=amount(w["native"], w["unit"]), delta_color="off", delta_arrow="off", + help="What the wallet holds in its own coin; this is what can be withdrawn.") + st.metric("In live positions", usd(w["positions_usd"]), delta=f"{amount(w['positions'], w['unit'])} · {w['n_open']} open", delta_color="off", + delta_arrow="off", help="The live book's open positions at its last mark, net of exit cost" + + (f" (marked {ago(w['marked_at'])})." if w["marked_at"] else "; at cost before the first mark.")) + st.metric("Total", usd(w["total_usd"]), delta=amount(None if w["native"] is None else w["native"] + w["positions"], w["unit"]), + delta_color="off", delta_arrow="off") + st.caption(f"1 {w['unit']} = {usd(w['price_usd'])} · {PRICE_SOURCE[chain]}, {ago(w['price_at'])}" if w["price_usd"] else f"No {w['unit']}/USD price yet.") + if w["error"]: + st.warning(f"Balance unavailable: {w['error']}") + + +def send(chain: str, amt: float | None, to: str) -> dict: + try: + return wallets.withdraw(chain, amt, to) + except wallets.REFUSALS as e: + return {"status": "refused", "error": str(e)} + except Exception as e: # noqa: BLE001 — shown; the outcome is unknown + return {"status": "error", "error": f"{type(e).__name__}: {e}"} + + +def open_withdraw(chain: str) -> None: + for k in ("result", "amt", "all"): # a fresh form each time it opens + st.session_state.pop(f"{chain}_wd_{k}", None) + st.session_state["withdraw_open"] = chain + + +def close_withdraw() -> None: + st.session_state.pop("withdraw_open", None) + + +def show_result(chain: str, r: dict) -> None: + link = f" · [transaction]({wallets.explorer(chain, 'tx', r['tx'])})" if r.get("tx") else "" + what = f"{amount(r.get('amount'), markets.MARKETS[chain].unit, 6)} to `{short(r.get('to'))}`" + if r["status"] == "confirmed": + st.success(f"Sent {what}{link}", icon=":material/check_circle:") + elif r["status"] == "pending": + st.info(f"Sent {what}, no receipt yet: it is booked when it lands. Look at the transaction before sending again{link}") + elif r["status"] == "expired": + st.warning("The transaction expired before it landed: nothing moved. You can try again.") + elif r["status"] == "failed": + st.error(f"The transaction failed on chain: only the fee moved{link}") + elif r["status"] == "refused": + st.error(f"Nothing was sent: {r['error']}") + elif r["status"] == "sending": + st.info("The withdrawal was still running when this window refreshed: its outcome is in the wallet's withdrawals.") + else: + st.error(f"{r['error']}. Whether it went out is unknown: look at the wallet on the explorer before trying again.") + if st.button("Close", key=f"{chain}_wd_close", on_click=close_withdraw): + st.rerun() + + +@st.dialog("Withdraw", width="medium", on_dismiss=close_withdraw) +def withdraw_dialog(chain: str) -> None: + key, unit, base = f"{chain}_wd", markets.MARKETS[chain].unit, BASE_UNITS[chain] + if (done := st.session_state.get(f"{key}_result")) is not None: # one withdrawal per opening: a second click never sends again + show_result(chain, done) + return + w = wallet_summary(chain) + to = st.selectbox("To", wallets.destinations(chain), key=f"{key}_to", help=f"Only your own wallets: {wallets.SETTINGS[chain]} in .env.") + if to is None: + st.warning(f"None of your own wallets is listed: add yours to {wallets.SETTINGS[chain]} in .env and restart the console.") + return + try: + lim = withdraw_limits(chain, to) + except Exception as e: # noqa: BLE001 + st.error(f"Cannot read the wallet: {type(e).__name__}: {e}") + return + everything = st.toggle(f"Everything withdrawable: {amount(lim['max'] / base, unit, 6)}", key=f"{key}_all") + amt = None if everything else st.number_input(f"Amount ({unit})", min_value=0.0, value=None, step=0.01 if chain == "sol" else 0.001, + format="%.6f", placeholder=f"at most {lim['max'] / base:.6f}", key=f"{key}_amt") + value, why = wallets.plan(chain, lim, amt) if everything or amt else (0, "enter an amount") + fee = f"fee ≈ {lim['fee'] / base:.6f} {unit}" if chain == "sol" else f"gas at most {lim['fee'] / base:.6f} {unit}" + keep = f" {lim['keep'] / base:g} {unit} stays for the exits of {lim['n_open']} open live position(s)." if lim["keep"] else "" + if why and (everything or amt): + st.warning(why) + elif not why: + px = w.get("price_usd") + st.markdown(f"Sends **{amount(value / base, unit, 6)}**" + (f" ({usd(value / base * px)})" if px else "") + f" to `{short(to)}`; {fee}.{keep}") + st.caption("This waits for the network's confirmation, up to about a minute.") + if st.button(f"Withdraw {amount(value / base, unit, 6)}" if not why else "Withdraw", type="primary", disabled=bool(why), icon=":material/send:", key=f"{key}_go"): + with st.spinner("Signing, sending, waiting for confirmation…"): + st.session_state[f"{key}_result"] = {"status": "sending"} + st.session_state[f"{key}_result"] = send(chain, amt, to) + wallet_summary.clear() + withdraw_limits.clear() + st.rerun() # the dialog stays open on its result; the page refreshes + + +def wallet_panel(chain: str) -> None: + with st.container(border=True): + wallet_balances(chain) + addr = wallets.address(chain) + if not addr: + return + blockers = wallets.withdraw_blockers(chain) + with st.container(horizontal=True, vertical_alignment="center"): + st.caption("Withdraw: " + "; ".join(blockers) + "." if blockers else f"Withdrawals go only to your own wallets ({wallets.SETTINGS[chain]} in .env).") + st.space("stretch") + st.button("Withdraw", key=f"{chain}_withdraw", icon=":material/move_up:", disabled=bool(blockers), on_click=open_withdraw, args=(chain,)) + hist = wallets.history(chain, addr) + if hist: + with st.expander(f"Withdrawals ({len(hist)})"): + df = pd.DataFrame(hist).assign(tx=lambda d: [wallets.explorer(chain, "tx", t) if t else None for t in d["tx"]]) + st.dataframe(df, hide_index=True, column_config={"amount": st.column_config.NumberColumn(markets.MARKETS[chain].unit, format="%.6f"), + "tx": st.column_config.LinkColumn("transaction", display_text=":material/open_in_new:")}) + if chain == "sol": + ev = q("SELECT ts, kind, pubkey FROM wallet_events ORDER BY id DESC LIMIT 20") + if ev: + with st.expander("Wallet events"): + st.dataframe(pd.DataFrame(ev), hide_index=True) @st.fragment(run_every="5s") -def rails_panel() -> None: - c = q1("SELECT * FROM circuit_state WHERE id = 1") or {} +def rails_panel(circuit_id: int = 1, title: str = "Trading rails", prefix: str = "") -> None: + c = q1("SELECT * FROM circuit_state WHERE id = %s", (circuit_id,)) or {} with st.container(border=True): - st.markdown(":material/shield: **Trading rails**") + st.markdown(f":material/shield: **{title}**") with st.container(horizontal=True, vertical_alignment="center"): st.badge("Kill switch tripped" if c.get("kill_switch") else "Kill switch armed", color="red" if c.get("kill_switch") else "green") st.caption("Trips when the live book falls too far below its peak wealth; blocks new entries until cleared." + (f" Reason: {c['kill_reason']}." if c.get("kill_reason") else "")) st.space("stretch") - if st.button("Clear kill switch", key="kill_clear", disabled=not c.get("kill_switch"), help="Clears the switch and re-bases the peak at current wealth."): - rails.reset_circuit(kill=True); st.toast("Kill switch cleared"); st.rerun() + if st.button("Clear kill switch", key=f"{prefix}kill_clear", disabled=not c.get("kill_switch"), help="Clears the switch and re-bases the peak at current wealth."): + rails.reset_circuit(kill=True, circuit_id=circuit_id); st.toast("Kill switch cleared"); st.rerun() with st.container(horizontal=True, vertical_alignment="center"): st.badge(f"Circuit tripped ({c.get('fail_count', 0)} failures)" if c.get("tripped") else f"Circuit ok ({c.get('fail_count', 0)} failures)", color="red" if c.get("tripped") else "green") st.caption("Trips after repeated failed or unverifiable live orders; blocks new live entries.") st.space("stretch") - if st.button("Reset circuit", key="circuit_reset", disabled=not c.get("tripped") and not c.get("fail_count")): - rails.reset_circuit(kill=False); st.toast("Circuit reset"); st.rerun() + if st.button("Reset circuit", key=f"{prefix}circuit_reset", disabled=not c.get("tripped") and not c.get("fail_count")): + rails.reset_circuit(kill=False, circuit_id=circuit_id); st.toast("Circuit reset"); st.rerun() with st.container(horizontal=True, vertical_alignment="center"): st.badge("New entries paused" if c.get("entries_paused") else "New entries allowed", color="orange" if c.get("entries_paused") else "green") st.caption("Operator switch: no new positions while paused; open positions still exit on schedule.") st.space("stretch") - if st.button("Resume entries" if c.get("entries_paused") else "Pause entries", key="entries_toggle"): - rails.set_entries_paused(not c.get("entries_paused")); st.rerun() - for b in q("SELECT book, halted, reason FROM book_state WHERE halted ORDER BY book"): + if st.button("Resume entries" if c.get("entries_paused") else "Pause entries", key=f"{prefix}entries_toggle"): + rails.set_entries_paused(not c.get("entries_paused"), circuit_id=circuit_id); st.rerun() + from fly_trader.markets import MARKETS + rh_books = [b for m in MARKETS.values() if m.circuit_id == rails.RH_CIRCUIT for b in m.books] + halted = q("SELECT book, halted, reason FROM book_state WHERE halted ORDER BY book") + mine = [b for b in halted if ("kalshi" in b["book"]) == (circuit_id == rails.KALSHI_CIRCUIT) and (b["book"] in rh_books) == (circuit_id == rails.RH_CIRCUIT)] + for b in mine: with st.container(horizontal=True, vertical_alignment="center"): st.badge(f"{b['book']} halted", color="red") st.caption(f"The paper book fell too far below its own peak ({b['reason']}); its entries are blocked, the other book trades on.") @@ -61,14 +212,57 @@ def settings_panel() -> None: st.caption("Set in the project .env; changes apply after the console restarts.") -def wallet_panel() -> None: - ev = q("SELECT ts, kind, pubkey FROM wallet_events ORDER BY id DESC LIMIT 20") +def rh_settings_panel() -> None: + missing = config.rh_live_prerequisites_missing() with st.container(border=True): - st.markdown(":material/account_balance_wallet: **Bot wallet**") - st.code(ev[0]["pubkey"] if ev else "No wallet yet: run `fly-trader wallet new`.", language=None) - if ev: - with st.expander("Wallet events"): - st.dataframe(pd.DataFrame(ev), hide_index=True) + st.markdown(":material/tune: **Robinhood Chain mode and limits**") + rows = [("Money", "paper; live mirror " + ("on" if not missing else "gated: " + ", ".join(missing)), "The RH race runs on paper books in ETH; after the RH handover the fly also trades the ETH wallet."), + ("RH_LIVE_ENABLED", "1" if config.RH_LIVE_ENABLED else "0", "Allows signing real Robinhood Chain transactions (live_rh)."), + ("Chain", f"{config.RH_EXPECTED_CHAIN_ID}" + (" (testnet)" if config.RH_TESTNET else ""), "The guard refuses any other chain id."), + ("Bot address", config.RH_BOT_ADDRESS or "not set", "RH_BOT_PRIVATE_KEY must derive to it."), + ("Starting capital", f"{config.RH_CAPITAL_ETH:g} ETH", "The USD value of 5 SOL at build time."), + ("Largest buy", f"{config.RH_LABEL_SIZE_ETH:g} ETH", "The label size (0.5 SOL in USD at build time): no RH buy is bigger."), + ("Smallest position", f"{config.RH_MIN_POSITION_ETH:g} ETH", "Smaller sized buys are skipped."), + ("Gas reserve", f"{config.RH_GAS_RESERVE_ETH:g} ETH", "Never spent on positions."), + ("Fee cap", f"{config.RH_MAX_FEE_GWEI:g} gwei", "A transaction never offers more per gas; a spike refuses instead."), + ("Kill switch", f"{config.RH_KILL_SWITCH_DRAWDOWN:.0%} drawdown", "On the live RH wealth (circuit 3)."), + ("Routes", "KyberSwap atomic, Uniswap v4 fallback, two legs through the base last", "Every round trip returns to ETH; no base inventory is kept.")] + st.dataframe(pd.DataFrame(rows, columns=["setting", "value", "meaning"]), hide_index=True) + + +def kalshi_settings_panel() -> None: + missing = config.kalshi_live_prerequisites_missing() + with st.container(border=True): + st.markdown(":material/tune: **Kalshi mode and limits**") + rows = [("Money", "paper; live mirror " + ("on" if not missing else "gated: " + ", ".join(missing)), "Both arms always trade paper books; with KALSHI_LIVE_ENABLED=1 and the prerequisites they mirror onto the subaccount."), + ("KALSHI_LIVE_ENABLED", "1" if config.KALSHI_LIVE_ENABLED else "0", "Allows real orders on the subaccount."), + ("Taker / maker live", f"{'on' if config.KALSHI_TAKER_LIVE else 'off'} / {'on' if config.KALSHI_MAKER_LIVE else 'off'}", "Per-arm live switches."), + ("Subaccount", str(config.KALSHI_SUBACCOUNT) if config.KALSHI_SUBACCOUNT else "none (fly-trader kalshi-subaccount create)", "The dedicated subaccount; the primary never trades."), + ("Capital cap", f"${config.KALSHI_CAPITAL_USD:g}", "Paper books start here; live never sizes above it. Position ≤ 10 %, slate ≤ 50 % of it."), + ("Cash floor", f"${config.KALSHI_CASH_FLOOR_USD:g}", "Never spent."), + ("Kill switch", f"{config.KALSHI_KILL_SWITCH_DRAWDOWN:.0%} drawdown", "On the combined live wealth (circuit 2); paper books halt on their own peaks."), + ("Entry window", f"{config.KALSHI_MIN_MINUTES_TO_CLOSE:g} min … {config.KALSHI_MAX_DAYS_TO_CLOSE:g} days to close", "Rows outside it are never scored."), + ("Gates", f"spread ≤ {config.KALSHI_MAX_SPREAD_CENTS}c, OI ≥ {config.KALSHI_MIN_OPEN_INTEREST}, 24 h volume ≥ {config.KALSHI_MIN_VOLUME_24H}", "Eligibility of a (market, side, minute)."), + ("Maker", f"quiet {config.KALSHI_MAKER_QUIET_MIN:g} min before close · TTL {config.KALSHI_MAKER_TTL_H:g} h · ≤ {config.KALSHI_MAKER_MAX_RESTING} resting", "Resting bids one tick inside the ask, post-only."), + ("Fees", f"taker {config.KALSHI_FEE_FACTOR:g} · maker {config.KALSHI_MAKER_FEE_FACTOR:g} (× series multiplier × p(1−p))", "The vendored fee model.")] + st.dataframe(pd.DataFrame(rows, columns=["setting", "value", "meaning"]), hide_index=True) + + +@st.fragment(run_every="30s") +def kalshi_account_panel() -> None: + with st.container(border=True): + st.markdown(":material/account_balance: **Kalshi subaccount**") + if not config.KALSHI_API_KEY_ID: + st.caption("No Kalshi key configured (KALSHI_API_KEY_ID / KALSHI_PRIVATE_KEY_PATH)."); return + st.caption(f"Key {config.KALSHI_API_KEY_ID[:8]}… · subaccount {config.KALSHI_SUBACCOUNT or 'none'} · fund it with `fly-trader kalshi-fund --usd N`; `fly-trader kalshi-status` prints balances, orders and settlements.") + resting = q1("SELECT count(*) AS n, COALESCE(sum(price_cents * count), 0) AS c FROM kalshi_orders WHERE book LIKE 'live%%' AND status IN ('resting','submitted')") or {} + opens = q1("SELECT count(*) AS n, COALESCE(sum(cost_cents + fee_cents), 0) AS c FROM kalshi_positions WHERE book LIKE 'live%%' AND status = 'open'") or {} + w = q1("SELECT wealth, sol_free AS cash, ts FROM wealth_marks WHERE book = 'live_kalshi' ORDER BY ts DESC LIMIT 1") or {} + with st.container(horizontal=True): + st.metric("Live wealth", f"${float(w['wealth']):.2f}" if w.get("wealth") is not None else "—", help=f"as of {ago(w.get('ts'))}" if w else None) + st.metric("Cash", f"${float(w['cash']):.2f}" if w.get("cash") is not None else "—") + st.metric("Open live positions", int(opens.get("n") or 0), delta=f"${float(opens.get('c') or 0) / 100:.2f} at cost", delta_color="off") + st.metric("Resting live orders", int(resting.get("n") or 0), delta=f"${float(resting.get('c') or 0) / 100:.2f} collateral", delta_color="off") def maintenance_panel() -> None: @@ -84,16 +278,40 @@ def maintenance_panel() -> None: from fly_trader.ops.reset import reset_training_state out = reset_training_state(reason="console button") st.success(f"Archived to {out['archived_to']}") + with st.container(horizontal=True, vertical_alignment="center"): + st.caption("Archive, then clear the visual fly's paper books, orders, scored rows and learning; it restarts from its bootstrap.") + st.space("stretch") + with st.popover("Reset Kalshi fly", disabled=sup.alive("kalshi_runner"), help="Stop the Kalshi engine first." if sup.alive("kalshi_runner") else None): + st.markdown("Archive and clear the Kalshi fly's books and learning now?") + if st.button("Reset", type="primary", key="kalshi_reset_confirm"): + from fly_trader.ops.reset import reset_kalshi_fly + st.success(f"Archived to {reset_kalshi_fly(reason='console button')['archived_to']}") errs = q("SELECT ts, service, endpoint, status, error FROM api_calls WHERE NOT ok ORDER BY id DESC LIMIT 30") if errs: with st.expander(f"Recent API errors ({len(errs)})"): st.dataframe(pd.DataFrame(errs), hide_index=True) +st.markdown("#### Wallets") +chains = [m.chain for m in markets.enabled()] +for col, chain in zip(st.columns(len(chains)), chains): + with col: + wallet_panel(chain) +if st.session_state.get("withdraw_open") in chains: # open until dismissed or closed, through full reruns + withdraw_dialog(st.session_state["withdraw_open"]) +st.markdown("#### Memecoins") rails_panel() +settings_panel() +if config.RH_ENABLED: + st.markdown("#### Robinhood Chain memecoins") + rails_panel(rails.RH_CIRCUIT, "Robinhood Chain trading rails", prefix="rh_") + rh_settings_panel() +st.markdown("#### Prediction markets") +rails_panel(rails.KALSHI_CIRCUIT, "Kalshi trading rails", prefix="k_") left, right = st.columns([3, 2]) with left: - settings_panel() + kalshi_settings_panel() with right: - wallet_panel() + kalshi_account_panel() +st.markdown("#### Maintenance") maintenance_panel() diff --git a/fly_trader/ui/app_pages/trades.py b/fly_trader/ui/app_pages/trades.py index d794bd16b7f38e0f26ae8979604abd278c75214d..a99e00bc1347933f1ff03d763442d78eaf12f7fa 100644 --- a/fly_trader/ui/app_pages/trades.py +++ b/fly_trader/ui/app_pages/trades.py @@ -1,15 +1,22 @@ -"""Trades: closed trades with their results, the decision log (every buy, sell and pick not taken) and fills.""" +"""Trades: closed trades with their results, the decision log (every buy, sell and pick not taken) and fills, for either memecoin chain.""" import pandas as pd import streamlit as st from fly_trader.db.queries import q, q1 -from fly_trader.ui.common import BOOK, labels, pct, sol +from fly_trader.markets import for_book +from fly_trader.ui.common import amount, chain_picker, labels, pct -books = [r["book"] for r in q("SELECT DISTINCT book FROM positions UNION SELECT DISTINCT book FROM fills ORDER BY 1")] or [BOOK] +mk = chain_picker() +seen = [r["book"] for r in q("SELECT DISTINCT book FROM positions UNION SELECT DISTINCT book FROM fills ORDER BY 1")] +books = list(mk.books) + [b for b in seen if b not in mk.books and for_book(b) is mk] # the chain's books even before their first trade +if st.session_state.get("trades_book") not in books: + st.session_state["trades_book"] = mk.selector_book with st.container(horizontal=True, vertical_alignment="bottom"): view = st.segmented_control("View", ["Closed trades", "Decision log", "Fills"], default="Closed trades", key="trades_view") - book = st.selectbox("Book", books, index=books.index(BOOK) if BOOK in books else 0, key="trades_book", width=220, - help="paper_selector is the selector's paper book; live is the bot wallet.") + book = st.selectbox("Book", books, key="trades_book", width=220, + help=f"{mk.selector_book} is the selector's paper book, {mk.fly_book} the fly's, {mk.live_book} the bot wallet; amounts in {mk.unit}.") +u = mk.unit +F3, F4 = ("%.3f", "%+.4f") if u == "SOL" else ("%.5f", "%+.6f") # ETH positions are ~20x smaller in number if view == "Closed trades": c = q1("SELECT count(*) AS n, sum(realized_sol) AS pnl, avg(realized_sol / NULLIF(cost_sol, 0)) AS avg_ret, " @@ -17,34 +24,35 @@ if view == "Closed trades": n = int(c.get("n") or 0) with st.container(horizontal=True): st.metric("Trades", n, border=True) - st.metric("Realized P&L", sol(c.get("pnl"), signed=True), border=True) + st.metric("Realized P&L", amount(c.get("pnl"), u, signed=True), border=True) st.metric("Average return", pct(c.get("avg_ret")), border=True, help="Per trade, after fees and price impact.") st.metric("Winners", f"{int(c.get('wins') or 0) / n:.0%}" if n else "—", border=True) rows = q("SELECT mint, opened_at, closed_at, cost_sol, realized_sol, forced_exit_kind FROM positions WHERE book = %s AND status = 'closed' ORDER BY closed_at DESC LIMIT 300", (book,)) if rows: names = labels([r["mint"] for r in rows]) df = pd.DataFrame([{"token": names.get(r["mint"], r["mint"][:6]), "opened": r["opened_at"], "closed": r["closed_at"], - "held (min)": (r["closed_at"] - r["opened_at"]).total_seconds() / 60 if r["closed_at"] else None, "cost (SOL)": float(r["cost_sol"] or 0), - "P&L (SOL)": float(r["realized_sol"] or 0), "return": float(r["realized_sol"] or 0) / float(r["cost_sol"]) * 100 if r["cost_sol"] else None, + "held (min)": (r["closed_at"] - r["opened_at"]).total_seconds() / 60 if r["closed_at"] else None, f"cost ({u})": float(r["cost_sol"] or 0), + f"P&L ({u})": float(r["realized_sol"] or 0), "return": float(r["realized_sol"] or 0) / float(r["cost_sol"]) * 100 if r["cost_sol"] else None, "exit": r["forced_exit_kind"] or "time"} for r in rows]) st.dataframe(df, hide_index=True, column_config={"opened": st.column_config.DatetimeColumn(format="MMM D HH:mm"), "closed": st.column_config.DatetimeColumn(format="MMM D HH:mm"), - "held (min)": st.column_config.NumberColumn(format="%.0f"), "cost (SOL)": st.column_config.NumberColumn(format="%.3f"), - "P&L (SOL)": st.column_config.NumberColumn(format="%+.4f"), "return": st.column_config.NumberColumn(format="%+.2f%%")}) + "held (min)": st.column_config.NumberColumn(format="%.0f"), f"cost ({u})": st.column_config.NumberColumn(format=F3), + f"P&L ({u})": st.column_config.NumberColumn(format=F4), "return": st.column_config.NumberColumn(format="%+.2f%%")}) else: st.caption("No closed trades in this book yet.") elif view == "Decision log": - kinds = {"Bought": ("selector_enter", "enter"), "Sold": ("selector_exit", "exit"), "Not taken": ("blocked",)} + kinds = {"Bought": ("selector_enter", "fly_enter", "enter"), "Sold": ("selector_exit", "fly_exit", "exit"), "Not taken": ("blocked",)} pick = st.pills("Show", list(kinds), selection_mode="multi", default=list(kinds), key="dec_kinds") want = [k for p in (pick or []) for k in kinds[p]] - rows = q("SELECT ts, kind, mint, m_hat, size_sol, rail, reason FROM decisions WHERE kind = ANY(%s) ORDER BY id DESC LIMIT 300", (want,)) if want else [] + rows = q("SELECT ts, kind, mint, m_hat, size_sol, rail, reason FROM decisions WHERE kind = ANY(%s) AND (detail->>'book' = %s OR %s = ANY(book_targets)) " + "ORDER BY id DESC LIMIT 300", (want, book, book)) if want else [] if rows: - names = labels([r["mint"] for r in rows]); action = {"selector_enter": "Bought", "enter": "Bought", "selector_exit": "Sold", "exit": "Sold", "blocked": "Not taken"} + names = labels([r["mint"] for r in rows]); action = {k: p for p, ks in kinds.items() for k in ks} df = pd.DataFrame([{"time": r["ts"], "action": action.get(r["kind"], r["kind"]), "token": names.get(r["mint"], (r["mint"] or "")[:6]), - "score": float(r["m_hat"]) if r["m_hat"] is not None else None, "size (SOL)": float(r["size_sol"] or 0), + "score": float(r["m_hat"]) if r["m_hat"] is not None else None, f"size ({u})": float(r["size_sol"] or 0), "why": (f"{r['rail']}: " if r["rail"] else "") + (r["reason"] or "")} for r in rows]) st.dataframe(df, hide_index=True, column_config={"time": st.column_config.DatetimeColumn(format="MMM D HH:mm:ss"), "score": st.column_config.NumberColumn(format="%.3f"), - "size (SOL)": st.column_config.NumberColumn(format="%.3f")}) + f"size ({u})": st.column_config.NumberColumn(format=F3)}) else: st.caption("No decisions recorded yet.") @@ -52,10 +60,10 @@ else: rows = q("SELECT ts, side, mint, sol_delta_lamports, price_sol, fee_lamports, verified_by, signature FROM fills WHERE book = %s ORDER BY id DESC LIMIT 300", (book,)) if rows: names = labels([r["mint"] for r in rows]) - df = pd.DataFrame([{"time": r["ts"], "side": r["side"], "token": names.get(r["mint"], r["mint"][:6]), "SOL": (r["sol_delta_lamports"] or 0) / 1e9, - "price (SOL)": float(r["price_sol"]) if r["price_sol"] is not None else None, "fee (SOL)": (r["fee_lamports"] or 0) / 1e9, + df = pd.DataFrame([{"time": r["ts"], "side": r["side"], "token": names.get(r["mint"], r["mint"][:6]), f"{u}": (r["sol_delta_lamports"] or 0) / 1e9, + f"price ({u})": float(r["price_sol"]) if r["price_sol"] is not None else None, f"fee ({u})": (r["fee_lamports"] or 0) / 1e9, "source": "simulated" if r["verified_by"] == "model" else r["verified_by"], "signature": r["signature"]} for r in rows]) - st.dataframe(df, hide_index=True, column_config={"time": st.column_config.DatetimeColumn(format="MMM D HH:mm:ss"), "SOL": st.column_config.NumberColumn(format="%+.4f"), - "price (SOL)": st.column_config.NumberColumn(format="%.3e"), "fee (SOL)": st.column_config.NumberColumn(format="%.5f")}) + st.dataframe(df, hide_index=True, column_config={"time": st.column_config.DatetimeColumn(format="MMM D HH:mm:ss"), f"{u}": st.column_config.NumberColumn(format=F4), + f"price ({u})": st.column_config.NumberColumn(format="%.3e"), f"fee ({u})": st.column_config.NumberColumn(format="%.5f")}) else: st.caption("No fills in this book yet.") diff --git a/fly_trader/ui/brain3d/__init__.py b/fly_trader/ui/brain3d/__init__.py index 606b473aab22ef658a3d446ad06b90519de447e5..8d22140edfcc27005ba95257d8a2b3147ed00c59 100644 --- a/fly_trader/ui/brain3d/__init__.py +++ b/fly_trader/ui/brain3d/__init__.py @@ -6,6 +6,12 @@ None, "rows": int, "candidates": int, "pathways": {"mode", "key", "reference", " "ratio", "strategy", "valence"}]}, "strategies": [...], "emphasis": [...], "message": str | None}`` — ``kc``/``mbon`` are sub-graph neuron indices. The geometry (``brain/geometry.py``) is fetched once per connectome from Streamlit's static directory ``ui/static/brain`` (``server.enableStaticServing``); everything else is small and sent per refresh. + +Version 2 (``payload_flies``): the whole brain with both flies overlaid — ``{"version": 2, "geometry": {... the full +geometry ...}, "flies": [{"name", "minute", "activity_b64", "n", "rows", "candidates", "pathways", "strategies", +"index": {"url", "n"}}, ...], "central_owner": "memecoin" | "kalshi", "emphasis", "message"}``. Each fly's activity +vector and pathway indices are over its own sub-graph; ``index`` (int32 sub → full) places them in the scene; neurons +two flies share (the central brain) show ``central_owner``'s activity. """ from __future__ import annotations @@ -14,13 +20,14 @@ from pathlib import Path import streamlit as st -from ...brain.geometry import build_geometry +from ...brain.geometry import build_full_geometry, build_geometry _DIR = Path(__file__).parent STATIC_DIR = _DIR.parent / "static" GEOMETRY_DIR = STATIC_DIR / "brain" EMPHASIS = ["KC", "MBON_APP", "MBON_AV", "MBON_OTHER"] VERSION = 1 +VERSION_FLIES = 2 _BRAIN = st.components.v2.component("fly_brain3d", html='
\n', css=(_DIR / "brain.css").read_text(), js=(_DIR / "brain.js").read_text()) @@ -31,6 +38,12 @@ def geometry() -> dict: return build_geometry(GEOMETRY_DIR) +@st.cache_resource(show_spinner="Placing every neuron of the brain…") +def geometry_full() -> dict: + """The whole brain's geometry with the per-fly index maps (built into the static directory on first use).""" + return build_full_geometry(GEOMETRY_DIR) + + def static_url(name: str, sha: str) -> str: base = (st.get_option("server.baseUrlPath") or "").strip("/") return f"/{base + '/' if base else ''}app/static/brain/{name}?v={sha[:12]}" @@ -50,5 +63,25 @@ def payload(meta: dict, act: dict | None, pathways: dict, strategies: list[str], "emphasis": EMPHASIS, "message": message} +def fly_payload(name: str, meta: dict, act: dict | None, pathways: dict, strategies: list[str]) -> dict: + """One fly's entry of a version-2 payload; ``meta`` is the full geometry (its ``flies[name]`` gives the sub-graph size and index map).""" + f = (meta.get("flies") or {}).get(name) or {}; sha = meta["connectome_sha256"]; b64 = None; minute = None; rows = cand = 0; note = None + if act is not None: + if len(act["h"]) != f.get("n") or (act["connectome"] and act["connectome"] != meta["connectome_file"]): + note = f"{name}: this minute's activity was recorded on another connectome" + else: + from datetime import datetime, timezone + b64 = base64.b64encode(act["h"].tobytes()).decode(); minute = datetime.fromtimestamp(act["t"], timezone.utc).isoformat() + rows, cand = act["n_rows"], act["n_cand"] + return {"name": name, "minute": minute, "activity_b64": b64, "n": int(f.get("n") or 0), "rows": rows, "candidates": cand, "pathways": pathways, "strategies": list(strategies), + "index": {"url": static_url(f["index"], sha), "n": int(f["n"])} if f else None, "note": note} + + +def payload_flies(meta: dict, flies: list[dict], central_owner: str = "kalshi", message: str | None = None) -> dict: + sha = meta["connectome_sha256"]; notes = [f["note"] for f in flies if f.get("note")] + return {"version": VERSION_FLIES, "geometry": {"url": static_url("geometry_full.bin", sha), "meta_url": static_url("geometry_full.json", sha), "sha": sha + ":full", "n": meta["n"]}, + "flies": flies, "central_owner": central_owner, "emphasis": EMPHASIS, "message": message or ("; ".join(notes) if notes else None)} + + def brain_view(data: dict, *, key: str = "brain3d", height: int = 560): return _BRAIN(data=data, key=key, height=height) diff --git a/fly_trader/ui/brain3d/brain.js b/fly_trader/ui/brain3d/brain.js index 87c52efd4ef0ceb2e97c3b184569445d4ee7d480..b698b3c92d5b5da9a987f844e00f1b6a598a977f 100644 --- a/fly_trader/ui/brain3d/brain.js +++ b/fly_trader/ui/brain3d/brain.js @@ -1,10 +1,14 @@ // The fly's brain in 3D: the neurons it runs on, coloured by population and lit by each minute's scoring, with the // KC→MBON synapses it has changed most drawn as lines. Standalone on purpose: it depends only on {data, parentElement} // (Streamlit's Custom Component v2 calls the default export with them; a web page can call it the same way with a -// fetched payload). The data contract is documented in ui/brain3d/__init__.py. +// fetched payload). The data contract is documented in ui/brain3d/__init__.py. Version 2 payloads carry ``flies``: two +// flies drawn into the whole brain, each fly's activity and pathway indices mapped through its ``index`` (sub → full); +// neurons both flies share (the central brain) show ``central_owner``'s activity. const THREE_URL = "https://cdn.jsdelivr.net/npm/three@0.170.0/build/three.module.js"; const EMPHASIS_SIZE = 2.4, BASE_SIZE = 1.1, DIM = 0.35, REST = 0.7; const WARM = [1.0, 0.55, 0.15], COOL = [0.25, 0.6, 1.0], GREEN = [0.2, 0.9, 0.35], RED = [1.0, 0.25, 0.2]; +const FLY_WARM = { memecoin: WARM, kalshi: [1.0, 0.35, 0.75] }, FLY_COOL = { memecoin: COOL, kalshi: [0.3, 0.9, 0.85] }; // the Kalshi fly lights magenta/teal +const PALETTE_EXTRA = { VISUAL: [0.55, 0.55, 0.75] }; const PALETTE = { KC: [1.0, 0.8, 0.2], MBON_APP: [0.3, 0.95, 0.4], MBON_AV: [1.0, 0.3, 0.3], MBON_OTHER: [1.0, 0.65, 0.2], DAN_PAM: [0.95, 0.3, 0.9], DAN_PPL1: [0.8, 0.25, 0.8], DAN_OTHER: [0.7, 0.3, 0.7], @@ -12,7 +16,7 @@ const PALETTE = { GRN_SWEET: [0.35, 0.85, 0.8], GRN_BITTER: [0.25, 0.65, 0.75], GRN_OTHER: [0.3, 0.7, 0.7], MECH_JO: [0.4, 0.6, 0.9], MECH_BRISTLE: [0.45, 0.55, 0.85], MECH_OTHER: [0.4, 0.5, 0.8], THERMO_WARM: [0.9, 0.5, 0.4], THERMO_COOL: [0.5, 0.6, 0.95], THERMO_OTHER: [0.6, 0.55, 0.8], - CX: [0.65, 0.45, 0.95], HUNGER: [0.9, 0.6, 0.6], DESCENDING: [0.92, 0.92, 0.95], OTHER: [0.45, 0.45, 0.5], + CX: [0.65, 0.45, 0.95], HUNGER: [0.9, 0.6, 0.6], DESCENDING: [0.92, 0.92, 0.95], OTHER: [0.45, 0.45, 0.5], VISUAL: [0.55, 0.55, 0.75], }; const POINT_VS = ` attribute float psize; attribute float alpha; attribute vec3 col; @@ -54,6 +58,15 @@ async function fetchGeometry(g) { return geom; } +const indexCache = new Map(); // url → Int32Array (sub-graph index → full-graph index) +async function fetchIndex(ix) { + if (!ix) return null; + if (indexCache.has(ix.url)) return indexCache.get(ix.url); + const buf = await fetch(ix.url).then(r => { if (!r.ok) throw new Error(`index ${r.status}`); return r.arrayBuffer(); }); + const arr = new Int32Array(buf); if (arr.length !== ix.n) throw new Error("index size mismatch"); + indexCache.set(ix.url, arr); return arr; +} + function el(tag, cls, parent) { const e = document.createElement(tag); if (cls) e.className = cls; parent.appendChild(e); return e; } function ensureDom(parent) { @@ -171,28 +184,53 @@ function hover(inst) { function decode(b64) { const s = atob(b64); const u = new Uint8Array(s.length); for (let i = 0; i < s.length; i++) u[i] = s.charCodeAt(i); return new Int8Array(u.buffer); } function recolor(inst, data) { - const meta = inst.geom.meta, n = meta.n, col = inst.colAttr.array, pop = inst.pop, hcol = inst.hcolAttr.array, hstr = inst.hstrAttr.array; - const base = meta.pop_order.map(p => PALETTE[p] || PALETTE.OTHER); - const act = data.activity_b64 ? decode(data.activity_b64) : null; inst.activity = act && act.length === n ? act : null; - let glowing = 0; + const meta = inst.geom.meta, n = meta.n, col = inst.colAttr.array, pop = inst.pop; + const base = meta.pop_order.map(p => PALETTE[p] || PALETTE_EXTRA[p] || PALETTE.OTHER); + let act = null, owner = new Uint8Array(0); // owner[i]: 0 none, 1 memecoin, 2 kalshi — which fly lit neuron i + if (data.flies) { + act = new Int8Array(n); owner = new Uint8Array(n); + const flies = data.flies.slice().sort((a, b) => (a.name === data.central_owner ? 1 : 0) - (b.name === data.central_owner ? 1 : 0)); // the owner writes last + for (const f of flies) { + const ix = inst.indices ? inst.indices.get(f.name) : null; if (!ix || !f.activity_b64) continue; + const a = decode(f.activity_b64); if (a.length !== ix.length) continue; + const tag = f.name === "kalshi" ? 2 : 1; + for (let k = 0; k < ix.length; k++) { const i = ix[k]; act[i] = a[k]; owner[i] = tag; } + } + inst.activity = owner.some(v => v) ? act : null; inst.owner = owner; + } else { const a = data.activity_b64 ? decode(data.activity_b64) : null; inst.activity = a && a.length === n ? a : null; inst.owner = null; } + const hcol = inst.hcolAttr.array, hstr = inst.hstrAttr.array; let glowing = 0; for (let i = 0; i < n; i++) { const b = base[pop[i]]; let r, g, bl, s = 0; - if (!inst.activity) { r = b[0] * REST; g = b[1] * REST; bl = b[2] * REST; } - else { - const v = inst.activity[i] / 127, mag = Math.abs(v), m = Math.pow(mag, 0.7), h = v >= 0 ? WARM : COOL; - r = b[0] * DIM * (1 - m) + h[0] * m; g = b[1] * DIM * (1 - m) + h[1] * m; bl = b[2] * DIM * (1 - m) + h[2] * m; - if (mag > GLOW_MIN) { // the glow: strength 0..1 above the threshold, a hot core toward white - s = Math.pow((mag - GLOW_MIN) / (1 - GLOW_MIN), 0.8); const w = 0.35 * s * s; - r = r * (1 - w) + w; g = g * (1 - w) + w; bl = bl * (1 - w) + w; - hcol[i * 3] = h[0] * 0.8 + 0.2 * s; hcol[i * 3 + 1] = h[1] * 0.8 + 0.2 * s; hcol[i * 3 + 2] = h[2] * 0.8 + 0.2 * s; glowing++; - } - } + if (!inst.activity || (inst.owner && !inst.owner[i])) { r = b[0] * REST; g = b[1] * REST; bl = b[2] * REST; } + else { const v = inst.activity[i] / 127, mag = Math.abs(v), m = Math.pow(mag, 0.7); const fly = inst.owner && inst.owner[i] === 2 ? "kalshi" : "memecoin"; + const h = v >= 0 ? FLY_WARM[fly] : FLY_COOL[fly]; + r = b[0] * DIM * (1 - m) + h[0] * m; g = b[1] * DIM * (1 - m) + h[1] * m; bl = b[2] * DIM * (1 - m) + h[2] * m; + if (mag > GLOW_MIN) { // the glow: strength 0..1 above the threshold, in that fly's hue, a hot core toward white + s = Math.pow((mag - GLOW_MIN) / (1 - GLOW_MIN), 0.8); const w = 0.35 * s * s; + r = r * (1 - w) + w; g = g * (1 - w) + w; bl = bl * (1 - w) + w; + hcol[i * 3] = h[0] * 0.8 + 0.2 * s; hcol[i * 3 + 1] = h[1] * 0.8 + 0.2 * s; hcol[i * 3 + 2] = h[2] * 0.8 + 0.2 * s; glowing++; + } } col[i * 3] = r; col[i * 3 + 1] = g; col[i * 3 + 2] = bl; hstr[i] = s; } inst.colAttr.needsUpdate = true; inst.hcolAttr.needsUpdate = true; inst.hstrAttr.needsUpdate = true; - inst.halo.visible = glowing > 0; inst.minute = data.minute; + inst.halo.visible = glowing > 0; inst.minute = minuteKey(data); +} + +function minuteKey(data) { return data.flies ? JSON.stringify([data.central_owner, data.flies.map(f => f.minute)]) : data.minute; } + +function pathwayItems(inst, data) { // [{...item, kc, mbon in scene indices, strategy}] over every fly (version 2) or the one payload + if (!data.flies) return ((data.pathways && data.pathways.items) || []).map(it => ({ ...it })); + const out = []; + for (const f of data.flies) { + const ix = inst.indices ? inst.indices.get(f.name) : null; if (!ix) continue; + for (const it of ((f.pathways && f.pathways.items) || [])) out.push({ ...it, kc: ix[it.kc], mbon: ix[it.mbon], strategy: `${f.name}:${it.strategy}` }); + } + return out; } +function strategyList(data) { return data.flies ? data.flies.flatMap(f => (f.strategies || []).map(s => `${f.name}:${s}`)) : (data.strategies || []); } +function pathwayKey(data) { return data.flies ? JSON.stringify(data.flies.map(f => f.pathways ? f.pathways.key : null)) : (data.pathways ? data.pathways.key : null); } + function strategyTints(inst, strategies) { const raw = cssVar(inst.dom.root, "--st-chart-categorical-colors", ""); const list = raw.split(",").map(hexToRgb).filter(Boolean); const out = {}; strategies.forEach((s, i) => { out[s] = list.length ? list[i % list.length] : [0.8, 0.8, 0.8]; }); out.shared = [0.8, 0.8, 0.8]; return out; @@ -201,7 +239,7 @@ function strategyTints(inst, strategies) { function rebuildLines(inst, data) { const { THREE, scene } = inst; const pos = inst.geom.pos; for (const l of inst.lines.values()) { scene.remove(l); l.geometry.dispose(); l.material.dispose(); } inst.lines.clear(); - const items = (data.pathways && data.pathways.items) || []; const tints = strategyTints(inst, data.strategies || []); + const items = pathwayItems(inst, data); const tints = strategyTints(inst, strategyList(data)); const groups = new Map(); for (const it of items) { if (!groups.has(it.strategy)) groups.set(it.strategy, []); groups.get(it.strategy).push(it); } const maxR = Math.max(1e-9, ...items.map(it => Math.abs(it.ratio))); for (const [strategy, its] of groups) { @@ -215,7 +253,7 @@ function rebuildLines(inst, data) { const l = new THREE.LineSegments(g, new THREE.LineBasicMaterial({ vertexColors: true, transparent: true, opacity: 0.9 })); l.visible = !inst.hiddenStrategies.has(strategy); scene.add(l); inst.lines.set(strategy, l); } - inst.pkey = data.pathways ? data.pathways.key : null; + inst.pkey = pathwayKey(data); } function legend(inst, data) { @@ -234,7 +272,7 @@ function legend(inst, data) { inst.alphaAttr.array.fill(on ? 1 : 0, lo, hi); inst.alphaAttr.needsUpdate = true; }); } - const strategies = data.strategies || []; + const strategies = strategyList(data); if (strategies.length) { el("div", "hd", box).textContent = "Learned pathways (green +, red −)"; const tints = strategyTints(inst, strategies); for (const s of [...strategies, "shared"]) { @@ -251,9 +289,9 @@ function message(inst, text) { const m = inst.dom.msg; m.textContent = text || " function apply(inst, data) { if (!inst.ready || !data) return; - const lk = JSON.stringify([inst.geom.meta.connectome_sha256, data.strategies || []]); if (inst.legendKey !== lk) legend(inst, data); - if (data.minute !== inst.minute || inst.minute === undefined) recolor(inst, data); - const pk = data.pathways ? data.pathways.key : null; if (pk !== inst.pkey) rebuildLines(inst, data); + const lk = JSON.stringify([inst.geom.meta.connectome_sha256, strategyList(data)]); if (inst.legendKey !== lk) legend(inst, data); + if (minuteKey(data) !== inst.minute || inst.minute === undefined) recolor(inst, data); + const pk = pathwayKey(data); if (pk !== inst.pkey) rebuildLines(inst, data); message(inst, data.message); inst.dom.hint.textContent = "drag to orbit · wheel to zoom · shift-drag to pan · double-click to recentre"; requestRender(inst); } @@ -264,6 +302,10 @@ async function boot(inst, data) { inst.emphasis = data.emphasis || []; const THREE = await three(); let geom; try { geom = await fetchGeometry(data.geometry); } catch (e) { message(inst, `the neuron geometry could not be fetched (${e.message}) — is static serving on?`); return; } + if (data.flies) { // version 2: each fly's sub-graph → scene index map + inst.indices = new Map(); + for (const f of data.flies) { try { inst.indices.set(f.name, await fetchIndex(f.index)); } catch (e) { message(inst, `${f.name}: index map could not be fetched (${e.message})`); } } + } if (inst.disposed) return; buildScene(inst, THREE, geom); inst.ready = true; apply(inst, inst.pending || data); inst.pending = null; } catch (e) { diff --git a/fly_trader/ui/brain3d/feed.py b/fly_trader/ui/brain3d/feed.py index 7e927a5da3ad16b4a06491dfaacb3e6135cc6b3b..5b0e6448870c41a706e76778c7d690e5f5cbfd65 100644 --- a/fly_trader/ui/brain3d/feed.py +++ b/fly_trader/ui/brain3d/feed.py @@ -11,25 +11,25 @@ import streamlit as st from ...brain import activity, pathways as pw from ...brain.connectome import CONNECTOME_DIR from ...db.queries import q, q1 -from ...ops.reset import activity_dir, fly_state_dir +from ...ops.reset import activity_dir, fly_state_dir, kalshi_activity_dir, kalshi_fly_state_dir # noqa: F401 (re-exported for the pages) MODES = {"since bootstrap": "bootstrap", "last 24 h": "24h"} N_PAIRS = 21438 -def activity_mtime() -> float: - d = activity_dir() +def activity_mtime(d: Path | None = None) -> float: + d = activity_dir() if d is None else Path(d) return d.stat().st_mtime if d.exists() else 0.0 @st.cache_data(ttl=4, show_spinner=False) -def minutes(mtime: float) -> list[str]: - return activity.listing(activity_dir()) +def minutes(mtime: float, d: str | None = None) -> list[str]: + return activity.listing(activity_dir() if d is None else Path(d)) @st.cache_data(ttl=600, max_entries=64, show_spinner=False) -def activity_at(stamp: str) -> dict | None: - p = activity.path_of(activity_dir(), stamp) +def activity_at(stamp: str, d: str | None = None) -> dict | None: + p = activity.path_of(activity_dir() if d is None else Path(d), stamp) try: return activity.load(p) except (FileNotFoundError, ValueError, OSError): @@ -51,9 +51,9 @@ def weights(boot_path: str): @st.cache_data(ttl=30, show_spinner=False) -def _pathways(mode: str, boot_id: int, boot_path: str, connectome_file: str, state_mtime: float, ref_path: str | None, mbon_offset: int) -> dict: +def _pathways(mode: str, boot_id: int, boot_path: str, connectome_file: str, state_mtime: float, ref_path: str | None, mbon_offset: int, state_dir: str | None = None) -> dict: kc, mbon, n_app, n_av = pairs(connectome_file); w0, learn, strategies = weights(boot_path) - state = fly_state_dir() / "state.pt" + state = (fly_state_dir() if state_dir is None else Path(state_dir)) / "state.pt" out = {"mode": mode, "key": f"{mode}:{boot_id}:{int(state_mtime)}:{ref_path or ''}", "reference": None, "items": [], "note": None, "changed": 0} if not state.exists(): out["note"] = "the fly has not written its state yet"; return out @@ -71,9 +71,15 @@ def _pathways(mode: str, boot_id: int, boot_path: str, connectome_file: str, sta return out -def pathways(mode_label: str, fly: dict, meta: dict) -> dict: - """The pathway payload for the chosen mode, or an empty one with the reason.""" +FLY_SNAP_KIND = {"memecoin": "fly_plastic", "kalshi": "kalshi_fly_plastic"} + + +def pathways(mode_label: str, fly: dict, meta: dict, fly_name: str = "memecoin") -> dict: + """The pathway payload for the chosen mode, or an empty one with the reason. ``fly_name`` picks the state directory + and snapshot kind (the Kalshi fly keeps its own); ``meta`` may be the sub-graph or the full geometry (the MBON offset + is the fly's own sub-graph offset either way, since KC/MBON rows precede every excluded population).""" mode = MODES.get(mode_label, "bootstrap"); boot_id = fly.get("bootstrap") + state_dir = kalshi_fly_state_dir() if fly_name == "kalshi" else fly_state_dir(); snap_kind = FLY_SNAP_KIND.get(fly_name, "fly_plastic") empty = {"mode": mode, "key": f"{mode}:none", "reference": None, "items": [], "note": None, "changed": 0} if not boot_id: return empty @@ -82,10 +88,10 @@ def pathways(mode_label: str, fly: dict, meta: dict) -> dict: return {**empty, "note": "the fly's bootstrap file is missing"} ref = None if mode == "24h": - rows = q("SELECT ts, path, note FROM brain_snapshots WHERE kind = 'fly_plastic' ORDER BY id DESC LIMIT 400") + rows = q("SELECT ts, path, note FROM brain_snapshots WHERE kind = %s ORDER BY id DESC LIMIT 400", (snap_kind,)) ref = pw.reference_snapshot(rows, time.time(), int(boot_id)) - state = fly_state_dir() / "state.pt"; mt = state.stat().st_mtime if state.exists() else 0.0 - return _pathways(mode, int(boot_id), r["path"], meta["connectome_file"], mt, ref, int(meta["pop_ranges"]["MBON_APP"][0])) + state = state_dir / "state.pt"; mt = state.stat().st_mtime if state.exists() else 0.0 + return _pathways(mode, int(boot_id), r["path"], meta["connectome_file"], mt, ref, int(meta["pop_ranges"]["MBON_APP"][0]), str(state_dir)) def caption(act: dict | None, pth: dict, fly: dict, meta: dict) -> str: diff --git a/fly_trader/ui/common.py b/fly_trader/ui/common.py index f11b3237c8daee7852a368437ca77293fb9a7db7..8e1047044e7e970bae8b5362713c4e0fa2e20d1f 100644 --- a/fly_trader/ui/common.py +++ b/fly_trader/ui/common.py @@ -8,7 +8,7 @@ from datetime import datetime, timezone import streamlit as st -from fly_trader import config +from fly_trader import config, markets from fly_trader.db.queries import q, q1 from fly_trader.ops.supervisor import WORKERS, get_supervisor from fly_trader.train.selector import DATA_VERSION, is_current, is_deployable @@ -22,9 +22,17 @@ WORKER_INFO = { # name → (title, icon, what it does, role) "train": ("Trainer", ":material/model_training:", "On demand: builds decision points from the archive, backtests day by day and saves a new model.", "train"), "replay": ("History archive", ":material/history:", "Downloads the hourly trade archive and builds the training feature set.", "train"), "discover": ("Token stats", ":material/query_stats:", "Records Jupiter stats (holders, organic score, top-holder share) every 10 minutes for every token the selector can trade: history for future model inputs.", "optional"), + "kalshi_runner": ("Kalshi trading engine", ":material/candlestick_chart:", "Scores every quoted Kalshi market once a minute with the visual fly, trades its taker and maker paper books (and the subaccount when live) and lets it learn from settlements.", "trade"), + "rh_stream": ("Robinhood Chain feed", ":material/sensors:", "Indexes Pons launches, graduations and Uniswap v4 swaps from Robinhood Chain into 1-minute rows in ETH (backfill from Pons V2, then live). The engine's Robinhood books read these.", "trade"), + "kalshi_stream": ("Kalshi market feed", ":material/sensors:", "Streams every Kalshi quote, trade and market lifecycle event into 1-minute rows. The Kalshi engine reads these.", "trade"), + "kalshi_train": ("Kalshi trainer", ":material/model_training:", "Fits the Kalshi strategy stack walk-forward by settlement day and bootstraps the visual fly when none exists.", "train"), + "kalshi_history": ("Kalshi history", ":material/history:", "Seeds settled markets and trades from the open dataset, fills 1-minute candles from the API and builds the feature rows.", "train"), } +WORKER_TYPE = {"runner": "memecoins", "pumpstream": "memecoins", "train": "memecoins", "replay": "memecoins", "discover": "memecoins", + "rh_stream": "memecoins", "kalshi_runner": "prediction markets", "kalshi_stream": "prediction markets", "kalshi_train": "prediction markets", "kalshi_history": "prediction markets"} ROLE_LABEL = {"trade": "needed to trade", "train": "needed to train", "optional": "optional"} -ORDER = [n for n in ("runner", "pumpstream", "train", "replay", "discover") if n in WORKERS] +ORDER = [n for n in ("runner", "pumpstream", "rh_stream", "train", "replay", "discover", "kalshi_runner", "kalshi_stream", "kalshi_train", "kalshi_history") if n in WORKERS] +KALSHI_BOOKS = {"paper_kalshi_taker": "Taker · paper", "paper_kalshi_maker": "Maker · paper", "live_kalshi_taker": "Taker · live", "live_kalshi_maker": "Maker · live"} # ---------------------------------------------------------------- formatting @@ -52,13 +60,37 @@ def ago(ts) -> str: def sol(x, nd: int = 4, signed: bool = False) -> str: - return "—" if x is None else f"{float(x):{'+' if signed else ''},.{nd}f} SOL" + return amount(x, "SOL", nd, signed) + + +def amount(x, unit: str = "SOL", nd: int = 4, signed: bool = False) -> str: + """A book amount in its market's unit (SOL on Solana, ETH on Robinhood Chain).""" + return "—" if x is None else f"{float(x):{'+' if signed else ''},.{nd}f} {unit}" + + +def chain_picker(key: str = "chain") -> markets.MarketSpec: + """The memecoin chain a page shows (Solana, or Robinhood Chain when RH_ENABLED): one choice shared by the pages.""" + ms = markets.enabled() + if len(ms) == 1: + return ms[0] + names = {m.name: m for m in ms} + st.session_state.setdefault(key, ms[0].name) + pick = st.segmented_control("Chain", list(names), key=key, label_visibility="collapsed") or ms[0].name + return names[pick] def pct(x, nd: int = 2, signed: bool = True) -> str: return "—" if x is None else f"{float(x) * 100:{'+' if signed else ''}.{nd}f}%" +def usd(x, nd: int = 2, signed: bool = False) -> str: + return "—" if x is None else f"{'+' if signed and float(x) >= 0 else ''}{'-' if float(x) < 0 else ''}${abs(float(x)):,.{nd}f}" + + +def cents(x, nd: int = 0) -> str: + return "—" if x is None else f"{float(x):.{nd}f}c" + + # ---------------------------------------------------------------- models _NOTE_OBJ = re.compile(r'"(walk_forward|random_baseline|fly|gbm|random)": (\{[^{}]*\})') _NOTE_SCALAR = re.compile(r'"(threshold|trained_through|horizon_min|line|costs|rows|days|fly_beats_gbm)": ("[^"]*"|[-0-9.eE]+|true|false)') @@ -138,14 +170,17 @@ def labels(mints) -> dict[str, str]: # ---------------------------------------------------------------- system state -def system_state() -> dict: +def system_state(market: markets.MarketSpec | None = None) -> dict: + """What one memecoin market (default Solana) is doing: its books' status keys, its feed, its circuit, its seat.""" + m = market or markets.SOL; rh = m.chain != "sol" sup = get_supervisor(); ws = sup.status(); now = datetime.now(timezone.utc) - sel, sel_at = setting("selector_status"); tr, tr_at = setting("training_status"); ps, _ = setting("pumpstream_status") - fly, fly_at = setting("fly_status"); ho, _ = setting("handover"); rp, _ = setting("fly_replay") - circuit = q1("SELECT kill_switch, kill_reason, tripped, fail_count, entries_paused FROM circuit_state WHERE id = 1") or {} + sel, sel_at = setting(m.selector_status_key); tr, tr_at = setting("training_status"); ps, _ = setting(m.stream_status_key) + fly, fly_at = setting(m.fly_status_key); ho, _ = setting(m.handover_key); rp, _ = setting("fly_replay") + circuit = q1("SELECT kill_switch, kill_reason, tripped, fail_count, entries_paused FROM circuit_state WHERE id = %s", (m.circuit_id,)) or {} ft = ps.get("flushed_through") feed_age = (now - datetime.fromisoformat(ft)).total_seconds() - 60 if ft else None # since the newest complete minute closed - feed_ok = ws["pumpstream"]["alive"] and feed_age is not None and feed_age < STALE_FEED_S + feed_worker = "rh_stream" if rh else "pumpstream" + feed_ok = ws.get(feed_worker, {}).get("alive", False) and feed_age is not None and feed_age < STALE_FEED_S runner = ws["runner"]["alive"] stage = str(tr.get("stage") or "") training = ws["train"]["alive"] or (tr_at is not None and (now - tr_at).total_seconds() < 180 and not stage.endswith(("saved", "done"))) @@ -164,9 +199,13 @@ def system_state() -> dict: trading, why = "starting", "Warming up on the last 24 hours of market minutes." else: trading, why = "trading", "Trading every minute." - from fly_trader.chain.cluster_guard import signing_allowed - live_money = bool(ho) and runner and fly.get("stage") == "trading" and signing_allowed() # the fly holds the seat and may sign (agent/fly_live.py) - return {"workers": ws, "runner": runner, "trading": trading, "trading_why": why, "live_money": live_money, + if rh: + signing = not config.rh_live_prerequisites_missing() + else: + from fly_trader.chain.cluster_guard import signing_allowed + signing = signing_allowed() + live_money = bool(ho) and runner and fly.get("stage") == "trading" and signing # the fly holds the seat and may sign (agent/fly_live.py, rh/live.py) + return {"market": m, "workers": ws, "runner": runner, "trading": trading, "trading_why": why, "live_money": live_money, "fly": fly, "fly_at": fly_at, "handover": ho or None, "replay": rp or None, "model": loaded_model() if runner else None, "latest": latest_snapshot("selector"), "deployable": latest_snapshot("selector", deployable=True), "training": training, "train_status": tr, "train_at": tr_at, @@ -174,6 +213,76 @@ def system_state() -> dict: "circuit": circuit} +# ---------------------------------------------------------------- prediction markets (Kalshi) +def kalshi_state() -> dict: + """What the Kalshi side is doing: feed, engine, the visual fly, training, history, rails (circuit 2) and books.""" + from fly_trader.kalshi import fly as KF, selector as kselector + sup = get_supervisor(); ws = sup.status(); now = datetime.now(timezone.utc) + fly, fly_at = setting("kalshi_fly_status"); st_, _ = setting("kalshi_stream_status"); tr, tr_at = setting("kalshi_training_pipeline") + rp, _ = setting("kalshi_fly_replay"); hs, hs_at = setting("kalshi_history_status"); ts_, ts_at = setting("training_status") + circuit = q1("SELECT kill_switch, kill_reason, tripped, fail_count, entries_paused FROM circuit_state WHERE id = 2") or {} + ft = st_.get("flushed_through") + feed_age = (now - datetime.fromisoformat(ft)).total_seconds() - 60 if ft else None + feed_ok = ws["kalshi_stream"]["alive"] and feed_age is not None and feed_age < STALE_FEED_S + runner = ws["kalshi_runner"]["alive"] + stage = str(fly.get("stage") or "") + if not runner: + err = (ws["kalshi_runner"].get("error") or "").splitlines() + trading, why = "stopped", ("The Kalshi engine is stopped: " + err[0].split(": ", 1)[-1]) if err else "The Kalshi engine is stopped." + elif stage in ("", "not trading"): + trading, why = "waiting", fly.get("detail") or "The visual fly has not qualified yet: it trades once its replay passed and a deployable bootstrap exists." + elif circuit.get("kill_switch"): + trading, why = "blocked", "Kalshi kill switch tripped" + (f" ({circuit['kill_reason']})" if circuit.get("kill_reason") else "") + ": no new entries." + elif circuit.get("entries_paused"): + trading, why = "paused", "New Kalshi entries are paused by the operator; open positions still run to settlement." + elif not feed_ok: + trading, why = "holding", "The Kalshi feed is stale: no entries until it recovers (the fly keeps learning)." + elif stage.startswith("catching"): + trading, why = "starting", "Catching up on the minutes the feed wrote while the engine was away." + else: + trading, why = "trading", "Trading both arms every minute: IOC entries and resting bids." + live = bool(config.KALSHI_LIVE_ENABLED) and not config.kalshi_live_prerequisites_missing() and runner and stage == "trading" + books = {b: q1("SELECT wealth, exposure, n_open, drawdown, ts FROM wealth_marks WHERE book = %s ORDER BY ts DESC LIMIT 1", (b,)) for b in KALSHI_BOOKS} + with __import__("fly_trader.db.connection", fromlist=["transaction"]).transaction() as conn: + sel = kselector.latest_current(conn); boot = KF.latest_current(conn); dep = KF.latest_deployable(conn) + training = ws["kalshi_train"]["alive"] and str(tr.get("stage") or "").startswith(("training", "bootstrapping")) + return {"workers": ws, "runner": runner, "trading": trading, "trading_why": why, "live_money": live, "fly": fly, "fly_at": fly_at, "replay": rp or None, + "feed": st_, "feed_ok": feed_ok, "feed_age": feed_age, "train": tr, "train_at": tr_at, "training": training, "train_progress": ts_ if training else {}, + "history": hs, "history_at": hs_at, "circuit": circuit, "books": books, "selector": sel, "bootstrap": boot, "deployable": dep, + "prerequisites_missing": config.kalshi_live_prerequisites_missing()} + + +KALSHI_BADGE = {"trading": ("Kalshi trading", "green"), "holding": ("Kalshi holding: feed stale", "orange"), "paused": ("Kalshi entries paused", "orange"), + "blocked": ("Kalshi kill switch", "red"), "starting": ("Kalshi starting", "blue"), "stopped": ("Kalshi not trading", "gray"), + "waiting": ("Kalshi waiting for the fly", "gray")} + + +def kalshi_strip() -> None: + """The prediction-markets row of the status strip.""" + k = kalshi_state(); fly = k["fly"] + with st.container(horizontal=True, gap="small"): + if k["live_money"]: + st.badge("Kalshi live money", icon=":material/payments:", color="red", help=f"Both arms mirror onto Kalshi subaccount {config.KALSHI_SUBACCOUNT} (cap ${config.KALSHI_CAPITAL_USD:g}).") + else: + st.badge("Kalshi paper", icon=":material/receipt:", color="blue", help="Prediction-market trades are simulated at the quoted book with real fees; " + + ("live mirroring is off (KALSHI_LIVE_ENABLED=0)." if not config.KALSHI_LIVE_ENABLED else "live gated: " + ", ".join(k["prerequisites_missing"]))) + text, color = KALSHI_BADGE[k["trading"]] + st.badge(text, icon=":material/candlestick_chart:", color=color, help=k["trading_why"]) + if fly.get("stage") and fly.get("stage") != "not trading": + st.badge(f"Visual fly learning · drift {float(fly.get('drift') or 0):.1%}" if not fly.get("learning_frozen") else "Visual fly: learning frozen", icon=":material/visibility:", + color="violet" if not fly.get("learning_frozen") else "orange", + help=f"The Kalshi fly on {fly.get('device') or '?'}: {fly.get('pending', 0)} settlements pending; edge lines " + ", ".join(f"{a} {float(v):.3f}" for a, v in (fly.get("lines") or {}).items())) + else: + st.badge("Visual fly not trading", icon=":material/visibility:", color="gray", help=fly.get("detail") or k["trading_why"]) + tr = k["train"] + st.badge(f"Kalshi training · {tr.get('stage', '')}" if k["training"] else "Kalshi not training", icon=":material/model_training:", color="blue" if k["training"] else "gray") + hs = k["history"] + if k["workers"]["kalshi_history"]["alive"]: + st.badge(f"Kalshi history · {hs.get('stage', '')}", icon=":material/history:", color="blue", help=f"{hs.get('done', '?')} of {hs.get('total', '?')} markets filled") + st.badge("Kalshi feed live" if k["feed_ok"] else "Kalshi feed stale", icon=":material/sensors:", color="green" if k["feed_ok"] else "red", + help=f"newest complete minute closed {k['feed_age']:.0f} s ago" if k["feed_age"] is not None else "no minute written yet") + + TRADING_BADGE = {"trading": ("Trading", "green"), "holding": ("Holding: feed stale", "orange"), "paused": ("Entries paused", "orange"), "blocked": ("Kill switch tripped", "red"), "starting": ("Starting", "blue"), "stopped": ("Not trading", "gray"), "waiting": ("Waiting for a qualified model", "gray")} @@ -181,6 +290,19 @@ TRADING_BADGE = {"trading": ("Trading", "green"), "holding": ("Holding: feed sta @st.fragment(run_every="5s") def status_strip() -> None: + memecoin_strip() + if config.RH_ENABLED: + try: + rh_strip() + except Exception as e: # the RH side must never take the Solana console down + st.caption(f"Robinhood Chain status unavailable: {type(e).__name__}: {e}") + try: + kalshi_strip() + except Exception as e: # the Kalshi side must never take the memecoin console down + st.caption(f"Kalshi status unavailable: {type(e).__name__}: {e}") + + +def memecoin_strip() -> None: s = system_state() with st.container(horizontal=True, gap="small"): if s["live_money"]: @@ -190,15 +312,18 @@ def status_strip() -> None: text, color = TRADING_BADGE[s["trading"]] st.badge(text, icon=":material/candlestick_chart:", color=color, help=s["trading_why"]) m, latest, dep = s["model"], s["latest"], s["deployable"] - st.badge(f"Model #{m['id']}" if m else "No model at work", icon=":material/psychology:", color="violet" if m else "gray", + st.badge(f"Selector #{m['id']}" if m else "No selector at work", icon=":material/psychology:", color="violet" if m else "gray", help=backtest_line(m["meta"]) if m else ("No model has been trained on the current data yet." if not latest else "No model has earned a place yet: " + (latest["meta"].get("deploy_reason") or "none qualified") if not dep else "The trading engine is stopped, so no model is in use.")) if latest and not is_deployable(latest["meta"]) and (not m or latest["id"] > m["id"]): - st.badge(f"Model #{latest['id']} not put to work", icon=":material/block:", color="orange", help=backtest_line(latest["meta"])) + st.badge(f"Selector #{latest['id']} not put to work", icon=":material/block:", color="orange", help=backtest_line(latest["meta"])) if m and dep and dep["id"] > m["id"]: - st.badge(f"Switching to model #{dep['id']}", icon=":material/upgrade:", color="blue", help="The trading engine picks it up within 10 minutes.") + st.badge(f"Switching to selector #{dep['id']}", icon=":material/upgrade:", color="blue", help="The trading engine picks it up within 10 minutes.") fly = s["fly"] + if fly.get("bootstrap"): # the fly's own model, beside the selector's: they are separate snapshots + st.badge(f"Fly #{fly['bootstrap']}", icon=":material/neurology:", color="violet", + help="The fly bootstrap trading the fly book (taught once by the selector, then learning on its own).") if fly.get("stage") and fly.get("stage") != "not trading": st.badge(f"Fly learning · drift {float(fly.get('drift') or 0):.1%}" if not fly.get("learning_frozen") else "Fly: learning frozen", icon=":material/neurology:", color="violet" if not fly.get("learning_frozen") else "orange", @@ -213,6 +338,31 @@ def status_strip() -> None: help=f"newest complete minute closed {s['feed_age']:.0f} s ago" if s["feed_age"] is not None else "no minute written yet") +def rh_strip() -> None: + """The Robinhood Chain row of the status strip: money, trading, the RH fly, its seat and the indexer.""" + s = system_state(markets.RH); fly = s["fly"]; ix = s["feed"] + with st.container(horizontal=True, gap="small"): + if s["live_money"]: + st.badge("RH live money", icon=":material/payments:", color="red", help="Trades spend real ETH from the Robinhood Chain bot wallet.") + else: + miss = config.rh_live_prerequisites_missing() + st.badge("RH paper — no real ETH", icon=":material/receipt:", color="blue", + help="Pons trades are simulated at market prices with the hook fee and impact" + (f"; live gated: {', '.join(miss)}." if miss else ".")) + text, color = TRADING_BADGE[s["trading"]] + st.badge(f"RH {text[0].lower() + text[1:]}", icon=":material/candlestick_chart:", color=color, help=s["trading_why"]) + if fly.get("stage") and fly.get("stage") != "not trading": + st.badge("RH fly trading", icon=":material/neurology:", color="violet", help="The fly on Robinhood Chain rows (one brain with the Solana fly).") + else: + st.badge("RH fly not trading", icon=":material/neurology:", color="gray", help=fly.get("detail") or s["trading_why"]) + if s["handover"]: + st.badge("RH fly holds the seat", icon=":material/swap_horiz:", color="green") + if ix.get("mode") == "backfill": + st.badge(f"RH indexer backfilling · {int(ix.get('behind_blocks') or 0):,} blocks behind", icon=":material/history:", color="blue") + else: + st.badge("RH feed live" if s["feed_ok"] else "RH feed stale", icon=":material/sensors:", color="green" if s["feed_ok"] else "red", + help=f"newest complete minute closed {s['feed_age']:.0f} s ago" if s["feed_age"] is not None else "no minute written yet") + + def restart_runner() -> None: """Stop the trading engine and start it again (it loads the newest model; RESET_ON_START clears the paper book).""" sup = get_supervisor() diff --git a/fly_trader/vault/backup.py b/fly_trader/vault/backup.py index e0dcc7f4920f3bd63635d577abc6c6d1a2bd7858..7931f2f00f519bf9749b7ffdfa6c76fac48d5a57 100644 --- a/fly_trader/vault/backup.py +++ b/fly_trader/vault/backup.py @@ -1,4 +1,4 @@ -"""Encrypted off-box backups: the wallet key (once) and the vault ledger (nightly), encrypted with ``age`` to Bryce's +"""Encrypted off-box backups: the two hot keys (once, encrypted by the signer) and the vault ledger (nightly), with ``age`` to Bryce's public key (``VAULT_BACKUP_RECIPIENT``: an age1… or ssh-ed25519 key) and uploaded to a private HF storage bucket with a token scoped to that bucket. Plaintext never leaves the server; only the holder of the private key can decrypt: @@ -40,19 +40,23 @@ def _upload(files: list[tuple[Path, str]]) -> None: batch_bucket_files(config.VAULT_BACKUP_BUCKET, add=[(str(p), remote) for p, remote in files], token=config.VAULT_BACKUP_HF_TOKEN) -def key_once() -> bool: - """Back up the wallet key the first time only; later runs are no-ops.""" +def key_once(signer) -> bool: + """Back up both hot keys the first time only; later runs are no-ops. The signer encrypts them to the operator's + public key itself, so the plaintext never leaves the signer; this process only uploads ciphertext.""" if state.get("backup_key_done") or not configured(): return False - from ..chain.keys import load_keypair - kp = load_keypair() - import base58 + import base64 + blobs = signer.call("backup_keys") with tempfile.TemporaryDirectory() as d: - out = Path(d) / "wallet-key.age" - _encrypt(base58.b58encode(bytes(kp)), out) - _upload([(out, f"keys/{kp.pubkey()}.age")]) - state.put("backup_key_done", {"ts": int(time.time()), "pubkey": str(kp.pubkey())}) - alerts.send(f"wallet key backup written (encrypted) for {kp.pubkey()}") + files = [] + for b in blobs: + out = Path(d) / f"{b['name']}-key.age" + out.write_bytes(base64.b64decode(b["age_b64"])) + files.append((out, f"keys/{b['name']}-{b['pubkey']}.age")) + _upload(files) + trading = next((b["pubkey"] for b in blobs if b["name"] == "trading"), None) + state.put("backup_key_done", {"ts": int(time.time()), "pubkey": trading}) + alerts.send(f"hot-key backups written (encrypted) for {', '.join(b['name'] for b in blobs)}") return True diff --git a/fly_trader/vault/claims.py b/fly_trader/vault/claims.py index b3831232049dc577706efb8df5f47379be45379b..102c4ef65a4daecfc71904cbcce1aa7338bfce8d 100644 --- a/fly_trader/vault/claims.py +++ b/fly_trader/vault/claims.py @@ -6,30 +6,26 @@ received -> verified -> (waiting_liquidity) -> sending -> paid | rejected | fail * One claim in flight per holder (partial unique index); the amount is fixed when the claim moves to ``sending``. * The payment row (signature, blockhash, last valid height) is written BEFORE broadcast; after a crash the claim is re-checked on chain and re-signed only once its blockhash can no longer land, so a holder is never paid twice. -* A payment never takes the wallet below the gas reserve; owed SOL is reserved out of the bankroll, so waiting is rare. +* Claims are paid from the Squads treasury under its weekly spending limit L2, which only the payout key may use; the + signer (fly_trader/signer) builds and signs the payment and keeps its own once-per-claim ledger. A claim the + treasury or this week's L2 cannot cover waits (``waiting_liquidity``) and alerts, never fails. """ from __future__ import annotations -import base64 import logging import time from psycopg.types.json import Jsonb -from solders.hash import Hash -from solders.instruction import Instruction -from solders.message import MessageV0 -from solders.pubkey import Pubkey -from solders.system_program import TransferParams, transfer -from solders.transaction import VersionedTransaction from .. import config from ..chain.cluster_guard import assert_vault_signing_allowed +from ..signer.client import SignerUnavailable +from ..signer.policy import PolicyError from ..db.apilog import record_event from ..db.connection import transaction -from . import alerts, claim_message as cm, evm, sigs, state, walletlock +from . import alerts, claim_message as cm, claims_eth, evm, sigs, state, walletlock log = logging.getLogger(__name__) -MEMO_PROGRAM = Pubkey.from_string("MemoSq4gqABAXKb96qnH8TysNcWxMyWCqXgDLGmfcHr") MAX_ATTEMPTS = 4 STALE_AFTER_S = 3600 # a claim first seen more than an hour after its expiry is refused (signed long ago) TERMINAL = ("paid", "rejected", "failed") @@ -82,11 +78,10 @@ def _owed(conn, evm_addr: str) -> int: return int(r["owed"]) if r else 0 -def _build(keypair, dest: str, lamports: int, claim_id: int, blockhash: str) -> VersionedTransaction: - payer = keypair.pubkey() - ixs = [transfer(TransferParams(from_pubkey=payer, to_pubkey=Pubkey.from_string(dest), lamports=int(lamports))), - Instruction(MEMO_PROGRAM, f"fly-vault-claim:{claim_id}".encode(), [])] - return VersionedTransaction(MessageV0.try_compile(payer, ixs, [], Hash.from_string(blockhash)), [keypair]) +def signer_claim_id(row: dict) -> str: + """The claim's key in the signer's own ledger: our id plus the holder's signed nonce, so a rebuilt database that + reuses an id can never make the signer pay a different claim under it.""" + return f"{int(row['id'])}:{row['nonce']}" def _record_paid(conn, cid: int, row: dict, sig: str, slot: int, fee: int) -> None: @@ -96,7 +91,7 @@ def _record_paid(conn, cid: int, row: dict, sig: str, slot: int, fee: int) -> No (sig, slot, row["sol"], int(row["lamports"]), fee, f"claim {cid} for {row['evm']}")) -def pay(cid: int, rpc, keypair, wait=None) -> str: +def pay(cid: int, rpc, signer, wait=None) -> str: """Send (or resume) the payment of a claim in ``sending``. Returns the resulting status.""" from ..execution.broker_live import await_confirmation assert_vault_signing_allowed() @@ -108,6 +103,12 @@ def pay(cid: int, rpc, keypair, wait=None) -> str: return row["status"] if row.get("tx_signature"): # resume after a crash: did the last attempt land? st = (rpc.get_signature_statuses([row["tx_signature"]], search_history=True) or [None])[0] + if st and st.get("err") is None and st.get("confirmationStatus") in ("confirmed", "finalized"): + # Landed. Record it from the history lookup: await_confirmation polls without history, so once the + # status has aged out of the recent cache it would report "expired" and we would pay again. + with transaction() as conn: + _record_paid(conn, cid, row, row["tx_signature"], int(st.get("slot") or 0), 5000) + return "paid" if st and st.get("err") is None: status, info = wait(rpc, row["tx_signature"], row.get("last_valid_block_height")) if status == "confirmed": @@ -125,24 +126,36 @@ def pay(cid: int, rpc, keypair, wait=None) -> str: _set(conn, cid, "failed", reason="payment did not land after several attempts") alerts.send(f"claim {cid} failed after {MAX_ATTEMPTS} attempts ({row['lamports']} lamports to {row['sol']})") return "failed" - native = rpc.get_balance(str(keypair.pubkey())) - gas = int(round(config.GAS_RESERVE_SOL * config.LAMPORTS_PER_SOL)) - if native - int(row["lamports"]) - 10_000 < gas: + try: + built = signer.call("pay_claim", claim_id=signer_claim_id(row), dest=row["sol"], lamports=int(row["lamports"])) + except PolicyError as e: + if e.code in ("cap", "liquidity"): + with transaction() as conn: + _set(conn, cid, "waiting_liquidity", reason=str(e)) + what = ("this week's payout limit (L2) is used up: raise it from the owner wallet in the Squads app" + if e.code == "cap" else "the treasury holds too little") + alerts.send(f"claim {cid} waiting ({int(row['lamports']) / config.LAMPORTS_PER_SOL:.4f} SOL): {what}", key="claim_liquidity") + return "waiting_liquidity" + if e.code == "in_flight": + return "sending" + if e.code == "paid" and row.get("tx_signature"): # the signer saw it land; our resume check will record it + return "sending" with transaction() as conn: - _set(conn, cid, "waiting_liquidity", reason="wallet below the gas reserve after this payment") - alerts.send(f"claim {cid} waiting: wallet {native} lamports cannot pay {row['lamports']} above the gas reserve", key="claim_liquidity") - return "waiting_liquidity" - bh = rpc.get_latest_blockhash() - tx = _build(keypair, row["sol"], int(row["lamports"]), cid, bh["blockhash"]) - sig = str(tx.signatures[0]) + _set(conn, cid, "failed", reason=f"the signer refused: {e}") + alerts.send(f"claim {cid}: the signer refused the payment: {e}") + return "failed" + except SignerUnavailable as e: + log.warning("claim %d: signer unavailable: %s", cid, e) + return "sending" + sig, lvbh = built["signature"], int(built["last_valid_block_height"]) with transaction() as conn: # persisted before it can land conn.execute("UPDATE vault_claims SET tx_signature = %s, blockhash = %s, last_valid_block_height = %s, attempts = attempts + 1, " - "updated_at = now() WHERE id = %s", (sig, bh["blockhash"], bh["last_valid_block_height"], cid)) + "updated_at = now() WHERE id = %s", (sig, built.get("blockhash"), lvbh, cid)) try: - rpc.send_transaction(base64.b64encode(bytes(tx)).decode()) + rpc.send_transaction(built["tx"]) except Exception as e: log.warning("claim %d send failed: %s", cid, type(e).__name__) - status, info = wait(rpc, sig, bh["last_valid_block_height"]) + status, info = wait(rpc, sig, lvbh) with transaction() as conn: if status == "confirmed": _record_paid(conn, cid, row, sig, int((info or {}).get("slot") or 0), 5000) @@ -154,15 +167,22 @@ def pay(cid: int, rpc, keypair, wait=None) -> str: return "sending" # expired unlanded: the next round re-signs -def process(relay, rpc, keypair, rh_rpc=None, now: float | None = None) -> dict: +def process(relay, rpc, signer, rh_rpc=None, now: float | None = None) -> dict: """One round: ingest new relay claims, verify, pay, report. Never raises into the worker loop.""" out = {"ingested": 0, "paid": 0, "rejected": 0} after = int(state.get("relay_claims_after") or 0) for c in relay.pending_claims(after=after): - with transaction() as conn: - if ingest(conn, c) is not None: - out["ingested"] += 1 - after = max(after, int(c["id"])) + try: + with transaction() as conn: + if ingest(conn, c) is not None: + out["ingested"] += 1 + except Exception as e: # a row we cannot store is skipped, not retried forever + log.warning("relay claim %s could not be stored: %s", c.get("id"), e) + alerts.send(f"relay claim {c.get('id')} was malformed and skipped: {e}", key="claim_malformed") + try: + after = max(after, int(c["id"])) + except (KeyError, TypeError, ValueError): + continue state.put("relay_claims_after", after) if state.halted(): return {**out, "halted": True} @@ -188,15 +208,25 @@ def process(relay, rpc, keypair, rh_rpc=None, now: float | None = None) -> dict: if row["status"] in ("verified", "waiting_liquidity"): with walletlock.exclusive(timeout_s=180), transaction() as conn: owed = _owed(conn, row["evm"]) + (int(row["lamports"] or 0) if row["status"] == "waiting_liquidity" else 0) + eth_leg = claims_eth.fix_amount(conn, cid, row["evm"]) # the ETH pot's leg, fixed with the SOL amount if owed < config.CLAIM_MIN_LAMPORTS: + if eth_leg: # ETH only: the SOL leg has nothing to pay + _set(conn, cid, "paid", lamports=0, reason="no SOL owed; ETH paid on Robinhood Chain") + continue _set(conn, cid, "rejected", reason=f"nothing to claim (owed {owed} lamports)"); out["rejected"] += 1 continue _set(conn, cid, "sending", lamports=owed) - if pay(cid, rpc, keypair) == "paid": + if pay(cid, rpc, signer) == "paid": out["paid"] += 1 except Exception as e: log.exception("claim %d", cid) record_event("error", "vault", f"claim {cid} error", {"error": type(e).__name__}) + if config.RH_ENABLED: + try: + out["eth"] = claims_eth.process() + except Exception as e: # the ETH legs never block the SOL payouts or the report + log.exception("ETH claim legs") + record_event("error", "vault", "ETH claim legs failed", {"error": type(e).__name__}) report(relay) return out @@ -208,7 +238,7 @@ def report(relay) -> int: if not rows: return 0 relay.report([{"id": int(r["relay_id"]), "status": r["status"], "reason": r["reason"], "lamports": r["lamports"], - "tx": r["tx_signature"] if r["status"] == "paid" else None} for r in rows]) + "tx": None} for r in rows]) # the payout tx names the fly's wallet: never published with transaction() as conn: conn.execute("UPDATE vault_claims SET reported_at = now() WHERE id = ANY(%s)", ([int(r["id"]) for r in rows],)) return len(rows) diff --git a/fly_trader/vault/claims_eth.py b/fly_trader/vault/claims_eth.py new file mode 100644 index 0000000000000000000000000000000000000000..acf1e20d0e4b00ee151f8434ed0e7eb2f590f7c8 --- /dev/null +++ b/fly_trader/vault/claims_eth.py @@ -0,0 +1,147 @@ +"""A claim's ETH leg: the holder's owed ETH (the vault's ETH pot, vault/settle_eth.py) paid in native ETH on Robinhood +Chain to the EVM address that signed the claim. The claim text (v1) is unchanged: the EVM address is already in it. + +none -> owed -> sending -> paid | failed (independent of the SOL leg's status) + +* The amount is fixed in the same database transaction that fixes the SOL amount (claims.process), from + ``vault_eth_accounts`` — so a later claim can never be paid the same wei again. +* The payment is a plain value transfer from the RH bot wallet (rh/wallet.RhWallet, the same key and guard as live RH + trading). Its ``rh_txs`` row and the claim's link to it are committed together before broadcast; after a crash the + row is settled on chain first (receipt, superseded at its nonce, or cancelled) and re-sent only if it can no longer + land, so a holder is never paid twice. +* Signing refused (RH_LIVE_ENABLED off, key or address missing) leaves the leg owed and alerts; it is never lost. +""" +from __future__ import annotations + +import logging +from datetime import datetime, timezone + +from .. import config +from ..db.connection import transaction +from . import alerts + +log = logging.getLogger(__name__) +MAX_ATTEMPTS = 4 +REBROADCAST_S = 300.0 +WEI = 10 ** 18 + + +def owed(conn, evm_addr: str) -> int: + r = conn.execute("SELECT owed FROM vault_eth_accounts WHERE evm = %s", (evm_addr,)).fetchone() + return int(r["owed"]) if r else 0 + + +def fix_amount(conn, cid: int, evm_addr: str) -> bool: + """Fix the ETH leg of claim ``cid`` (once). True when there is an ETH leg to pay.""" + if not config.RH_ENABLED: + return False + row = conn.execute("SELECT eth_status FROM vault_claims WHERE id = %s", (cid,)).fetchone() + if row and row["eth_status"] != "none": + return row["eth_status"] in ("owed", "sending", "paid") + eth = owed(conn, evm_addr) + if eth < config.CLAIM_MIN_WEI: + return False + conn.execute("UPDATE vault_claims SET eth_wei = %s, eth_status = 'owed', updated_at = now() WHERE id = %s AND eth_status = 'none'", (eth, cid)) + return True + + +def _wallet(): + from ..rh.rpc import RhRpc + from ..rh.wallet import RhWallet + return RhWallet(RhRpc()) + + +def _paid(conn, row: dict, tx_id: int) -> None: + conn.execute("UPDATE vault_claims SET eth_status = 'paid', eth_reason = NULL, updated_at = now(), reported_at = NULL WHERE id = %s", (row["id"],)) + conn.execute("UPDATE rh_txs SET applied_at = now() WHERE id = %s", (tx_id,)) + conn.execute("INSERT INTO rh_wallet_flows (direction, kind, wei, note) VALUES ('out', 'payout', %s, %s)", (int(row["eth_wei"]), f"claim {row['id']} for {row['evm']}")) + + +def _retry(conn, row: dict, why: str) -> None: + conn.execute("UPDATE vault_claims SET eth_status = 'owed', eth_tx_id = NULL, eth_reason = %s, updated_at = now() WHERE id = %s", (why[:200], row["id"])) + + +def pay_one(wallet, row: dict) -> str: + """Advance one claim's ETH leg. Returns its status.""" + from ..rh.wallet import SendFailed + cid = int(row["id"]) + with wallet.lock: + if row.get("eth_tx_id"): # settle the last attempt before anything else + with transaction() as conn: + tx = dict(conn.execute("SELECT * FROM rh_txs WHERE id = %s", (row["eth_tx_id"],)).fetchone()) + rc = wallet.rpc.receipt(tx["hash"]) + if rc is None and tx["status"] in ("signed", "sent"): + age = (datetime.now(timezone.utc) - tx["created_at"]).total_seconds() + if wallet.rpc.tx_count(wallet.address, "latest") <= int(tx["nonce"]): + if age < REBROADCAST_S: + rc = wallet.wait(tx["hash"], timeout_s=30.0) + if rc is None: + return "sending" + else: # cannot land any more once its nonce is taken: take it + c = wallet.cancel(int(tx["nonce"]), int(tx["max_fee_wei"] or 0)) + rc_c = wallet.wait(c["hash"], timeout_s=120.0) + if rc_c is not None: # its gas is a wallet flow like any other + with transaction() as conn: + wallet.record_receipt(conn, int(c["id"]), rc_c) + conn.execute("UPDATE rh_txs SET applied_at = now() WHERE id = %s", (c["id"],)) + rc = wallet.rpc.receipt(tx["hash"]) + with transaction() as conn: + if rc is not None and rc.get("blockNumber"): + res = wallet.record_receipt(conn, int(tx["id"]), rc) + if res["ok"]: + _paid(conn, row, int(tx["id"])); return "paid" + conn.execute("UPDATE rh_txs SET applied_at = now() WHERE id = %s", (tx["id"],)) + _retry(conn, row, "payout reverted") + else: + conn.execute("UPDATE rh_txs SET status = 'replaced', applied_at = now(), updated_at = now() WHERE id = %s AND status IN ('signed', 'sent')", (tx["id"],)) + _retry(conn, row, "payout did not land") + row = {**row, "eth_tx_id": None} + if int(row.get("eth_attempts") or 0) >= MAX_ATTEMPTS: + with transaction() as conn: + conn.execute("UPDATE vault_claims SET eth_status = 'failed', eth_reason = 'the ETH payout did not land after several attempts', updated_at = now(), " + "reported_at = NULL WHERE id = %s", (cid,)) + alerts.send(f"claim {cid}: ETH payout failed after {MAX_ATTEMPTS} attempts ({int(row['eth_wei']) / WEI:.6f} ETH to {row['evm']})") + return "failed" + def link(conn, tid): + conn.execute("UPDATE vault_claims SET eth_tx_id = %s, eth_status = 'sending', eth_attempts = eth_attempts + 1, updated_at = now() WHERE id = %s", (tid, cid)) + try: + s = wallet.send(to=row["evm"], data=b"", value=int(row["eth_wei"]), kind="payout", on_persist=link) + except SendFailed as e: + with transaction() as conn: + _retry(conn, row, f"send failed: {e}") + return "owed" + rc = wallet.wait(s["hash"], timeout_s=120.0) + if rc is None: + return "sending" + with transaction() as conn: + res = wallet.record_receipt(conn, s["id"], rc) + if res["ok"]: + _paid(conn, row, s["id"]); return "paid" + conn.execute("UPDATE rh_txs SET applied_at = now() WHERE id = %s", (s["id"],)) + _retry(conn, row, "payout reverted") + return "owed" + + +def process(wallet=None) -> dict: + """Pay every owed or unfinished ETH leg. Never raises into the claims round.""" + out = {"paid": 0, "waiting": 0} + with transaction() as conn: + todo = [dict(r) for r in conn.execute("SELECT * FROM vault_claims WHERE eth_status IN ('owed', 'sending') ORDER BY id").fetchall()] + if not todo: + return out + if wallet is None: + from ..rh.guard import SigningRefused + try: + wallet = _wallet() + except SigningRefused as e: + out["waiting"] = len(todo) + alerts.send(f"{len(todo)} ETH claim payout(s) wait: Robinhood Chain signing is off ({e})", key="eth_claims_signing", cooldown_s=3600) + return out + for row in todo: + try: + st = pay_one(wallet, row) + out["paid"] += st == "paid"; out["waiting"] += st in ("owed", "sending") + except Exception as e: # noqa: BLE001 — one leg's failure never blocks the rest + log.exception("claim %s ETH leg", row["id"]) + alerts.send(f"claim {row['id']}: ETH leg error {type(e).__name__}", key=f"eth_leg_{row['id']}", cooldown_s=1800) + return out diff --git a/fly_trader/vault/cli.py b/fly_trader/vault/cli.py index fd5eff715937bc585d10d8e5cda28baa74a07e4e..fa2a24ea6fcd78480605a0442a2c8d0cc697b7d8 100644 --- a/fly_trader/vault/cli.py +++ b/fly_trader/vault/cli.py @@ -5,7 +5,7 @@ import json def add(sub) -> None: - v = sub.add_parser("vault", help="the $FLY vault: status, scan, settle, reclassify, withdraw, claims, resume") + v = sub.add_parser("vault", help="the $FLY vault: status, scan, settle, reclassify, withdraw, claims, resume, custody") vs = v.add_subparsers(dest="vault_cmd", required=True) vs.add_parser("status", help="ledger, index, halt state, last settlement") vs.add_parser("scan", help="classify new wallet transactions and index Robinhood Chain once") @@ -13,8 +13,9 @@ def add(sub) -> None: st.add_argument("--dry-run", action="store_true", help="print what the current period would allocate, write nothing") rc = vs.add_parser("reclassify", help="relabel one transaction's flow") rc.add_argument("signature"); rc.add_argument("kind", choices=["deposit", "profit", "internal"]); rc.add_argument("--note") - wd = vs.add_parser("withdraw", help="send principal back to a funding address") - wd.add_argument("--sol", type=float, required=True); wd.add_argument("--to", required=True) + vs.add_parser("withdraw", help="how much principal you may take out of the treasury, and how (Squads app)") + vs.add_parser("custody", help="read and check the treasury multisig and its spending limits L1/L2") + vs.add_parser("squads-setup", help="the Squads app steps that create the treasury's spending limits for this server's keys") vs.add_parser("claims", help="recent claims") vs.add_parser("resume", help="clear a vault halt (after you checked why it halted)") v.set_defaults(fn=run) @@ -26,21 +27,32 @@ def _rpc(): return HttpSolanaRpc(config.vault_solana_rpc_url()) +def _keys() -> dict: + """The server's public keys, from the signer (the private keys never enter this process).""" + from ..signer import client + return client.get().call("pubkeys") + + def run(args) -> None: from .. import config - from ..chain.keys import load_keypair from ..db.connection import transaction - from . import flows, rh_index, settle, state + from . import custody, flows, rh_index, settle, state cmd = args.vault_cmd + k = _keys() if cmd not in ("reclassify", "claims", "resume") else {} + wallet, ours = k.get("trading"), {x for x in (k.get("trading"), k.get("payout")) if x} if cmd == "status": from . import publish with transaction() as conn: - s = publish.stats(conn, str(load_keypair().pubkey())) + s = publish.stats(conn, wallet) print(json.dumps({k: s[k] for k in ("fly", "wallet", "ledger", "vault", "settlement", "index")}, indent=2, default=str)) print("halt:", state.halted()) elif cmd == "scan": - print(flows.scan(_rpc(), str(load_keypair().pubkey()))) + print(flows.scan_all(_rpc(), wallet, ours)) print(rh_index.run_once()) + elif cmd == "custody": + print(json.dumps(custody.check(_rpc(), wallet, k.get("payout")), indent=2, default=str)) + elif cmd == "squads-setup": + print(squads_setup(k)) elif cmd == "settle": if args.dry_run: import time @@ -48,19 +60,25 @@ def run(args) -> None: t1 = settle.period_end_at_or_before(time.time()) + int(config.VAULT_PERIOD_S) start, evs = settle.event_inputs(conn, config.RH_CHAIN_ID, t1 - int(config.VAULT_PERIOD_S), int(time.time())) w = settle.weights(start, evs, t1 - int(config.VAULT_PERIOD_S), int(time.time())) - native = _rpc().get_balance(str(load_keypair().pubkey())) + t = custody.treasury_address() + native = _rpc().get_balance(wallet) + (_rpc().get_balance(t) if t else 0) cost, _ = settle.open_positions(conn) f = settle.flow_totals(conn, 2**62) r = settle.realized(native, 0, cost, f["deposits"], f["withdrawals"], f["payouts"]) p = settle.plan(r, settle.allocated_total(conn), f["payouts"], native, int(config.GAS_RESERVE_SOL * config.LAMPORTS_PER_SOL)) print(json.dumps({"period_end": t1, "realized_so_far": r, "plan": p, "allocation_if_now": settle.allocate(p["amount"], w)}, indent=2)) else: - print(settle.run_once(_rpc(), str(load_keypair().pubkey()), rh_index.run_once())) + print(settle.run_once(_rpc(), wallet, rh_index.run_once(), treasury=custody.treasury_address(), our_keys=ours)) elif cmd == "reclassify": print(flows.reclassify(args.signature, args.kind, args.note), "row(s) changed") elif cmd == "withdraw": from . import withdraw - print(withdraw.withdraw(int(round(args.sol * config.LAMPORTS_PER_SOL)), args.to, _rpc(), load_keypair())) + t = custody.treasury_address() + if not t: + raise SystemExit("no treasury configured (VAULT_MULTISIG)") + rpc = _rpc() + with transaction() as conn: + print(withdraw.instructions(withdraw.cap(conn, rpc.get_balance(wallet), rpc.get_balance(t)), t)) elif cmd == "claims": with transaction() as conn: for r in conn.execute("SELECT id, relay_id, evm, sol, status, reason, lamports, tx_signature, updated_at FROM vault_claims ORDER BY id DESC LIMIT 30"): @@ -68,3 +86,37 @@ def run(args) -> None: elif cmd == "resume": state.resume() print("vault halt cleared; entries stay paused until you resume them (fly-trader resume-entries)") + + +def squads_setup(k: dict) -> str: + """Step by step, for the owner's wallet in the Squads app. Prints; signs nothing.""" + from .. import config + ms = config.VAULT_MULTISIG + lines = ["Treasury setup, signed by your owner wallet (Solflare now, the Ledger later). The easy way: open", + "https://fly-trader.app/owner.html, paste the two keys below, and use its buttons. The Squads app (app.squads.so)", + "may not let a spending limit name a key that is not a member; the owner page builds the transactions itself.", + "The server's keys must NEVER be members of the multisig; they only get the two spending limits below.", "", + f"trading key (bot_key): {k.get('trading')}", f"payout key (payout_key): {k.get('payout')}", ""] + if not ms: + lines += ["1. Create a multisig: members = your owner wallet only, threshold 1, time lock 0.", + " Then set VAULT_MULTISIG= in /srv/fly/custody.env, run sudo python3 /usr/local/lib/fly/fly_update.py restart,", + " and run this again.", ""] + else: + from . import custody + lines += [f"Multisig {ms}; treasury (vault 0) {custody.treasury_address()} -- fund the treasury, not the trading wallet.", ""] + lines += ["2. Settings -> Spending limits -> Add (L1: the trading float's refill):", + " token SOL, vault 0, period Daily, amount " + f"{config.TREASURY_FLOAT_SOL:g} SOL,", + f" member (who may use it): {k.get('trading')} <- the trading key", + f" destination (ONLY this): {k.get('trading')} <- the trading wallet itself", + " Put its address in VAULT_LIMIT_TRADING (/srv/fly/custody.env).", + "3. Add another (L2: claims):", + " token SOL, vault 0, period Weekly, amount 1 SOL (raise it when a settlement alert says owed > cap),", + f" member: {k.get('payout')} <- the payout key", + " destinations: none (any holder)", + " Put its address in VAULT_LIMIT_PAYOUT (/srv/fly/custody.env).", + "4. sudo python3 /usr/local/lib/fly/fly_update.py restart, then: sudo fly vault custody (every check must pass).", + "", + "Emergency: in the Squads app remove L1 and L2 (Settings -> Spending limits -> Remove). The server can then move", + "nothing out of the treasury, whatever it runs; the trading wallet keeps only its float."] + return "\n".join(lines) diff --git a/fly_trader/vault/custody.py b/fly_trader/vault/custody.py new file mode 100644 index 0000000000000000000000000000000000000000..9973ac60146b75264fc6e0576a0853f51e112a28 --- /dev/null +++ b/fly_trader/vault/custody.py @@ -0,0 +1,126 @@ +"""The treasury's on-chain setup, checked every minute (docs/vault/SPEC.md "Custody"). + +What must hold, and what the operator is told when it does not or when it changes: + +* the multisig's members are the owner's wallets only: neither server key may ever be a member (a member could vote + itself any power; the server gets only the two spending limits) +* L1: SOL, per day, member = the trading key, the ONLY destination = the trading wallet +* L2: SOL, per week, member = the payout key (any destination: holders) + +A limit that disappears is the owner revoking it (or someone who took the owner wallet): the signer then refuses and +this alerts once. Every change to members, threshold or limits is reported with before and after. +""" +from __future__ import annotations + +import logging + +from solders.pubkey import Pubkey + +from .. import config +from ..signer import squads +from . import alerts, state + +log = logging.getLogger(__name__) + + +def treasury_address() -> str | None: + return str(squads.vault_pda(Pubkey.from_string(config.VAULT_MULTISIG), 0)) if config.VAULT_MULTISIG else None + + +def _account(rpc, addr: str) -> bytes | None: + info = rpc.call("getAccountInfo", [addr, {"encoding": "base64", "commitment": "confirmed"}]) + v = (info or {}).get("value") + if not v or v.get("owner") != str(squads.PROGRAM_ID): + return None + import base64 + return base64.b64decode(v["data"][0]) + + +def read(rpc) -> dict: + """The current setup, as plain JSON (for state, alerts and the console).""" + out: dict = {"multisig": config.VAULT_MULTISIG, "treasury": treasury_address()} + data = _account(rpc, config.VAULT_MULTISIG) + if data is None: + out["error"] = "the multisig account is missing or not a Squads account" + return out + ms = squads.decode_multisig(data) + out.update({"threshold": ms["threshold"], "time_lock": ms["time_lock"], "members": sorted(str(k) for k, _ in ms["members"]), + "config_authority": str(ms["config_authority"])}) + for name, addr in (("L1", config.VAULT_LIMIT_TRADING), ("L2", config.VAULT_LIMIT_PAYOUT)): + d = _account(rpc, addr) if addr else None + if d is None: + out[name] = None + continue + lim = squads.decode_spending_limit(d) + out[name] = {"address": addr, "multisig": str(lim["multisig"]), "mint": str(lim["mint"]), "amount": lim["amount"], "period": lim["period"], + "remaining": lim["remaining"], "last_reset": lim["last_reset"], "members": [str(k) for k in lim["members"]], + "destinations": [str(k) for k in lim["destinations"]]} + return out + + +def problems(c: dict, trading: str, payout: str) -> list[str]: + """What is wrong with the setup (pure).""" + if c.get("error"): + return [c["error"]] + bad = [] + for who, key in (("trading", trading), ("payout", payout)): + if key in c.get("members", []): + bad.append(f"the {who} key is a MEMBER of the treasury multisig: remove it (the server must only hold spending limits)") + if str(c.get("config_authority")) not in ("11111111111111111111111111111111", "None"): + bad.append(f"the multisig has a config authority ({c['config_authority']}): it can change members without a vote") + l1, l2 = c.get("L1"), c.get("L2") + if l1 is None: + bad.append("L1 (the trading float's daily refill) is missing: the float cannot refill") + else: + if l1["multisig"] != c["multisig"] or l1["mint"] != "11111111111111111111111111111111": + bad.append("L1 is not a SOL limit on this multisig") + if l1["members"] != [trading]: + bad.append(f"L1 members are {l1['members']}, expected only the trading key") + if l1["destinations"] != [trading]: + bad.append("L1 must have the trading wallet as its ONLY destination (else a stolen trading key could send treasury SOL anywhere)") + if l2 is None: + bad.append("L2 (weekly claims) is missing: claims will wait") + else: + if l2["multisig"] != c["multisig"] or l2["mint"] != "11111111111111111111111111111111": + bad.append("L2 is not a SOL limit on this multisig") + if l2["members"] != [payout]: + bad.append(f"L2 members are {l2['members']}, expected only the payout key") + return bad + + +def changes(before: dict | None, now: dict) -> list[str]: + """Human-readable differences between two readings (pure).""" + if not before: + return [] + out = [] + for k, label in (("members", "multisig members"), ("threshold", "threshold"), ("time_lock", "time lock"), ("config_authority", "config authority")): + if before.get(k) != now.get(k): + out.append(f"{label}: {before.get(k)} -> {now.get(k)}") + for name in ("L1", "L2"): + b, n = before.get(name), now.get(name) + if (b is None) != (n is None): + out.append(f"{name} {'removed' if n is None else 'created'}") + elif b and n: + for k in ("amount", "period", "members", "destinations"): + if b.get(k) != n.get(k): + fmt = (lambda v: f"{v / config.LAMPORTS_PER_SOL:g} SOL") if k == "amount" else str + out.append(f"{name} {k}: {fmt(b.get(k))} -> {fmt(n.get(k))}") + return out + + +def check(rpc, trading: str, payout: str) -> dict: + """Read, compare with the last reading, alert on problems and changes; stores the reading in vault_kv.""" + c = read(rpc) + before = state.get("custody") + for line in changes(before, c): + alerts.send(f"treasury setup changed: {line}. If you did not do this in the Squads app, revoke L1 and L2 now.") + bad = problems(c, trading, payout) + if bad: + alerts.send("treasury setup: " + "; ".join(bad), key="custody_problem", cooldown_s=6 * 3600) + state.put("custody", c) + return {**c, "problems": bad} + + +def owners() -> set[str]: + """The multisig members as last read (their outbound treasury transfers are principal withdrawals).""" + return set((state.get("custody") or {}).get("members") or []) diff --git a/fly_trader/vault/eth_schema.py b/fly_trader/vault/eth_schema.py new file mode 100644 index 0000000000000000000000000000000000000000..eec0c3d23450e899d5d080919513feae7c4a1236 --- /dev/null +++ b/fly_trader/vault/eth_schema.py @@ -0,0 +1,38 @@ +"""The vault's ETH pot (Robinhood Chain memecoin profits, paid in native ETH on Robinhood Chain to the locker's EVM +address). Idempotent and additive like vault/schema.py; applied after it (db/schema.apply_schema). Amounts in wei.""" +from __future__ import annotations + +ETH_DDL: list[str] = [ + """CREATE TABLE IF NOT EXISTS vault_eth_settlements ( + id bigserial PRIMARY KEY, + settlement_id bigint NOT NULL UNIQUE REFERENCES vault_settlements(id), + period_start timestamptz NOT NULL, period_end timestamptz NOT NULL, + status text NOT NULL CHECK (status IN ('allocated', 'failed')), + book text NOT NULL, + native numeric(78,0), open_cost numeric(78,0), lots numeric(78,0), deposits numeric(78,0), withdrawals numeric(78,0), payouts numeric(78,0), + realized numeric(78,0), allocated_before numeric(78,0), pot numeric(78,0), liquid_cap numeric(78,0), allocated numeric(78,0), carried numeric(78,0), + earners int, inputs_sha256 text, created_at timestamptz NOT NULL DEFAULT now())""", + """CREATE TABLE IF NOT EXISTS vault_eth_allocations ( + eth_settlement_id bigint NOT NULL REFERENCES vault_eth_settlements(id), + evm text NOT NULL, + weight numeric(100) NOT NULL, + wei numeric(78,0) NOT NULL CHECK (wei >= 0), + PRIMARY KEY (eth_settlement_id, evm))""", + # a claim's ETH leg: the amount fixed with the SOL amount, then paid by the RH wallet (rh_txs row eth_tx_id) + "ALTER TABLE vault_claims ADD COLUMN IF NOT EXISTS eth_wei numeric(78,0)", + "ALTER TABLE vault_claims ADD COLUMN IF NOT EXISTS eth_status text NOT NULL DEFAULT 'none'", + "ALTER TABLE vault_claims ADD COLUMN IF NOT EXISTS eth_tx_id bigint", + "ALTER TABLE vault_claims ADD COLUMN IF NOT EXISTS eth_attempts int NOT NULL DEFAULT 0", + "ALTER TABLE vault_claims ADD COLUMN IF NOT EXISTS eth_reason text", + "ALTER TABLE vault_claims DROP CONSTRAINT IF EXISTS vault_claims_eth_status_check", + "ALTER TABLE vault_claims ADD CONSTRAINT vault_claims_eth_status_check CHECK (eth_status IN ('none', 'owed', 'sending', 'paid', 'failed'))", + # owed ETH per holder: allocated − paid − in flight + """CREATE OR REPLACE VIEW vault_eth_accounts AS + SELECT a.evm, a.allocated, COALESCE(c.claimed, 0)::numeric(78,0) AS claimed, COALESCE(c.in_flight, 0)::numeric(78,0) AS in_flight, + (a.allocated - COALESCE(c.claimed, 0) - COALESCE(c.in_flight, 0))::numeric(78,0) AS owed + FROM (SELECT evm, SUM(wei)::numeric(78,0) AS allocated FROM vault_eth_allocations GROUP BY evm) a + LEFT JOIN (SELECT evm, + SUM(eth_wei) FILTER (WHERE eth_status = 'paid') AS claimed, + SUM(eth_wei) FILTER (WHERE eth_status IN ('owed', 'sending')) AS in_flight + FROM vault_claims GROUP BY evm) c USING (evm)""", +] diff --git a/fly_trader/vault/flows.py b/fly_trader/vault/flows.py index 58782cdb61e0f85cf2708ddc4fe03d1af1e15c10..ed38cee27339e95b700ad6a33681159f049e2364 100644 --- a/fly_trader/vault/flows.py +++ b/fly_trader/vault/flows.py @@ -1,14 +1,16 @@ -"""Every lamport that enters or leaves the trading wallet other than by trading (plan "Core accounting"). +"""Every lamport that enters or leaves the book other than by trading (plan "Core accounting"). The book is two +accounts: the trading wallet and the Squads treasury (vault/payout.py); each has its own scan cursor. -The scanner walks the wallet's finalized transaction history and classifies each transaction by who signed it: +The scanner walks each account's finalized transaction history and classifies each transaction by who signed it: -* **ours** (our key signed): swaps, token-account closes, claim payouts, principal withdrawals. Claims and withdrawals - write their own ``vault_flows`` rows when they are sent; the rest move lamports inside R (fees, rent) and need no row. - A transaction our key signed that no table knows about is ``unknown_outbound``: a possible key leak, so entries and - claims halt until the operator looks (``fly-trader vault reclassify``). +* **ours** (the trading or the payout key signed): swaps, token-account closes, claim payouts, treasury top-ups and + sweeps. Those write their own ``vault_flows`` rows (or orders/fills rows) when they are sent; the rest move lamports + inside R (fees, rent) and need no row. A transaction our keys signed that no table knows about is + ``unknown_outbound``: a possible key leak, so entries and claims halt until the operator looks + (``fly-trader vault reclassify``). * **theirs**: SOL (or wSOL) arriving is a ``deposit`` when the sender is in ``FUNDING_ADDRESSES``, else ``profit`` (a - gift counts for the lockers). Lamports leaving without our signature cannot happen on a system account; if they do, - the row is an ``anomaly`` and the vault halts. + gift counts for the lockers). SOL leaving the treasury with a signature of one of its multisig members is the owner + taking principal out (``withdrawal``, alerted). Lamports leaving any other way are an ``anomaly`` and the vault halts. Signatures are 88-char base58 strings, which the log scrubber redacts as secrets; logs here carry 12-char prefixes. """ @@ -26,6 +28,7 @@ from ..db.connection import transaction log = logging.getLogger(__name__) SCAN_NAME = "solana_wallet" +SCAN_TREASURY = "solana_treasury" PAGE = 1000 @@ -55,8 +58,11 @@ def _wsol_delta(meta: dict, wallet: str) -> int: return total(meta.get("postTokenBalances")) - total(meta.get("preTokenBalances")) -def classify(tx: dict, wallet: str, known_ours: bool, funding: set[str]) -> Flow | None: - """The flow one finalized transaction makes, or None when it is trading/internal or moves nothing.""" +def classify(tx: dict, wallet: str, known_ours: bool, funding: set[str], our_keys: set[str] | None = None, + owners: set[str] | None = None) -> Flow | None: + """The flow one finalized transaction makes to ``wallet`` (the trading wallet or the treasury), or None when it is + trading/internal or moves nothing. ``our_keys``: the server's signing keys (default: ``wallet`` itself); + ``owners``: the treasury multisig's members, whose outbound transfers are principal withdrawals.""" keys = _keys(tx) meta = tx.get("meta") or {} sig = ((tx.get("transaction") or {}).get("signatures") or [None])[0] @@ -66,7 +72,8 @@ def classify(tx: dict, wallet: str, known_ours: bool, funding: set[str]) -> Flow return None pre, post = meta.get("preBalances") or [], meta.get("postBalances") or [] delta = (int(post[idx]) - int(pre[idx])) if idx < len(pre) and idx < len(post) else 0 - ours = bool(keys[idx].get("signer")) + signers = {k.get("pubkey") for k in keys if k.get("signer")} + ours = bool(signers & set(our_keys or {wallet})) if ours: if known_ours: return None @@ -76,6 +83,12 @@ def classify(tx: dict, wallet: str, known_ours: bool, funding: set[str]) -> Flow return None # a failed transaction someone else paid for moves nothing of ours amount = delta + _wsol_delta(meta, wallet) if amount < 0: + if signers & set(owners or ()): # the owner moved principal out of the treasury (Squads app) + dest, gain = None, 0 + for i, k in enumerate(keys): + if i != idx and i < len(pre) and i < len(post) and int(post[i]) - int(pre[i]) > gain: + dest, gain = k.get("pubkey"), int(post[i]) - int(pre[i]) + return Flow(sig, slot, bt, "out", "withdrawal", dest, -amount, 0, "owner withdrawal from the treasury") return Flow(sig, slot, bt, "out", "anomaly", keys[0].get("pubkey"), -amount, 0, "lamports left without our signature") if amount == 0: return None @@ -96,7 +109,7 @@ def known_ours(conn, signatures: list[str]) -> set[str]: "SELECT signature AS s FROM orders WHERE signature = ANY(%(s)s) " "UNION SELECT signature FROM fills WHERE signature = ANY(%(s)s) " "UNION SELECT tx_signature FROM vault_claims WHERE tx_signature = ANY(%(s)s) " - "UNION SELECT signature FROM vault_flows WHERE signature = ANY(%(s)s) AND kind IN ('withdrawal', 'claim') " + "UNION SELECT signature FROM vault_flows WHERE signature = ANY(%(s)s) AND kind IN ('withdrawal', 'claim', 'sweep', 'topup') " "UNION SELECT detail->>'signature' FROM wallet_events WHERE detail->>'signature' = ANY(%(s)s) " "UNION SELECT (value->>'signature') FROM vault_kv WHERE key LIKE 'ours:%%' AND value->>'signature' = ANY(%(s)s)", {"s": list(signatures)}).fetchall() @@ -111,22 +124,27 @@ def insert(conn, f: Flow, classified_by: str = "auto") -> bool: return row is not None -def cursor() -> dict: +def cursor(name: str = SCAN_NAME) -> dict: with transaction() as conn: - r = conn.execute("SELECT cursor, slot, detail FROM vault_scan WHERE name = %s", (SCAN_NAME,)).fetchone() + r = conn.execute("SELECT cursor, slot, detail FROM vault_scan WHERE name = %s", (name,)).fetchone() return dict(r) if r else {"cursor": None, "slot": None, "detail": None} def scanned_through() -> int: - """Every flow with slot <= this is in vault_flows.""" - return int(cursor().get("slot") or 0) + """Every flow with slot <= this is in vault_flows (both accounts: the lower of the two cursors).""" + slots = [int(cursor(SCAN_NAME).get("slot") or 0)] + if config.VAULT_MULTISIG: + slots.append(int(cursor(SCAN_TREASURY).get("slot") or 0)) + return min(slots) -def scan(rpc, wallet: str, funding: set[str] | None = None, max_pages: int = 50) -> dict: +def scan(rpc, wallet: str, funding: set[str] | None = None, max_pages: int = 50, name: str = SCAN_NAME, + our_keys: set[str] | None = None, owners: set[str] | None = None) -> dict: """Classify finalized transactions newer than the cursor. Idempotent; stops at the first unreadable transaction.""" funding = set(funding if funding is not None else config.FUNDING_ADDRESSES) + SCAN = name head = int(rpc.call("getSlot", [{"commitment": "finalized"}])) - cur = cursor() + cur = cursor(SCAN) until = cur.get("cursor") newest: list[dict] = [] before = None @@ -142,7 +160,7 @@ def scan(rpc, wallet: str, funding: set[str] | None = None, max_pages: int = 50) log.warning("vault flow scan: more than %d pages behind; continuing next round", max_pages) head = None # not caught up: do not advance the scanned-through slot counts: dict[str, int] = {} - halted = [] + halted, withdrawals = [], [] for entry in reversed(newest): # oldest first so the cursor only ever moves forward sig = entry["signature"] tx = rpc.get_transaction(sig) @@ -151,28 +169,66 @@ def scan(rpc, wallet: str, funding: set[str] | None = None, max_pages: int = 50) head = None break with transaction() as conn: - f = classify(tx, wallet, sig in known_ours(conn, [sig]), funding) + f = classify(tx, wallet, sig in known_ours(conn, [sig]), funding, our_keys, owners) if f is not None and insert(conn, f): counts[f.kind] = counts.get(f.kind, 0) + 1 if f.kind in ("unknown_outbound", "anomaly"): halted.append(f) + elif f.kind == "withdrawal" and name == SCAN_TREASURY: + withdrawals.append(f) conn.execute( "INSERT INTO vault_scan (name, cursor, slot, detail, updated_at) VALUES (%s,%s,%s,%s,now()) " "ON CONFLICT (name) DO UPDATE SET cursor = EXCLUDED.cursor, detail = EXCLUDED.detail, updated_at = now()", - (SCAN_NAME, sig, cur.get("slot"), Jsonb({"last_slot": int(entry.get("slot") or 0)}))) + (SCAN, sig, cur.get("slot"), Jsonb({"last_slot": int(entry.get("slot") or 0)}))) if head is not None: with transaction() as conn: conn.execute( "INSERT INTO vault_scan (name, slot, updated_at) VALUES (%s,%s,now()) " "ON CONFLICT (name) DO UPDATE SET slot = GREATEST(COALESCE(vault_scan.slot, 0), EXCLUDED.slot), updated_at = now()", - (SCAN_NAME, head)) + (SCAN, head)) + chk = None + if halted: + from ..chain.rpc import check_rpc + chk = check_rpc() for f in halted: + if chk is not None and not _confirmed_elsewhere(chk, f, wallet, funding, our_keys, owners): + from . import alerts + alerts.send(f"the RPC providers disagree about transaction {f.signature[:12]}… ({f.kind} on the primary): not halting, " + "look at it on an explorer", key="rpc_disagree", cooldown_s=1800) + continue _halt(f) + for f in withdrawals: + from . import alerts + alerts.send(f"owner withdrawal from the treasury: {f.lamports / config.LAMPORTS_PER_SOL:.4f} SOL to {f.counterparty}" + + ("" if f.counterparty in funding else " (NOT a funding address: if this was not you, the owner wallet is compromised)")) if counts: log.info("vault flow scan: %s", counts) return {"new": counts, "through_slot": head, "seen": len(newest)} +def scan_all(rpc, trading: str, our_keys: set[str]) -> dict: + """Both accounts of the book: the trading wallet and (when configured) the treasury.""" + from .custody import owners, treasury_address + out = {"wallet": scan(rpc, trading, our_keys=our_keys)} + t = treasury_address() + if t: + out["treasury"] = scan(rpc, t, name=SCAN_TREASURY, our_keys=our_keys, owners=owners()) + return out + + +def _confirmed_elsewhere(chk, f: Flow, wallet: str, funding: set[str], our_keys, owners) -> bool: + """The second provider classifies the same transaction the same way (else the primary may be lying).""" + try: + tx = chk.get_transaction(f.signature) + except Exception: + return True # the check provider is down: the primary's word stands + if tx is None: + return False + with transaction() as conn: + g = classify(tx, wallet, f.signature in known_ours(conn, [f.signature]), funding, our_keys, owners) + return g is not None and g.kind == f.kind and g.lamports == f.lamports + + def _halt(f: Flow) -> None: from . import state reason = f"{f.kind} transaction {f.signature[:12]}… ({f.lamports} lamports)" diff --git a/fly_trader/vault/nav.py b/fly_trader/vault/nav.py index 0e6890af927f811f1d9dfd18df491b4582228dda..e1386030fd704701ec06b0394535cc15590ec716 100644 --- a/fly_trader/vault/nav.py +++ b/fly_trader/vault/nav.py @@ -18,6 +18,19 @@ WINDOW_DAYS = 30 LAMPORTS = config.LAMPORTS_PER_SOL +def reserved_in_trading(conn) -> int: + """What the TRADING wallet must hold back: owed SOL the treasury does not hold (vault/payout.py returns it).""" + from . import payout + return max(0, reserved_lamports(conn) - payout.cached_balance()) + + +def treasury_free(conn) -> int: + """The treasury's idle bankroll: its SOL beyond what it owes. Part of the bankroll trades are sized on, though the + trading wallet can only spend its own float (refilled from here through L1).""" + from . import payout + return max(0, payout.cached_balance() - reserved_lamports(conn)) + + def reserved_lamports(conn) -> int: """Owed and not yet paid: everything allocated minus claims paid (in-flight claims are still in the wallet).""" r = conn.execute("SELECT COALESCE((SELECT sum(allocated) FROM vault_settlements WHERE status = 'allocated'), 0) - " diff --git a/fly_trader/vault/payout.py b/fly_trader/vault/payout.py new file mode 100644 index 0000000000000000000000000000000000000000..19f1c1d6a11ab4eb47e139a58c317e71fe38103c --- /dev/null +++ b/fly_trader/vault/payout.py @@ -0,0 +1,121 @@ +"""The treasury and the trading float (docs/vault/SPEC.md "Custody"). + +Everything beyond a small float lives in a Squads v4 treasury (the multisig's vault 0) owned by Bryce's wallet. The +server's keys can take SOL out of it only through two on-chain spending limits: L1 lets the trading key refill ITS +OWN wallet (per day), L2 lets the payout key pay holders (per week, claims.py). A stolen server can therefore lose at +most the float plus what is left of those two limits, until the owner revokes them from his phone. + +Every minute ``rebalance`` keeps the trading wallet's SOL near ``TREASURY_FLOAT_SOL``: a top-up (treasury -> trading, +L1) when it is low, a sweep (trading -> treasury) when it holds more, or when the treasury could not cover what is +owed. The signer (fly_trader/signer) builds and signs both; the flow row is written before broadcast so the scanner +knows the transfer is ours. + +Accounting treats trading wallet + treasury as one book: R, NAV and the performance index count their SOL together, +so neither move is profit, loss or a flow. The trading bankroll holds back only the owed SOL the treasury lacks. +""" +from __future__ import annotations + +import logging + +from .. import config +from ..chain.cluster_guard import assert_vault_signing_allowed +from ..db.apilog import record_event +from ..db.connection import transaction +from ..signer.client import SignerUnavailable +from ..signer.policy import PolicyError +from . import alerts, state, walletlock + +log = logging.getLogger(__name__) +FEE = 5000 +TREASURY_RENT = 890_880 # the treasury PDA is a 0-byte system account: it keeps this much + + +def cached_balance() -> int: + """The treasury's lamports as the vault worker last read them (0 until then: reserves more, never less).""" + return int((state.get("treasury_balance") or {}).get("lamports") or 0) + + +def refresh_balance(rpc, treasury: str) -> int: + lam = int(rpc.get_balance(treasury)) + state.put("treasury_balance", {"lamports": lam}) + return lam + + +def owed_total(conn) -> int: + """Allocated to lockers and not yet paid out (in-flight claims included: their SOL has not left yet).""" + r = conn.execute("SELECT COALESCE((SELECT sum(allocated) FROM vault_settlements WHERE status = 'allocated'), 0) - " + "COALESCE((SELECT sum(lamports) FROM vault_flows WHERE kind = 'claim'), 0) AS r").fetchone() + return max(0, int(r["r"] or 0)) + + +def plan(trading: int, treasury: int, owed: int, float_target: int, gas_reserve: int, min_move: int) -> tuple[str | None, int]: + """What to move now. Pure. The treasury must hold everything owed first; then the trading wallet is brought to + ``float_target``. Returns ('sweep' | 'topup' | None, lamports).""" + spare_trading = max(0, trading - gas_reserve - FEE) # what trading can give without touching gas + short = max(0, owed + TREASURY_RENT - treasury) # the treasury cannot pay what it owes + excess = max(0, trading - float_target) + give = min(spare_trading, max(excess, short)) + if give >= min_move or (short > 0 and give > 0): + return "sweep", give + want = float_target - trading + available = max(0, treasury - TREASURY_RENT - owed) # never lend out what holders are owed + take = min(want, available) + if want >= min_move and take >= min_move: + return "topup", take + return None, 0 + + +def rebalance(rpc, signer, trading: str, treasury: str, wait=None) -> dict: + """One round under the wallet lock: plan, ask the signer, record, broadcast, confirm.""" + from ..execution.broker_live import await_confirmation + assert_vault_signing_allowed() + wait = wait or await_confirmation + L = config.LAMPORTS_PER_SOL + with walletlock.exclusive(timeout_s=180): + tb = refresh_balance(rpc, treasury) + native = int(rpc.get_balance(trading)) + with transaction() as conn: + owed = owed_total(conn) + target = 0 if state.get("panic") else int(config.TREASURY_FLOAT_SOL * L) # panic: everything but gas goes back + action, amount = plan(native, tb, owed, target, int(config.GAS_RESERVE_SOL * L), int(config.TOPUP_MIN_SOL * L)) + if action is None: + return {"action": None, "trading": native, "treasury": tb, "owed": owed} + try: + built = signer.call("topup" if action == "topup" else "return_float", lamports=amount) + except PolicyError as e: + if action == "topup" and e.code == "cap": + alerts.send(f"the trading float cannot refill: {e}", key="topup_cap", cooldown_s=6 * 3600) + elif action == "topup" and e.code == "liquidity": + alerts.send("the treasury has nothing left to lend the trading float", key="topup_liquidity", cooldown_s=6 * 3600) + else: + alerts.send(f"treasury {action} refused by the signer: {e}", key=f"{action}_refused", cooldown_s=3600) + return {"action": action, "refused": str(e), "code": e.code} + except SignerUnavailable as e: + log.warning("treasury %s: signer unavailable: %s", action, e) + return {"action": action, "refused": str(e), "code": "error"} + if native < 2 * FEE: # the trading wallet pays its own top-up's fee + alerts.send("the trading wallet has no SOL for fees, so it cannot refill itself from the treasury: send it 0.05 SOL " + "(from a funding address)", key="trading_no_fee", cooldown_s=6 * 3600) + return {"action": action, "refused": "no SOL for the fee", "code": "liquidity"} + sig = built["signature"] + direction, counterparty, note = (("in", treasury, "float refill from the treasury (L1)") if action == "topup" + else ("out", treasury, "float excess to the treasury")) + with transaction() as conn: + conn.execute("INSERT INTO vault_flows (signature, slot, block_time, direction, kind, counterparty, lamports, fee_lamports, classified_by, note) " + "VALUES (%s, 0, now(), %s, %s, %s, %s, %s, 'sender', %s)", + (sig, direction, action, counterparty, amount, FEE, note + " (pending)")) + try: + rpc.send_transaction(built["tx"]) + except Exception as e: + log.warning("treasury %s send failed: %s", action, type(e).__name__) + status, info = wait(rpc, sig, int(built["last_valid_block_height"])) + with transaction() as conn: + if status == "confirmed": + conn.execute("UPDATE vault_flows SET slot = %s, note = %s WHERE signature = %s AND kind = %s", + (int((info or {}).get("slot") or 0), note, sig, action)) + else: + conn.execute("DELETE FROM vault_flows WHERE signature = %s AND kind = %s", (sig, action)) + refresh_balance(rpc, treasury) + record_event("info" if status == "confirmed" else "warning", "vault", f"treasury {action} {status}", + {"lamports": amount, "owed": owed, "signature_prefix": sig[:12]}) + return {"action": action, "lamports": amount if status == "confirmed" else 0, "status": status, "owed": owed} diff --git a/fly_trader/vault/prices.py b/fly_trader/vault/prices.py index 018d502dfd0eec17da09c3dcd73646dc0f64adf4..17298ba78de42cf8c08dd0068c6411aa6a818f64 100644 --- a/fly_trader/vault/prices.py +++ b/fly_trader/vault/prices.py @@ -1,5 +1,6 @@ -"""USD prices for the public stats: SOL from Jupiter Price v3, $FLY from its GeckoTerminal pool. Cached 60 s; a -failed fetch keeps the last good value (and its timestamp, so the page can show its age).""" +"""USD prices for the public stats and the console: SOL from Jupiter Price v3, $FLY from its GeckoTerminal pool, ETH from a +KyberSwap quote of 1 ETH into USDG on Robinhood Chain. Cached 60 s; a failed fetch keeps the last good value (and its +timestamp, so the page can show its age).""" from __future__ import annotations import logging @@ -12,6 +13,7 @@ from .. import config log = logging.getLogger(__name__) TTL_S = 60.0 +USDG_DECIMALS = 6 _cache: dict[str, tuple[float, float]] = {} _lock = threading.Lock() @@ -47,10 +49,21 @@ def _fly_usd() -> float: return float(r.json()["data"]["attributes"]["base_token_price_usd"]) +def _eth_usd() -> float: + from ..rh import kyber as K + with httpx.Client(timeout=10, headers={"x-client-id": config.KYBER_CLIENT_ID}) as http: + out = K.Kyber(client=http).get_route(K.NATIVE, config.RH_USDG, 10 ** 18)["routeSummary"]["amountOut"] + return int(out) / 10 ** USDG_DECIMALS + + def sol_usd() -> tuple[float, float]: return _cached("sol", _sol_usd) +def eth_usd() -> tuple[float, float]: + return _cached("eth", _eth_usd) + + def fly_usd() -> tuple[float, float]: return _cached("fly", _fly_usd) diff --git a/fly_trader/vault/publish.py b/fly_trader/vault/publish.py index 6349530e2ee9a69b439d35ff36e7053be53bf818..3c24b7143a98982706ecfeae38b6b40e3931c32b 100644 --- a/fly_trader/vault/publish.py +++ b/fly_trader/vault/publish.py @@ -1,6 +1,6 @@ -"""The public stats (docs/vault/SPEC.md §4): one snapshot every minute plus history deltas and holder accounts, pushed -to the relay. Built only from the database and cached prices, so a push never waits on the chain; any failure is -logged and retried next minute and never reaches trading.""" +"""The public stats (docs/vault/SPEC.md §4): one snapshot every vault-worker loop (~10 s) plus history deltas and holder +accounts, pushed to the relay. Built only from the database and cached prices, so a push never waits on the chain; any +failure is logged and retried on the next loop and never reaches trading.""" from __future__ import annotations import json @@ -10,11 +10,34 @@ import time from .. import config from ..agent import handover, rails from ..db.connection import transaction -from . import nav, prices, rh_index, settle, state +from . import prices, rh_index, settle, state log = logging.getLogger(__name__) LAMPORTS = config.LAMPORTS_PER_SOL NAV_EVERY_S = 300 # history points: one NAV mark per 5 min is plenty for a chart +# Nothing public may help anyone front-run the fly. Wallet SOL and NAV are live but coarse: taken at the last 5-minute +# mark and rounded to 0.1 SOL, so a single buy cannot be matched to an on-chain trade by its size and minute (which +# would reveal the hidden wallet). Realized profit is taken at that same mark, so it cannot flicker when a buy lands. +# Open positions, the performance index (how close the kill switch is) and the next settlement time are never +# published; a trade appears when it closes. The wallet address is never published, nor anything that leads to it +# (transaction signatures, deposit senders). +STEP_S = 300 +ROUND_LAMPORTS = 100_000_000 + + +def coarse(lamports: int) -> int: + return int(round(int(lamports) / ROUND_LAMPORTS)) * ROUND_LAMPORTS + + +def realized_at(conn, ts, native: int) -> int: + """R at a past mark: its SOL, the cost of positions open at that moment, and the flows before it.""" + cost = conn.execute("SELECT COALESCE(sum(cost_sol), 0) AS c FROM positions WHERE book = %s AND opened_at <= %s " + "AND (closed_at IS NULL OR closed_at > %s)", (config.VAULT_BOOK, ts, ts)).fetchone()["c"] + f = conn.execute( + "SELECT COALESCE(sum(lamports) FILTER (WHERE kind = 'deposit'), 0) AS d, " + "COALESCE(sum(lamports + fee_lamports) FILTER (WHERE kind = 'withdrawal'), 0) AS wd, " + "COALESCE(sum(lamports) FILTER (WHERE kind = 'claim'), 0) AS p FROM vault_flows WHERE block_time <= %s", (ts,)).fetchone() + return settle.realized(native, 0, int(round(float(cost) * LAMPORTS)), int(f["d"]), int(f["wd"]), int(f["p"])) def _ui(conn, key: str): @@ -37,15 +60,19 @@ def settlement_item(r) -> dict: def stats(conn, wallet: str, now: float | None = None) -> dict: now = time.time() if now is None else now - m = conn.execute("SELECT * FROM vault_nav ORDER BY ts DESC LIMIT 1").fetchone() - w = {k: int(m[k]) if m else 0 for k in ("native", "token_acct", "positions_value", "exit_cost")} - w["nav"] = int(m["wealth"]) if m else 0 - cost, _mints = settle.open_positions(conn) - w["open_cost"] = cost + m = conn.execute("SELECT * FROM vault_nav WHERE ts <= to_timestamp(%s) ORDER BY ts DESC LIMIT 1", (int(now) // STEP_S * STEP_S,)).fetchone() + w = {"native": coarse(m["native"]) if m else 0, "nav": coarse(m["wealth"]) if m else 0} f = settle.flow_totals(conn, 2**62) - r_now = settle.realized(w["native"], w["token_acct"], cost, f["deposits"], f["withdrawals"], f["payouts"]) + if settle.paper(): # a paper book: R is its closed-trade profit since the vault started + start = int(state.get("vault_started_at", conn=conn) or 0) + rs = conn.execute("SELECT COALESCE(sum(realized_sol), 0) AS s FROM positions WHERE book = %s AND status = 'closed' " + "AND closed_at >= to_timestamp(%s)", (config.VAULT_BOOK, start)).fetchone()["s"] + r_now = int(round(float(rs) * LAMPORTS)) + f["deposits"] = int(round(config.CAPITAL_SOL * LAMPORTS)) + else: + r_now = realized_at(conn, m["ts"], int(m["native"])) if m else 0 a = settle.allocated_total(conn) - booked = conn.execute("SELECT COALESCE(sum(realized_sol), 0) AS s FROM positions WHERE book = 'live' AND status = 'closed'").fetchone()["s"] + booked = conn.execute("SELECT COALESCE(sum(realized_sol), 0) AS s FROM positions WHERE book = %s AND status = 'closed'", (config.VAULT_BOOK,)).fetchone()["s"] c = rails.load_circuit(conn) ho = handover.state(conn) fs = _ui(conn, "fly_status") or {} @@ -54,19 +81,10 @@ def stats(conn, wallet: str, now: float | None = None) -> dict: ix = rh_index.index_state() tot = rh_index.totals(conn) last = conn.execute("SELECT * FROM vault_settlements WHERE status = 'allocated' ORDER BY period_end DESC LIMIT 1").fetchone() - from ..execution import ledger - opens = ledger.open_positions(conn, "live") # the same rows, decimals and holds the live book trades on - positions = [] - for p in opens: - px = float(p["last_mark_price"] or p["entry_price"] or 0.0) - positions.append({"mint": p["mint"], "symbol": _symbol(conn, p["mint"]), "opened_at": _t(p["opened_at"]), - "cost": int(round(float(p["cost_sol"]) * LAMPORTS)), - "value": int(round(int(p["qty"]) / 10 ** int(p["decimals"] or 6) * px * LAMPORTS)), - "entry_price": float(p["entry_price"] or 0.0), "mark_price": px, "hold_min": int((p["hold_s"] or 0) // 60)}) up = (ix.get("upgrade_scheduled") or {}).get("next") return { - "v": 1, "ts": int(now), "cluster": "mainnet" if config.VAULT_CLUSTER == "mainnet-beta" else config.VAULT_CLUSTER, - "fly": {"state": fly_state, "wallet": wallet, "handover": bool(ho), "kill_switch": bool(c.kill_switch), + "v": 1, "ts": int(now) // 3600 * 3600, "book": config.VAULT_BOOK, "cluster": "mainnet" if config.VAULT_CLUSTER == "mainnet-beta" else config.VAULT_CLUSTER, + "fly": {"state": fly_state, "handover": bool(ho), "kill_switch": bool(c.kill_switch), "entries_paused": bool(c.entries_paused), "model": {"fly": fs.get("bootstrap"), "selector": (_ui(conn, "pinned_selector_snapshot") or {}).get("id"), "release": (_ui(conn, "release") or {}).get("seq")}}, @@ -78,13 +96,13 @@ def stats(conn, wallet: str, now: float | None = None) -> dict: "vault": {"address": config.VAULT_ADDRESS, "chain_id": config.RH_CHAIN_ID, "total_locked": str(tot["total_locked"]), "total_pending": str(tot["total_pending"]), "earners": tot["earners"], "finalized_block": ix["through_block"], "paused": ix["paused"], "impl": ix["impl"], "upgrade_scheduled": {"eta": up["eta"], "id": up["id"]} if up else None}, - "settlement": {"next_at": settle.period_end_at_or_before(now) + int(config.VAULT_PERIOD_S), - "last": settlement_item(last) if last else None}, - "positions": positions, - "index": nav.status(conn, since=rails.kill_rebase_at(conn)), + "settlement": {"last": settlement_item(last) if last else None}, } + + + def _symbol(conn, mint: str) -> str: r = conn.execute("SELECT symbol FROM tokens WHERE mint = %s", (mint,)).fetchone() return (r["symbol"] if r and r["symbol"] else mint[:4] + "…") @@ -94,18 +112,22 @@ def history(conn, cur: dict) -> tuple[dict, dict]: """New history items since the cursors in ``cur``; returns (history, new cursors).""" out: dict = {} nav_since = float(cur.get("nav", 0)) - rows = conn.execute("SELECT ts, wealth FROM vault_nav WHERE extract(epoch FROM ts) >= %s AND consistent ORDER BY ts", (nav_since + NAV_EVERY_S,)).fetchall() - series = dict(nav.load(conn)) + # one point per 5-minute boundary: the newest consistent mark at or before it + rows = conn.execute( + "SELECT DISTINCT ON (b) to_timestamp(b) AS ts, wealth FROM (" + " SELECT (ceil(extract(epoch FROM ts) / %(s)s) * %(s)s)::bigint AS b, ts AS mark_ts, wealth FROM vault_nav WHERE consistent" + ") x WHERE b > %(since)s AND b <= %(now)s ORDER BY b, mark_ts DESC", + {"s": STEP_S, "since": int(nav_since), "now": int(time.time()) // STEP_S * STEP_S}).fetchall() sol = prices.sol_usd()[0] pts, last = [], nav_since for r in rows: t = r["ts"].timestamp() - if t - last >= NAV_EVERY_S: - pts.append({"ts": int(t), "nav": int(r["wealth"]), "index": series.get(t, 1.0), "sol_usd": sol}); last = t + if t - last >= NAV_EVERY_S: # boundaries are STEP_S apart, so every one qualifies + pts.append({"ts": int(t), "nav": coarse(r["wealth"]), "sol_usd": sol}); last = t if pts: out["nav"] = pts trades = conn.execute("SELECT id, mint, opened_at, closed_at, cost_sol, realized_sol, forced_exit_kind FROM positions " - "WHERE book = 'live' AND status = 'closed' AND id > %s ORDER BY id LIMIT 500", (int(cur.get("trades", 0)),)).fetchall() + "WHERE book = %s AND status = 'closed' AND id > %s ORDER BY id LIMIT 500", (config.VAULT_BOOK, int(cur.get("trades", 0)))).fetchall() if trades: out["trades"] = [{"id": int(t["id"]), "mint": t["mint"], "symbol": _symbol(conn, t["mint"]), "opened_at": _t(t["opened_at"]), "closed_at": _t(t["closed_at"]), "cost": int(round(float(t["cost_sol"]) * LAMPORTS)), @@ -115,8 +137,9 @@ def history(conn, cur: dict) -> tuple[dict, dict]: fl = conn.execute("SELECT id, block_time, signature, direction, kind, counterparty, lamports FROM vault_flows " "WHERE id > %s AND kind IN ('deposit', 'profit', 'withdrawal', 'claim') ORDER BY id LIMIT 500", (int(cur.get("flows", 0)),)).fetchall() if fl: - out["flows"] = [{"id": int(x["id"]), "ts": _t(x["block_time"]), "signature": x["signature"], "direction": x["direction"], - "kind": x["kind"], "counterparty": x["counterparty"], "lamports": int(x["lamports"])} for x in fl] + # no signature or counterparty: either leads straight to the fly's wallet, which is not published + out["flows"] = [{"id": int(x["id"]), "ts": _t(x["block_time"]), "direction": x["direction"], + "kind": x["kind"], "lamports": int(x["lamports"])} for x in fl] st = conn.execute("SELECT * FROM vault_settlements WHERE status = 'allocated' AND id > %s ORDER BY id", (int(cur.get("settlements", 0)),)).fetchall() if st: out["settlements"] = [settlement_item(s) for s in st] @@ -129,21 +152,31 @@ def history(conn, cur: dict) -> tuple[dict, dict]: return out, new +def eth_accounts(conn) -> dict[str, dict]: + """The ETH pot per holder (wei as strings: they exceed 2**53).""" + rows = conn.execute("SELECT * FROM vault_eth_accounts").fetchall() + return {r["evm"]: {k: str(int(r[k])) for k in ("allocated", "claimed", "in_flight", "owed")} for r in rows} + + def accounts(conn) -> list[dict]: - rows = conn.execute("SELECT * FROM vault_accounts ORDER BY evm").fetchall() + rows = [dict(r) for r in conn.execute("SELECT * FROM vault_accounts ORDER BY evm").fetchall()] + eth = eth_accounts(conn) + rows += [{"evm": e, "allocated": 0, "claimed": 0, "in_flight": 0, "owed": 0} for e in sorted(set(eth) - {r["evm"] for r in rows})] out = [] for r in rows: allocs = conn.execute("SELECT s.period_end, a.lamports, a.weight, s.total_weight FROM vault_allocations a JOIN vault_settlements s " "ON s.id = a.settlement_id WHERE a.evm = %s ORDER BY s.period_end DESC LIMIT 52", (r["evm"],)).fetchall() - cl = conn.execute("SELECT id, status, lamports, sol, tx_signature, created_at, updated_at FROM vault_claims WHERE evm = %s " + cl = conn.execute("SELECT relay_id, status, lamports, sol, created_at, updated_at FROM vault_claims WHERE evm = %s AND relay_id IS NOT NULL " "ORDER BY id DESC LIMIT 20", (r["evm"],)).fetchall() out.append({"evm": r["evm"], "allocated": int(r["allocated"]), "claimed": int(r["claimed"]), "in_flight": int(r["in_flight"]), "owed": int(r["owed"]), "allocations": [{"period_end": _t(a["period_end"]), "lamports": int(a["lamports"]), "weight": str(a["weight"]), "share": float(a["weight"]) / float(a["total_weight"]) if a["total_weight"] else 0.0} for a in allocs], - "claims": [{"id": int(c["id"]), "status": c["status"], "lamports": int(c["lamports"] or 0), "sol": c["sol"], - "tx": c["tx_signature"] if c["status"] == "paid" else "", "created_at": _t(c["created_at"]), + "claims": [{"id": int(c["relay_id"]), "status": c["status"], # the relay's id: GET /api/claim/ "lamports": int(c["lamports"] or 0), "sol": c["sol"], + "tx": "", "created_at": _t(c["created_at"]), "updated_at": _t(c["updated_at"])} for c in cl]}) + if r["evm"] in eth: + out[-1]["eth"] = eth[r["evm"]] return out diff --git a/fly_trader/vault/rh_index.py b/fly_trader/vault/rh_index.py index 82d3ffe080e7c405863a51fb568721b53cdadcd0..5c3d1313e2b75e11ff813dee2fa21837d82d8a35 100644 --- a/fly_trader/vault/rh_index.py +++ b/fly_trader/vault/rh_index.py @@ -83,25 +83,44 @@ def _block_times(rpc: evm.EvmRpc, logs: list[dict]) -> dict[int, int]: def _timelock_ops(rpc: evm.EvmRpc, frm: int, to: int, ops: dict, times: dict) -> dict: - """Pending timelock operations aimed at the vault proxy: id -> {eta}; executed/cancelled ones drop out.""" + """Pending timelock operations aimed at the vault proxy or at the timelock itself (role changes, `updateDelay`): + id -> {eta, id, target}; executed/cancelled ones drop out. Anything the owner key schedules is public here, so + holders and the operator see it during the delay.""" if not config.VAULT_TIMELOCK: return ops logs = rpc.get_logs(config.VAULT_TIMELOCK, frm, to, [[T_SCHEDULED, T_EXECUTED, T_TL_CANCELLED]]) times.update(_block_times(rpc, logs)) - vault = (config.VAULT_ADDRESS or "").lower() + watched = {(config.VAULT_ADDRESS or "").lower(), config.VAULT_TIMELOCK.lower()} for lg in logs: t0, op = lg["topics"][0].lower(), lg["topics"][1] if t0 == T_SCHEDULED: + # CallScheduled(bytes32 indexed id, uint256 indexed index, address target, uint256 value, bytes data, + # bytes32 predecessor, uint256 delay): words 0 target, 1 value, 2 offset of data, 3 predecessor, + # 4 delay; the dynamic tail (data length, data) starts at word 5. w = evm.data_words(lg["data"]) target = ("0x" + w[0][-40:]).lower() - delay = evm.word_to_int(w[5]) if len(w) > 5 else 0 - if target == vault: - ops[op] = {"eta": times[int(lg["blockNumber"], 16)] + delay, "id": op} + delay = evm.word_to_int(w[4]) if len(w) > 4 else 0 + if target in watched: + ops[op] = {"eta": times[int(lg["blockNumber"], 16)] + delay, "id": op, "target": target} else: ops.pop(op, None) return ops +def _announce(before: dict, after: dict) -> None: + """Telegram the operator once per newly scheduled operation (docs/vault/RUNBOOK.md promises this).""" + from datetime import datetime, timezone + from . import alerts + for op, o in after.items(): + if op in before: + continue + what = "the vault (upgrade or role change)" if o.get("target") == (config.VAULT_ADDRESS or "").lower() else "the timelock itself (delay or proposer change)" + eta = datetime.fromtimestamp(int(o["eta"]), tz=timezone.utc).strftime("%Y-%m-%d %H:%M UTC") + alerts.send(f"timelock operation {op} scheduled on {what}; executable from {eta}. If you did not schedule it: " + "Blockscout, the timelock, Write contract, cancel() from the owner wallet.", + key=f"tl_op:{op}", cooldown_s=30 * 86400) + + def run_once(rpc: evm.EvmRpc | None = None, max_steps: int = 20) -> dict: """Index finalized blocks after the cursor. Returns index_state().""" if not config.VAULT_ADDRESS: @@ -147,6 +166,7 @@ def run_once(rpc: evm.EvmRpc | None = None, max_steps: int = 20) -> dict: through_time = head_time if to == head else int(rpc.block(to)["timestamp"], 16) frm = to + 1 rng = min(MAX_RANGE, rng * 2) + _announce((st.get("upgrade_scheduled") or {}).get("ops") or {}, ops) impl = rpc.implementation(config.VAULT_ADDRESS, hex(through) if through else "finalized") detail = {"through_time": through_time, "impl": impl, "impl_codehash": rpc.code_hash(impl, "finalized"), "paused": paused, "range": rng, "upgrade_scheduled": {"ops": ops, "next": min(ops.values(), key=lambda o: o["eta"]) if ops else None}} diff --git a/fly_trader/vault/schema.py b/fly_trader/vault/schema.py index 94665d046078d68d5a74f234e6a2438b5ec647a1..0eafe081d813dfb3d1b8f068fe343f7ad1352270 100644 --- a/fly_trader/vault/schema.py +++ b/fly_trader/vault/schema.py @@ -25,6 +25,11 @@ VAULT_DDL: list[str] = [ created_at timestamptz NOT NULL DEFAULT now(), UNIQUE (signature, direction, kind))""", "CREATE INDEX IF NOT EXISTS vault_flows_slot ON vault_flows (slot)", + # internal moves between the trading wallet and the Squads treasury (vault/payout.py), counted in neither direction: + # 'sweep' trading -> treasury (the float's excess), 'topup' treasury -> trading (spending limit L1) + "ALTER TABLE vault_flows DROP CONSTRAINT IF EXISTS vault_flows_kind_check", + "ALTER TABLE vault_flows ADD CONSTRAINT vault_flows_kind_check CHECK (kind IN " + "('deposit', 'profit', 'withdrawal', 'claim', 'sweep', 'topup', 'unknown_outbound', 'anomaly'))", """CREATE TABLE IF NOT EXISTS vault_scan ( name text PRIMARY KEY, cursor text, diff --git a/fly_trader/vault/settle.py b/fly_trader/vault/settle.py index c9749ae0a3c8d5be32192b6001568df0aedadbeb..48bdeb0b93f27682bc01a161b0acdb8e1880d009 100644 --- a/fly_trader/vault/settle.py +++ b/fly_trader/vault/settle.py @@ -92,13 +92,13 @@ def token_account_lamports(accounts: list[dict], open_mints: set[str]) -> int: def open_positions(conn) -> tuple[int, set[str]]: - rows = conn.execute("SELECT mint, cost_sol FROM positions WHERE book = 'live' AND status = 'open'").fetchall() + rows = conn.execute("SELECT mint, cost_sol FROM positions WHERE book = %s AND status = 'open'", (config.VAULT_BOOK,)).fetchall() return int(round(sum(float(r["cost_sol"] or 0.0) for r in rows) * LAMPORTS)), {r["mint"] for r in rows} def last_own_slot(conn) -> int: r = conn.execute("SELECT GREATEST((SELECT max(slot) FROM fills WHERE book = 'live'), " - "(SELECT max(slot) FROM vault_flows WHERE kind IN ('claim', 'withdrawal')), " + "(SELECT max(slot) FROM vault_flows WHERE kind IN ('claim', 'withdrawal', 'sweep', 'topup')), " "(SELECT max((detail->>'slot')::bigint) FROM wallet_events WHERE kind = 'ata_closed' AND detail ? 'slot')) AS s").fetchone() return int(r["s"] or 0) @@ -113,7 +113,37 @@ def flow_totals(conn, through_slot: int) -> dict: return {"deposits": int(r["d"]), "withdrawals": int(r["wd"]), "payouts": int(r["p"]), "gifts": int(r["gifts"])} -def snapshot(rpc, wallet: str, wait_s: float = FINALITY_WAIT_S) -> dict: +class RpcDisagree(RuntimeError): + pass + + +def agree_with_check_rpc(accounts: list[str], native: int, slot: int, tries: int = 3) -> None: + """The second provider (SOLANA_CHECK_RPC_URL) must see the same lamports at the same finalized slot: a lying or + lagging primary could otherwise inflate (or shrink) what a settlement allocates. Raises RpcDisagree (and alerts).""" + from ..chain.rpc import check_rpc + chk = check_rpc() + if chk is None: + return + seen = None + for i in range(tries): + try: + total = 0 + for a in accounts: + v, s = chk.get_balance_ctx(a, commitment="finalized", min_context_slot=slot) + total += v + if total == native: + return + seen = total + except Exception as e: # a check node behind the slot refuses (minContextSlot) + seen = f"{type(e).__name__}" + time.sleep(2.0 * (i + 1)) + from . import alerts + alerts.send(f"settlement paused: the RPC providers disagree on the book's balance at slot {slot} ({native} vs {seen})", + key="rpc_disagree", cooldown_s=1800) + raise RpcDisagree(f"providers disagree at slot {slot}: {native} vs {seen}") + + +def snapshot(rpc, wallet: str, wait_s: float = FINALITY_WAIT_S, treasury: str | None = None, our_keys: set[str] | None = None) -> dict: """The wallet at a finalized slot S that already contains every transaction our own code sent, read while no swap, claim or withdrawal can run, plus every flow up to S.""" from . import flows, walletlock @@ -129,6 +159,9 @@ def snapshot(rpc, wallet: str, wait_s: float = FINALITY_WAIT_S) -> dict: continue # a fill landed while we waited for the lock cost, mints = open_positions(conn) native, slot = rpc.get_balance_ctx(wallet, commitment="finalized", min_context_slot=max(last, 1)) + if treasury: # the treasury is part of the same book (vault/payout.py) + native += rpc.get_balance_ctx(treasury, commitment="finalized", min_context_slot=max(last, 1))[0] + agree_with_check_rpc([a for a in (wallet, treasury) if a], native, slot) accts = rpc.get_token_accounts_by_owner(wallet, commitment="finalized") break if time.monotonic() > deadline: @@ -138,7 +171,7 @@ def snapshot(rpc, wallet: str, wait_s: float = FINALITY_WAIT_S) -> dict: while flows.scanned_through() < slot: if time.monotonic() > deadline + wait_s: raise TimeoutError(f"flow scan has not reached slot {slot}") - flows.scan(rpc, wallet) + flows.scan_all(rpc, wallet, our_keys or {wallet}) if flows.scanned_through() < slot: time.sleep(3.0) with transaction() as conn: @@ -156,21 +189,47 @@ def allocated_total(conn) -> int: def due(now: float | None = None) -> tuple[int, int] | None: - """(T0, T1) of the newest period that ended after the vault started and has no settlement yet.""" + """(T0, T1) to settle next: T1 is the newest period end after the vault started; T0 is where the last settlement + ended (the first settlement covers the one period before T1). Normally that is one period. If periods were missed + (worker down, snapshot timeouts), it is one window over all of them: the profit is cumulative, so weighting it over + the whole window is exact, whereas settling only the newest period would hand the missed weeks' profit to that + period's holders.""" now = time.time() if now is None else now start = int(state.get("vault_started_at") or 0) t1 = period_end_at_or_before(now) if not start or t1 <= start: return None with transaction() as conn: - if conn.execute("SELECT 1 FROM vault_settlements WHERE period_end = %s", (_ts(t1),)).fetchone(): - return None - return t1 - int(config.VAULT_PERIOD_S), t1 + last = conn.execute("SELECT max(period_end) AS e FROM vault_settlements").fetchone()["e"] + t0 = int(last.timestamp()) if last is not None else t1 - int(config.VAULT_PERIOD_S) + return (t0, t1) if t0 < t1 else None -def take_snapshot(rpc, wallet: str, t0: int, t1: int) -> int: - snap = snapshot(rpc, wallet) - r = realized(snap["native"], snap["token_acct"], snap["open_cost"], snap["deposits"], snap["withdrawals"], snap["payouts"]) +def paper() -> bool: + return config.VAULT_BOOK != "live" + + +def paper_snapshot() -> dict: + """A paper book has no wallet: R is its closed-trade profit since the vault started (fees are inside each trade), + the latest wealth mark stands in for the wallet, and nothing flows in or out.""" + start = int(state.get("vault_started_at") or 0) + with transaction() as conn: + r = conn.execute("SELECT COALESCE(sum(realized_sol), 0) AS s FROM positions WHERE book = %s AND status = 'closed' AND closed_at >= to_timestamp(%s)", + (config.VAULT_BOOK, start)).fetchone() + m = conn.execute("SELECT sol_free FROM wealth_marks WHERE book = %s ORDER BY ts DESC LIMIT 1", (config.VAULT_BOOK,)).fetchone() + cost, _ = open_positions(conn) + paid = int(conn.execute("SELECT COALESCE(sum(lamports), 0) AS p FROM vault_flows WHERE kind = 'claim'").fetchone()["p"]) + realized_ = int(round(float(r["s"]) * LAMPORTS)) + native = int(round(float(m["sol_free"]) * LAMPORTS)) if m else 0 + # realized() adds back payouts; here R is already the book's profit, so present it as native-only terms + return {"slot": 0, "at": int(time.time()), "native": native, "token_acct": 0, "open_cost": cost, "deposits": 0, "withdrawals": 0, + "payouts": paid, "realized_override": realized_} + + +def take_snapshot(rpc, wallet: str, t0: int, t1: int, treasury: str | None = None, our_keys: set[str] | None = None) -> int: + snap = paper_snapshot() if paper() else snapshot(rpc, wallet, treasury=treasury, our_keys=our_keys) + r = snap["realized_override"] if "realized_override" in snap else \ + realized(snap["native"], snap["token_acct"], snap["open_cost"], snap["deposits"], snap["withdrawals"], snap["payouts"]) with transaction() as conn: sid = conn.execute( "INSERT INTO vault_settlements (period_start, period_end, status, snapshot_slot, snapshot_at, native, token_acct, open_cost, " @@ -235,14 +294,15 @@ def allocate_settlement(sid: int, index_state: dict) -> dict: return {"allocated": total, "pot": p["pot"], "earners": len(alloc), "carried": p["pot"] - total, "sha256": digest} -def run_once(rpc, wallet: str, index_state: dict, now: float | None = None) -> dict: +def run_once(rpc, wallet: str, index_state: dict, now: float | None = None, treasury: str | None = None, + our_keys: set[str] | None = None) -> dict: """Advance settlement by at most one step per phase; safe to call every minute.""" if state.halted(): return {"halted": True} out: dict = {} d = due(now) if d: - sid = take_snapshot(rpc, wallet, *d) + sid = take_snapshot(rpc, wallet, *d, treasury=treasury, our_keys=our_keys) out["snapshot"] = sid with transaction() as conn: pend = conn.execute("SELECT id FROM vault_settlements WHERE status = 'snapshotted' ORDER BY period_end").fetchall() @@ -255,4 +315,19 @@ def run_once(rpc, wallet: str, index_state: dict, now: float | None = None) -> d f"(pot {res['pot'] / LAMPORTS:.4f}, carried {res['carried'] / LAMPORTS:.4f})") else: break # settle in period order + if config.RH_ENABLED: # the ETH pot follows each allocated SOL settlement + try: + from . import settle_eth + out["eth"] = settle_eth.run_once(None if paper() else _rh_balance) + except Exception as e: + log.exception("ETH settlement") + record_event("error", "vault", "ETH settlement failed", {"error": type(e).__name__}) return out + + +def _rh_balance() -> int: + """The RH bot wallet's native wei (the address is pinned in config; no key is needed to read it).""" + from ..rh.rpc import RhRpc + if not config.RH_BOT_ADDRESS: + raise RuntimeError("RH_BOT_ADDRESS is not set") + return RhRpc().get_balance(config.RH_BOT_ADDRESS.lower()) diff --git a/fly_trader/vault/settle_eth.py b/fly_trader/vault/settle_eth.py new file mode 100644 index 0000000000000000000000000000000000000000..013107d408fc825e22361197d974a9967bb6e30e --- /dev/null +++ b/fly_trader/vault/settle_eth.py @@ -0,0 +1,132 @@ +"""The weekly settlement's ETH pot: Robinhood Chain memecoin profit shared with the same lockers, by the same weights, +with the same formulas as the SOL pot (vault/settle.py) — in wei, paid in native ETH on Robinhood Chain. + + R = N + C + L - D + Wd + P N native ETH of the RH bot wallet, C cost of open live_rh positions, L basis of + open base lots, D deposits, Wd withdrawals, P ETH claims paid + pot = max(0, R - A) A = all ETH ever allocated (losses and dust carry forward) + out = min(pot, N - (A - P) - gas) never allocate ETH that is not liquid + +It runs after each SOL settlement is allocated (Robinhood Chain has then finalized past the period end, so the weights +are final): one ETH row per SOL settlement, taken while no RH intent is in flight (the wallet then matches the books). +A paper vault (``VAULT_BOOK`` not 'live') shares the paper RH fly book's closed-trade profit instead. +""" +from __future__ import annotations + +import hashlib +import json +import logging + +from .. import config +from ..db.apilog import record_event +from ..db.connection import transaction +from ..markets import RH +from . import settle, state + +log = logging.getLogger(__name__) +WEI = 10 ** 18 + + +def book() -> str: + return RH.live_book if not settle.paper() else RH.fly_book + + +def _w(x) -> int: + return int(round(float(x or 0.0) * WEI)) + + +def allocated_total(conn) -> int: + return int(conn.execute("SELECT COALESCE(sum(allocated), 0) AS a FROM vault_eth_settlements WHERE status = 'allocated'").fetchone()["a"]) + + +def paid_total(conn) -> int: + return int(conn.execute("SELECT COALESCE(sum(eth_wei), 0) AS p FROM vault_claims WHERE eth_status = 'paid'").fetchone()["p"]) + + +def reserved_in_trading(conn) -> int: + """ETH owed to lockers (allocated, not yet paid): the live RH book never trades it (rh/live.reserved_wei).""" + if settle.paper(): + return 0 + return max(0, allocated_total(conn) - paid_total(conn)) + + +def in_flight(conn) -> bool: + a = conn.execute("SELECT count(*) AS n FROM rh_intents WHERE state NOT IN ('done', 'failed', 'aborted')").fetchone()["n"] + b = conn.execute("SELECT count(*) AS n FROM rh_txs WHERE applied_at IS NULL AND status IN ('signed', 'sent') AND kind <> 'payout'").fetchone()["n"] + return bool(a or b) + + +def snapshot(conn, native_wei: int | None) -> dict: + """R's terms now. ``native_wei``: the RH wallet's balance (None for a paper vault).""" + bk = book() + cost = _w(conn.execute("SELECT COALESCE(sum(cost_sol), 0) AS c FROM positions WHERE book = %s AND status = 'open'", (bk,)).fetchone()["c"]) + paid = paid_total(conn) + if settle.paper(): + start = int(state.get("vault_started_at", conn=conn) or 0) + r = conn.execute("SELECT COALESCE(sum(realized_sol), 0) AS s FROM positions WHERE book = %s AND status = 'closed' AND closed_at >= to_timestamp(%s)", + (bk, start)).fetchone()["s"] + m = conn.execute("SELECT sol_free FROM wealth_marks WHERE book = %s ORDER BY ts DESC LIMIT 1", (bk,)).fetchone() + return {"native": _w(m["sol_free"]) if m else 0, "open_cost": cost, "lots": 0, "deposits": 0, "withdrawals": 0, "payouts": paid, "realized": _w(r)} + lots = _w(conn.execute("SELECT COALESCE(sum(basis_eth), 0) AS b FROM rh_base_lots WHERE status = 'open'").fetchone()["b"]) + f = conn.execute("SELECT COALESCE(sum(wei) FILTER (WHERE kind = 'deposit'), 0) AS d, COALESCE(sum(wei) FILTER (WHERE kind = 'withdrawal'), 0) AS wd " + "FROM rh_wallet_flows").fetchone() + d, wd = int(f["d"]), int(f["wd"]) + return {"native": int(native_wei or 0), "open_cost": cost, "lots": lots, "deposits": d, "withdrawals": wd, "payouts": paid, + "realized": settle.realized(int(native_wei or 0), lots, cost, d, wd, paid)} + + +def settle_one(sid: int, native_wei: int | None) -> dict: + """The ETH pot of SOL settlement ``sid`` (already allocated).""" + with transaction() as conn: + s = conn.execute("SELECT * FROM vault_settlements WHERE id = %s", (sid,)).fetchone() + if s is None or s["status"] != "allocated": + return {"skipped": "the SOL settlement is not allocated"} + if conn.execute("SELECT 1 FROM vault_eth_settlements WHERE settlement_id = %s", (sid,)).fetchone(): + return {"skipped": "done"} + if not settle.paper() and in_flight(conn): + return {"waiting": "an RH intent is in flight"} + t0, t1 = int(s["period_start"].timestamp()), int(s["period_end"].timestamp()) + snap = snapshot(conn, native_wei) + start, evs = settle.event_inputs(conn, config.RH_CHAIN_ID, t0, t1) + w = settle.weights(start, evs, t0, t1) + before = allocated_total(conn) + p = settle.plan(snap["realized"], before, snap["payouts"], snap["native"], 0 if settle.paper() else int(config.VAULT_RH_GAS_RESERVE_WEI)) + alloc = settle.allocate(p["amount"], w) + total = sum(alloc.values()) + inputs = {"settlement": sid, "period": [t0, t1], "book": book(), "snapshot": snap, "allocated_before": before, "plan": p, + "weights": {a: str(v) for a, v in sorted(w.items())}, "allocations": {a: str(v) for a, v in sorted(alloc.items())}} + digest = hashlib.sha256(json.dumps(inputs, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + eid = int(conn.execute( + "INSERT INTO vault_eth_settlements (settlement_id, period_start, period_end, status, book, native, open_cost, lots, deposits, withdrawals, payouts, " + "realized, allocated_before, pot, liquid_cap, allocated, carried, earners, inputs_sha256) " + "VALUES (%s,%s,%s,'allocated',%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) RETURNING id", + (sid, s["period_start"], s["period_end"], book(), snap["native"], snap["open_cost"], snap["lots"], snap["deposits"], snap["withdrawals"], + snap["payouts"], snap["realized"], before, p["pot"], p["liquid_cap"], total, p["pot"] - total, len(alloc), digest)).fetchone()["id"]) + for a, wei in alloc.items(): + conn.execute("INSERT INTO vault_eth_allocations (eth_settlement_id, evm, weight, wei) VALUES (%s,%s,%s,%s)", (eid, a, w[a], wei)) + state.put(f"settlement_eth_inputs:{sid}", inputs, conn=conn) + record_event("info", "vault", "ETH settlement allocated", {"settlement": sid, "pot_wei": str(p["pot"]), "allocated_wei": str(total), "earners": len(alloc)}) + return {"allocated": total, "pot": p["pot"], "earners": len(alloc), "carried": p["pot"] - total, "sha256": digest} + + +def run_once(balance=None) -> dict: + """Settle the ETH pot of every allocated SOL settlement that has none, in period order. ``balance()``: the RH + wallet's native wei (live vault); the step waits while it is unavailable.""" + if not config.RH_ENABLED or state.halted(): + return {} + out = {} + with transaction() as conn: + todo = [int(r["id"]) for r in conn.execute("SELECT s.id FROM vault_settlements s LEFT JOIN vault_eth_settlements e ON e.settlement_id = s.id " + "WHERE s.status = 'allocated' AND e.id IS NULL ORDER BY s.period_end").fetchall()] + for sid in todo: + native = None + if not settle.paper(): + if balance is None: + out[sid] = {"waiting": "no RH wallet"}; break + native = int(balance()) + res = settle_one(sid, native); out[sid] = res + if "allocated" not in res and res.get("skipped") != "done": + break + if "allocated" in res: + from . import alerts + alerts.send(f"settlement #{sid} ETH: {res['allocated'] / WEI:.6f} ETH to {res['earners']} holders (pot {res['pot'] / WEI:.6f}, carried {res['carried'] / WEI:.6f})") + return out diff --git a/fly_trader/vault/sigs.py b/fly_trader/vault/sigs.py index 4c35240b67f0890491bade0f996d238f5885d771..217dd3e58df0c45aeb6f4b0d8d09bb04c4ea3faa 100644 --- a/fly_trader/vault/sigs.py +++ b/fly_trader/vault/sigs.py @@ -86,7 +86,9 @@ def verify_evm(text: str, sig_hex: str, address: str, rpc: "evm.EvmRpc | None" = out = rpc.eth_call(want, data, gas=EIP1271_GAS) except Exception: raise SignatureInvalid("contract wallet rejected the signature") from None - if not out or out[:10].lower() != EIP1271_MAGIC: + # exactly one ABI word: bytes4 magic, left-aligned, zero padded (a fallback that echoes calldata starts with the + # selector too, and must not count as a yes) + if not out or out.lower() != EIP1271_MAGIC + "0" * 56: raise SignatureInvalid("contract wallet rejected the signature") return "eip1271" diff --git a/fly_trader/vault/state.py b/fly_trader/vault/state.py index 5b82ee9abdf3ad8f6e40b3630aa9898bafb412b6..10173e27a248625f92a9b9dcbf070d14059ebc9f 100644 --- a/fly_trader/vault/state.py +++ b/fly_trader/vault/state.py @@ -61,3 +61,4 @@ def halted() -> dict | None: def resume() -> None: put("halt", None) + put("panic", None) diff --git a/fly_trader/vault/telegram_cmd.py b/fly_trader/vault/telegram_cmd.py new file mode 100644 index 0000000000000000000000000000000000000000..3da4277311694e106c2d8781e59421b20314c26b --- /dev/null +++ b/fly_trader/vault/telegram_cmd.py @@ -0,0 +1,122 @@ +"""Two Telegram commands for the operator, read by polling (outbound only: the server still accepts no connection). + +A message counts only when it comes from ``TELEGRAM_CHAT_ID`` AND from the user ``TELEGRAM_ADMIN_USER_ID``; anything +else is ignored. A hijacked Telegram account can therefore at worst stop the fly and pull its float back to the +treasury -- never move SOL anywhere else, never resume it (resume is SSH-only). + + /status halt and kill-switch state, the float, the treasury, what is owed, what L1/L2 have left, the release + /panic halt the vault (claims, settlement, entries), tell the signer to refuse buys, top-ups and claims, trip + the kill switch with liquidation, and return the float to the treasury as positions sell +""" +from __future__ import annotations + +import logging +import time + +import httpx + +from .. import config +from . import alerts, state + +log = logging.getLogger(__name__) +HELP = "commands: /status, /panic (halts everything, sells, returns the float to the treasury; resume only over SSH)" + + +def _updates(offset: int) -> list[dict]: + r = httpx.get(f"https://api.telegram.org/bot{config.TELEGRAM_BOT_TOKEN}/getUpdates", + params={"offset": offset, "timeout": 0, "allowed_updates": '["message"]'}, timeout=10) + r.raise_for_status() + return (r.json() or {}).get("result") or [] + + +def authorised(msg: dict) -> bool: + chat, sender = str((msg.get("chat") or {}).get("id")), str((msg.get("from") or {}).get("id")) + return bool(config.TELEGRAM_CHAT_ID and config.TELEGRAM_ADMIN_USER_ID) and chat == str(config.TELEGRAM_CHAT_ID) \ + and sender == str(config.TELEGRAM_ADMIN_USER_ID) + + +def status_text(signer=None) -> str: + from ..db.connection import transaction + from . import nav, payout + L = config.LAMPORTS_PER_SOL + with transaction() as conn: + owed = payout.owed_total(conn) + k = conn.execute("SELECT kill_switch, entries_paused FROM circuit_state WHERE id = 1").fetchone() or {} + rel = conn.execute("SELECT value FROM ui_settings WHERE key = 'release'").fetchone() + last = conn.execute("SELECT wealth FROM vault_nav ORDER BY ts DESC LIMIT 1").fetchone() + c = state.get("custody") or {} + lim = lambda n: (f"{(c[n]['remaining']) / L:.3f}/{c[n]['amount'] / L:g} SOL per {c[n]['period'].lower()}" if c.get(n) else "missing") # noqa: E731 + sig = "" + if signer is not None: + try: + s = signer.call("status") + sig = f"\nsigner: {'PANIC' if s.get('panic') else 'ok'}, buys 24 h {s.get('buys_24h_lamports', 0) / L:.3f} SOL" + except Exception as e: + sig = f"\nsigner: UNREACHABLE ({type(e).__name__})" + h = state.halted() + return (f"{'HALTED: ' + '; '.join(h.get('reasons', [])) if h else 'running'}; kill switch {'ON' if k.get('kill_switch') else 'off'}" + f"{', entries paused' if k.get('entries_paused') else ''}\n" + f"book {(int(last['wealth']) / L) if last else 0:.4f} SOL, treasury {payout.cached_balance() / L:.4f} SOL, owed {owed / L:.4f} SOL\n" + f"L1 {lim('L1')}; L2 {lim('L2')}\n" + f"release {((rel or {}).get('value') or {}).get('seq', '?')}{sig}") + + +def panic(signer, who: str = "telegram") -> str: + from ..db.connection import transaction + reason = f"PANIC from {who}" + state.halt(reason, {"at": int(time.time())}) + state.put("panic", {"at": int(time.time()), "by": who}) + out = ["vault halted (claims, settlement, entries)"] + try: + signer.call("panic", reason=reason) + out.append("signer refuses buys, top-ups and claims") + except Exception as e: + out.append(f"signer NOT reached ({type(e).__name__}): revoke L1 and L2 in the owner wallet now") + with transaction() as conn: + conn.execute("UPDATE circuit_state SET kill_switch = true, kill_reason = %s, updated_at = now() WHERE id = 1", (reason,)) + out.append("kill switch on: open positions are sold within a minute, then the float goes back to the treasury") + return "; ".join(out) + ". Resume only over SSH: fly-trader vault resume, reset-circuit --kill, and the signer's clear-panic." + + +def poll(signer) -> int: + """Handle new commands; returns how many were obeyed. Never raises into the worker loop's caller beyond its job.""" + if not (config.TELEGRAM_BOT_TOKEN and config.TELEGRAM_CHAT_ID): + return 0 + offset = int(state.get("telegram_offset") or 0) + n = 0 + for u in _updates(offset): + offset = max(offset, int(u["update_id"]) + 1) + state.put("telegram_offset", offset) # each update is handled at most once + msg = u.get("message") or {} + text = (msg.get("text") or "").strip().split("@")[0].lower() + if not text.startswith("/"): + continue + if not authorised(msg): + log.warning("telegram command %r from an unauthorised chat/user ignored", text[:20]) + continue + if text == "/panic": + alerts.send(panic(signer)) + elif text == "/status": + alerts.send(status_text(signer)) + else: + alerts.send(HELP) + continue + n += 1 + return n + + +def heartbeat(signer, now: float | None = None) -> bool: + """The daily all-quiet message at 09:00 UTC (a silent day then means something is wrong).""" + now = time.time() if now is None else now + day = time.strftime("%Y-%m-%d", time.gmtime(now)) + if time.gmtime(now).tm_hour < 9 or state.get("heartbeat_day") == day: + return False + state.put("heartbeat_day", day) + alerts.send("daily report\n" + status_text(signer)) + return True + + +def deadman_ping() -> None: + """healthchecks.io-style ping: its own Telegram integration alerts when these stop (the box or this worker died).""" + if config.HEALTHCHECK_URL: + httpx.get(config.HEALTHCHECK_URL, timeout=10) diff --git a/fly_trader/vault/withdraw.py b/fly_trader/vault/withdraw.py index 477e4522ccc0c194a86eafa22975adde50e9a33f..9dbb40e4005248a4623adc7af53d54432d3e840d 100644 --- a/fly_trader/vault/withdraw.py +++ b/fly_trader/vault/withdraw.py @@ -1,72 +1,35 @@ -"""Principal withdrawal: the operator takes deposited SOL back out, only to one of his funding addresses. +"""Principal withdrawal: the operator takes deposited SOL back out of the treasury, from his own wallet in the Squads +app (the server's keys cannot: the treasury's spending limits only refill the trading float and pay holders). - cap = min(D - Wd + min(0, R - A), N - reserved - gas) + cap = min(D - Wd + min(0, R - A), treasury - owed - rent) The first term is his principal net of any loss not yet earned back (so he never takes the lockers' profit or makes -them fund his loss twice); the second keeps owed SOL, open positions and the gas reserve untouched. The flow row is -written before broadcast and the transfer's fee is charged to the withdrawal, not to the lockers. +them fund his loss twice); the second leaves what holders are owed in the treasury. The flow scanner books the +transfer as a withdrawal when it sees it (a multisig member signed it) and alerts; a withdrawal to an address outside +``FUNDING_ADDRESSES`` alerts louder. """ from __future__ import annotations -import base64 -import logging - -from solders.hash import Hash -from solders.message import MessageV0 -from solders.pubkey import Pubkey -from solders.system_program import TransferParams, transfer -from solders.transaction import VersionedTransaction - from .. import config -from ..chain.cluster_guard import assert_vault_signing_allowed -from ..db.apilog import record_event -from ..db.connection import transaction -from . import nav, settle, walletlock - -log = logging.getLogger(__name__) -FEE = 5000 +from . import nav, payout, settle -def cap(conn, native: int, token_acct: int = 0) -> dict: +def cap(conn, trading_native: int, treasury_native: int, token_acct: int = 0) -> dict: cost, _ = settle.open_positions(conn) f = settle.flow_totals(conn, 2**62) + native = int(trading_native) + int(treasury_native) r = settle.realized(native, token_acct, cost, f["deposits"], f["withdrawals"], f["payouts"]) a = settle.allocated_total(conn) principal = f["deposits"] - f["withdrawals"] + min(0, r - a) - liquid = native - nav.reserved_lamports(conn) - int(round(config.GAS_RESERVE_SOL * config.LAMPORTS_PER_SOL)) - FEE - return {"principal": principal, "liquid": liquid, "cap": max(0, min(principal, liquid)), "realized": r, "allocated": a} - - -def withdraw(lamports: int, to: str, rpc, keypair, wait=None) -> dict: - from ..execution.broker_live import await_confirmation - if to not in set(config.FUNDING_ADDRESSES): - raise ValueError("withdrawals go only to an address in FUNDING_ADDRESSES") - assert_vault_signing_allowed() - wait = wait or await_confirmation - wallet = str(keypair.pubkey()) - with walletlock.exclusive(timeout_s=180): - native = rpc.get_balance(wallet) - with transaction() as conn: - c = cap(conn, native) - if lamports <= 0 or lamports > c["cap"]: - raise ValueError(f"amount must be 1..{c['cap']} lamports (principal {c['principal']}, liquid {c['liquid']})") - bh = rpc.get_latest_blockhash() - msg = MessageV0.try_compile(keypair.pubkey(), [transfer(TransferParams(from_pubkey=keypair.pubkey(), to_pubkey=Pubkey.from_string(to), - lamports=int(lamports)))], [], Hash.from_string(bh["blockhash"])) - tx = VersionedTransaction(msg, [keypair]) - sig = str(tx.signatures[0]) - with transaction() as conn: # known before it can land: the flow scanner never mistakes it - conn.execute("INSERT INTO vault_flows (signature, slot, block_time, direction, kind, counterparty, lamports, fee_lamports, classified_by, note) " - "VALUES (%s, 0, now(), 'out', 'withdrawal', %s, %s, %s, 'sender', 'principal withdrawal (pending)')", - (sig, to, int(lamports), FEE)) - rpc.send_transaction(base64.b64encode(bytes(tx)).decode()) - status, info = wait(rpc, sig, bh["last_valid_block_height"]) - with transaction() as conn: - if status == "confirmed": - conn.execute("UPDATE vault_flows SET slot = %s, note = 'principal withdrawal' WHERE signature = %s AND kind = 'withdrawal'", - (int((info or {}).get("slot") or 0), sig)) - else: - conn.execute("DELETE FROM vault_flows WHERE signature = %s AND kind = 'withdrawal'", (sig,)) - record_event("info" if status == "confirmed" else "error", "vault", f"principal withdrawal {status}", - {"lamports": lamports, "to": to, "signature_prefix": sig[:12]}) - return {"status": status, "signature": sig, "lamports": lamports, "cap_before": c["cap"]} + liquid = int(treasury_native) - nav.reserved_lamports(conn) - payout.TREASURY_RENT + return {"principal": principal, "treasury_liquid": liquid, "cap": max(0, min(principal, liquid)), "realized": r, "allocated": a} + + +def instructions(c: dict, treasury: str) -> str: + sol = c["cap"] / config.LAMPORTS_PER_SOL + to = ", ".join(config.FUNDING_ADDRESSES) or "(set FUNDING_ADDRESSES)" + return (f"You may withdraw up to {sol:.6f} SOL (principal {c['principal'] / config.LAMPORTS_PER_SOL:.6f}, " + f"treasury free {c['treasury_liquid'] / config.LAMPORTS_PER_SOL:.6f}).\n" + f"In the Squads app (app.squads.so) with your owner wallet: open the multisig, vault 0 ({treasury}),\n" + f"Send -> SOL -> at most {sol:.6f} -> to one of your funding addresses: {to}. Approve and execute.\n" + "The fly books it as a withdrawal when it lands (and tells you on Telegram).") diff --git a/fly_trader/vault/worker.py b/fly_trader/vault/worker.py index 8f1096bc7a8ce6f1b5276e88fa7b39633d2dbec4..5db61ac3cccb160499e356ef6aaa908a572594f9 100644 --- a/fly_trader/vault/worker.py +++ b/fly_trader/vault/worker.py @@ -1,8 +1,11 @@ """The ``vault`` worker: one thread inside the console (ops/supervisor.py) that runs the vault's periodic jobs. -Every loop (~10 s): pull and pay claims. Every minute: scan the wallet's flows, index Robinhood Chain, mark NAV when -the live book is not doing it (paper warm-up), advance settlement, push the public stats. Hourly: close empty token -accounts. Each job is isolated: one failing never stops the others, and nothing here can raise into trading. +Every loop (~10 s): pull and pay claims, push the public stats. Every minute: check the treasury's on-chain setup, +scan the trading wallet's and the treasury's flows, index Robinhood Chain, keep the trading float (top-up / sweep), +mark NAV when the live book is not doing it (paper warm-up), advance settlement. Hourly: close empty token accounts, +backups. Each job is isolated: one failing never stops the others, and nothing here can raise into trading. + +No key is loaded here: every signature comes from the signer (fly_trader/signer; its own container on the server). """ from __future__ import annotations @@ -15,7 +18,7 @@ from .. import config from ..db.apilog import record_event from ..db.connection import transaction from ..logging_setup import setup -from . import alerts, backup, claims, flows, nav, publish, rh_index, settle, state, walletlock +from . import alerts, backup, claims, custody, flows, nav, payout, publish, rh_index, settle, state, telegram_cmd, walletlock log = logging.getLogger(__name__) LOOP_S = 10.0 @@ -24,13 +27,18 @@ HOUR_S = 3600.0 class Jobs: - def __init__(self): - from ..chain.keys import load_keypair + def __init__(self, signer=None): from ..chain.rpc import HttpSolanaRpc + from ..signer import client as signer_client from .evm import EvmRpc from .relay_client import RelayClient - self.keypair = load_keypair() - self.wallet = str(self.keypair.pubkey()) + self.signer = signer or signer_client.get() + keys = self.signer.call("pubkeys") + self.wallet, self.payout_key = keys["trading"], keys.get("payout") + if not self.payout_key or self.payout_key == self.wallet: + raise RuntimeError("the payout key must exist and differ from the trading key") + self.our_keys = {self.wallet, self.payout_key} + self.treasury = custody.treasury_address() self.rpc = HttpSolanaRpc(config.vault_solana_rpc_url()) self.rh = EvmRpc(config.RH_RPC_URL, config.RH_CHAIN_ID) self.relay = RelayClient() @@ -60,54 +68,86 @@ class Jobs: # ---- jobs ---- def claims(self): - claims.process(self.relay, self.rpc, self.keypair, self.rh) + claims.process(self.relay, self.rpc, self.signer, self.rh) + + def custody(self): + if not settle.paper() and self.treasury: + custody.check(self.rpc, self.wallet, self.payout_key) def scan(self): - flows.scan(self.rpc, self.wallet) + if not settle.paper(): # a paper book has no wallet to scan + flows.scan_all(self.rpc, self.wallet, self.our_keys) def index(self): self.index_state = rh_index.run_once(self.rh) def mark(self): - """The live book marks NAV every trade minute; before the handover (or while the feed is stale) this does.""" + """The live book marks NAV every trade minute; before the handover (or while the feed is stale) this does. + A paper vault copies its book's own wealth marks: the running fly's real NAV, just not real SOL.""" + if settle.paper(): + with transaction() as conn: + for r in conn.execute("SELECT ts, sol_free, positions_value, exit_cost FROM wealth_marks WHERE book = %s AND ts > " + "COALESCE((SELECT max(ts) FROM vault_nav), to_timestamp(0)) ORDER BY ts LIMIT 5000", (config.VAULT_BOOK,)).fetchall(): + L = config.LAMPORTS_PER_SOL + nav.mark(conn, r["ts"], int(round(float(r["sol_free"]) * L)), int(round((float(r["positions_value"]) + float(r["exit_cost"])) * L)), + int(round(float(r["exit_cost"]) * L))) + return with transaction() as conn: r = conn.execute("SELECT max(ts) AS t FROM vault_nav").fetchone() - if r["t"] and (datetime.now(timezone.utc) - r["t"]).total_seconds() < 150: + if r["t"] and (datetime.now(timezone.utc) - r["t"]).total_seconds() < 50: return with walletlock.try_shared() as consistent: - native = self.rpc.get_balance(self.wallet) + native = self.rpc.get_balance(self.wallet) + payout.cached_balance() # trading wallet + treasury: one book with transaction() as conn: cost, _ = settle.open_positions(conn) m1 = datetime.fromtimestamp(int(time.time() // 60 * 60), timezone.utc) nav.mark(conn, m1, native, cost, 0, consistent=consistent) - if native < int(config.GAS_RESERVE_SOL * config.LAMPORTS_PER_SOL): - alerts.send(f"wallet {native / config.LAMPORTS_PER_SOL:.4f} SOL is below the gas reserve", key="low_gas", cooldown_s=6 * 3600) + trading = self.rpc.get_balance(self.wallet) + if trading < int(config.GAS_RESERVE_SOL * config.LAMPORTS_PER_SOL): + alerts.send(f"trading wallet {trading / config.LAMPORTS_PER_SOL:.4f} SOL is below the gas reserve", key="low_gas", cooldown_s=6 * 3600) def settle(self): st = getattr(self, "index_state", None) or rh_index.index_state() - settle.run_once(self.rpc, self.wallet, st) + settle.run_once(self.rpc, self.wallet, st, treasury=None if settle.paper() else self.treasury, our_keys=self.our_keys) + + def rebalance(self): + """Keep the trading float near TREASURY_FLOAT_SOL and the treasury able to pay everything owed.""" + if settle.paper() or not self.treasury: + return + payout.rebalance(self.rpc, self.signer, self.wallet, self.treasury) def publish(self): publish.push(self.relay, self.wallet) + def telegram(self): + telegram_cmd.poll(self.signer) + + def heartbeat(self): + telegram_cmd.heartbeat(self.signer) + + def deadman(self): + telegram_cmd.deadman_ping() + def backup(self): + if settle.paper(): + return if not backup.configured(): - alerts.send("encrypted backups are not configured (VAULT_BACKUP_*): losing the server would lose the wallet", + alerts.send("encrypted backups are not configured (VAULT_BACKUP_*): losing the server would lose the ledger and the float's keys", key="backup_missing", cooldown_s=24 * 3600) return - backup.key_once() + backup.key_once(self.signer) backup.ledger_nightly() def close_atas(self): from ..chain.cluster_guard import assert_vault_signing_allowed from ..execution.broker_live import close_empty_atas - if config.VAULT_CLUSTER != "mainnet-beta": - return # the devnet rehearsal holds no token accounts worth closing + if config.VAULT_CLUSTER != "mainnet-beta" or settle.paper(): + return # the devnet rehearsal / a paper vault holds no token accounts worth closing assert_vault_signing_allowed() with walletlock.exclusive(timeout_s=120): with transaction() as conn: _, mints = settle.open_positions(conn) - res = close_empty_atas(rpc=self.rpc, keypair=self.keypair, skip_mints=mints | {config.WSOL_MINT}) + res = close_empty_atas(rpc=self.rpc, signer=self.signer, skip_mints=mints | {config.WSOL_MINT}) if res.get("closed"): log.info("closed %d empty token accounts", res["closed"]) @@ -122,8 +162,12 @@ def main(stop_event: threading.Event | None = None) -> None: log.info("vault worker: wallet %s, relay %s, vault %s on chain %s", jobs.wallet, config.RELAY_URL, config.VAULT_ADDRESS, config.RH_CHAIN_ID) while not stop_event.is_set(): jobs.run("claims", jobs.claims) + jobs.run("publish", jobs.publish) # every loop (10 s): the site is live + jobs.run("telegram", jobs.telegram) # /panic answers within a loop + if jobs.due("deadman", 300.0): + jobs.run("deadman", jobs.deadman) if jobs.due("minute", MINUTE_S): - for name in ("scan", "index", "mark", "settle", "publish"): + for name in ("custody", "scan", "index", "rebalance", "mark", "settle", "rebalance", "heartbeat"): if stop_event.is_set(): break jobs.run(name, getattr(jobs, name)) diff --git a/models/policies/fly_20260924T185123Z.pt b/models/policies/fly_20260924T185123Z.pt deleted file mode 100644 index 101abe1c28ccd92a8b399d006893cb176e2a8bbe..0000000000000000000000000000000000000000 --- a/models/policies/fly_20260924T185123Z.pt +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:5f2313748341ee8e2c8b49c0057d9cb5209e67a249510f3a2cff226cbca2e36d -size 28435697 diff --git a/models/policies/fly_20261006T023153Z.pt b/models/policies/fly_20261006T023153Z.pt new file mode 100644 index 0000000000000000000000000000000000000000..3bd92a86196c33ee8dacd26fa8f9fcef8e2bb62d --- /dev/null +++ b/models/policies/fly_20261006T023153Z.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:73141bf59bd4321c66f28e4c5a7c83f994adf744574de8b17fe5d3ca94fcc1e7 +size 28525445 diff --git a/models/selectors/selector_20260924T083620Z.joblib b/models/selectors/selector_20260924T083620Z.joblib deleted file mode 100644 index d6ac697dad1e2acf67bb363d3dea6a8392515c5b..0000000000000000000000000000000000000000 --- a/models/selectors/selector_20260924T083620Z.joblib +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:3c7fa68272b3f27af603c2b617938b2225f392b671238ca09d2786a95f4f359e -size 1510942 diff --git a/models/selectors/selector_20261003T201413Z.skops b/models/selectors/selector_20261003T201413Z.skops new file mode 100644 index 0000000000000000000000000000000000000000..275453d28780399c88f892d6f60dd3f615cbc3c7 --- /dev/null +++ b/models/selectors/selector_20261003T201413Z.skops @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b38f0633ee4a9b693bfb9a01a5e5de044e2ce317e08fb9726e77b48dfbb603fe +size 40812282 diff --git a/models/wallet_skill/2026-09-23.parquet b/models/wallet_skill/2026-09-23.parquet deleted file mode 100644 index 8245cd2311fd149432f22bc2665c3bf712576ae3..0000000000000000000000000000000000000000 --- a/models/wallet_skill/2026-09-23.parquet +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:7768436ea3d81d5c77d65e760948eb5eeb2eae0f5b7502babfcee74b22ce67fd -size 308471154 diff --git a/models/wallet_skill/2026-09-25.parquet b/models/wallet_skill/2026-09-25.parquet deleted file mode 100644 index e04beaa157c441f00b7e76749b8f7d15574037cc..0000000000000000000000000000000000000000 --- a/models/wallet_skill/2026-09-25.parquet +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:79168a715b3359580468a5d31d27a23748409e310368d7ad828f8c84f30d628b -size 304577818 diff --git a/models/wallet_skill/2026-10-11.parquet b/models/wallet_skill/2026-10-11.parquet new file mode 100644 index 0000000000000000000000000000000000000000..c1e4e76ae14c913830a25550db6aa8a7634184b0 --- /dev/null +++ b/models/wallet_skill/2026-10-11.parquet @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34c0faf085f068c217a98d21722aa44a57907b466547f723ffa3fc1e6ee6287c +size 281123254 diff --git a/models/wallet_skill/TABLE b/models/wallet_skill/TABLE index d2f93c77b375ff704491b532a2bf44a4c7237646..a9d117ae39de31be02bee750281b44da8d9e71be 100644 --- a/models/wallet_skill/TABLE +++ b/models/wallet_skill/TABLE @@ -1 +1 @@ -2026-09-25.parquet +2026-10-11.parquet diff --git a/models/wallet_skill/TABLE.sha256 b/models/wallet_skill/TABLE.sha256 new file mode 100644 index 0000000000000000000000000000000000000000..9b86d399bcff0d9fa4f7de96d38915d1592becab --- /dev/null +++ b/models/wallet_skill/TABLE.sha256 @@ -0,0 +1 @@ +34c0faf085f068c217a98d21722aa44a57907b466547f723ffa3fc1e6ee6287c diff --git a/pyproject.toml b/pyproject.toml index 0b273de0a090ddb37122a7a60c4fdd42d305168d..2d499b46dc39553d33785416a22733689315d2da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,9 +22,14 @@ dependencies = [ "streamlit>=1.40", "altair", "psutil", + "kalshi-python-async>=3.10,<3.11", + "cryptography", + "aiohttp", "coincurve>=20", # secp256k1: vault claim signatures (EIP-191 / EIP-1271) "pycryptodome>=3.20", # keccak-256 for EIP-55 checksums and Robinhood Chain log topics "huggingface_hub>=1.0", # vault backups to a storage bucket; tools/publish_release.py + "skops>=0.16", + "zstandard>=0.25.0", ] [project.optional-dependencies] diff --git a/release.json b/release.json new file mode 100644 index 0000000000000000000000000000000000000000..254512ae2b6705cee7f57732c4c2213ec8c71d1d --- /dev/null +++ b/release.json @@ -0,0 +1,2074 @@ +{ + "code_digest": "194a1fa58d0873e2e58137a05f94cde3f00f00087714562b4ca57f0ede203b19", + "created_at": 1791708831, + "files": [ + { + "class": "infra", + "path": ".dockerignore", + "sha256": "41d39dea23ebaedd4b1b7c19f976843b6919ef4af200df35fdf7d47a119ce5ab", + "size": 107 + }, + { + "class": "code", + "path": ".env.example", + "sha256": "3e65b8c14fb7f950a12031b3580a3a9ee994912e7f3b164a74205863ea95af99", + "size": 5081 + }, + { + "class": "code", + "path": "CHANGELOG.md", + "sha256": "ababb35cf4b972ad0aea25a3abff50daee05ee586088bc3b944e606a5f7940b1", + "size": 7542 + }, + { + "class": "infra", + "path": "Dockerfile", + "sha256": "071d1e416c08cd9f6a6117e036c5642fabe0f0c2b9682bcdc6b4a46fcd296816", + "size": 3580 + }, + { + "class": "code", + "path": "LICENSE", + "sha256": "ddf152a9c8531501797023c7ad275e5454931f23d95f948905286cc6e4c43911", + "size": 1080 + }, + { + "class": "code", + "path": "README.md", + "sha256": "65c9f60b9733a6e635513a664993248e18977e421d5b9d32a61871fc86503dd7", + "size": 15469 + }, + { + "class": "code", + "path": "config/mbon_valence.json", + "sha256": "c519a0ca043d88359dee5ebfa980b1d9ea9377f18beb9eb557b40da3fbd3d8af", + "size": 1406 + }, + { + "class": "infra", + "path": "deploy/bootstrap-host.sh", + "sha256": "2c2ab0ac26f2da83127ddfdcb146f239d208639450d3681a8a367951a245f23f", + "size": 23939 + }, + { + "class": "infra", + "path": "deploy/check-host.sh", + "sha256": "6e127ccafd843b701cfcc473a0cac2de92afef32e8f341e2801d1636fbbc19d5", + "size": 7902 + }, + { + "class": "infra", + "path": "deploy/custody.env.example", + "sha256": "9e0438e31ebf8acaaa262dba153fc394a911847340847033dc56de6569e5ee66", + "size": 1682 + }, + { + "class": "infra", + "path": "deploy/docker-compose.vault.yml", + "sha256": "36c1cb5b1d3ab7441383cbe348d537b555236f0a773815cc93c958ddc8e01382", + "size": 3804 + }, + { + "class": "infra", + "path": "deploy/fly", + "sha256": "826dc2ef97387e33c017dd7bd113bfef0fbbd9a88f0f7e2b3b42bfc810556e23", + "size": 379 + }, + { + "class": "infra", + "path": "deploy/fly-aide.service", + "sha256": "e67ec7ceade79c9e747da787c055581bec5992e3301a032867d9c9e6e0c4decf", + "size": 216 + }, + { + "class": "infra", + "path": "deploy/fly-aide.timer", + "sha256": "f1579d0d479cbd6cdb5ad24888a8b420c7d3df4164d5da51e0179272b20f0fe2", + "size": 133 + }, + { + "class": "infra", + "path": "deploy/fly-audit-watch.service", + "sha256": "0a4b6f672880a30c955627283f0ac0640b005da28e300529879006ac350da638", + "size": 225 + }, + { + "class": "infra", + "path": "deploy/fly-audit-watch.timer", + "sha256": "9716f6dfa2b186fbfb3bbaa96836e60f10f84bfdce08b416794b405e468910df", + "size": 155 + }, + { + "class": "infra", + "path": "deploy/fly-update.service", + "sha256": "65d66d916657701e5736fda86320e6e3258c2b802b7f82353120c035d36ce2bc", + "size": 315 + }, + { + "class": "infra", + "path": "deploy/fly-update.timer", + "sha256": "245d0253e5fbb3bd6a626bb585a6b74390b859b28bae11bc43e4fa77ada9459d", + "size": 160 + }, + { + "class": "infra", + "path": "deploy/fly_audit_watch.py", + "sha256": "64851a6d6fe433fa92b12ee25c53d529fba888143fa10fcad1c5a1f474006b5c", + "size": 5055 + }, + { + "class": "infra", + "path": "deploy/fly_update.py", + "sha256": "3897986b5df7079fc1021abf2beb8b721dbf49b3b16b1fbacbb44c62b850c3c2", + "size": 19342 + }, + { + "class": "infra", + "path": "deploy/signer.env.example", + "sha256": "47c26188d1fcb08d2f16b2efd6394c9c1b8cc64024ded2619e55fcb8ee4d6584", + "size": 354 + }, + { + "class": "infra", + "path": "deploy/update.json.example", + "sha256": "120eaa9f0bab88ecf6cdf1b5a308b716dea98ff581b071681094c41ae6b96817", + "size": 335 + }, + { + "class": "infra", + "path": "deploy/vault.env.example", + "sha256": "56d303a454effe477f27b4131f05ab022c30dd03e3e000d825d8e88eb6bd0fec", + "size": 2759 + }, + { + "class": "infra", + "path": "docker-compose.cuda.yml", + "sha256": "eabe65a65006d942239147b582c67ded501c5817e296368b8123a9828a203fc4", + "size": 868 + }, + { + "class": "infra", + "path": "docker-compose.yml", + "sha256": "80c11bee226ee894aceefb05bb0d1c14530208ab653b9c30bf7762aeaae41274", + "size": 1776 + }, + { + "class": "infra", + "path": "docker/bootstrap.sh", + "sha256": "b0b6627a99da9736e580d9d1308bb8f2be829d28b867335d6901940d32bdd74d", + "size": 3546 + }, + { + "class": "infra", + "path": "docker/entrypoint.sh", + "sha256": "92b22d46a9c854bebdb2c46822d2b9a0092465022bab6c7f36029864dab09c7a", + "size": 531 + }, + { + "class": "infra", + "path": "docker/signer-requirements.txt", + "sha256": "33c79b9c2ce8010500a61b5806043bacac1c2b97d33b6bb6545a0cbc3f4534b4", + "size": 2874 + }, + { + "class": "code", + "path": "docs/vault/RUNBOOK.md", + "sha256": "185a3f3ef96c369acac37f4b053f6e7054c2bb22ade56e42bf43ad9a747bb8b4", + "size": 30309 + }, + { + "class": "code", + "path": "docs/vault/SPEC.md", + "sha256": "a56960c7b8122cd4647c42f4df8598173a1c545523251fdcb41f85b21e4adc9b", + "size": 28160 + }, + { + "class": "code", + "path": "fly_trader/__init__.py", + "sha256": "cb47698ef01343b5269b966c91a3222f83a91b6688690edc397020f8476ea246", + "size": 89 + }, + { + "class": "code", + "path": "fly_trader/__main__.py", + "sha256": "4c97fafce95b6290ec6ae20a7452a98ce743f7a04632c1ce27dd8ae7c73e550e", + "size": 18872 + }, + { + "class": "code", + "path": "fly_trader/agent/__init__.py", + "sha256": "01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b", + "size": 1 + }, + { + "class": "code", + "path": "fly_trader/agent/fly_live.py", + "sha256": "8ea34965281ac10291b9f7aad83252ad38c69dd7c2721f90ff93b145c2e4dd30", + "size": 13682 + }, + { + "class": "code", + "path": "fly_trader/agent/fly_session.py", + "sha256": "a04e962c0d9cf0a839d68f91d5f9b54c1a019faeb5017c7a0890d2ea0ab7f9ff", + "size": 50751 + }, + { + "class": "code", + "path": "fly_trader/agent/handover.py", + "sha256": "4fd30eb334079114e91ad5c03578407499f023b5f017113aeab4c333258419af", + "size": 1323 + }, + { + "class": "code", + "path": "fly_trader/agent/minute_engine.py", + "sha256": "4a77f6221f0d64a44fcbc09c48e42b87f2c83b2f989f4fcd0ded5243bd07af51", + "size": 26406 + }, + { + "class": "code", + "path": "fly_trader/agent/paper_trading.py", + "sha256": "046594c635112cee2cab7544e6ce02d945b638a8ecceca6f20174447c9c0a0e8", + "size": 8293 + }, + { + "class": "code", + "path": "fly_trader/agent/rails.py", + "sha256": "3f11b82b000fb2dc8198d883898649269cc6a72f439341a58f87d870f4522fcf", + "size": 8696 + }, + { + "class": "code", + "path": "fly_trader/agent/runner.py", + "sha256": "814908a23370f8b3ba415ad2d628a49e8d3a0b4859bea0335ab1240bbfe875b2", + "size": 2651 + }, + { + "class": "code", + "path": "fly_trader/agent/selector_session.py", + "sha256": "7209b5173f47f4401b4254d14fba521e62a9f04a1d7de8df7b9cff30cad62f64", + "size": 11356 + }, + { + "class": "code", + "path": "fly_trader/agent/sizing.py", + "sha256": "7161f892c672a03637eee7287362e7abcc5bdd4714f6c42fc2b56b108dd50586", + "size": 6994 + }, + { + "class": "code", + "path": "fly_trader/brain/__init__.py", + "sha256": "01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b", + "size": 1 + }, + { + "class": "code", + "path": "fly_trader/brain/activity.py", + "sha256": "776d72ae2ec2e5b6d6daca92ba7d3b7fe664f2ab6652c42dadfdb4f77e046e95", + "size": 3004 + }, + { + "class": "code", + "path": "fly_trader/brain/connectome.py", + "sha256": "fd18df6b040dd8bb759f01457152dd0844c02f10ec229d226d5f544a7d5a27c5", + "size": 10944 + }, + { + "class": "code", + "path": "fly_trader/brain/connectome_build.py", + "sha256": "1b427adf35a5e95fd02e8d2528ba1b80e5846ebaf588fe2dac3626018ac4efa2", + "size": 17457 + }, + { + "class": "code", + "path": "fly_trader/brain/device.py", + "sha256": "15fa10cc1904d267570c31feef78bae3921efeee416fb2096a4e66c1bca5f387", + "size": 6224 + }, + { + "class": "code", + "path": "fly_trader/brain/geometry.py", + "sha256": "ab70a904f9e782dd7e926f9ae3ca126c681bfdf263671941f6cae0ca2daf53d2", + "size": 8030 + }, + { + "class": "code", + "path": "fly_trader/brain/pathways.py", + "sha256": "c9ba4b80491252a3f493a532521364190ff73c6fbb28756d455fb59799a73154", + "size": 4742 + }, + { + "class": "code", + "path": "fly_trader/brain/plastic.py", + "sha256": "3e78090b68982e1674334bc034002a648de41d51d229b7a5be93b0747409112b", + "size": 17174 + }, + { + "class": "code", + "path": "fly_trader/brain/populations.py", + "sha256": "b9f4060e2abd1de40fe95a1b7328b20e526b479742fea5f0ce71651de50e10c2", + "size": 27687 + }, + { + "class": "code", + "path": "fly_trader/chain/__init__.py", + "sha256": "01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b", + "size": 1 + }, + { + "class": "code", + "path": "fly_trader/chain/balances.py", + "sha256": "bf5a5dce16518b100b68b2e5e9b5b75e7dfbc852985cba6553efe6ad1e934e39", + "size": 3047 + }, + { + "class": "code", + "path": "fly_trader/chain/cluster_guard.py", + "sha256": "2120367d2226e50ec648c72abb80e91bf2e9442aab4eacf95b386b6508775b22", + "size": 3595 + }, + { + "class": "code", + "path": "fly_trader/chain/jupiter_swap.py", + "sha256": "987bcca58370c7b20fd58cc6e31ca00ea4d6431a7f9295cf08359581b36ac2ac", + "size": 11881 + }, + { + "class": "code", + "path": "fly_trader/chain/jupiter_tokens.py", + "sha256": "0df7e297c7ce0f59d21f5114656a3ffc777d5dec942e65b9adca2659938482e9", + "size": 3100 + }, + { + "class": "code", + "path": "fly_trader/chain/keys.py", + "sha256": "1df5a29d36508abdd8164957cf7d7504061bf9e14f3d52ce3909bff23277a560", + "size": 6977 + }, + { + "class": "code", + "path": "fly_trader/chain/rpc.py", + "sha256": "628f97d81fc90b3d88228e96a1ca184d591619727d0be664e1b983d9d448ba96", + "size": 11855 + }, + { + "class": "code", + "path": "fly_trader/chain/signing.py", + "sha256": "10c2635121d3c8b8ef7fb352720dad115ea4e84ca7c1cad2d1ced1f222656d58", + "size": 3587 + }, + { + "class": "code", + "path": "fly_trader/chain/withdraw.py", + "sha256": "39a1de86b29e226310eb53977579e14e15735f840255939b2cb39aeb87c6caa8", + "size": 7393 + }, + { + "class": "code", + "path": "fly_trader/config.py", + "sha256": "17c3d4a4fc2c39e60639dcb16ef77121ae3c805cf333bf1c52f1066822764927", + "size": 28817 + }, + { + "class": "code", + "path": "fly_trader/db/__init__.py", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "size": 0 + }, + { + "class": "code", + "path": "fly_trader/db/apilog.py", + "sha256": "9047b67d17a7ca94c553b505990cc2f76b967b5f6d43ba8ce7da1802da8aebb8", + "size": 2058 + }, + { + "class": "code", + "path": "fly_trader/db/connection.py", + "sha256": "9bc44e3d76323971a352a991603c32cf4ad9d270fbcb1a0a104b27e3b0ea5073", + "size": 2736 + }, + { + "class": "code", + "path": "fly_trader/db/queries.py", + "sha256": "eb5caab4f715e86f5a8d3f6992e5a954fcfab299547d3fa1b3ba3aeccc72a850", + "size": 1802 + }, + { + "class": "code", + "path": "fly_trader/db/schema.py", + "sha256": "24ee76b4422f8ecda150679f39d9f86f03f7722e0fb3f5dd3d697f9b038c0d0d", + "size": 38657 + }, + { + "class": "code", + "path": "fly_trader/execution/__init__.py", + "sha256": "01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b", + "size": 1 + }, + { + "class": "code", + "path": "fly_trader/execution/broker_live.py", + "sha256": "890bb93b04d3fa85964e55e2074fd039284f037c72905557bae787e137ccf5f3", + "size": 26977 + }, + { + "class": "code", + "path": "fly_trader/execution/broker_paper.py", + "sha256": "11dbfa13cf02cdb65f3ac651b4c79143aefaa0c135df98a8b4802d659a522b93", + "size": 6571 + }, + { + "class": "code", + "path": "fly_trader/execution/ledger.py", + "sha256": "2a298b48f217334143debdb5d1d54d923e6927e82337323d40faafb0e9445632", + "size": 8925 + }, + { + "class": "code", + "path": "fly_trader/execution/worker.py", + "sha256": "318df1fa1e689193e95a6ac6cf8554bdf7ac30dc6706a82cee5aecb8976cebd5", + "size": 6602 + }, + { + "class": "code", + "path": "fly_trader/ingest/__init__.py", + "sha256": "01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b", + "size": 1 + }, + { + "class": "code", + "path": "fly_trader/ingest/corpus_pull.py", + "sha256": "5ff09d3fcdd597aee797bdd47d5e655aa50e0db5980acd3f3f2961cc5562fbd1", + "size": 24985 + }, + { + "class": "code", + "path": "fly_trader/ingest/discovery.py", + "sha256": "702ef1add301bd8802d5ac2f787690ca87fef98844eb27af4a210373f003b32d", + "size": 3922 + }, + { + "class": "code", + "path": "fly_trader/ingest/pumpstream.py", + "sha256": "565d339ddc9d6503e46e74a9cfc845cc6d4645b9a8f4df01057555286291c0df", + "size": 32120 + }, + { + "class": "code", + "path": "fly_trader/ingest/replay_pull.py", + "sha256": "600f27f86689770c504a01c6ace2367f2b49f489e4ec39ac025c5d38b6bc537f", + "size": 19067 + }, + { + "class": "code", + "path": "fly_trader/kalshi/__init__.py", + "sha256": "975b38b905ac2d7bd2b2bccbec7e2abeef18a5e9ca6981d13628573ea9c65e30", + "size": 113 + }, + { + "class": "code", + "path": "fly_trader/kalshi/client.py", + "sha256": "1e913d9a6f224a3e2dd00112dc9b618e2b65397ef3e7bd03996aae8d2e92722f", + "size": 18214 + }, + { + "class": "code", + "path": "fly_trader/kalshi/data.py", + "sha256": "b58a1485804ca601443b83c4902bdae1a192fbf5978596a575ec653524af606c", + "size": 10812 + }, + { + "class": "code", + "path": "fly_trader/kalshi/decisions.py", + "sha256": "26be3c7a435569c0712168f7e0ee416c2e8a8efe204a1b94ea7b05fd03cd87bc", + "size": 10869 + }, + { + "class": "code", + "path": "fly_trader/kalshi/engine.py", + "sha256": "59adb30b1e1777a05c83fa96932463d72ed001980fbf724c5dad3a96bc148c36", + "size": 15329 + }, + { + "class": "code", + "path": "fly_trader/kalshi/features.py", + "sha256": "e40de6167fbd56feda0229d9542395bac0697fad5e1ddf7bfae833de9ccf2e55", + "size": 15069 + }, + { + "class": "code", + "path": "fly_trader/kalshi/fly.py", + "sha256": "a02c37e9cc963a3a540b63bc28eeb34b21a3ff9e845c43a88e4b6ef22a0b936f", + "size": 16685 + }, + { + "class": "code", + "path": "fly_trader/kalshi/fly_replay.py", + "sha256": "2fab46f85eafa048d7a4b17f23f5fff007ee1c9e95f262ef4d1bed97262163fd", + "size": 17438 + }, + { + "class": "code", + "path": "fly_trader/kalshi/fly_session.py", + "sha256": "88674757704a59e293ca46d38e8b38c9d178eca8935d6e0b3f585935a44a3fc9", + "size": 36507 + }, + { + "class": "code", + "path": "fly_trader/kalshi/history.py", + "sha256": "e2d823fe468fb505de39ea1654fd8b2ed83a841d83a6a676dc672d597de8d199", + "size": 38741 + }, + { + "class": "code", + "path": "fly_trader/kalshi/live.py", + "sha256": "76557921f8235006c118c805e45bf9c668f4d2b95a2e71e9dabd822d0d5c37ff", + "size": 23552 + }, + { + "class": "code", + "path": "fly_trader/kalshi/maker.py", + "sha256": "a3288ad71db6852ebfaff175535308dd0485cb4f3da628a79a0055e16e041241", + "size": 10660 + }, + { + "class": "code", + "path": "fly_trader/kalshi/mature.py", + "sha256": "66c9469f01d98b80573ed5ad2710f3cd6b30d7f0dcc3f39d2d92ad88f608ea3d", + "size": 19157 + }, + { + "class": "code", + "path": "fly_trader/kalshi/paper.py", + "sha256": "e4886bc2c48b14f0a25a203d1bdd0481c3a0c8bb3d2131d3398484944383d506", + "size": 14941 + }, + { + "class": "code", + "path": "fly_trader/kalshi/pipeline.py", + "sha256": "71d7fb1867d36b3f36579aa978beaf3ca007652501689f46f623fd7dccc0649f", + "size": 5418 + }, + { + "class": "code", + "path": "fly_trader/kalshi/selector.py", + "sha256": "49a3a413d72047d9a426b9bd3f3a18ef972394ee2c1098ddf9743f5da539897a", + "size": 9437 + }, + { + "class": "code", + "path": "fly_trader/kalshi/strategies.py", + "sha256": "8c2e4320052944fcb1764b977b7fbd8e1e8df7a46191e9d00b37cd85635f435e", + "size": 5940 + }, + { + "class": "code", + "path": "fly_trader/kalshi/stream.py", + "sha256": "be88523786ddb5bcf132902233d7b92f5b5d678ab6ec18b691086b181f85a88e", + "size": 27467 + }, + { + "class": "code", + "path": "fly_trader/kalshi/vendor/__init__.py", + "sha256": "fd3f17e2adc47cedd977d17b25a25da96c28315d2ac609b02007a3ea13509779", + "size": 340 + }, + { + "class": "code", + "path": "fly_trader/kalshi/vendor/book_sizing.py", + "sha256": "1a916ad3162bd13bb7d429074c3a8fe344d10766d07b5d143c9db61693906939", + "size": 13994 + }, + { + "class": "code", + "path": "fly_trader/kalshi/vendor/config.py", + "sha256": "c3af3dd54a6d21707925a5b7260e30070386c63a444fa17b1171d74106e46a24", + "size": 657 + }, + { + "class": "code", + "path": "fly_trader/kalshi/vendor/database.py", + "sha256": "32b029c46468da3481d5e1e165f0afc8b2da59addf41097079ef6734b2676b61", + "size": 834 + }, + { + "class": "code", + "path": "fly_trader/kalshi/vendor/db_connection.py", + "sha256": "43216deff6bfcfee5656f0660b7ebe56290db8410836bb0c4bd3115d8e963d71", + "size": 290 + }, + { + "class": "code", + "path": "fly_trader/kalshi/vendor/gate_register.py", + "sha256": "121d4ef99408a2574fc677c0b3de22e71740e93f39a48f20b762cd9eba03a37a", + "size": 209 + }, + { + "class": "code", + "path": "fly_trader/kalshi/vendor/kalshi_client.py", + "sha256": "7c73abc09f56bfa006467255ddaa08ce758133aafa395a015576804b7c3588d1", + "size": 45970 + }, + { + "class": "code", + "path": "fly_trader/kalshi/vendor/kalshi_stream.py", + "sha256": "e092d58ffcae8ac90284308d20f6d79a02abc66d6ecdca14ed35928c0fc1ab12", + "size": 14293 + }, + { + "class": "code", + "path": "fly_trader/kalshi/vendor/prediction_utils.py", + "sha256": "c867bcccfd566fdb59ab0759d4994e4fe73e1053d07204a6ff7137e163c54b77", + "size": 247 + }, + { + "class": "code", + "path": "fly_trader/kalshi/vendor/venue_kalshi.py", + "sha256": "ff88c7698a78dff9ccd1a1bae00b30a31f9cd967654fc2c80f51bd79ca698d94", + "size": 16572 + }, + { + "class": "code", + "path": "fly_trader/kalshi/vendor/venue_register.py", + "sha256": "e266224b8e104f0d4dedbff8352bdadb8615e5a4b4321998bdcd59a93726a79b", + "size": 13298 + }, + { + "class": "code", + "path": "fly_trader/logging_setup.py", + "sha256": "222426b3fb8096a408e4bfcceee561d5200d4908c85a54744984c292b820527d", + "size": 4514 + }, + { + "class": "code", + "path": "fly_trader/market/__init__.py", + "sha256": "01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b", + "size": 1 + }, + { + "class": "code", + "path": "fly_trader/market/exit_cost.py", + "sha256": "4c8757dae7daf29e757c1d50986c1b0c41aeca03e6fed42959a248ea6d78ee8b", + "size": 7041 + }, + { + "class": "code", + "path": "fly_trader/market/features.py", + "sha256": "350b1036e67502c31a40404bd713ef48bdce3ff84900b5caf784a6b93c576d55", + "size": 13467 + }, + { + "class": "code", + "path": "fly_trader/markets.py", + "sha256": "c588e6297d92d69bb3d93c1619bae4cd0394f7677688ab809b9240260799c257", + "size": 4699 + }, + { + "class": "code", + "path": "fly_trader/ops/__init__.py", + "sha256": "01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b", + "size": 1 + }, + { + "class": "code", + "path": "fly_trader/ops/autoupdate.py", + "sha256": "6f6e4052630b7f979369e239578c24c2a559d27a812efdfd1adf606262ed5d8e", + "size": 10438 + }, + { + "class": "code", + "path": "fly_trader/ops/procs.py", + "sha256": "a235e69f7e06490f2ae4001088819ffd8f8b0a8c25b8b383529d3f1d3830f215", + "size": 5665 + }, + { + "class": "code", + "path": "fly_trader/ops/release.py", + "sha256": "bd84cd5a02ce9df1f166ffc7622f8dce41f4e20ea21455c5150374176af93c40", + "size": 10794 + }, + { + "class": "code", + "path": "fly_trader/ops/reset.py", + "sha256": "40662d513e6608ff1943e3587a155a9f92fc4daebfaa855018f78daeb2c34f79", + "size": 11996 + }, + { + "class": "code", + "path": "fly_trader/ops/supervisor.py", + "sha256": "c91ab4a08c2366411900fbd4ad5c8e22eb0b3d766c2d5058464681620b0eefcf", + "size": 11805 + }, + { + "class": "code", + "path": "fly_trader/ops/wallets.py", + "sha256": "10bdb62ace3444b6d80aa0e2ca85be241d371392b85bca0ba23376d4753f9811", + "size": 7821 + }, + { + "class": "code", + "path": "fly_trader/rh/__init__.py", + "sha256": "36356716ff4e4aae40c278b644b2758c7fd9f7cece58cf924affa41cd236be2b", + "size": 226 + }, + { + "class": "code", + "path": "fly_trader/rh/abi.py", + "sha256": "151c5857a2933e7d2a00bc9bae15191df57f48b4f4363aad8d9c7f6a111b933b", + "size": 8335 + }, + { + "class": "code", + "path": "fly_trader/rh/accounting.py", + "sha256": "8fb2a30ad091ec38e75fdf99c76a6eca084fbb44b18433d9e32963102fdc5ed0", + "size": 16726 + }, + { + "class": "code", + "path": "fly_trader/rh/blocktime.py", + "sha256": "61c9c71cd6d43e3106639d14217c60d9f2b2ec12a6bfa701d6e3134ada44ea24", + "size": 5587 + }, + { + "class": "code", + "path": "fly_trader/rh/corpus.py", + "sha256": "cd19c128695eff54887bbcc6743b10c57e586f7d81d0fec94f45f68b1b6423a0", + "size": 5395 + }, + { + "class": "code", + "path": "fly_trader/rh/exec.py", + "sha256": "04d338e2602496e46db21876b649de9b959edc9e87ebc6fcc77a99438555bd69", + "size": 23834 + }, + { + "class": "code", + "path": "fly_trader/rh/guard.py", + "sha256": "4228a50e2cc9bed98b6edb4b3d07321aa94b55dd6d4fba5cd04ed948f4df28e4", + "size": 2993 + }, + { + "class": "code", + "path": "fly_trader/rh/hypersync.py", + "sha256": "b282dd71f39edab204958e821cb0dbd2596c5d70cec6a0eb3efdc7369d3917cb", + "size": 9443 + }, + { + "class": "code", + "path": "fly_trader/rh/index.py", + "sha256": "14a5d09f2ce85dd7d99f241773273f708182b9f99bb3eedcce08a7270f03d88a", + "size": 21402 + }, + { + "class": "code", + "path": "fly_trader/rh/kyber.py", + "sha256": "4755c946c90f54db924633df776e43202860a603d349679e130006dacf5f084f", + "size": 7337 + }, + { + "class": "code", + "path": "fly_trader/rh/live.py", + "sha256": "07583e9557c9dcc1abd98f60a28d8f1ca41c333487474452b74302370ebb9849", + "size": 13856 + }, + { + "class": "code", + "path": "fly_trader/rh/meta.py", + "sha256": "f7935db3bd7ed1af186abcc0d34731918005130d750ce815c95362de399c2b83", + "size": 7919 + }, + { + "class": "code", + "path": "fly_trader/rh/minutes.py", + "sha256": "01413e92ceb0292e73272faa3b047a5b1d2e0789df0d1693f2ab2137a209b11a", + "size": 9176 + }, + { + "class": "code", + "path": "fly_trader/rh/pons.py", + "sha256": "152f4a34a5e6c0f2fbc399eb5fb6f8528e1a2a5ad05794bde21bb793788896f4", + "size": 5283 + }, + { + "class": "code", + "path": "fly_trader/rh/prefetch.py", + "sha256": "35653641c48dea9f1f6bbfeafaebc6746436ddd4aeee0bd811a005ed97e90e56", + "size": 9569 + }, + { + "class": "code", + "path": "fly_trader/rh/prices.py", + "sha256": "561523f5386c7cf0a67a4e790d7053d9ba328cfa2cbe9223161f725123de24c1", + "size": 13020 + }, + { + "class": "code", + "path": "fly_trader/rh/probe.py", + "sha256": "33258ac688c1fc067aed55e6ca135dc549c7a605f62db91cd6e1871160858a37", + "size": 3829 + }, + { + "class": "code", + "path": "fly_trader/rh/router.py", + "sha256": "148f622f3cd1b13028df370568db94b66a4e904692dae7573f1d5320470b8338", + "size": 3904 + }, + { + "class": "code", + "path": "fly_trader/rh/rpc.py", + "sha256": "517791921a831cd4ef89a9281cfa01fd4dee23d26946e9f67c1a241a24f0bbca", + "size": 6217 + }, + { + "class": "code", + "path": "fly_trader/rh/scan.py", + "sha256": "4441de970206a0a862484ab2380b060946ee19b14faa3809eb122bfc49f45368", + "size": 2567 + }, + { + "class": "code", + "path": "fly_trader/rh/schema.py", + "sha256": "8bc59a2cd1e4fe876ebe1b2bb98032c080584f5cd420cdf5f3f45b2596fe4b84", + "size": 13142 + }, + { + "class": "code", + "path": "fly_trader/rh/smoke.py", + "sha256": "2d86a89999042c075c9e0523e16ce3006eec2453549f93b8eebc605f55f7f172", + "size": 5765 + }, + { + "class": "code", + "path": "fly_trader/rh/stream.py", + "sha256": "87b68980a4998519fe902ce1ec67cc9974665f9fe851b15a003b8fbb9afb4824", + "size": 8000 + }, + { + "class": "code", + "path": "fly_trader/rh/tx.py", + "sha256": "fe3b881727b1668cf01739b1398ffbcb7f11f05aedd89f3d802eb39258bdce6e", + "size": 5644 + }, + { + "class": "code", + "path": "fly_trader/rh/univ4.py", + "sha256": "a9686cae06ded463e0dd6582da4333099c0c5e087ef0a469b8d4412d152756eb", + "size": 4599 + }, + { + "class": "code", + "path": "fly_trader/rh/v4.py", + "sha256": "2dff4e807133b458cace46daafe565898ef6655931a04c0a88d57602eb124da2", + "size": 4199 + }, + { + "class": "code", + "path": "fly_trader/rh/wallet.py", + "sha256": "6a925cd070ecaab27baef90dc65fe18c29d39d17ec467997c26e55af7aefa284", + "size": 13756 + }, + { + "class": "code", + "path": "fly_trader/rh/wallet_skill.py", + "sha256": "58b5207c3e90a9870ffb5b90bd4a809c35ef4e0b9dcb12a05a68d76b3376df48", + "size": 7425 + }, + { + "class": "code", + "path": "fly_trader/rh/withdraw.py", + "sha256": "b0874c0dee6ee5211ef13d891a89ff4ef07fe351eb77c83e6d82681c7a192204", + "size": 6683 + }, + { + "class": "code", + "path": "fly_trader/signer/__init__.py", + "sha256": "d98ed5a9e30e14723337366b99a98f6e244c81d12a4adbccc38654d878df45d6", + "size": 105 + }, + { + "class": "code", + "path": "fly_trader/signer/__main__.py", + "sha256": "b8c97ede1a3c63c55f97dcae99b10f139df2488c1fb850329cf73b7f1da49a91", + "size": 2196 + }, + { + "class": "code", + "path": "fly_trader/signer/client.py", + "sha256": "5ccc3a2d57d354f1170777beb507ebac1ea6de0e187039a2b54302d109efa07b", + "size": 3314 + }, + { + "class": "code", + "path": "fly_trader/signer/core.py", + "sha256": "4bd41ece00a9fada02b45806c741f646469c4a3759e2416091a0a8ee6c9519c9", + "size": 16435 + }, + { + "class": "code", + "path": "fly_trader/signer/ledger.py", + "sha256": "585c2b2a7218a2bffb267f423a925f9758e89c8b669ba8078a8090f47bacd237", + "size": 3051 + }, + { + "class": "code", + "path": "fly_trader/signer/policy.py", + "sha256": "4daecbe7428b60935c87eb519be9069b47d76edb396ad9744ee32fa1cda8319c", + "size": 10333 + }, + { + "class": "code", + "path": "fly_trader/signer/rpc.py", + "sha256": "357985c4f40e6dc941562cbeb7a957c5cde7824cbdff1a5ddc09cffe3f0a3637", + "size": 3414 + }, + { + "class": "code", + "path": "fly_trader/signer/server.py", + "sha256": "b61476f6edd4e973ea32ad51a45dd5f0830be5a9ce8f1a9269c840a5b97ad5bc", + "size": 2240 + }, + { + "class": "code", + "path": "fly_trader/signer/squads.py", + "sha256": "b16d95cff4ed368b5b26f33216387c95c8884c42cea06774c321a467520b7d2f", + "size": 6142 + }, + { + "class": "code", + "path": "fly_trader/train/__init__.py", + "sha256": "01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b", + "size": 1 + }, + { + "class": "code", + "path": "fly_trader/train/corpus_backtest.py", + "sha256": "4349748dbfce28410aeba6ad8d3db1e4995a81717631b7bc3489204b0f69948a", + "size": 15836 + }, + { + "class": "code", + "path": "fly_trader/train/corpus_features.py", + "sha256": "43d8c676e34f9cf909e41e71e08a6d51f3cace72a10744ff41a0f3b456553708", + "size": 14954 + }, + { + "class": "code", + "path": "fly_trader/train/corpus_meta.py", + "sha256": "f43081ad1f9a223bc82ddfd08683da714c3c808f9e219f92e2a757f154f5a5a3", + "size": 23616 + }, + { + "class": "code", + "path": "fly_trader/train/decisions.py", + "sha256": "8d9f20f09783a49ae134d228aa08cda340979141fd44b2db7bae3ddbd853d0c6", + "size": 20909 + }, + { + "class": "code", + "path": "fly_trader/train/flow.py", + "sha256": "93f672aeb6b9e0c58f8e88a0d765372226ef975ed70f269915e7be5ebcf3db15", + "size": 7298 + }, + { + "class": "code", + "path": "fly_trader/train/fly_calibrate.py", + "sha256": "ac708508ceb44d92ddb05e6d6546770190fef94fdbc9a314b8c54ef395842bbc", + "size": 6647 + }, + { + "class": "code", + "path": "fly_trader/train/fly_governance.py", + "sha256": "1261cfb68217791cabce57d4f6daef1599f5adb9d166a2ddd7b02f77aa435d8a", + "size": 3148 + }, + { + "class": "code", + "path": "fly_trader/train/fly_replay.py", + "sha256": "f0ce82a0f7334cbc093bca94bed6d5715ee09078efdb37bb8ed3b0d8aefdaa09", + "size": 40719 + }, + { + "class": "code", + "path": "fly_trader/train/fly_selector.py", + "sha256": "ff98348f8d4b4142ce7836755db4c51dd992b844083745b7da33733d6ec8226f", + "size": 54150 + }, + { + "class": "code", + "path": "fly_trader/train/mature.py", + "sha256": "70f97740d9e73767a00a3cc8fe4141628ffaf8d82ed9efbec7d9e3da6d44af11", + "size": 25513 + }, + { + "class": "code", + "path": "fly_trader/train/model_io.py", + "sha256": "d7a843a47fba5f7affcaaaf70b1a01d82b47736d74efb4a5ace7365b6dbe7c12", + "size": 4320 + }, + { + "class": "code", + "path": "fly_trader/train/pipeline.py", + "sha256": "7a8bca1dc4c8c8149dc90914db344d189a499232b755505c21e8e1b528f28b1d", + "size": 5615 + }, + { + "class": "code", + "path": "fly_trader/train/progress.py", + "sha256": "e16c8932b9f262fa751e9f3310b5d7c5830c505ae4c082ab4b78a28d460d76b1", + "size": 2292 + }, + { + "class": "code", + "path": "fly_trader/train/replay_assemble.py", + "sha256": "23f3409b48bcbd9ba9db806bdcbec12ec06226956d7f10a08b9704ed05824e5e", + "size": 10672 + }, + { + "class": "code", + "path": "fly_trader/train/scaling.py", + "sha256": "ad6723031da5f30626b800889d6c18e86fa4f489512e41dbec9edf9cce64444e", + "size": 2551 + }, + { + "class": "code", + "path": "fly_trader/train/selector.py", + "sha256": "03d7037ed879fc362eaf5a3558c194432fa2f3699a2cfe872a747c944a503eda", + "size": 25427 + }, + { + "class": "code", + "path": "fly_trader/train/selector_eval.py", + "sha256": "16cdbf7da30715dfd58d17f357c7c5e47586fb3cc64bc4f5b6cdb31643a54746", + "size": 3243 + }, + { + "class": "code", + "path": "fly_trader/train/strategies.py", + "sha256": "be87d4e69d258f08e8c6f53458b3ffc9c955528692c85b86101ad14905442ab1", + "size": 54241 + }, + { + "class": "code", + "path": "fly_trader/train/wallet_skill.py", + "sha256": "498ecc62df27ae11450bae7965786edd298389ac1fbfc1e7694148f15f735585", + "size": 17534 + }, + { + "class": "code", + "path": "fly_trader/ui/__init__.py", + "sha256": "01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b", + "size": 1 + }, + { + "class": "code", + "path": "fly_trader/ui/app.py", + "sha256": "7444df73fa09ac8b54274930855727cc29c2c2a29c33551a6636ff2649e28b91", + "size": 2873 + }, + { + "class": "code", + "path": "fly_trader/ui/app_pages/data.py", + "sha256": "80ccc44cc4b8e85dd7ae6c7dbf22decc7cfc269f30c1b295c09fa91180cb6d81", + "size": 6559 + }, + { + "class": "code", + "path": "fly_trader/ui/app_pages/kalshi_data.py", + "sha256": "5302e0c24256deebbce959a232a39e96be6f8781e4e42ac4c19d14f35d7469df", + "size": 5551 + }, + { + "class": "code", + "path": "fly_trader/ui/app_pages/kalshi_model.py", + "sha256": "a2ece4b38a87892da8491f7c5a1bc0f80bc60f1bcab29b6ffbb396fdcea22738", + "size": 14701 + }, + { + "class": "code", + "path": "fly_trader/ui/app_pages/kalshi_overview.py", + "sha256": "0b482404905720605d88d98c59f66cfb2881feb4f7cc928a6d560a21f7861ef0", + "size": 11634 + }, + { + "class": "code", + "path": "fly_trader/ui/app_pages/kalshi_trades.py", + "sha256": "4777b86cea2bf2c286777aec09ecbca42824a45fc8d9e362c13314baae832d6b", + "size": 7137 + }, + { + "class": "code", + "path": "fly_trader/ui/app_pages/model.py", + "sha256": "357e52fc0e7e0b6f1cc819fad0a1e79292155bb4bacc3b0be55d77c201d4c40d", + "size": 25276 + }, + { + "class": "code", + "path": "fly_trader/ui/app_pages/overview.py", + "sha256": "677d66377dae3c6f6fc35ba88b92125cc95f2bbe360141c00e709a62fdc30f19", + "size": 13155 + }, + { + "class": "code", + "path": "fly_trader/ui/app_pages/processes.py", + "sha256": "775c284197759d923123492f3187117d462c6cdb01d545bd40e76887b66c15a8", + "size": 5278 + }, + { + "class": "code", + "path": "fly_trader/ui/app_pages/rh.py", + "sha256": "0aa09694547a205f066640a39d85adb41a71db6d22e6bb575a4fd6f9c6e0164b", + "size": 7839 + }, + { + "class": "code", + "path": "fly_trader/ui/app_pages/safety.py", + "sha256": "c20d9d19f935364834987af330f13addb57e26cdca78360cbfd8a678702dc37c", + "size": 22406 + }, + { + "class": "code", + "path": "fly_trader/ui/app_pages/trades.py", + "sha256": "dfbb13cbddffd47b1d4ac0d56ddbcc35300920a8f0534d4e27b3dd3e41bf03d9", + "size": 5888 + }, + { + "class": "code", + "path": "fly_trader/ui/app_pages/vault.py", + "sha256": "79def871ae20e65835292a0a8466c18b44064d6f036538898ad53f0a433c5da9", + "size": 3857 + }, + { + "class": "code", + "path": "fly_trader/ui/brain3d/__init__.py", + "sha256": "b1ad5ec132501a2ddadb5ba67708b36a3d228ac22d8af65627690546d30939b8", + "size": 5361 + }, + { + "class": "code", + "path": "fly_trader/ui/brain3d/brain.css", + "sha256": "41a8a3f4e0d1ce2080288a219ba3f3e87f093f75885da5d2f6f2031b977ae451", + "size": 1787 + }, + { + "class": "code", + "path": "fly_trader/ui/brain3d/brain.js", + "sha256": "9ad6f2c87d6b8f3548e3e109d8773f929843d2962f4461a364056ffb98237c38", + "size": 23096 + }, + { + "class": "code", + "path": "fly_trader/ui/brain3d/feed.py", + "sha256": "b8726c0906819c2ae8590fce3e29ce72b0e3e4edeba59a3ca755e41698a400da", + "size": 5915 + }, + { + "class": "code", + "path": "fly_trader/ui/common.py", + "sha256": "5fb04a6a3875a0d33d04670d1183d8f3d4975704fa348bc5a6a7aa92876fee0f", + "size": 25565 + }, + { + "class": "code", + "path": "fly_trader/vault/__init__.py", + "sha256": "33995db17911bb1ec108eb0ff4ec8c5c987be2e78f46d435058114958fe2801d", + "size": 506 + }, + { + "class": "code", + "path": "fly_trader/vault/alerts.py", + "sha256": "4ce6f55ec442ea3e7d9487e6e1ab89760b06f09bcd8ffefd348a0e33f7c9e6f7", + "size": 1435 + }, + { + "class": "code", + "path": "fly_trader/vault/backup.py", + "sha256": "381fcbde6f55be2c6f41c8fee1e78db7a1ef8df158c5530417f17240b562fa58", + "size": 3224 + }, + { + "class": "code", + "path": "fly_trader/vault/claim_message.py", + "sha256": "1c1adeda60ae9e6739578c65ab03df97f68bfa16e2fc16a6e621c4caccc44abc", + "size": 3188 + }, + { + "class": "code", + "path": "fly_trader/vault/claims.py", + "sha256": "95b1a59b0d0cb7ad8ea128b9b3ea9f0c28eac775f0a6a8045efe99f6fc0d0f6a", + "size": 14446 + }, + { + "class": "code", + "path": "fly_trader/vault/claims_eth.py", + "sha256": "e64a1a9af1283a7f73b4278d59bef91c3a3c92aa93c680cd82b31ef44fe9d774", + "size": 7898 + }, + { + "class": "code", + "path": "fly_trader/vault/cli.py", + "sha256": "ee62fce36b5412fd39a972a7e3f3d42ab9626a34ad82e63eacb5e4d086c2e8f4", + "size": 7328 + }, + { + "class": "code", + "path": "fly_trader/vault/custody.py", + "sha256": "237085b140958f107e2640b8eb113f690bccf3d83bb302f293a9ed44a517962e", + "size": 6041 + }, + { + "class": "code", + "path": "fly_trader/vault/eth_schema.py", + "sha256": "331a7c796c56ea8a3c6f9e308195c7904070a8d5492ba96bb8415719152cf05f", + "size": 2856 + }, + { + "class": "code", + "path": "fly_trader/vault/evm.py", + "sha256": "420ebdef83dc09bea812ddcda35797c0f2b8aee7cc637ee59740533a5fd3442b", + "size": 6172 + }, + { + "class": "code", + "path": "fly_trader/vault/flows.py", + "sha256": "fa14ac1f9e5bee5180ff0edae02e2c8a8d4a7f16e3430d599903947209b9719c", + "size": 13848 + }, + { + "class": "code", + "path": "fly_trader/vault/nav.py", + "sha256": "439bb6a155fe4dcabd3545ca53c612c57098b5f399121c36d8042bff66b0ec0d", + "size": 4687 + }, + { + "class": "code", + "path": "fly_trader/vault/payout.py", + "sha256": "95a54ef3e2a9ec6f6b7aba6d29a857d262369719d79da1d56cc531934cc0c209", + "size": 7222 + }, + { + "class": "code", + "path": "fly_trader/vault/prices.py", + "sha256": "ad7f9a23f742d0feffbb5ec19e08509f9ed1ba01c55a9d4d7a0eb0f223744d75", + "size": 2384 + }, + { + "class": "code", + "path": "fly_trader/vault/publish.py", + "sha256": "bc794aaa2e74902b04a0dba67f37551f42502c90136e2e263d10f6a664762dd7", + "size": 11729 + }, + { + "class": "code", + "path": "fly_trader/vault/relay_client.py", + "sha256": "597fa5eab1735ee93b653c0e847446e91038934bc9da3e3d7034b06808b48e37", + "size": 3581 + }, + { + "class": "code", + "path": "fly_trader/vault/rh_index.py", + "sha256": "6b830e7e0dbc99b8b4654b2ae49488510deeb4b2a6d68961e396cf634bd0aeff", + "size": 10028 + }, + { + "class": "code", + "path": "fly_trader/vault/schema.py", + "sha256": "ca9ce74baaf1668a3e7fe35ec316d050bbcc82e6ddc960a6591c04100a6f5f67", + "size": 6402 + }, + { + "class": "code", + "path": "fly_trader/vault/settle.py", + "sha256": "41ccba7bc56764f4ebe4d2c0bf0e2b123ca21deba484568650ae1c580f86c99e", + "size": 18880 + }, + { + "class": "code", + "path": "fly_trader/vault/settle_eth.py", + "sha256": "520659a355982e0f62f63ddbd1bdcff4e4449999f9f02a64c07913c08a0ae070", + "size": 7814 + }, + { + "class": "code", + "path": "fly_trader/vault/sigs.py", + "sha256": "f5afa72c5943a857f515f904d6ced0f08c7b9a7a788a38f5f47543ab6206136d", + "size": 4978 + }, + { + "class": "code", + "path": "fly_trader/vault/state.py", + "sha256": "b4a60a4b806771ed59c139bb3d529116139deffcc2ed37512b412a242ec2fa7a", + "size": 2115 + }, + { + "class": "code", + "path": "fly_trader/vault/telegram_cmd.py", + "sha256": "b6d6ad7997736a1e070b5a490d1825efe191d8b47132daebf0c7aee3518eb034", + "size": 5860 + }, + { + "class": "code", + "path": "fly_trader/vault/walletlock.py", + "sha256": "f837fbf54d560c15ac28e53e4f9d704869238d3d7d4465c55c75e6c3cd023d9b", + "size": 2133 + }, + { + "class": "code", + "path": "fly_trader/vault/withdraw.py", + "sha256": "d834c9b36fe967cbb7e7349b3831e01b782eecc4de79db7e295fdc3be014019b", + "size": 2106 + }, + { + "class": "code", + "path": "fly_trader/vault/worker.py", + "sha256": "e5599f41f802edc4bc6b123e0a3b567556ac568fa8fbe495fe8ce86d9bfaf328", + "size": 8478 + }, + { + "class": "model", + "path": "models/connectome/current.txt", + "sha256": "5028fc0b2c8d6e5f8f7220229101150e0de1207a7f6873d3709793d2fa87fb19", + "size": 25 + }, + { + "class": "model", + "path": "models/connectome/fafb783_d1bad40e798b.npz", + "sha256": "9435f08f175b1b569e6d6d2ea338f906d4a43349e6e3216213f511c5082436d0", + "size": 14403111 + }, + { + "class": "model", + "path": "models/connectome/scale.json", + "sha256": "c6df7317236399b54e606a3c43808fa91972a18bb333926ddca7bc9c6939b720", + "size": 751 + }, + { + "class": "model", + "path": "models/geometry/geometry.bin", + "sha256": "bd2cb3f56f6df852d1dc2fc1c4ce86f4e9d55c40d387f02608e22817bdce962d", + "size": 501072 + }, + { + "class": "model", + "path": "models/geometry/geometry.json", + "sha256": "d8c3c4c6e6473adaf050b59e1003ebd7d56678f1abf0a597f326b9b0e7b5b321", + "size": 333680 + }, + { + "class": "model", + "path": "models/policies/fly_20261006T023153Z.pt", + "sha256": "73141bf59bd4321c66f28e4c5a7c83f994adf744574de8b17fe5d3ca94fcc1e7", + "size": 28525445 + }, + { + "class": "model", + "path": "models/selectors/selector_20261003T201413Z.skops", + "sha256": "b38f0633ee4a9b693bfb9a01a5e5de044e2ce317e08fb9726e77b48dfbb603fe", + "size": 40812282 + }, + { + "class": "model", + "path": "models/wallet_skill/2026-10-11.parquet", + "sha256": "34c0faf085f068c217a98d21722aa44a57907b466547f723ffa3fc1e6ee6287c", + "size": 281123254 + }, + { + "class": "model", + "path": "models/wallet_skill/TABLE", + "sha256": "068d40ac4cd6395a31eacf1724b8a13d8eb4922a046d086458325ff11762ff1c", + "size": 19 + }, + { + "class": "model", + "path": "models/wallet_skill/TABLE.sha256", + "sha256": "3e73a7afd95b3488e8654955e3a9c32b83868bb4b554a4a34ad0560c0935f29d", + "size": 65 + }, + { + "class": "model", + "path": "models/wallet_skill/config.json", + "sha256": "d19fa8e03b9ea51da2651b6e02231542b7c651a366d7205ae057d81d39436c7a", + "size": 871 + }, + { + "class": "model", + "path": "models/wallet_skill/fit.json", + "sha256": "5f8fd66801e6e169802c40e5c217d7107fdcb0986b12a2d4699a61f84656e27a", + "size": 19291 + }, + { + "class": "code", + "path": "pyproject.toml", + "sha256": "9b559d42f4ebfd4fbcc4e25c5f733534e5f2f0134e210dbf531a026d8b0633b8", + "size": 2326 + }, + { + "class": "code", + "path": "release/allowed_signers", + "sha256": "0e84ff7a16fad3300c2ed3edf5da7e5e0536faacfd983991ba924889db2fd539", + "size": 894 + }, + { + "class": "seed", + "path": "seed/release_seed.json", + "sha256": "f30e3b97b067d91fd5a22f73bd74e17a2d6be8e056b7b6b297962fd867962f13", + "size": 146466 + }, + { + "class": "seed", + "path": "seed/seed.sql", + "sha256": "a84919bc31e58b0d363c5262c51265a210851123fb250fe5eb389b0b74f7ed86", + "size": 120890 + }, + { + "class": "code", + "path": "tests/conftest.py", + "sha256": "195c7c2f7c9e6ed6dec51bd075b84b99472e92ce8391c0a9ab1b475bcf24f4a2", + "size": 879 + }, + { + "class": "code", + "path": "tests/fixtures/rh_logs.json", + "sha256": "55aa6f69263520cb4298c4582faff68dedede5caa7eb56779e45f46483de7f8c", + "size": 9142 + }, + { + "class": "code", + "path": "tests/fixtures/solana_golden.npz", + "sha256": "ce8546ac191f518c89aa6160dd546f282d10ca20248c9c9fe882e6c3136983fc", + "size": 227838 + }, + { + "class": "code", + "path": "tests/fixtures/ultra_orders.json", + "sha256": "72cbe6f92079ae3e0a808cd2a6de5d24143604bfc1f17728336599f60bd0b1a4", + "size": 86661 + }, + { + "class": "code", + "path": "tests/test_audit_watch.py", + "sha256": "f19e70e5c2ed474312e8de6817c23f05d19af28cd29ae14fe49274e904de1275", + "size": 1326 + }, + { + "class": "code", + "path": "tests/test_autoupdate.py", + "sha256": "6d6c34b8449db1a29e49a9ba378715d60b460bba476f0ee3be2578e4b7565a50", + "size": 5472 + }, + { + "class": "code", + "path": "tests/test_balances.py", + "sha256": "5e894b913fd82a6768b375a2c7c96fd22115cc2c1b104b985a573e26c8df6253", + "size": 1354 + }, + { + "class": "code", + "path": "tests/test_book_halt.py", + "sha256": "e9221c490b1cb35a13d13b622e17328491245d2b773590dc751d23468aa12eaa", + "size": 1246 + }, + { + "class": "code", + "path": "tests/test_broker_live.py", + "sha256": "2f800bb1d16666897aae54ec13e83a7c9ade587ba1659bd4181044c884b9dd82", + "size": 14801 + }, + { + "class": "code", + "path": "tests/test_cluster_guard.py", + "sha256": "9d87a91e1bc939cceea4c86b0e6df39e20c7d58402a570993828e0078221fdfe", + "size": 1279 + }, + { + "class": "code", + "path": "tests/test_connectome.py", + "sha256": "cd9058dbcef01e35c4ba7f065b510c615833bf6022d241b328f28d75aece00b9", + "size": 3110 + }, + { + "class": "code", + "path": "tests/test_curve_parity.py", + "sha256": "57a7b480303241218f4b9399986a20aef546b4821b8cc06ef6519b6251cd0901", + "size": 4720 + }, + { + "class": "code", + "path": "tests/test_custody_e2e.py", + "sha256": "4cc6fa10c0cbb74ca5ccf4fa6d4cbc3e57b7892469980a45651d3b419073e95a", + "size": 3990 + }, + { + "class": "code", + "path": "tests/test_custom_pools.py", + "sha256": "51d364246448b11a0aee05833d6dc6035141c4280cf0f6666e83e9375d52d3cd", + "size": 7101 + }, + { + "class": "code", + "path": "tests/test_decisions.py", + "sha256": "24cce59236533ad9402972cc763b2b03fe8a07e09b6af3865787d5fe20332b7a", + "size": 5899 + }, + { + "class": "code", + "path": "tests/test_device.py", + "sha256": "76f06a3e3234dcba28a75db40eb7e613992baf640d9c4910fb73cf1f7684cb87", + "size": 5544 + }, + { + "class": "code", + "path": "tests/test_features.py", + "sha256": "c2dc6f2a911cf96b07d0190c1ac99bf2dadaae25c090bbc028aba9a23d1d9c7e", + "size": 1878 + }, + { + "class": "code", + "path": "tests/test_fix_ingest.py", + "sha256": "a62b97650674194df5a4831e98e65039582e97228f52a2a0d47c31b9410f0547", + "size": 5303 + }, + { + "class": "code", + "path": "tests/test_fix_live.py", + "sha256": "51dfb364af15f59ee42db0344b67f025ef1c3659ea11ee584dd84a3be1d1ccec", + "size": 7373 + }, + { + "class": "code", + "path": "tests/test_fix_train.py", + "sha256": "7badc36947c572e4dbcf3146886703c6501cf6b6d9c0347935800b964980bd72", + "size": 13569 + }, + { + "class": "code", + "path": "tests/test_flow_parity.py", + "sha256": "f330e9106ada810975cf91d19a5e3c2d8a5784a4b38e1327e19d8ff07c1e0961", + "size": 9111 + }, + { + "class": "code", + "path": "tests/test_fly_brain_view.py", + "sha256": "d3ae73618a89e0e71b635ac8ff1425ef61723b0b21f49fe9af04ed2d89c7f706", + "size": 7910 + }, + { + "class": "code", + "path": "tests/test_fly_calibration.py", + "sha256": "cf4e02c856cc602c4f69bce734e5c69306b1726285cc8dfdca0ec27d659781dc", + "size": 5278 + }, + { + "class": "code", + "path": "tests/test_fly_channels.py", + "sha256": "01b09d86b173b8178818cc6e415ca5491b970fe2fef5adfebc330f2ad85547c3", + "size": 3695 + }, + { + "class": "code", + "path": "tests/test_fly_distill.py", + "sha256": "ecfe5d4c65df378a74a2b145eaa23a92f166911e716e94b6c7a40e4149fdc41f", + "size": 15425 + }, + { + "class": "code", + "path": "tests/test_fly_live.py", + "sha256": "03eb383fa9b36012bf217f50395662fb1bae2010aed4fda9d32a1ab91dfccedc", + "size": 14067 + }, + { + "class": "code", + "path": "tests/test_fly_plastic.py", + "sha256": "06d02897d4df7c3559dd90abe6d7f0070b2b4f7a0fa4e4c1433a26a02a9f8ac4", + "size": 6492 + }, + { + "class": "code", + "path": "tests/test_fly_replay.py", + "sha256": "44e57bf2f2b42e9d3b51f3fcbe648d39f6f699152464782ecf03ed2e08975342", + "size": 11598 + }, + { + "class": "code", + "path": "tests/test_fly_residual.py", + "sha256": "3c1db6e8816f426810c4af5ff2e592d50099e42e4523418be8eb894c5db7c887", + "size": 4250 + }, + { + "class": "code", + "path": "tests/test_fly_session.py", + "sha256": "2abe70c3da549670e74524a0bd25962770d25940a9bb50f5cc0ce4d8892126d2", + "size": 3679 + }, + { + "class": "code", + "path": "tests/test_fly_strategies.py", + "sha256": "8bff2e2c64d06998c3c45dc9435761e668a806e2e381697d95f1e7eaef0b8c73", + "size": 6523 + }, + { + "class": "code", + "path": "tests/test_fly_teacher_gate.py", + "sha256": "f351b5fb9c57b025202f665a68f574f505f910672cb422c4b0871acfabf577a1", + "size": 5606 + }, + { + "class": "code", + "path": "tests/test_guards.py", + "sha256": "59a3c203e5f59d7a94a480de7a31f1e076846c5d160ec27a1d6fc25d9521fbef", + "size": 1307 + }, + { + "class": "code", + "path": "tests/test_kalshi_engine.py", + "sha256": "ea0bbcce71dd41b8451d711f2aacf55b7cc1f90f2799ef130e48b3ca163108a4", + "size": 14372 + }, + { + "class": "code", + "path": "tests/test_kalshi_fly.py", + "sha256": "06d3d90ceeb0f43c89835a44e79f55b0eb3b6547e33d7d8e9971eca8407ba820", + "size": 13922 + }, + { + "class": "code", + "path": "tests/test_kalshi_history.py", + "sha256": "30d70e991413436304e23489076f4a2522e4bf381a2542f5301ef14920db1d99", + "size": 15116 + }, + { + "class": "code", + "path": "tests/test_kalshi_live.py", + "sha256": "4fb1996b0e2073adbd871faa59d2d6493a7d76c4628e6a26b5e1ae77ac49be97", + "size": 12595 + }, + { + "class": "code", + "path": "tests/test_kalshi_maker_label.py", + "sha256": "e855dda9a62075a76425dcf236fa59f96a70f85c82803672fc1bedafe58749b6", + "size": 6555 + }, + { + "class": "code", + "path": "tests/test_kalshi_paper.py", + "sha256": "ed08bba9edebce3c5a0245059298dfde091f7c7e210650ede6ad3ecd29d6bdbf", + "size": 11055 + }, + { + "class": "code", + "path": "tests/test_kalshi_rails.py", + "sha256": "1b716dea82a4c545b2ca089820b1c7fb8eecb86aed072064dd6d5404e28790c6", + "size": 1578 + }, + { + "class": "code", + "path": "tests/test_kalshi_stream.py", + "sha256": "9d1c468d8ed2b0e810f787049674da6f4895dade9f2943469cebf8f7fdfc5cde", + "size": 7224 + }, + { + "class": "code", + "path": "tests/test_kalshi_ui.py", + "sha256": "7b67130847a4b4daaf8564aaa7f725ab404a34782e23375d48dd6da56c7ec1b2", + "size": 4448 + }, + { + "class": "code", + "path": "tests/test_kalshi_vendor.py", + "sha256": "f834c6e915d54555c839972845e9848328a4724ceabee86612ab7874a46da486", + "size": 1762 + }, + { + "class": "code", + "path": "tests/test_keys.py", + "sha256": "d9eeec7fca9bfd93ab26ae6a3c8769d9411c0c8c14b100e08c8cf4484e23abea", + "size": 3429 + }, + { + "class": "code", + "path": "tests/test_markets.py", + "sha256": "4c5e38437d73b46113256a83b5bf5d72dc07f266100515bb7ed8a5c965306198", + "size": 1841 + }, + { + "class": "code", + "path": "tests/test_minute_engine.py", + "sha256": "5ad71b13a91c636bd9c9c280a4f86746929777c490af53b4f77ab20ef232772d", + "size": 6162 + }, + { + "class": "code", + "path": "tests/test_model_io.py", + "sha256": "faa842a017c3aa458e570b04269e359aed656ddcd541b9394d5967b6bfdecdc9", + "size": 2647 + }, + { + "class": "code", + "path": "tests/test_multi_hold.py", + "sha256": "5bfc7aa332712661d1f25f3f36d4b5d62b8a5ac2c65a8be765870c53c6451c4a", + "size": 6110 + }, + { + "class": "code", + "path": "tests/test_pipeline.py", + "sha256": "6908d23119729e84e39a23bb8c9fc7b0ce121d93972691d052640e6fe4c235f3", + "size": 5636 + }, + { + "class": "code", + "path": "tests/test_pool_id_guard.py", + "sha256": "a75a505e8a81800878816b44f2886dba367f33c1042b56d55fa0c619f89cf2a2", + "size": 1983 + }, + { + "class": "code", + "path": "tests/test_position_hold.py", + "sha256": "414bbd86ab2fd8afe375cae187d318afb2be4c1ca3a8e39f7e8b3d24d81fdff9", + "size": 3970 + }, + { + "class": "code", + "path": "tests/test_pumpstream.py", + "sha256": "381d48bbea1164768a2c3c6a527ae2387e95fa0cc8197ee638f6437d3f93da0d", + "size": 6175 + }, + { + "class": "code", + "path": "tests/test_rails.py", + "sha256": "d26c3e651968171be7eca6b39a602bc26145e8fc7f5c8bf786117e481f772a92", + "size": 736 + }, + { + "class": "code", + "path": "tests/test_release.py", + "sha256": "5f0c425e655fd3699e2638701a7e078a4d7273976d842102f8b86b7acf501a6f", + "size": 12671 + }, + { + "class": "code", + "path": "tests/test_reset.py", + "sha256": "8e1805ff67733b77232ad6a7773fbc91896c4c2171ac67a502adac42e4cfbb51", + "size": 8639 + }, + { + "class": "code", + "path": "tests/test_review_fixes.py", + "sha256": "59c2613ae8d15d69f57da42e140fdf6590f937f1684c432194aee58a2662415b", + "size": 5522 + }, + { + "class": "code", + "path": "tests/test_rh_anvil.py", + "sha256": "943ebdf33e40fb2ca75979aff36905eda2c878465a0c41dcc4b04980977ab8ff", + "size": 5970 + }, + { + "class": "code", + "path": "tests/test_rh_blocktime.py", + "sha256": "b19c1258a94cd0123357f193d1b87b192eb0d3a3d3f45cfee2e4954e5f6bf4bd", + "size": 1979 + }, + { + "class": "code", + "path": "tests/test_rh_decisions.py", + "sha256": "972967314844a109f7ac9ab30493a67e0e2c31194465c65c657880a5bc0aa2d8", + "size": 4739 + }, + { + "class": "code", + "path": "tests/test_rh_endpoints.py", + "sha256": "98500e7f9440b46dd7e72d4a8af5d476db68a235c6d2d3f6f33bb5a1ff079515", + "size": 2496 + }, + { + "class": "code", + "path": "tests/test_rh_engine.py", + "sha256": "adf64bf23ddc1b28ace638d0dcf819069166ad1121eb37a04b18cae22d50aa00", + "size": 10079 + }, + { + "class": "code", + "path": "tests/test_rh_evm.py", + "sha256": "3f1734a35deca8c75022d5c83c32aa63b26ab8458c577d373f3ec409d5ce21d4", + "size": 3641 + }, + { + "class": "code", + "path": "tests/test_rh_exec.py", + "sha256": "30a9a0b4e06edeb15a0ced548e2aaacccb5ea1c7a4844d4e192eaf4a6c917abe", + "size": 16713 + }, + { + "class": "code", + "path": "tests/test_rh_fork.py", + "sha256": "df6b25dd5ecf547602031cf06e7c49833e0afb2e8284206f4a92257147eb698d", + "size": 10032 + }, + { + "class": "code", + "path": "tests/test_rh_hypersync.py", + "sha256": "8cab5be6ad72133b9adbcb71c1d79ec141f87a2e90f973419feb45f99c185f84", + "size": 3647 + }, + { + "class": "code", + "path": "tests/test_rh_index.py", + "sha256": "4faf174761db4e8829e27fbce8899aae973d619e1c6f87d03aec2511eccc17cd", + "size": 10856 + }, + { + "class": "code", + "path": "tests/test_rh_kyber.py", + "sha256": "e854e73f90913981bf1635f7eb08842489c4270950566947dd74991022cdafda", + "size": 3330 + }, + { + "class": "code", + "path": "tests/test_rh_prefetch.py", + "sha256": "0630547802644d61aff4d213739a7b3d756000bd5be8d1bf4772c534b68d7f84", + "size": 10912 + }, + { + "class": "code", + "path": "tests/test_rh_prices_lockstep.py", + "sha256": "409c912c7934d9f9f3e68425dedc24ddbf4d1e1284fc0e8ee9024ad33b6e7702", + "size": 3349 + }, + { + "class": "code", + "path": "tests/test_rh_router.py", + "sha256": "8abd6c34157ae4215ca9220bf12d905bd5fc0023d4b55d950b5e3c5dd4b5747e", + "size": 4403 + }, + { + "class": "code", + "path": "tests/test_rh_schema.py", + "sha256": "afa2e70af74974cb659a6059bd90a4d9b2270d597eafbf7e39fab435377e6324", + "size": 1619 + }, + { + "class": "code", + "path": "tests/test_rh_training.py", + "sha256": "d075582461615b079160662a21dc420677cf04e8a93aee3d5af6c4caff1fd736", + "size": 5754 + }, + { + "class": "code", + "path": "tests/test_rh_ui.py", + "sha256": "b22b250d4874db1e8a159099f6d98a6d7dd88ce38cbbe58c30a6446a6ccd4cd5", + "size": 2985 + }, + { + "class": "code", + "path": "tests/test_rh_wallet.py", + "sha256": "9e1b2a1470efdff990e113bb9dd809df334e00d40c3e47c18be14061ad561ec6", + "size": 7486 + }, + { + "class": "code", + "path": "tests/test_schema_apply.py", + "sha256": "cc626369e87e83d5fba53da2726a93da1ec2d834e93d700d547da81e7c7073b8", + "size": 2298 + }, + { + "class": "code", + "path": "tests/test_signer.py", + "sha256": "4c4de835d5278663f5d057da1687262b5749da30a443eb623416e6534fdb5e41", + "size": 18181 + }, + { + "class": "code", + "path": "tests/test_signing.py", + "sha256": "475255b29b58e806672ecbafe126386ee4983fd45e7f4669f4f2043dc54ceacd", + "size": 4025 + }, + { + "class": "code", + "path": "tests/test_sizing.py", + "sha256": "ac27051378d46c56d64f83c6d848a9a27bda966fc0fc76ee881a28b2b0d30a13", + "size": 7142 + }, + { + "class": "code", + "path": "tests/test_solana_identity.py", + "sha256": "9b949d0b7729643713a71829ed44c80a3034b84b69f1797ccaff300dae4a3e42", + "size": 7707 + }, + { + "class": "code", + "path": "tests/test_strategies.py", + "sha256": "ad4124ad14d2121abd80b8d90b2cfd77075953d8b81a28380443073d0794b766", + "size": 11882 + }, + { + "class": "code", + "path": "tests/test_supervisor.py", + "sha256": "4d13b3812a2fc6536c14cb6fc72b1b39f41911a5677aad34dc249a08f9b1aac4", + "size": 5611 + }, + { + "class": "code", + "path": "tests/test_vault_claims.py", + "sha256": "aa71bc5aa4116712d45f3b3348c02f06ff8f939870b44685bbdfae18e65c1c79", + "size": 6069 + }, + { + "class": "code", + "path": "tests/test_vault_core.py", + "sha256": "11f47354ca3793d36f44ad1538ec6d5f7ab435e9f17f2376991090acf812e083", + "size": 12049 + }, + { + "class": "code", + "path": "tests/test_vault_custody.py", + "sha256": "0f071a0fc58264f9b5ec162ea28a51ff89cfac62654b1e10f801415e14826b52", + "size": 5415 + }, + { + "class": "code", + "path": "tests/test_vault_eth.py", + "sha256": "30a62b1549a96a6a24de32fb3897dc5a9959dbec9413fd29659c4d6206f4d005", + "size": 10334 + }, + { + "class": "code", + "path": "tests/test_vault_payout.py", + "sha256": "7eb07c086448a0576080a075dea174cc48c2e123ca124382a388dd7907df3f78", + "size": 4809 + }, + { + "class": "code", + "path": "tests/test_vault_redteam_offchain.py", + "sha256": "231472d311627aea2bd19efaa11da8f445be0cff5838ed1e1c09690bfc60aa2d", + "size": 28613 + }, + { + "class": "code", + "path": "tests/test_vault_relay_e2e.py", + "sha256": "f49542a280c6553f1076108f95810d5914188ad95b724844b163f2617163d6a6", + "size": 5813 + }, + { + "class": "code", + "path": "tests/test_vault_settle_db.py", + "sha256": "07dd763de1e9993d646202cf78b07488ffaf0611e2c5979d13cfb621fa790c8c", + "size": 4287 + }, + { + "class": "code", + "path": "tests/test_vault_telegram.py", + "sha256": "8f2b52dc68ae40cc51c122d39dce54501be97390c6033be9fc3ecdc6ceead222", + "size": 3068 + }, + { + "class": "code", + "path": "tests/test_wallet_skill.py", + "sha256": "b87f69ce55bb85dc9f68bd6fdf53a0910966a4b1ddec95cda662963f8835d7e4", + "size": 4764 + }, + { + "class": "code", + "path": "tests/test_withdraw.py", + "sha256": "f721d939d3726604567918b498e1e0be10c045e73013c405995a110198e42f2e", + "size": 18229 + }, + { + "class": "code", + "path": "tests/vectors/claim_v1.json", + "sha256": "73287bc36d343ece236cc387572984fed0dcfcebbe67564fae9f8263e1a58dd8", + "size": 3987 + }, + { + "class": "code", + "path": "tests/vectors/evm_tx_v1.json", + "sha256": "c3948c14c70e936c4aadda62d0e85bbe593e6fd61d0c1197aa3a08ef5e3a4475", + "size": 23002 + }, + { + "class": "code", + "path": "tools/custody_e2e.sh", + "sha256": "e2770c772dfe3176d78714188a07bb6f2458775962f09b0db31054c6a7f2edb6", + "size": 2096 + }, + { + "class": "code", + "path": "tools/ipv6_forward.py", + "sha256": "00134fd8d03ac5987f1e1d86cd841d4f6d3a220d81f57928ead8f45d5af6f4a9", + "size": 1151 + }, + { + "class": "code", + "path": "tools/kalshi_fly_after_selector.sh", + "sha256": "17e3d123e5894702471a5d1368a06d1465c578edd16935d55ef6e0f7c568e16f", + "size": 2155 + }, + { + "class": "code", + "path": "tools/kalshi_selector_run.sh", + "sha256": "4bd5dfa21534b58e90c2ef3bf0f2e86ed52feecbed79d6dbde5209682d11cf73", + "size": 1437 + }, + { + "class": "code", + "path": "tools/kalshi_selector_when_room.sh", + "sha256": "f12621657c49a0a24f32dd1e066c75f381245b93c72e9fa29bb0e2772dae49ad", + "size": 1658 + }, + { + "class": "code", + "path": "tools/make_seed.py", + "sha256": "7cb87911b674ad3a53ac94712bb013946ff8d1704f29173985602dd4f7813dbb", + "size": 9835 + }, + { + "class": "code", + "path": "tools/publish_release.py", + "sha256": "a3695b825f6a76cefcfd19a42ec68e235622409c0b1d118b0ce92fb3d2d884cc", + "size": 9136 + }, + { + "class": "code", + "path": "tools/rh_constants.py", + "sha256": "ef72d38aa0b1b9c932874a20d27c3ddb28197d49fa8d57253cec8319784d4b0d", + "size": 1683 + }, + { + "class": "code", + "path": "tools/vault_demo.py", + "sha256": "64aa0c28e12dc55f72db948b0bde8a4ed7b5d45b8084e981f89ac4e850b51de8", + "size": 20205 + }, + { + "class": "code", + "path": "tools/vendor_kalshi.py", + "sha256": "def1344dd74a092c51746ac2967528e174a741d03956a291a5fae2f1fd84485b", + "size": 4909 + }, + { + "class": "code", + "path": "uv.lock", + "sha256": "47989813f747ad1f9d597ec2a84d55be4390d4909de78c0b7650a96eb0e64059", + "size": 341752 + } + ], + "fly_version": { + "agg": 4, + "calib": "match", + "chains": "sol+rh-1", + "cols": "86151dda", + "costs": "real-fees-sized-1", + "features": 3, + "fly": "flynet-3-ch", + "lines": "per-chain-1", + "meta": "rug-1", + "plastic": "mb4-ch", + "rh_agg": 1, + "rh_costs": "pons-kyber-1:0.0445583:0.0222791:7.2e-06:0.0099", + "rh_skill": "rh-h10-L30", + "selector": "multi-1", + "skill": "h10-L30", + "teacher": "deployed-1" + }, + "git_commit": "7ab2355a0c2b27f5553c3bbedfb319483e9df5d1", + "prev_sha256": null, + "schema": 1, + "seq": 1 +} diff --git a/release.json.sig b/release.json.sig new file mode 100644 index 0000000000000000000000000000000000000000..f8e7ea914c292e4464622c62ef30f932353cf3bb --- /dev/null +++ b/release.json.sig @@ -0,0 +1,7 @@ +-----BEGIN SSH SIGNATURE----- +U1NIU0lHAAAAAQAAAGgAAAATZWNkc2Etc2hhMi1uaXN0cDI1NgAAAAhuaXN0cDI1NgAAAE +EE5f3chQ5e85sy87ILNPgJjCov/+MkpuM4Kblkep+wYTcylLoH8x6i9b5dKOTW9cvSBR9L +e/dwxqIg+z74sBIV6AAAABJmbHktdHJhZGVyLXJlbGVhc2UAAAAAAAAABnNoYTUxMgAAAG +MAAAATZWNkc2Etc2hhMi1uaXN0cDI1NgAAAEgAAAAgOEzUVG70q06LeGameF6olxSA4xoc +ofdWNNfFh39BrmcAAAAgBEI1wfmMleNAWo1ktxreMaZzFqWB2BeL1Nuuu4uSq+c= +-----END SSH SIGNATURE----- diff --git a/release/allowed_signers b/release/allowed_signers new file mode 100644 index 0000000000000000000000000000000000000000..28325a47f1fc365c0ab227816c079579331aa9f5 --- /dev/null +++ b/release/allowed_signers @@ -0,0 +1,8 @@ +# The release keys this install trusts (fly_trader/ops/autoupdate.py, AUTO_UPDATE=1): an update from Hugging Face is +# installed only if its release.json is signed by one of these keys, for the fly-trader-release namespace. With no key +# line here, automatic updates are off. A signed release may change this file (key rotation). +# +# One line per key: namespaces="fly-trader-release" (principals: bryce, bryce-recovery) +# docs/vault/RUNBOOK.md section 0 makes these keys; tools/publish_release.py refuses to publish unless its key is here. +bryce namespaces="fly-trader-release" ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBOX93IUOXvObMvOyCzT4CYwqL//jJKbjOCm5ZHqfsGE3MpS6B/MeovW+XSjk1vXL0gUfS3v3cMaiIPs++LASFeg= +bryce-recovery namespaces="fly-trader-release" ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILR8F0AAO/jveVruQofn9rvzqoWf8KD/F4XzhRN89Jlq diff --git a/seed/release_seed.json b/seed/release_seed.json index 0b43a3bbd5f612ffce684cf848cc6ed022937f24..6ddc4f7701191d1bcd6a9b4b07c3dbba58b53666 100644 --- a/seed/release_seed.json +++ b/seed/release_seed.json @@ -2,82 +2,151 @@ "snapshots": [ { "kind": "selector", - "file": "selector_20260919T201310Z.joblib", - "sha256": "3997c13242e1ef3abcd9a4105cdb58c39a29a72f55c55d2585166d522d948ac1", - "note": "{\"threshold\": 0.048561317734419324, \"horizon_min\": 120, \"trained_through\": \"2026-09-18\", \"walk_forward\": {\"n\": 3172, \"win\": 0.4735182849936948, \"pf\": 1.3978238516274475, \"total\": 128.41733735687012, \"mean\": 0.04048465868753787, \"weeks\": 10, \"weeks_pos\": 9, \"sign_ok\": true, \"admissible\": false, \"base_ok\": true, \"random_mean\": -0.04681328311562538, \"days\": null}, \"selection\": {\"n\": 6652, \"win\": 0.45625375826819003, \"pf\": 1.5568584699178285, \"total\": 359.86430143312464, \"mean\": 0.054098662271967024, \"weeks\": 10, \"weeks_pos\": 10, \"sign_ok\": true, \"admissible\": false, \"base_ok\": true}, \"random_baseline\": {\"mean\": -0.04681328311562538}, \"components\": [{\"name\": \"inputs: rug+flow+skill\", \"passed\": false, \"reason\": \"improves the ev strategy on the selection half, fails on the evaluation half\", \"selection\": {\"n\": 5190, \"win\": 0.43641618497109824, \"pf\": 1.3862310369585873, \"total\": 231.10611372606309, \"mean\": 0.04452911632486765, \"weeks\": 10, \"weeks_pos\": 9, \"sign_ok\": true, \"admissible\": false, \"base_ok\": true}, \"evaluation\": {\"n\": 1845, \"win\": 0.5116531165311653, \"pf\": 1.2823937362697073, \"total\": 62.2172390420601, \"mean\": 0.033722080781604394, \"weeks\": 10, \"weeks_pos\": 6, \"sign_ok\": false, \"admissible\": false, \"base_ok\": false}, \"params\": {\"groups\": [\"rug\", \"flow\", \"skill\"]}, \"trials\": 30, \"best_seen\": {\"n\": 5190, \"win\": 0.43641618497109824, \"pf\": 1.3862310369585873, \"total\": 231.10611372606309, \"mean\": 0.04452911632486765, \"weeks\": 10, \"weeks_pos\": 9, \"sign_ok\": true, \"admissible\": false, \"base_ok\": true}}, {\"name\": \"inputs: rug\", \"passed\": true, \"reason\": \"improves the ev strategy on the selection half\", \"selection\": {\"n\": 4856, \"win\": 0.4433690280065898, \"pf\": 1.4199964576000768, \"total\": 234.3598922135643, \"mean\": 0.048261921790272715, \"weeks\": 10, \"weeks_pos\": 8, \"sign_ok\": false, \"admissible\": false, \"base_ok\": true}, \"evaluation\": {\"n\": 1687, \"win\": 0.4866627148784825, \"pf\": 1.3549911389698504, \"total\": 72.38500758860027, \"mean\": 0.04290753265477195, \"weeks\": 10, \"weeks_pos\": 8, \"sign_ok\": false, \"admissible\": false, \"base_ok\": true}, \"params\": {\"groups\": [\"rug\"]}, \"trials\": 30, \"best_seen\": {\"n\": 12166, \"win\": 0.43021535426598717, \"pf\": 1.1943888942509129, \"total\": 249.60527102982905, \"mean\": 0.02051662592716004, \"weeks\": 10, \"weeks_pos\": 7, \"sign_ok\": false, \"admissible\": false, \"base_ok\": false}}, {\"name\": \"inputs: flow\", \"passed\": false, \"reason\": \"does not improve the ev strategy on the selection half\", \"selection\": {\"n\": 5089, \"win\": 0.4409510709373158, \"pf\": 1.3535782472945181, \"total\": 205.49713800986956, \"mean\": 0.040380651996437325, \"weeks\": 10, \"weeks_pos\": 8, \"sign_ok\": false, \"admissible\": false, \"base_ok\": true}, \"evaluation\": null, \"params\": {\"groups\": [\"flow\"]}, \"trials\": 30, \"best_seen\": {\"n\": 12908, \"win\": 0.42872637124264024, \"pf\": 1.183703489312015, \"total\": 244.49831808073122, \"mean\": 0.018941611255092285, \"weeks\": 10, \"weeks_pos\": 8, \"sign_ok\": false, \"admissible\": false, \"base_ok\": false}}, {\"name\": \"inputs: skill\", \"passed\": false, \"reason\": \"does not improve the ev strategy on the selection half\", \"selection\": {\"n\": 4944, \"win\": 0.436084142394822, \"pf\": 1.3860342884823982, \"total\": 212.80902668083218, \"mean\": 0.04304389698236897, \"weeks\": 10, \"weeks_pos\": 9, \"sign_ok\": true, \"admissible\": false, \"base_ok\": true}, \"evaluation\": null, \"params\": {\"groups\": [\"skill\"]}, \"trials\": 30, \"best_seen\": {\"n\": 30285, \"win\": 0.4312365857685323, \"pf\": 1.1148965895956877, \"total\": 282.3862591761157, \"mean\": 0.009324294508044104, \"weeks\": 10, \"weeks_pos\": 7, \"sign_ok\": false, \"admissible\": false, \"base_ok\": false}}, {\"name\": \"strategy: ev\", \"passed\": true, \"reason\": \"profit fallback: below 65 % winners, PF \\u22651.3 on both halves\", \"selection\": {\"n\": 4856, \"win\": 0.4433690280065898, \"pf\": 1.4199964576000768, \"total\": 234.3598922135643, \"mean\": 0.048261921790272715, \"weeks\": 10, \"weeks_pos\": 8, \"sign_ok\": false, \"admissible\": false, \"base_ok\": true}, \"evaluation\": {\"n\": 1687, \"win\": 0.4866627148784825, \"pf\": 1.3549911389698504, \"total\": 72.38500758860027, \"mean\": 0.04290753265477195, \"weeks\": 10, \"weeks_pos\": 8, \"sign_ok\": false, \"admissible\": false, \"base_ok\": true}, \"params\": {\"hold_min\": 120, \"line\": 0.048561317734419324, \"thr\": {}, \"high\": null}, \"trials\": 254, \"best_seen\": {\"n\": 12166, \"win\": 0.43021535426598717, \"pf\": 1.1943888942509129, \"total\": 249.60527102982905, \"mean\": 0.02051662592716004, \"weeks\": 10, \"weeks_pos\": 7, \"sign_ok\": false, \"admissible\": false, \"base_ok\": false}}, {\"name\": \"strategy: capitulation\", \"passed\": true, \"reason\": \"profit fallback: below 65 % winners, PF \\u22651.3 on both halves\", \"selection\": {\"n\": 7437, \"win\": 0.4318945811483125, \"pf\": 1.3572278876582393, \"total\": 260.93817915331056, \"mean\": 0.03508648368338182, \"weeks\": 10, \"weeks_pos\": 9, \"sign_ok\": true, \"admissible\": false, \"base_ok\": true}, \"evaluation\": {\"n\": 3616, \"win\": 0.46432522123893805, \"pf\": 1.369075903134249, \"total\": 131.28512194135692, \"mean\": 0.036306726200596495, \"weeks\": 10, \"weeks_pos\": 9, \"sign_ok\": true, \"admissible\": false, \"base_ok\": true}, \"params\": {\"hold_min\": 120, \"line\": 0.02170279622077948, \"thr\": {\"neg_ret_1m\": 0.000229347959975712, \"top_sell_share_1m\": 0.9872784614562988, \"buyers_ret\": -0.9555114507675171}, \"high\": null}, \"trials\": 326, \"best_seen\": {\"n\": 25141, \"win\": 0.43594129111809393, \"pf\": 1.1396617105231777, \"total\": 300.13287771363275, \"mean\": 0.011937984873856758, \"weeks\": 10, \"weeks_pos\": 8, \"sign_ok\": false, \"admissible\": false, \"base_ok\": false}}, {\"name\": \"strategy: breakout\", \"passed\": false, \"reason\": \"fails on the evaluation half\", \"selection\": {\"n\": 806, \"win\": 0.41811414392059554, \"pf\": 1.3704845024089776, \"total\": 37.82851162024781, \"mean\": 0.046933637245965026, \"weeks\": 10, \"weeks_pos\": 5, \"sign_ok\": false, \"admissible\": false, \"base_ok\": true}, \"evaluation\": {\"n\": 577, \"win\": 0.6949740034662045, \"pf\": 1.1277748365607487, \"total\": 13.312870207955712, \"mean\": 0.023072565351743003, \"weeks\": 10, \"weeks_pos\": 6, \"sign_ok\": false, \"admissible\": false, \"base_ok\": false}, \"params\": {\"hold_min\": 240, \"line\": 0.07295550033450127, \"thr\": {\"org_imb_5m\": null, \"buyers_slope\": null, \"log_org_vol_15m\": 2.082864999771118}, \"high\": \"dd_1h\"}, \"trials\": 334, \"best_seen\": {\"n\": 5418, \"win\": 0.3549280177187154, \"pf\": 1.1214306198017956, \"total\": 68.40240365069758, \"mean\": 0.012625028359301878, \"weeks\": 10, \"weeks_pos\": 7, \"sign_ok\": false, \"admissible\": false, \"base_ok\": false}}, {\"name\": \"combination\", \"passed\": true, \"reason\": \"'score' made more on the selection half\", \"selection\": null, \"evaluation\": null, \"params\": {\"rule\": \"score\"}, \"trials\": 0, \"best_seen\": null}, {\"name\": \"meta-label: ev\", \"passed\": true, \"reason\": \"improves the book on the selection half, passes on the evaluation half and beats random removal\", \"selection\": {\"n\": 8538, \"win\": 0.44038416490981497, \"pf\": 1.37928681358971, \"total\": 328.3758206743696, \"mean\": 0.03846050839474931, \"weeks\": 10, \"weeks_pos\": 10, \"sign_ok\": true, \"admissible\": false, \"base_ok\": true}, \"evaluation\": {\"n\": 4178, \"win\": 0.4691239827668741, \"pf\": 1.371375156148074, \"total\": 161.47078334595426, \"mean\": 0.03864786580803118, \"weeks\": 10, \"weeks_pos\": 9, \"sign_ok\": true, \"admissible\": false, \"base_ok\": true, \"random_removal\": {\"filtered_total\": 161.47079467773438, \"random_total\": 144.83329350066924, \"removed\": 135}}, \"params\": {\"cut\": 0.4034840017557144, \"line\": 0.048561317734419324}, \"trials\": 210, \"best_seen\": {\"n\": 13052, \"win\": 0.46276432730615996, \"pf\": 1.1850528442487627, \"total\": 220.3587098019707, \"mean\": 0.016883137435026868, \"weeks\": 9, \"weeks_pos\": 8, \"sign_ok\": true, \"admissible\": false, \"base_ok\": false}}, {\"name\": \"meta-label: capitulation\", \"passed\": false, \"reason\": \"rejected (selection, evaluation or random-removal test)\", \"selection\": {\"n\": 8867, \"win\": 0.4710725160708244, \"pf\": 1.3000808722110484, \"total\": 238.6872466020177, \"mean\": 0.02691860230089294, \"weeks\": 9, \"weeks_pos\": 9, \"sign_ok\": true, \"admissible\": false, \"base_ok\": true}, \"evaluation\": {\"n\": 4930, \"win\": 0.4776876267748479, \"pf\": 1.308051162803864, \"total\": 146.70473431929713, \"mean\": 0.02975755260026311, \"weeks\": 10, \"weeks_pos\": 9, \"sign_ok\": true, \"admissible\": false, \"base_ok\": true, \"random_removal\": {\"filtered_total\": 146.70474243164062, \"random_total\": 161.47078334595426, \"removed\": 0}}, \"params\": {\"cut\": 0.45691339671611786, \"line\": 0.011362157016992563}, \"trials\": 210, \"best_seen\": {\"n\": 11365, \"win\": 0.46889573251209854, \"pf\": 1.2577842532528587, \"total\": 220.86113635363654, \"mean\": 0.01943344798536177, \"weeks\": 9, \"weeks_pos\": 9, \"sign_ok\": true, \"admissible\": false, \"base_ok\": false}}, {\"name\": \"dump veto\", \"passed\": true, \"reason\": \"improves the book on the selection half, passes on the evaluation half and beats random removal\", \"selection\": {\"n\": 8374, \"win\": 0.44124671602579413, \"pf\": 1.4118872282744042, \"total\": 345.0511686700811, \"mean\": 0.041205059549806675, \"weeks\": 10, \"weeks_pos\": 10, \"sign_ok\": true, \"admissible\": false, \"base_ok\": true}, \"evaluation\": {\"n\": 4058, \"win\": 0.47042878265155247, \"pf\": 1.3931662708134354, \"total\": 161.12030342951766, \"mean\": 0.039704362599684, \"weeks\": 10, \"weeks_pos\": 9, \"sign_ok\": true, \"admissible\": false, \"base_ok\": true, \"random_removal\": {\"filtered_total\": 161.12030029296875, \"random_total\": 156.41600681219424, \"removed\": 120}}, \"params\": {\"dump\": -0.3325011432170868, \"cut\": 0.1505558244884014}, \"trials\": 56, \"best_seen\": {\"n\": 8374, \"win\": 0.44124671602579413, \"pf\": 1.4118872282744042, \"total\": 345.0511686700811, \"mean\": 0.041205059549806675, \"weeks\": 10, \"weeks_pos\": 10, \"sign_ok\": true, \"admissible\": false, \"base_ok\": true}}, {\"name\": \"hour and market gates\", \"passed\": true, \"reason\": \"improves the book, passes on the evaluation half and beats random removal\", \"selection\": {\"n\": 6652, \"win\": 0.45625375826819003, \"pf\": 1.5568584699178285, \"total\": 359.86430143312464, \"mean\": 0.054098662271967024, \"weeks\": 10, \"weeks_pos\": 10, \"sign_ok\": true, \"admissible\": false, \"base_ok\": true}, \"evaluation\": {\"n\": 3172, \"win\": 0.4735182849936948, \"pf\": 1.3978238516274475, \"total\": 128.41733735687012, \"mean\": 0.04048465868753787, \"weeks\": 10, \"weeks_pos\": 9, \"sign_ok\": true, \"admissible\": false, \"base_ok\": true, \"random_removal\": {\"filtered_total\": 128.41734313964844, \"random_total\": 123.31061138069417, \"removed\": 886}}, \"params\": {\"ev\": {\"hours\": [3], \"regimes\": [[11.075756072998047, 11.180865287780762]]}, \"capitulation\": {\"hours\": [2, 4, 10, 12, 13, 16, 17, 20, 21, 23], \"regimes\": [[11.4250337600708, 11.614822673797608]]}}, \"trials\": 0, \"best_seen\": null}], \"strategies\": {\"ev\": {\"line\": 0.048561317734419324, \"hold_min\": 120, \"thr\": {}, \"high\": null, \"hours\": [3], \"regimes\": [[11.075756072998047, 11.180865287780762]], \"sizing\": [{\"lo\": 0.0, \"n\": 1214, \"mean\": 0.043260690297942606, \"win\": 0.44233937397034595, \"kelly\": 0.25}, {\"lo\": 0.003545435145497322, \"n\": 1214, \"mean\": 0.030302551236103654, \"win\": 0.46622734761120266, \"kelly\": 0.16}, {\"lo\": 0.009100683033466339, \"n\": 1214, \"mean\": 0.030798823842828667, \"win\": 0.44069192751235586, \"kelly\": 0.17}, {\"lo\": 0.021355299279093742, \"n\": 1214, \"mean\": 0.08868562178421593, \"win\": 0.4242174629324547, \"kelly\": 0.06}], \"selection\": {\"n\": 4856, \"win\": 0.4433690280065898, \"pf\": 1.4199964576000768, \"total\": 234.3598922135643, \"mean\": 0.048261921790272715, \"weeks\": 10, \"weeks_pos\": 8, \"sign_ok\": false, \"admissible\": false, \"base_ok\": true}, \"evaluation\": {\"n\": 1687, \"win\": 0.4866627148784825, \"pf\": 1.3549911389698504, \"total\": 72.38500758860027, \"mean\": 0.04290753265477195, \"weeks\": 10, \"weeks_pos\": 8, \"sign_ok\": false, \"admissible\": false, \"base_ok\": true}}, \"capitulation\": {\"line\": 0.02170279622077948, \"hold_min\": 120, \"thr\": {\"neg_ret_1m\": 0.000229347959975712, \"top_sell_share_1m\": 0.9872784614562988, \"buyers_ret\": -0.9555114507675171}, \"high\": null, \"hours\": [2, 4, 10, 12, 13, 16, 17, 20, 21, 23], \"regimes\": [[11.4250337600708, 11.614822673797608]], \"sizing\": [{\"lo\": 0.0, \"n\": 1859, \"mean\": 0.02187701561079286, \"win\": 0.45293168370091447, \"kelly\": 0.26}, {\"lo\": 0.00394807755947113, \"n\": 1859, \"mean\": 0.06614231310249848, \"win\": 0.4190424959655729, \"kelly\": 0.11}, {\"lo\": 0.010377474129199982, \"n\": 1859, \"mean\": 0.026808915657258844, \"win\": 0.4389456697149005, \"kelly\": 0.22}, {\"lo\": 0.02529596909880638, \"n\": 1860, \"mean\": 0.02552283487551493, \"win\": 0.4166666666666667, \"kelly\": 0.11}], \"selection\": {\"n\": 7437, \"win\": 0.4318945811483125, \"pf\": 1.3572278876582393, \"total\": 260.93817915331056, \"mean\": 0.03508648368338182, \"weeks\": 10, \"weeks_pos\": 9, \"sign_ok\": true, \"admissible\": false, \"base_ok\": true}, \"evaluation\": {\"n\": 3616, \"win\": 0.46432522123893805, \"pf\": 1.369075903134249, \"total\": 131.28512194135692, \"mean\": 0.036306726200596495, \"weeks\": 10, \"weeks_pos\": 9, \"sign_ok\": true, \"admissible\": false, \"base_ok\": true}}}, \"groups\": [\"rug\"], \"combine\": \"score\", \"veto\": {\"level_index\": 1, \"dump\": -0.3325011432170868, \"cut\": 0.1505558244884014, \"levels\": [-0.43370068073272705, -0.3325011432170868, -0.21201005578041077, -0.13198548555374146], \"selection\": {\"n\": 8374, \"win\": 0.44124671602579413, \"pf\": 1.4118872282744042, \"total\": 345.0511686700811, \"mean\": 0.041205059549806675, \"weeks\": 10, \"weeks_pos\": 10, \"sign_ok\": true, \"admissible\": false, \"base_ok\": true}, \"trials\": 56, \"best_seen\": {\"n\": 8374, \"win\": 0.44124671602579413, \"pf\": 1.4118872282744042, \"total\": 345.0511686700811, \"mean\": 0.041205059549806675, \"weeks\": 10, \"weeks_pos\": 10, \"sign_ok\": true, \"admissible\": false, \"base_ok\": true}}, \"fallback\": true, \"line\": 0.048561317734419324, \"costs\": \"real fees (pool fee by market cap, Jupiter 10 bps, network fee) + impact\", \"data\": {\"agg\": 4, \"features\": 3, \"costs\": \"real-fees-1\", \"selector\": \"multi-1\", \"cols\": \"1ff02dc9\", \"skill\": \"h10-L30\", \"meta\": \"rug-1\"}, \"deployable\": true, \"deploy_reason\": \"its backtest made +4.05% per trade after costs (random -4.68%); profit fallback (below 65 % winners)\", \"sizing\": [{\"lo\": 0.0, \"n\": 1214, \"mean\": 0.043260690297942606, \"win\": 0.44233937397034595, \"kelly\": 0.25}, {\"lo\": 0.003545435145497322, \"n\": 1214, \"mean\": 0.030302551236103654, \"win\": 0.46622734761120266, \"kelly\": 0.16}, {\"lo\": 0.009100683033466339, \"n\": 1214, \"mean\": 0.030798823842828667, \"win\": 0.44069192751235586, \"kelly\": 0.17}, {\"lo\": 0.021355299279093742, \"n\": 1214, \"mean\": 0.08868562178421593, \"win\": 0.4242174629324547, \"kelly\": 0.06}], \"model\": {\"kind\": \"stack\", \"min_age_h\": 6.0, \"min_resq_sol\": 10.0, \"min_vol_15m_sol\": 5.0}, \"rows\": 29361102, \"days\": 153, \"first_day\": \"2026-04-19\", \"last_day\": \"2026-09-18\"}", - "source_id": 59 + "file": "selector_20261003T201413Z.skops", + "sha256": "b38f0633ee4a9b693bfb9a01a5e5de044e2ce317e08fb9726e77b48dfbb603fe", + "note": "{\"threshold\": 0.11529686208814378, \"horizon_min\": 120, \"trained_through\": \"2026-10-02\", \"walk_forward\": {\"n\": 2933, \"win\": 0.480054551653597, \"pf\": 1.6792328308220534, \"total\": 260.0641021499687, \"mean\": 0.0886682925843739, \"weeks\": 10, \"weeks_pos\": 8, \"sign_ok\": false, \"admissible\": false, \"base_ok\": true, \"random_mean\": -0.04572119563817978, \"days\": null}, \"selection\": {\"n\": 2579, \"win\": 0.423419930205506, \"pf\": 1.425860298729596, \"total\": 152.31196360703325, \"mean\": 0.059058535714243215, \"weeks\": 9, \"weeks_pos\": 7, \"sign_ok\": false, \"admissible\": false, \"base_ok\": true}, \"random_baseline\": {\"mean\": -0.04572119563817978}, \"components\": [{\"name\": \"inputs: rug+flow+skill\", \"passed\": true, \"reason\": \"improves the ev strategy on the selection half\", \"selection\": {\"n\": 4629, \"win\": 0.42989846619140204, \"pf\": 1.3337552712670528, \"total\": 177.56542640196858, \"mean\": 0.038359348974285716, \"weeks\": 9, \"weeks_pos\": 7, \"sign_ok\": false, \"admissible\": false, \"base_ok\": true}, \"evaluation\": {\"n\": 5304, \"win\": 0.46587481146304677, \"pf\": 1.3353569732988675, \"total\": 201.20124764325192, \"mean\": 0.03793387021931597, \"weeks\": 10, \"weeks_pos\": 8, \"sign_ok\": false, \"admissible\": false, \"base_ok\": true}, \"params\": {\"groups\": [\"rug\", \"flow\", \"skill\"]}, \"trials\": 30, \"best_seen\": {\"n\": 4629, \"win\": 0.42989846619140204, \"pf\": 1.3337552712670528, \"total\": 177.56542640196858, \"mean\": 0.038359348974285716, \"weeks\": 9, \"weeks_pos\": 7, \"sign_ok\": false, \"admissible\": false, \"base_ok\": true}}, {\"name\": \"strategy: ev\", \"passed\": true, \"reason\": \"profit fallback: below 65 % winners, PF \\u22651.3 on both halves\", \"selection\": {\"n\": 4629, \"win\": 0.42989846619140204, \"pf\": 1.3337552712670528, \"total\": 177.56542640196858, \"mean\": 0.038359348974285716, \"weeks\": 9, \"weeks_pos\": 7, \"sign_ok\": false, \"admissible\": false, \"base_ok\": true}, \"evaluation\": {\"n\": 5304, \"win\": 0.46587481146304677, \"pf\": 1.3353569732988675, \"total\": 201.20124764325192, \"mean\": 0.03793387021931597, \"weeks\": 10, \"weeks_pos\": 8, \"sign_ok\": false, \"admissible\": false, \"base_ok\": true}, \"params\": {\"hold_min\": 120, \"line\": 0.03766093432903288, \"thr\": {}, \"high\": null}, \"trials\": 254, \"best_seen\": {\"n\": 7226, \"win\": 0.4046498754497647, \"pf\": 1.2213627086551027, \"total\": 203.35054916350782, \"mean\": 0.02814150970986823, \"weeks\": 9, \"weeks_pos\": 6, \"sign_ok\": false, \"admissible\": false, \"base_ok\": false}}, {\"name\": \"strategy: capitulation\", \"passed\": true, \"reason\": \"profit fallback: below 65 % winners, PF \\u22651.3 on both halves\", \"selection\": {\"n\": 2550, \"win\": 0.4164705882352941, \"pf\": 1.3969661280720116, \"total\": 144.03126928402344, \"mean\": 0.05648285069961703, \"weeks\": 9, \"weeks_pos\": 7, \"sign_ok\": false, \"admissible\": false, \"base_ok\": true}, \"evaluation\": {\"n\": 2917, \"win\": 0.4796023311621529, \"pf\": 1.6604187318564194, \"total\": 252.55368486529187, \"mean\": 0.08657993996067599, \"weeks\": 10, \"weeks_pos\": 7, \"sign_ok\": false, \"admissible\": false, \"base_ok\": true}, \"params\": {\"hold_min\": 240, \"line\": 0.06913333304226396, \"thr\": {\"neg_ret_1m\": 0.00023369600821752107, \"top_sell_share_1m\": null, \"buyers_ret\": -0.9555114507675171}, \"high\": null}, \"trials\": 326, \"best_seen\": {\"n\": 6612, \"win\": 0.41696914700544463, \"pf\": 1.253637254659857, \"total\": 216.06627043115805, \"mean\": 0.032677899339255606, \"weeks\": 9, \"weeks_pos\": 6, \"sign_ok\": false, \"admissible\": false, \"base_ok\": false}}, {\"name\": \"strategy: breakout\", \"passed\": false, \"reason\": \"fails on the evaluation half\", \"selection\": {\"n\": 194, \"win\": 0.38144329896907214, \"pf\": 2.0283563531346855, \"total\": 28.084343573420483, \"mean\": 0.14476465759495094, \"weeks\": 9, \"weeks_pos\": 8, \"sign_ok\": true, \"admissible\": false, \"base_ok\": true}, \"evaluation\": {\"n\": 203, \"win\": 0.7241379310344828, \"pf\": 0.7808668785674713, \"total\": -6.714265794442326, \"mean\": -0.03307520095784397, \"weeks\": 10, \"weeks_pos\": 6, \"sign_ok\": false, \"admissible\": false, \"base_ok\": false}, \"params\": {\"hold_min\": 120, \"line\": 0.0904355053529145, \"thr\": {\"org_imb_5m\": null, \"buyers_slope\": null, \"log_org_vol_15m\": 2.083093547821045}, \"high\": \"dd_1h\"}, \"trials\": 334, \"best_seen\": {\"n\": 2081, \"win\": 0.38875540605478137, \"pf\": 1.193609019909915, \"total\": 47.299630237832844, \"mean\": 0.022729279306983586, \"weeks\": 9, \"weeks_pos\": 5, \"sign_ok\": false, \"admissible\": false, \"base_ok\": false}}, {\"name\": \"combination\", \"passed\": true, \"reason\": \"'score' made more on the selection half\", \"selection\": null, \"evaluation\": null, \"params\": {\"rule\": \"score\"}, \"trials\": 0, \"best_seen\": null}, {\"name\": \"meta-label: ev\", \"passed\": true, \"reason\": \"improves the book on the selection half, passes on the evaluation half and beats random removal\", \"selection\": {\"n\": 2658, \"win\": 0.4179834462001505, \"pf\": 1.3834151621947472, \"total\": 144.59336371760583, \"mean\": 0.0543993091488359, \"weeks\": 9, \"weeks_pos\": 8, \"sign_ok\": true, \"admissible\": false, \"base_ok\": true}, \"evaluation\": {\"n\": 2989, \"win\": 0.47607895617263296, \"pf\": 1.6329352687746375, \"total\": 249.40963365992502, \"mean\": 0.08344250038806458, \"weeks\": 10, \"weeks_pos\": 7, \"sign_ok\": false, \"admissible\": false, \"base_ok\": true, \"random_removal\": {\"filtered_total\": 249.40963745117188, \"random_total\": 136.43414531027412, \"removed\": 2861}}, \"params\": {\"cut\": 0.46649493277072906, \"line\": 0.11529686208814378}, \"trials\": 210, \"best_seen\": {\"n\": 11698, \"win\": 0.4624722174730723, \"pf\": 1.1285983328516533, \"total\": 113.18220021868092, \"mean\": 0.00967534623172174, \"weeks\": 8, \"weeks_pos\": 7, \"sign_ok\": true, \"admissible\": false, \"base_ok\": false}}, {\"name\": \"meta-label: capitulation\", \"passed\": false, \"reason\": \"rejected (selection, evaluation or random-removal test)\", \"selection\": {\"n\": 3559, \"win\": 0.4520932846305142, \"pf\": 1.2846180967169816, \"total\": 126.99779967030554, \"mean\": 0.03568356270590209, \"weeks\": 8, \"weeks_pos\": 6, \"sign_ok\": false, \"admissible\": false, \"base_ok\": false}, \"evaluation\": {\"n\": 6601, \"win\": 0.47295864262990456, \"pf\": 1.457000534598307, \"total\": 373.215928148049, \"mean\": 0.05653930134041039, \"weeks\": 10, \"weeks_pos\": 9, \"sign_ok\": true, \"admissible\": false, \"base_ok\": true, \"random_removal\": {\"filtered_total\": 373.2159423828125, \"random_total\": 249.40963365992502, \"removed\": 0}}, \"params\": {\"cut\": 0.4471900165081024, \"line\": 0.030003319680690765}, \"trials\": 210, \"best_seen\": {\"n\": 13029, \"win\": 0.45076368101926473, \"pf\": 1.170110820407643, \"total\": 185.11671483521832, \"mean\": 0.014208052408873921, \"weeks\": 8, \"weeks_pos\": 8, \"sign_ok\": true, \"admissible\": false, \"base_ok\": false}}, {\"name\": \"dump veto\", \"passed\": true, \"reason\": \"improves the book on the selection half, passes on the evaluation half and beats random removal\", \"selection\": {\"n\": 2579, \"win\": 0.423419930205506, \"pf\": 1.425860298729596, \"total\": 152.31196360703325, \"mean\": 0.059058535714243215, \"weeks\": 9, \"weeks_pos\": 7, \"sign_ok\": false, \"admissible\": false, \"base_ok\": true}, \"evaluation\": {\"n\": 2933, \"win\": 0.480054551653597, \"pf\": 1.6792328308220534, \"total\": 260.0641021499687, \"mean\": 0.0886682925843739, \"weeks\": 10, \"weeks_pos\": 8, \"sign_ok\": false, \"admissible\": false, \"base_ok\": true, \"random_removal\": {\"filtered_total\": 260.0640869140625, \"random_total\": 243.56071725736874, \"removed\": 56}}, \"params\": {\"dump\": -0.16902928948402404, \"cut\": 0.4313309609889984}, \"trials\": 56, \"best_seen\": {\"n\": 2579, \"win\": 0.423419930205506, \"pf\": 1.425860298729596, \"total\": 152.31196360703325, \"mean\": 0.059058535714243215, \"weeks\": 9, \"weeks_pos\": 7, \"sign_ok\": false, \"admissible\": false, \"base_ok\": true}}], \"strategies\": {\"ev\": {\"line\": 0.11529686208814378, \"hold_min\": 120, \"thr\": {}, \"high\": null, \"hours\": [], \"regimes\": [], \"sizing\": [{\"lo\": 0.0, \"n\": 1157, \"mean\": 0.03137388963998088, \"win\": 0.45980985306828004, \"kelly\": 0.24}, {\"lo\": 0.0029479563236236572, \"n\": 1157, \"mean\": 0.002502596321285927, \"win\": 0.4373379429559205, \"kelly\": 0.01}, {\"lo\": 0.008467614650726318, \"n\": 1157, \"mean\": 0.03428360684746643, \"win\": 0.4278305963699222, \"kelly\": 0.12}, {\"lo\": 0.020865503698587418, \"n\": 1158, \"mean\": 0.08523678672043543, \"win\": 0.3946459412780656, \"kelly\": 0.05}], \"selection\": {\"n\": 4629, \"win\": 0.42989846619140204, \"pf\": 1.3337552712670528, \"total\": 177.56542640196858, \"mean\": 0.038359348974285716, \"weeks\": 9, \"weeks_pos\": 7, \"sign_ok\": false, \"admissible\": false, \"base_ok\": true}, \"evaluation\": {\"n\": 5304, \"win\": 0.46587481146304677, \"pf\": 1.3353569732988675, \"total\": 201.20124764325192, \"mean\": 0.03793387021931597, \"weeks\": 10, \"weeks_pos\": 8, \"sign_ok\": false, \"admissible\": false, \"base_ok\": true}}, \"capitulation\": {\"line\": 0.06913333304226396, \"hold_min\": 240, \"thr\": {\"neg_ret_1m\": 0.00023369600821752107, \"top_sell_share_1m\": null, \"buyers_ret\": -0.9555114507675171}, \"high\": null, \"hours\": [], \"regimes\": [], \"sizing\": [{\"lo\": 0.0, \"n\": 638, \"mean\": 0.0436464974508308, \"win\": 0.4373040752351097, \"kelly\": 0.13}, {\"lo\": 0.004838019609451294, \"n\": 637, \"mean\": 0.00892687059909758, \"win\": 0.39717425431711145, \"kelly\": 0.04}, {\"lo\": 0.012772876769304276, \"n\": 637, \"mean\": 0.06240325061294793, \"win\": 0.41444270015698587, \"kelly\": 0.22}, {\"lo\": 0.02930697798728943, \"n\": 638, \"mean\": 0.11088952460551786, \"win\": 0.4169278996865204, \"kelly\": 0.15}], \"selection\": {\"n\": 2550, \"win\": 0.4164705882352941, \"pf\": 1.3969661280720116, \"total\": 144.03126928402344, \"mean\": 0.05648285069961703, \"weeks\": 9, \"weeks_pos\": 7, \"sign_ok\": false, \"admissible\": false, \"base_ok\": true}, \"evaluation\": {\"n\": 2917, \"win\": 0.4796023311621529, \"pf\": 1.6604187318564194, \"total\": 252.55368486529187, \"mean\": 0.08657993996067599, \"weeks\": 10, \"weeks_pos\": 7, \"sign_ok\": false, \"admissible\": false, \"base_ok\": true}}}, \"groups\": [\"rug\", \"flow\", \"skill\"], \"combine\": \"score\", \"veto\": {\"level_index\": 3, \"dump\": -0.16902928948402404, \"cut\": 0.4313309609889984, \"levels\": [-0.5416608870029449, -0.4038370966911316, -0.266157066822052, -0.16902928948402404], \"selection\": {\"n\": 2579, \"win\": 0.423419930205506, \"pf\": 1.425860298729596, \"total\": 152.31196360703325, \"mean\": 0.059058535714243215, \"weeks\": 9, \"weeks_pos\": 7, \"sign_ok\": false, \"admissible\": false, \"base_ok\": true}, \"trials\": 56, \"best_seen\": {\"n\": 2579, \"win\": 0.423419930205506, \"pf\": 1.425860298729596, \"total\": 152.31196360703325, \"mean\": 0.059058535714243215, \"weeks\": 9, \"weeks_pos\": 7, \"sign_ok\": false, \"admissible\": false, \"base_ok\": true}}, \"fallback\": true, \"holdout\": {\"days\": [\"2026-09-12\", \"2026-10-02\"], \"rows\": 4379737, \"n\": 2156, \"win\": 0.48283858998144713, \"pf\": 1.4519683896655893, \"total\": 111.96499740793661, \"mean\": 0.05193181697956244, \"weeks\": 3, \"weeks_pos\": 3, \"sign_ok\": false, \"admissible\": false, \"base_ok\": true, \"random_mean\": -0.045112401247024536}, \"chains\": {\"rh\": {\"days\": [\"2026-09-12\", \"2026-10-02\"], \"rows\": 1239101, \"n\": 2072, \"win\": 0.48166023166023164, \"pf\": 1.4511607633956578, \"total\": 106.26783827815962, \"mean\": 0.05128756673656352, \"weeks\": 3, \"weeks_pos\": 3, \"sign_ok\": false, \"admissible\": false, \"base_ok\": true, \"random_mean\": -0.04212987422943115, \"deployable\": true}, \"sol\": {\"days\": [\"2026-09-12\", \"2026-10-02\"], \"rows\": 3140636, \"n\": 84, \"win\": 0.5119047619047619, \"pf\": 1.467581146721967, \"total\": 5.697159129776992, \"mean\": 0.06782332297353562, \"weeks\": 3, \"weeks_pos\": 3, \"sign_ok\": false, \"admissible\": false, \"base_ok\": false, \"random_mean\": -0.05558569356799126, \"deployable\": true}}, \"line\": 0.11529686208814378, \"costs\": \"real fees (pool fee by market cap, Jupiter 10 bps, network fee) + impact\", \"data\": {\"agg\": 4, \"features\": 3, \"costs\": \"real-fees-sized-1\", \"selector\": \"multi-1\", \"cols\": \"86151dda\", \"skill\": \"h10-L30\", \"meta\": \"rug-1\", \"chains\": \"sol+rh-1\", \"rh_agg\": 1, \"rh_skill\": \"rh-h10-L30\", \"rh_costs\": \"pons-kyber-1:0.0445583:0.0222791:7.2e-06:0.0099\"}, \"deployable\": true, \"deploy_reason\": \"its backtest made +8.87% per trade after costs (random -4.57%); profit fallback (below 65 % winners)\", \"sizing\": [{\"lo\": 0.0, \"n\": 1157, \"mean\": 0.03137388963998088, \"win\": 0.45980985306828004, \"kelly\": 0.24}, {\"lo\": 0.0029479563236236572, \"n\": 1157, \"mean\": 0.002502596321285927, \"win\": 0.4373379429559205, \"kelly\": 0.01}, {\"lo\": 0.008467614650726318, \"n\": 1157, \"mean\": 0.03428360684746643, \"win\": 0.4278305963699222, \"kelly\": 0.12}, {\"lo\": 0.020865503698587418, \"n\": 1158, \"mean\": 0.08523678672043543, \"win\": 0.3946459412780656, \"kelly\": 0.05}], \"model\": {\"kind\": \"stack\", \"min_age_h\": 6.0, \"min_resq_sol\": 10.0, \"min_vol_15m_sol\": 5.0}, \"rows\": 36259577, \"days\": 167, \"first_day\": \"2026-04-19\", \"last_day\": \"2026-10-02\"}", + "source_id": 154 }, { "kind": "fly_selector", - "file": "fly_20260921T160246Z.pt", - "sha256": "d18ef071d4106cea6846b494340ce9518e5e5ac55cde31b79670fbead406c744", - "note": "{\"S\": \"2026-09-21\", \"train_through\": \"2026-09-12\", \"calibration_days\": [\"2026-09-14\", \"2026-09-20\"], \"line\": 0.04060962647199624, \"sizing\": [{\"lo\": 0.0, \"n\": 81, \"mean\": 0.03765792501975357, \"win\": 0.49382716049382713, \"kelly\": 0.26}, {\"lo\": 0.0020544106140733473, \"n\": 81, \"mean\": 0.002718701303281166, \"win\": 0.4567901234567901, \"kelly\": 0.02}, {\"lo\": 0.006942897662520479, \"n\": 81, \"mean\": 0.0007167349764962255, \"win\": 0.48148148148148145, \"kelly\": 0.01}, {\"lo\": 0.015385258570313524, \"n\": 81, \"mean\": 0.07037295356829777, \"win\": 0.6296296296296297, \"kelly\": 0.5700000000000001}], \"lines\": {\"ev\": 0.04060962647199624, \"capitulation\": 0.04896915547549754}, \"calibration\": {\"trades\": 325, \"mean\": 0.027888946775346993, \"total\": 9.063907701987773, \"per_strategy\": {\"ev\": {\"line\": 0.04060962647199624, \"trades\": 324, \"mean\": 0.027866578716957184, \"total\": 9.028771504294127, \"hold_min\": 120}, \"capitulation\": {\"line\": 0.04896915547549754, \"trades\": 110, \"mean\": 0.04093937277370556, \"total\": 4.503331005107611, \"hold_min\": 120}}}, \"strategies\": [\"ev\", \"capitulation\"], \"rules\": {\"ev\": {\"thr\": {}, \"high\": null, \"hold_min\": 120}, \"capitulation\": {\"thr\": {\"neg_ret_1m\": 0.000229347959975712, \"top_sell_share_1m\": 0.9872784614562988, \"buyers_ret\": -0.9555114507675171}, \"high\": null, \"hold_min\": 120}}, \"diagnostics\": {\"mbon_share\": 0.3961058327857704, \"mbon_saturated\": 0.010416666977107525, \"kc_active\": 0.09959564357995987, \"rows\": 50000, \"slope\": 1.005056561531653, \"rank_corr\": 0.8535549287862947}, \"gates_ok\": true, \"gate_failures\": [], \"fit\": {\"epochs\": [{\"epoch\": 0, \"mse\": 0.3367231594663317, \"secs\": 3108.421549797058, \"rows\": 3912704, \"val_mse\": 0.24399760291454034, \"val_se\": 0.0038252531609198506}, {\"epoch\": 1, \"mse\": 0.20434178183626922, \"secs\": 6239.6601638793945, \"rows\": 3912704, \"val_mse\": 0.21627898232944742, \"val_se\": 0.0041017738607596445}, {\"epoch\": 2, \"mse\": 0.1676639474282961, \"secs\": 9364.127672672272, \"rows\": 3912704, \"val_mse\": 0.239599813865461, \"val_se\": 0.003708129521054708}], \"stopped\": false, \"rows\": 3913129, \"training_rows\": 22262569, \"top_weight\": 10.0, \"strategies\": [\"ev\", \"capitulation\"], \"epochs_run\": 3, \"epoch_cap\": 8, \"val_rows\": 200000, \"val_mse\": 0.21627898232944742, \"early_stop\": \"epoch 2 moved the held-out error by -0.0233, inside its standard error 0.0037\"}, \"teacher\": \"deployed selector #59: ev (line 0.0486, 120 min), capitulation (line 0.0217, 120 min)\", \"teacher_line\": 0.048561317734419324, \"teacher_stack\": null, \"teacher_snapshot\": 59, \"data\": {\"agg\": 4, \"features\": 3, \"costs\": \"real-fees-sized-1\", \"selector\": \"multi-1\", \"cols\": \"1ff02dc9\", \"skill\": \"h10-L30\", \"meta\": \"rug-1\", \"fly\": \"flynet-3-ch\", \"plastic\": \"mb4-ch\", \"teacher\": \"deployed-1\"}}", - "source_id": 62 + "file": "fly_20261006T023153Z.pt", + "sha256": "73141bf59bd4321c66f28e4c5a7c83f994adf744574de8b17fe5d3ca94fcc1e7", + "note": "{\"S\": \"2026-10-05\", \"train_through\": \"2026-09-26\", \"calibration_days\": [\"2026-09-28\", \"2026-10-04\"], \"line\": 0.16517850856645203, \"sizing\": [], \"lines\": {\"ev\": 0.16517850856645203, \"capitulation\": 0.11595480144023895}, \"chain_lines\": {\"rh\": {\"ev\": 0.1612473428249359, \"capitulation\": 0.10683617740869522}}, \"calibration\": {\"trades\": 527, \"mean\": 0.045142892621126446, \"total\": 23.790304411333636, \"per_strategy\": {\"ev\": {\"line\": 0.16517850856645203, \"trades\": 0, \"mean\": null, \"total\": 0.0, \"hold_min\": 120, \"rule\": \"match\", \"teacher_trades\": 0, \"chain\": \"sol\"}, \"capitulation\": {\"line\": 0.11595480144023895, \"trades\": 41, \"mean\": 0.12749095906208202, \"total\": 5.2271293215453625, \"hold_min\": 240, \"rule\": \"match\", \"teacher_trades\": 41, \"chain\": \"sol\"}, \"ev@rh\": {\"line\": 0.1612473428249359, \"trades\": 24, \"mean\": 0.09742691635741114, \"total\": 2.3382459925778676, \"hold_min\": 120, \"rule\": \"match\", \"teacher_trades\": 24, \"chain\": \"rh\"}, \"capitulation@rh\": {\"line\": 0.10683617740869522, \"trades\": 486, \"mean\": 0.03819583351808287, \"total\": 18.563175089788274, \"hold_min\": 240, \"rule\": \"match\", \"teacher_trades\": 486, \"chain\": \"rh\"}}}, \"strategies\": [\"ev\", \"capitulation\"], \"rules\": {\"ev\": {\"thr\": {}, \"high\": null, \"hold_min\": 120}, \"capitulation\": {\"thr\": {\"neg_ret_1m\": 0.00023369600821752107, \"top_sell_share_1m\": null, \"buyers_ret\": -0.9555114507675171}, \"high\": null, \"hold_min\": 240}}, \"diagnostics\": {\"mbon_share\": 0.40377666358338027, \"mbon_saturated\": 0.02083333395421505, \"kc_active\": 0.10001585632562637, \"rows\": 50000, \"slope\": 1.2467359339584205, \"rank_corr\": 0.908825040513757}, \"gates_ok\": true, \"gate_failures\": [], \"fit\": {\"epochs\": [{\"epoch\": 0, \"mse\": 0.4948816107000309, \"secs\": 3383.3052699565887, \"rows\": 4158464, \"val_mse\": 0.3124759687605339, \"val_se\": 0.005248413752788144}, {\"epoch\": 1, \"mse\": 0.3147444514031629, \"secs\": 6833.467020750046, \"rows\": 4158464, \"val_mse\": 0.2591359709877761, \"val_se\": 0.0030850056886689205}, {\"epoch\": 2, \"mse\": 0.2590653069447602, \"secs\": 10234.36536693573, \"rows\": 4158464, \"val_mse\": 0.21819832194501979, \"val_se\": 0.0024959954740391343}, {\"epoch\": 3, \"mse\": 0.22754745311416802, \"secs\": 13664.291687965393, \"rows\": 4158464, \"val_mse\": 0.20198258384915535, \"val_se\": 0.0024399089474643077}, {\"epoch\": 4, \"mse\": 0.2057440036793584, \"secs\": 17083.494806051254, \"rows\": 4158464, \"val_mse\": 0.19749039311960737, \"val_se\": 0.002670306554215004}, {\"epoch\": 5, \"mse\": 0.19020645904756772, \"secs\": 20524.88501882553, \"rows\": 4158464, \"val_mse\": 0.1723157133250382, \"val_se\": 0.0023445692390604107}, {\"epoch\": 6, \"mse\": 0.1785365387162561, \"secs\": 23951.43882393837, \"rows\": 4158464, \"val_mse\": 0.19201832960320905, \"val_se\": 0.0017933120401605726}], \"stopped\": false, \"rows\": 4158669, \"training_rows\": 27173380, \"top_weight\": 10.0, \"strategies\": [\"ev\", \"capitulation\"], \"epochs_run\": 7, \"epoch_cap\": 8, \"val_rows\": 200000, \"val_mse\": 0.1723157133250382, \"early_stop\": \"epoch 6 moved the held-out error by -0.0197, inside its standard error 0.0018\"}, \"teacher\": \"deployed selector #154: ev (line 0.1153, 120 min), capitulation (line 0.0691, 240 min), refit through 2026-09-11 (blind to the calibration week from 2026-09-28)\", \"teacher_line\": 0.11529686208814378, \"teacher_stack\": null, \"teacher_snapshot\": 154, \"data\": {\"agg\": 4, \"features\": 3, \"costs\": \"real-fees-sized-1\", \"selector\": \"multi-1\", \"cols\": \"86151dda\", \"skill\": \"h10-L30\", \"meta\": \"rug-1\", \"chains\": \"sol+rh-1\", \"rh_agg\": 1, \"rh_skill\": \"rh-h10-L30\", \"rh_costs\": \"pons-kyber-1:0.0445583:0.0222791:7.2e-06:0.0099\", \"fly\": \"flynet-3-ch\", \"plastic\": \"mb4-ch\", \"teacher\": \"deployed-1\", \"calib\": \"match\", \"lines\": \"per-chain-1\"}}", + "source_id": 157 } ], "verdict": { - "S": "2026-07-27", + "S": "2026-09-09", "data": { "agg": 4, "fly": "flynet-3-ch", - "cols": "1ff02dc9", + "cols": "86151dda", "meta": "rug-1", + "calib": "match", "costs": "real-fees-sized-1", + "lines": "per-chain-1", + "score": "teacher", "skill": "h10-L30", + "chains": "sol+rh-1", + "rh_agg": 1, "plastic": "mb4-ch", "teacher": "deployed-1", "features": 3, + "rh_costs": "pons-kyber-1:0.0445583:0.0222791:7.2e-06:0.0099", + "rh_skill": "rh-h10-L30", "selector": "multi-1" }, - "secs": 12592.698919296265, - "alpha": 0.0003, + "secs": 38337.81323695183, + "alpha": 0.0001, + "chains": { + "rh": { + "edge_p": 0.0005, + "frozen": { + "n": 1477, + "pf": 1.2590082935365987, + "win": 0.4685172647257955, + "days": 16, + "mean": 0.026347791776061058, + "median": -0.012595674954354763, + "days_positive": 11 + }, + "passed": true, + "random": { + "n": 26586, + "pf": 0.5751959068792711, + "win": 0.3041450387421951, + "mean": -0.046134307980537415, + "median": -0.06115628033876419 + }, + "reason": "its backtest made +2.66% per trade after costs over 1477 trades (random -4.61%; 0.1% of day resamples at or below zero)", + "evaluation": { + "n": 1477, + "pf": 1.2612278943579527, + "win": 0.4685172647257955, + "days": 16, + "mean": 0.02657903917133808, + "median": -0.012595674954354763, + "days_positive": 11 + } + }, + "sol": { + "edge_p": 0.004, + "frozen": { + "n": 308, + "pf": 1.4669271249617488, + "win": 0.512987012987013, + "days": 16, + "mean": 0.0377873033285141, + "median": 0.005911512766033411, + "days_positive": 11 + }, + "passed": true, + "random": { + "n": 6091, + "pf": 0.458730469095968, + "win": 0.5076342144147102, + "mean": -0.04378599673509598, + "median": 0.00037788774352520704 + }, + "reason": "its backtest made +3.78% per trade after costs over 308 trades (random -4.38%; 0.4% of day resamples at or below zero)", + "evaluation": { + "n": 308, + "pf": 1.4669271249617488, + "win": 0.512987012987013, + "days": 16, + "mean": 0.0377873033285141, + "median": 0.005911512766033411, + "days_positive": 11 + } + } + }, "chosen": { - "mean": 0.023196974769234657, - "alpha": 0.0003, - "total": 75.43656158447266, - "config": 5, - "trades": 3252, + "mean": 0.030173424631357193, + "alpha": 0.0001, + "total": 89.58489990234375, + "config": 1, + "trades": 2969, "governance": { - "ic": 0, - "any": 0, + "ic": 35, + "any": 35, "drift": 0, - "checks": 1365, + "checks": 1482, "shadow": 0 }, "half_life_days": 1.0 }, "frozen": { - "n": 2048, - "pf": 1.0961381659648606, - "win": 0.44140625, - "days": 29, - "mean": 0.010135887190699577, - "median": -0.019196834415197372, + "n": 1785, + "pf": 1.2885933084122407, + "win": 0.47619047619047616, + "days": 16, + "mean": 0.02832167036831379, + "median": -0.009349760599434376, "random": { - "n": 39636, - "pf": 0.46682219913822093, - "win": 0.4265314360682208, - "mean": -0.05251633748412132, - "median": -0.0052637094631791115 + "n": 34226, + "pf": 0.5107005230232796, + "win": 0.47361654882253257, + "mean": -0.0425419919192791, + "median": -0.0014032165054231882 }, - "days_positive": 18 + "days_positive": 12 }, "passed": true, "random": { - "n": 32155, - "pf": 0.46781214203894617, - "win": 0.42220494479863163, - "mean": -0.052701301872730255, - "median": -0.005381436087191105 + "n": 34226, + "pf": 0.5107005230232796, + "win": 0.47361654882253257, + "mean": -0.0425419919192791, + "median": -0.0014032165054231882 }, - "reason": "its backtest made +1.52% per trade after costs (random -5.27%)", + "reason": "sol: passed \u2014 its backtest made +3.78% per trade after costs over 308 trades (random -4.38%; 0.4% of day resamples at or below zero); rh: passed \u2014 its backtest made +2.66% per trade after costs over 1477 trades (random -4.61%; 0.1% of day resamples at or below zero)", "configs": [ { - "mean": 0.016807934269309044, + "mean": 0.03036709874868393, "alpha": 0.0, - "total": 65.02989959716797, + "total": 91.28350067138672, "config": 0, - "trades": 3869, + "trades": 3006, "governance": { "ic": 0, "any": 0, @@ -88,492 +157,401 @@ "half_life_days": null }, { - "mean": 0.01756637543439865, + "mean": 0.030173424631357193, "alpha": 0.0001, - "total": 63.76594543457031, + "total": 89.58489990234375, "config": 1, - "trades": 3630, + "trades": 2969, "governance": { - "ic": 0, - "any": 0, + "ic": 35, + "any": 35, "drift": 0, - "checks": 1365, + "checks": 1482, "shadow": 0 }, "half_life_days": 1.0 }, { - "mean": 0.020793743431568146, + "mean": 0.032286640256643295, "alpha": 0.0001, - "total": 68.8896713256836, + "total": 67.99566650390625, "config": 2, - "trades": 3313, + "trades": 2106, "governance": { - "ic": 0, - "any": 0, + "ic": 28, + "any": 29, "drift": 0, - "checks": 1365, - "shadow": 0 + "checks": 1482, + "shadow": 1 }, "half_life_days": 3.0 }, { - "mean": 0.02111992798745632, + "mean": 0.03781268000602722, "alpha": 0.0001, - "total": 61.73354721069336, + "total": 55.92495346069336, "config": 3, - "trades": 2923, + "trades": 1479, "governance": { - "ic": 0, - "any": 0, + "ic": 17, + "any": 69, "drift": 0, - "checks": 1365, - "shadow": 0 + "checks": 1482, + "shadow": 52 }, "half_life_days": 7.0 }, { - "mean": 0.020111948251724243, + "mean": 0.04063548520207405, "alpha": 0.0001, - "total": 50.62177276611328, + "total": 48.437496185302734, "config": 4, - "trades": 2517, + "trades": 1192, "governance": { - "ic": 0, - "any": 68, + "ic": 7, + "any": 40, "drift": 0, - "checks": 1365, - "shadow": 68 + "checks": 1482, + "shadow": 33 }, "half_life_days": 30.0 }, { - "mean": 0.023196974769234657, + "mean": 0.03773646801710129, "alpha": 0.0003, - "total": 75.43656158447266, + "total": 79.32205200195312, "config": 5, - "trades": 3252, + "trades": 2102, "governance": { - "ic": 0, - "any": 0, + "ic": 30, + "any": 31, "drift": 0, - "checks": 1365, - "shadow": 0 + "checks": 1482, + "shadow": 1 }, "half_life_days": 1.0 }, { - "mean": 0.020104866474866867, + "mean": 0.04369469732046127, "alpha": 0.0003, - "total": 52.694854736328125, + "total": 42.51494216918945, "config": 6, - "trades": 2621, + "trades": 973, "governance": { - "ic": 0, - "any": 34, + "ic": 16, + "any": 53, "drift": 0, - "checks": 1365, - "shadow": 34 + "checks": 1482, + "shadow": 37 }, "half_life_days": 3.0 }, { - "mean": 0.015242534689605236, + "mean": 0.05426740273833275, "alpha": 0.0003, - "total": 31.61301612854004, + "total": 37.17317199707031, "config": 7, - "trades": 2074, + "trades": 685, "governance": { - "ic": 0, - "any": 93, + "ic": 8, + "any": 62, "drift": 0, - "checks": 1365, - "shadow": 93 + "checks": 1482, + "shadow": 54 }, "half_life_days": 7.0 }, { - "mean": 0.014115910045802593, + "mean": 0.034157898277044296, "alpha": 0.0003, - "total": 23.277135848999023, + "total": 20.494739532470703, "config": 8, - "trades": 1649, + "trades": 600, "governance": { "ic": 0, - "any": 111, + "any": 58, "drift": 0, - "checks": 1365, - "shadow": 111 + "checks": 1482, + "shadow": 58 }, "half_life_days": 30.0 }, { - "mean": 0.014811468310654163, + "mean": 0.04557060822844505, "alpha": 0.001, - "total": 39.4577522277832, + "total": 51.90492248535156, "config": 9, - "trades": 2664, + "trades": 1139, "governance": { - "ic": 0, - "any": 74, + "ic": 20, + "any": 20, "drift": 0, - "checks": 1365, - "shadow": 74 + "checks": 1482, + "shadow": 0 }, "half_life_days": 1.0 }, { - "mean": 0.011060998775064945, + "mean": 0.03615965694189072, "alpha": 0.001, - "total": 18.847942352294922, + "total": 22.129711151123047, "config": 10, - "trades": 1704, + "trades": 612, "governance": { - "ic": 0, - "any": 139, + "ic": 5, + "any": 53, "drift": 0, - "checks": 1365, - "shadow": 139 + "checks": 1482, + "shadow": 48 }, "half_life_days": 3.0 }, { - "mean": 0.002477346919476986, + "mean": 0.04262426868081093, "alpha": 0.001, - "total": 3.146230697631836, + "total": 21.3121337890625, "config": 11, - "trades": 1270, + "trades": 500, "governance": { "ic": 0, - "any": 184, + "any": 78, "drift": 0, - "checks": 1365, - "shadow": 184 + "checks": 1482, + "shadow": 78 }, "half_life_days": 7.0 }, { - "mean": 0.018671352416276932, + "mean": 0.023908229544758797, "alpha": 0.001, - "total": 19.77296257019043, + "total": 10.639162063598633, "config": 12, - "trades": 1059, + "trades": 445, "governance": { "ic": 0, - "any": 144, + "any": 66, "drift": 0, - "checks": 1365, - "shadow": 144 + "checks": 1482, + "shadow": 66 }, "half_life_days": 30.0 }, { - "mean": 0.01424670871347189, + "mean": 0.04726389795541763, "alpha": 0.003, - "total": 28.10875701904297, + "total": 46.224090576171875, "config": 13, - "trades": 1973, + "trades": 978, "governance": { - "ic": 0, - "any": 203, + "ic": 6, + "any": 58, "drift": 0, - "checks": 1365, - "shadow": 203 + "checks": 1482, + "shadow": 52 }, "half_life_days": 1.0 }, { - "mean": 0.027894049882888794, + "mean": 0.03016720339655876, "alpha": 0.003, - "total": 38.716941833496094, + "total": 20.634366989135742, "config": 14, - "trades": 1388, + "trades": 684, "governance": { "ic": 0, - "any": 71, + "any": 137, "drift": 0, - "checks": 1365, - "shadow": 71 + "checks": 1482, + "shadow": 137 }, "half_life_days": 3.0 }, { - "mean": 0.01914161443710327, + "mean": 0.034875430166721344, "alpha": 0.003, - "total": 18.816207885742188, + "total": 21.832019805908203, "config": 15, - "trades": 983, + "trades": 626, "governance": { "ic": 0, - "any": 112, + "any": 136, "drift": 0, - "checks": 1365, - "shadow": 112 + "checks": 1482, + "shadow": 136 }, "half_life_days": 7.0 }, { - "mean": 0.028120551258325577, + "mean": 0.03963538631796837, "alpha": 0.003, - "total": 22.524560928344727, + "total": 23.345243453979492, "config": 16, - "trades": 801, + "trades": 589, "governance": { "ic": 0, - "any": 196, + "any": 199, "drift": 0, - "checks": 1365, - "shadow": 196 + "checks": 1482, + "shadow": 199 }, "half_life_days": 30.0 } ], "per_day": { - "2026-08-24": { - "n": 96, - "pf": 1.0117903231205847, - "win": 0.4479166666666667, - "mean": 0.0016304353484883904, - "median": -0.04346885532140732 - }, - "2026-08-25": { - "n": 92, - "pf": 1.5657616101643343, - "win": 0.5217391304347826, - "mean": 0.06755560636520386, - "median": 0.007279674988240004 - }, - "2026-08-26": { - "n": 263, - "pf": 1.165655794982386, - "win": 0.44866920152091255, - "mean": 0.014974601566791534, - "median": -0.012878135778009892 - }, - "2026-08-27": { - "n": 170, - "pf": 0.6946020453820136, - "win": 0.40588235294117647, - "mean": -0.03204870969057083, - "median": -0.03053605556488037 - }, - "2026-08-28": { - "n": 56, - "pf": 1.4862921757026535, - "win": 0.5178571428571429, - "mean": 0.06174355000257492, - "median": 0.010778131894767284 - }, - "2026-08-29": { - "n": 213, - "pf": 1.011389332575889, - "win": 0.43661971830985913, - "mean": 0.0011059067910537124, - "median": -0.026353871449828148 + "2026-09-24": { + "n": 120, + "pf": 1.2096282469078616, + "win": 0.4666666666666667, + "mean": 0.019853290170431137, + "median": -0.008774115703999996 }, - "2026-08-30": { - "n": 50, - "pf": 1.3834646670134814, - "win": 0.38, - "mean": 0.0530250258743763, - "median": -0.08351919054985046 + "2026-09-25": { + "n": 130, + "pf": 1.2343648703740158, + "win": 0.46923076923076923, + "mean": 0.022507645189762115, + "median": -0.015695948153734207 }, - "2026-08-31": { - "n": 41, - "pf": 1.6108359675631871, - "win": 0.43902439024390244, - "mean": 0.07970548421144485, - "median": -0.03631661832332611 + "2026-09-26": { + "n": 159, + "pf": 0.9110357851981143, + "win": 0.4528301886792453, + "mean": -0.010142914019525051, + "median": -0.02541925199329853 }, - "2026-09-01": { - "n": 43, - "pf": 0.7714416116969681, - "win": 0.4883720930232558, - "mean": -0.031781017780303955, - "median": -0.0033531002700328827 + "2026-09-27": { + "n": 134, + "pf": 1.4929109749231875, + "win": 0.44029850746268656, + "mean": 0.046415600925683975, + "median": -0.021206816658377647 }, - "2026-09-02": { - "n": 31, - "pf": 1.8960080329396884, - "win": 0.5161290322580645, - "mean": 0.09989593923091888, - "median": 0.056023046374320984 + "2026-09-28": { + "n": 146, + "pf": 1.4797542117441214, + "win": 0.3904109589041096, + "mean": 0.06142570450901985, + "median": -0.05863966420292854 }, - "2026-09-03": { - "n": 39, - "pf": 0.47435046931837, - "win": 0.38461538461538464, - "mean": -0.0845755785703659, - "median": -0.10096535086631775 + "2026-09-29": { + "n": 128, + "pf": 0.8905144422024887, + "win": 0.4453125, + "mean": -0.012050645425915718, + "median": -0.02929481491446495 }, - "2026-09-04": { - "n": 44, - "pf": 2.124818757306114, - "win": 0.36363636363636365, - "mean": 0.17666810750961304, - "median": -0.04468941688537598 + "2026-09-30": { + "n": 130, + "pf": 2.164358561365056, + "win": 0.5615384615384615, + "mean": 0.0869135931134224, + "median": 0.03113124519586563 }, - "2026-09-05": { - "n": 33, - "pf": 3.159536891732615, - "win": 0.45454545454545453, - "mean": 0.11582708358764648, - "median": -0.02216796576976776 + "2026-10-01": { + "n": 94, + "pf": 1.0760464155183325, + "win": 0.48936170212765956, + "mean": 0.008363611996173859, + "median": -0.0028990297578275204 }, - "2026-09-06": { - "n": 34, - "pf": 0.8038220998481828, - "win": 0.47058823529411764, - "mean": -0.028142571449279785, - "median": -0.04691909998655319 + "2026-10-02": { + "n": 105, + "pf": 0.9110992827614596, + "win": 0.45714285714285713, + "mean": -0.0093840342015028, + "median": -0.004047797992825508 }, - "2026-09-07": { - "n": 25, - "pf": 0.538075679014026, - "win": 0.36, - "mean": -0.07897567003965378, - "median": -0.057357002049684525 + "2026-10-03": { + "n": 89, + "pf": 1.5717055834211324, + "win": 0.5393258426966292, + "mean": 0.046940337866544724, + "median": 0.02129359170794487 }, - "2026-09-08": { - "n": 24, - "pf": 0.5229846306363845, - "win": 0.4583333333333333, - "mean": -0.0886785164475441, - "median": -0.027842504903674126 + "2026-10-04": { + "n": 111, + "pf": 1.5649065404247118, + "win": 0.4864864864864865, + "mean": 0.049075424671173096, + "median": -0.005090842954814434 }, - "2026-09-09": { - "n": 6, - "pf": 1.775177140941807, - "win": 0.3333333333333333, - "mean": 0.064696304500103, - "median": -0.0772169902920723 + "2026-10-05": { + "n": 113, + "pf": 1.2647730391586305, + "win": 0.5309734513274337, + "mean": 0.0248811524361372, + "median": 0.017877638339996338 }, - "2026-09-10": { - "n": 10, - "pf": 0.0, - "win": 0.0, - "mean": -0.4276605248451233, - "median": -0.40527138113975525 + "2026-10-06": { + "n": 101, + "pf": 1.4999371942039894, + "win": 0.48514851485148514, + "mean": 0.04013614356517792, + "median": -0.002721513854339719 }, - "2026-09-11": { - "n": 9, - "pf": 0.4853454520861287, - "win": 0.3333333333333333, - "mean": -0.07680502533912659, - "median": -0.1473425328731537 - }, - "2026-09-12": { - "n": 104, - "pf": 0.5093969547294807, - "win": 0.4326923076923077, - "mean": -0.043387047946453094, - "median": -0.009646685793995857 - }, - "2026-09-13": { - "n": 13, - "pf": 0.9683888718696267, - "win": 0.46153846153846156, - "mean": -0.003039261559024453, - "median": -0.011067567393183708 - }, - "2026-09-14": { - "n": 19, - "pf": 2.3648619908537536, - "win": 0.631578947368421, - "mean": 0.19480086863040924, - "median": 0.07961719483137131 - }, - "2026-09-15": { - "n": 19, - "pf": 1.9207007670652974, - "win": 0.42105263157894735, - "mean": 0.12944886088371277, - "median": -0.0763954222202301 - }, - "2026-09-16": { - "n": 20, - "pf": 1.989758339975232, - "win": 0.6, - "mean": 0.07185642421245575, - "median": 0.06811605393886566 - }, - "2026-09-17": { - "n": 23, - "pf": 2.308382394662165, - "win": 0.5652173913043478, - "mean": 0.07706345617771149, - "median": 0.08088493347167969 - }, - "2026-09-18": { - "n": 63, - "pf": 1.357841853369037, - "win": 0.47619047619047616, - "mean": 0.030619792640209198, - "median": -0.004897014237940311 - }, - "2026-09-19": { - "n": 34, - "pf": 1.4133537729578933, - "win": 0.4117647058823529, - "mean": 0.04197608679533005, - "median": -0.05088033154606819 + "2026-10-07": { + "n": 84, + "pf": 1.1800436231035252, + "win": 0.5, + "mean": 0.01683460921049118, + "median": -0.005321653559803963 }, - "2026-09-20": { - "n": 40, - "pf": 1.2252380260800315, - "win": 0.425, - "mean": 0.03793734312057495, - "median": -0.07361158728599548 + "2026-10-08": { + "n": 79, + "pf": 1.9083646078747787, + "win": 0.4936708860759494, + "mean": 0.07138421386480331, + "median": -0.004599324893206358 }, - "2026-09-21": { - "n": 37, - "pf": 1.2097114398883206, - "win": 0.5675675675675675, - "mean": 0.016092097386717796, - "median": 0.01574813388288021 + "2026-10-09": { + "n": 62, + "pf": 0.8436231572986813, + "win": 0.46774193548387094, + "mean": -0.017560668289661407, + "median": -0.032155461609363556 } }, "learning": [ { - "n": 931, - "day": "2026-07-27", + "n": 2360, + "day": "2026-09-09", "step": [ 0.0, - 0.06421024, - 0.06420731, - 0.06420637, - 0.06420581, - 0.18729984, - 0.18720574, - 0.18719876, - 0.1871946, - 0.58122008, - 0.57589885, - 0.57586091, - 0.57583875, - 1.56960882, - 1.57073448, - 1.57049312, - 1.56501226 + 0.19114103, + 0.19111604, + 0.19110858, + 0.19110422, + 0.56869623, + 0.56852255, + 0.56847102, + 0.56844206, + 1.84675575, + 1.8459054, + 1.84566744, + 1.84553071, + 4.07084698, + 4.07035153, + 4.07023325, + 4.07017128 ], "drift": [ 0.0, - 0.00123, - 0.00152, - 0.00163, - 0.00169, - 0.00348, - 0.00437, - 0.00467, - 0.00485, - 0.01091, - 0.01395, - 0.01494, - 0.01554, - 0.02869, - 0.03589, - 0.03833, - 0.03947 + 0.00364, + 0.00466, + 0.005, + 0.00521, + 0.01001, + 0.01279, + 0.01373, + 0.01431, + 0.0231, + 0.02928, + 0.03136, + 0.03262, + 0.03277, + 0.04133, + 0.04424, + 0.046 ], "capped": [ 0, @@ -585,94 +563,94 @@ 0, 0, 0, - 0, - 0, - 0, - 0, - 44, - 45, - 45, - 45 + 8, + 8, + 8, + 8, + 331, + 331, + 331, + 331 ], "mean_delta": [ - -0.039099, - -0.038733, - -0.038703, - -0.038693, - -0.038688, - -0.038012, - -0.037925, - -0.037898, - -0.037881, - -0.035483, - -0.035206, - -0.035118, - -0.035067, - -0.030323, - -0.029782, - -0.029599, - -0.029538 + -0.03834, + -0.036569, + -0.03649, + -0.036466, + -0.036452, + -0.033291, + -0.033073, + -0.033008, + -0.032969, + -0.02497, + -0.024521, + -0.024387, + -0.024309, + -0.018631, + -0.018057, + -0.017887, + -0.017788 ], "mean_abs_delta": [ - 0.189807, - 0.189711, - 0.189705, - 0.189703, - 0.189702, - 0.189527, - 0.18951, - 0.189504, - 0.189501, - 0.188888, - 0.188834, - 0.188818, - 0.188808, - 0.187597, - 0.187515, - 0.187486, - 0.187462 + 0.243879, + 0.243245, + 0.243225, + 0.243219, + 0.243215, + 0.242084, + 0.242034, + 0.24202, + 0.242011, + 0.239226, + 0.239148, + 0.239126, + 0.239114, + 0.236935, + 0.236862, + 0.236843, + 0.236831 ] }, { - "n": 1044, - "day": "2026-07-28", + "n": 1490, + "day": "2026-09-10", "step": [ 0.0, - 0.07277379, - 0.07218625, - 0.07189107, - 0.07176147, - 0.20257254, - 0.19562552, - 0.19169733, - 0.19072503, - 0.53300112, - 0.4800792, - 0.46328981, - 0.45527648, - 1.08817285, - 1.01850277, - 0.99633343, - 0.98662623 + 0.1282473, + 0.12781086, + 0.1276504, + 0.12754914, + 0.37799113, + 0.37522809, + 0.37424898, + 0.37364627, + 1.22299966, + 1.21300152, + 1.2096912, + 1.20772753, + 3.09826104, + 3.08997546, + 3.08711205, + 3.085262 ], "drift": [ 0.0, - 0.00112, - 0.0018, - 0.0021, - 0.00231, - 0.00342, - 0.00533, - 0.00625, - 0.00678, - 0.00927, - 0.01447, - 0.0171, - 0.01855, - 0.02377, - 0.03621, - 0.04178, - 0.04546 + 0.00529, + 0.00792, + 0.00897, + 0.00966, + 0.01383, + 0.02016, + 0.02263, + 0.02421, + 0.03039, + 0.04162, + 0.04599, + 0.04882, + 0.04772, + 0.06524, + 0.072, + 0.07637 ], "capped": [ 0, @@ -684,94 +662,94 @@ 0, 0, 0, - 0, - 0, - 0, - 0, - 40, - 36, - 34, - 32 + 2, + 2, + 2, + 2, + 152, + 149, + 148, + 147 ], "mean_delta": [ - -0.027195, - -0.026147, - -0.025859, - -0.025755, - -0.025687, - -0.02413, - -0.023407, - -0.023065, - -0.0229, - -0.01855, - -0.016465, - -0.015533, - -0.015078, - -0.006329, - -0.001372, - 0.000907, - 0.001877 + -0.048624, + -0.043967, + -0.042428, + -0.041858, + -0.041498, + -0.03632, + -0.032529, + -0.031133, + -0.030258, + -0.022787, + -0.016992, + -0.014919, + -0.013626, + -0.017012, + -0.012647, + -0.011101, + -0.010144 ], "mean_abs_delta": [ - 0.190608, - 0.190481, - 0.190461, - 0.190455, - 0.190451, - 0.190261, - 0.190232, - 0.190217, - 0.190211, - 0.189725, - 0.18969, - 0.189675, - 0.189664, - 0.189226, - 0.189369, - 0.189431, - 0.189449 + 0.203057, + 0.201621, + 0.201182, + 0.201021, + 0.20092, + 0.199419, + 0.198512, + 0.198193, + 0.197993, + 0.195899, + 0.194927, + 0.194612, + 0.194431, + 0.193714, + 0.192914, + 0.192643, + 0.192477 ] }, { - "n": 863, - "day": "2026-07-29", + "n": 2708, + "day": "2026-09-11", "step": [ 0.0, - 0.06034305, - 0.05845988, - 0.05776155, - 0.05721584, - 0.16231128, - 0.14620346, - 0.1383591, - 0.13605227, - 0.41583466, - 0.35392232, - 0.33466695, - 0.32878579, - 0.90389129, - 0.76926308, - 0.70084347, - 0.65879158 + 0.25055709, + 0.24931297, + 0.24878305, + 0.24843247, + 0.7385869, + 0.73237111, + 0.73011377, + 0.72871685, + 2.37396603, + 2.35785869, + 2.35226077, + 2.34895474, + 5.01956554, + 4.99887312, + 4.99164725, + 4.98743835 ], "drift": [ 0.0, - 0.00083, - 0.00173, - 0.00219, - 0.00252, - 0.00233, - 0.0047, - 0.00605, - 0.00698, - 0.00685, - 0.01407, - 0.01877, - 0.02179, - 0.02133, - 0.03601, - 0.04508, - 0.05245 + 0.00886, + 0.01362, + 0.01567, + 0.01707, + 0.02142, + 0.03018, + 0.03374, + 0.0362, + 0.04396, + 0.05978, + 0.06625, + 0.07057, + 0.06365, + 0.08646, + 0.09587, + 0.10222 ], "capped": [ 0, @@ -783,94 +761,94 @@ 0, 0, 0, - 0, - 0, - 0, - 0, - 20, - 18, - 15, - 13 + 23, + 22, + 21, + 22, + 431, + 428, + 426, + 426 ], "mean_delta": [ - -0.012039, - -0.011268, - -0.010674, - -0.010403, - -0.010234, - -0.009831, - -0.008321, - -0.007575, - -0.007109, - -0.006105, - -0.002358, - -7e-05, - 0.001157, - 0.001562, - 0.00964, - 0.014695, - 0.017686 + -0.066137, + -0.060051, + -0.057097, + -0.055824, + -0.054972, + -0.05114, + -0.045154, + -0.042783, + -0.041284, + -0.038975, + -0.032338, + -0.029703, + -0.028001, + -0.031565, + -0.024916, + -0.022423, + -0.020842 ], "mean_abs_delta": [ - 0.188678, - 0.188475, - 0.188319, - 0.188247, - 0.188202, - 0.188106, - 0.187716, - 0.187524, - 0.187409, - 0.187209, - 0.186402, - 0.18595, - 0.18572, - 0.185977, - 0.184891, - 0.184281, - 0.184005 + 0.251312, + 0.249431, + 0.248641, + 0.248307, + 0.24809, + 0.246854, + 0.245486, + 0.244963, + 0.244638, + 0.243016, + 0.241709, + 0.241248, + 0.240971, + 0.24051, + 0.239109, + 0.238597, + 0.238282 ] }, { - "n": 1236, - "day": "2026-07-30", + "n": 2001, + "day": "2026-09-12", "step": [ 0.0, - 0.10988152, - 0.10774172, - 0.10559816, - 0.10466212, - 0.31706643, - 0.29190488, - 0.28316394, - 0.27648895, - 0.9586995, - 0.81282441, - 0.75331462, - 0.72414899, - 2.05303031, - 1.7496128, - 1.59959924, - 1.50315613 + 0.1826168, + 0.18207247, + 0.18186614, + 0.1817378, + 0.54391342, + 0.54233645, + 0.54200476, + 0.54189227, + 1.79139739, + 1.78749173, + 1.7869684, + 1.78686687, + 4.26189446, + 4.246626, + 4.24025719, + 4.23618915 ], "drift": [ 0.0, - 0.00353, - 0.00502, - 0.00575, - 0.00627, - 0.01019, - 0.01367, - 0.01591, - 0.01733, - 0.03045, - 0.0398, - 0.04493, - 0.04817, - 0.06619, - 0.08122, - 0.08852, - 0.09422 + 0.005, + 0.01102, + 0.01416, + 0.01644, + 0.011, + 0.02139, + 0.02646, + 0.03017, + 0.0247, + 0.04295, + 0.05191, + 0.05836, + 0.04704, + 0.07637, + 0.08993, + 0.09947 ], "capped": [ 0, @@ -882,94 +860,94 @@ 0, 0, 0, - 0, - 0, - 0, - 0, - 102, - 87, - 80, - 79 + 5, + 8, + 8, + 9, + 286, + 288, + 286, + 287 ], "mean_delta": [ - -0.091839, - -0.090642, - -0.089971, - -0.089642, - -0.089387, - -0.088579, - -0.087192, - -0.086045, - -0.085377, - -0.082853, - -0.07795, - -0.074143, - -0.072069, - -0.066969, - -0.05794, - -0.052086, - -0.04854 + -0.030598, + -0.022891, + -0.01746, + -0.014995, + -0.013291, + -0.013717, + -0.005212, + -0.001838, + 0.000439, + -0.007286, + 0.002196, + 0.006156, + 0.008769, + -0.003335, + 0.004922, + 0.008127, + 0.010161 ], "mean_abs_delta": [ - 0.237217, - 0.236632, - 0.236356, - 0.236219, - 0.23611, - 0.235612, - 0.235062, - 0.234589, - 0.234319, - 0.232781, - 0.231053, - 0.229731, - 0.229042, - 0.226722, - 0.22439, - 0.222879, - 0.222044 + 0.240174, + 0.238265, + 0.237107, + 0.236598, + 0.236251, + 0.236032, + 0.234395, + 0.233809, + 0.233435, + 0.233989, + 0.232292, + 0.23168, + 0.231305, + 0.232431, + 0.23083, + 0.230277, + 0.229931 ] }, { - "n": 371, - "day": "2026-07-31", + "n": 2012, + "day": "2026-09-13", "step": [ 0.0, - 0.04234476, - 0.03978243, - 0.03882724, - 0.03834538, - 0.10608156, - 0.09335886, - 0.09183339, - 0.08744498, - 0.23009973, - 0.18223433, - 0.16370089, - 0.15161757, - 0.3607691, - 0.27135012, - 0.22326726, - 0.20295452 + 0.21982043, + 0.21626672, + 0.21434172, + 0.21294512, + 0.63683065, + 0.62368678, + 0.61723266, + 0.61262904, + 1.99879227, + 1.96794189, + 1.95389722, + 1.9444037, + 4.28735923, + 4.26309931, + 4.25257243, + 4.24607554 ], "drift": [ 0.0, - 0.00223, - 0.00455, - 0.00569, - 0.00657, - 0.00623, - 0.01209, - 0.01551, - 0.01774, - 0.01846, - 0.03446, - 0.04292, - 0.04901, - 0.03862, - 0.06792, - 0.08091, - 0.09142 + 0.01208, + 0.01992, + 0.02411, + 0.02707, + 0.02774, + 0.04059, + 0.04736, + 0.05236, + 0.05817, + 0.08155, + 0.0927, + 0.10083, + 0.08725, + 0.11862, + 0.1336, + 0.14449 ], "capped": [ 0, @@ -981,94 +959,94 @@ 0, 0, 0, - 0, - 0, - 0, - 0, - 7, - 2, - 1, - 1 + 17, + 13, + 11, + 11, + 333, + 324, + 321, + 321 ], "mean_delta": [ - -0.052608, - -0.050835, - -0.049833, - -0.049374, - -0.049009, - -0.047575, - -0.045204, - -0.043704, - -0.04279, - -0.037831, - -0.031311, - -0.027866, - -0.025479, - -0.019581, - -0.008466, - -0.00396, - -0.00054 + -0.119832, + -0.110132, + -0.103852, + -0.100391, + -0.09784, + -0.097303, + -0.088654, + -0.084247, + -0.081038, + -0.076759, + -0.068202, + -0.064189, + -0.061405, + -0.05926, + -0.05074, + -0.047498, + -0.045481 ], "mean_abs_delta": [ - 0.276031, - 0.275416, - 0.275123, - 0.274987, - 0.274879, - 0.27429, - 0.273585, - 0.273156, - 0.272904, - 0.271009, - 0.269555, - 0.268781, - 0.268237, - 0.265625, - 0.263527, - 0.26297, - 0.262476 + 0.264323, + 0.26012, + 0.257681, + 0.256385, + 0.255446, + 0.254951, + 0.251953, + 0.250481, + 0.249426, + 0.247341, + 0.244868, + 0.243866, + 0.243215, + 0.242131, + 0.240389, + 0.239686, + 0.239265 ] }, { - "n": 54, - "day": "2026-08-01", + "n": 2133, + "day": "2026-09-14", "step": [ 0.0, - 0.00741917, - 0.00741597, - 0.00676187, - 0.00605164, - 0.0194067, - 0.01646108, - 0.01194924, - 0.0110565, - 0.03528636, - 0.01351464, - 0.0101124, - 0.00974537, - 0.03056685, - 0.01951861, - 0.0138812, - 0.01356826 + 0.23188563, + 0.22810843, + 0.22625628, + 0.22503369, + 0.67246073, + 0.65975472, + 0.65391736, + 0.64989842, + 2.1162268, + 2.08203617, + 2.06818091, + 2.05947671, + 4.17329574, + 4.17041343, + 4.16754732, + 4.16457737 ], "drift": [ 0.0, - 0.00138, - 0.00387, - 0.00522, - 0.00637, - 0.00379, - 0.01028, - 0.0145, - 0.01744, - 0.01216, - 0.03019, - 0.04078, - 0.0486, - 0.02671, - 0.06085, - 0.0774, - 0.09115 + 0.01416, + 0.02469, + 0.02995, + 0.03382, + 0.03014, + 0.04773, + 0.05683, + 0.06383, + 0.05762, + 0.08725, + 0.10166, + 0.11207, + 0.07328, + 0.1115, + 0.13109, + 0.1461 ], "capped": [ 0, @@ -1080,94 +1058,94 @@ 0, 0, 0, - 0, - 0, - 0, - 0, - 1, - 0, - 0, - 0 + 21, + 20, + 19, + 18, + 393, + 388, + 383, + 385 ], "mean_delta": [ - 0.021166, - 0.022723, - 0.024943, - 0.026034, - 0.026946, - 0.025496, - 0.030874, - 0.034262, - 0.036473, - 0.034028, - 0.047387, - 0.055297, - 0.060891, - 0.046447, - 0.071219, - 0.081994, - 0.090024 + -0.09698, + -0.082646, + -0.07423, + -0.069849, + -0.06679, + -0.065887, + -0.053981, + -0.048011, + -0.043628, + -0.041042, + -0.025054, + -0.017945, + -0.013094, + -0.03023, + -0.014329, + -0.008031, + -0.003997 ], "mean_abs_delta": [ - 0.26533, - 0.265417, - 0.265613, - 0.265685, - 0.265759, - 0.265548, - 0.265895, - 0.26615, - 0.266267, - 0.265943, - 0.266645, - 0.267432, - 0.267988, - 0.266451, - 0.269451, - 0.271657, - 0.273752 + 0.262175, + 0.258569, + 0.256876, + 0.25607, + 0.255537, + 0.255085, + 0.253301, + 0.252476, + 0.25193, + 0.250899, + 0.249185, + 0.248541, + 0.248172, + 0.249131, + 0.247707, + 0.247219, + 0.24697 ] }, { - "n": 242, - "day": "2026-08-02", + "n": 1767, + "day": "2026-09-15", "step": [ 0.0, - 0.01872033, - 0.01859977, - 0.0181289, - 0.01802677, - 0.0558438, - 0.05201761, - 0.04925188, - 0.04794973, - 0.17378149, - 0.14824457, - 0.12593751, - 0.1140936, - 0.47729315, - 0.33874456, - 0.24572129, - 0.17729059 + 0.13082462, + 0.12807035, + 0.12691447, + 0.1262311, + 0.38357165, + 0.37626038, + 0.37457562, + 0.37416996, + 1.2576157, + 1.24848204, + 1.25087272, + 1.25437331, + 3.13282058, + 3.10825873, + 3.10193769, + 3.09932395 ], "drift": [ 0.0, - 0.00088, - 0.00314, - 0.00483, - 0.00643, - 0.00248, - 0.00828, - 0.01332, - 0.01741, - 0.00756, - 0.02334, - 0.03559, - 0.04639, - 0.01838, - 0.04457, - 0.06531, - 0.0875 + 0.00964, + 0.02176, + 0.02879, + 0.03416, + 0.01986, + 0.03942, + 0.05074, + 0.05976, + 0.03512, + 0.06398, + 0.08102, + 0.09456, + 0.05027, + 0.08732, + 0.11055, + 0.13017 ], "capped": [ 0, @@ -1179,94 +1157,94 @@ 0, 0, 0, - 0, - 0, - 0, - 0, - 6, - 5, - 3, - 2 + 2, + 2, + 2, + 2, + 194, + 186, + 188, + 189 ], "mean_delta": [ - -0.036087, - -0.035588, - -0.034009, - -0.032926, - -0.031938, - -0.034705, - -0.030626, - -0.027369, - -0.024855, - -0.031729, - -0.020271, - -0.012292, - -0.005555, - -0.026377, - -0.005737, - 0.007242, - 0.018867 + -0.049886, + -0.036129, + -0.023739, + -0.017324, + -0.012671, + -0.022447, + -0.00489, + 0.004062, + 0.010737, + -0.00716, + 0.011367, + 0.019659, + 0.02526, + -0.002027, + 0.010032, + 0.0152, + 0.018675 ], "mean_abs_delta": [ - 0.214218, - 0.214074, - 0.213654, - 0.213363, - 0.213101, - 0.213812, - 0.212741, - 0.211945, - 0.211312, - 0.212985, - 0.210212, - 0.20864, - 0.207606, - 0.211498, - 0.207818, - 0.206297, - 0.20526 + 0.198057, + 0.194771, + 0.192451, + 0.191491, + 0.190933, + 0.192258, + 0.190222, + 0.189701, + 0.189472, + 0.190494, + 0.189427, + 0.189326, + 0.189398, + 0.190068, + 0.189223, + 0.188959, + 0.188849 ] }, { - "n": 508, - "day": "2026-08-03", + "n": 2081, + "day": "2026-09-16", "step": [ 0.0, - 0.04134416, - 0.0371876, - 0.03266551, - 0.0305573, - 0.11778908, - 0.08932215, - 0.08048781, - 0.07565604, - 0.31825422, - 0.24333699, - 0.19889814, - 0.18343096, - 0.76600191, - 0.54425102, - 0.4655616, - 0.33341111 + 0.20784134, + 0.20151466, + 0.19820528, + 0.19590611, + 0.59886797, + 0.5767962, + 0.5654473, + 0.55716899, + 1.87655532, + 1.81595846, + 1.78898973, + 1.7715539, + 3.97739167, + 3.94241426, + 3.92871029, + 3.91951185 ], "drift": [ 0.0, - 0.00086, - 0.00272, - 0.0045, - 0.00642, - 0.00251, - 0.0068, - 0.01191, - 0.0169, - 0.00584, - 0.01774, - 0.03055, - 0.04362, - 0.01263, - 0.03241, - 0.05504, - 0.08091 + 0.01545, + 0.02817, + 0.03538, + 0.04147, + 0.03353, + 0.05398, + 0.06625, + 0.07651, + 0.06853, + 0.09982, + 0.11663, + 0.12991, + 0.09462, + 0.13238, + 0.15425, + 0.17315 ], "capped": [ 0, @@ -1278,94 +1256,94 @@ 0, 0, 0, - 0, - 0, - 0, - 0, - 22, - 15, - 13, - 5 + 11, + 7, + 6, + 5, + 346, + 332, + 328, + 323 ], "mean_delta": [ - -0.011377, - -0.010753, - -0.009203, - -0.007918, - -0.006615, - -0.009584, - -0.005852, - -0.001992, - 0.001368, - -0.007048, - 0.003835, - 0.013027, - 0.021712, - -0.001486, - 0.016475, - 0.030461, - 0.046891 + -0.130949, + -0.118029, + -0.106376, + -0.099784, + -0.094935, + -0.102888, + -0.087331, + -0.07841, + -0.071425, + -0.079286, + -0.063549, + -0.055869, + -0.050599, + -0.060387, + -0.046477, + -0.040718, + -0.037114 ], "mean_abs_delta": [ - 0.206632, - 0.206521, - 0.206251, - 0.206031, - 0.205803, - 0.20633, - 0.205706, - 0.205081, - 0.204565, - 0.205893, - 0.204323, - 0.203515, - 0.203302, - 0.204999, - 0.203303, - 0.203833, - 0.206042 + 0.248057, + 0.242605, + 0.238024, + 0.23569, + 0.234087, + 0.236914, + 0.231772, + 0.229171, + 0.227336, + 0.229563, + 0.225384, + 0.2236, + 0.222481, + 0.224475, + 0.221428, + 0.220284, + 0.219627 ] }, { - "n": 692, - "day": "2026-08-04", + "n": 987, + "day": "2026-09-17", "step": [ 0.0, - 0.04460403, - 0.03859602, - 0.03441494, - 0.03139943, - 0.12362088, - 0.09626486, - 0.06885854, - 0.05953138, - 0.38367219, - 0.20308522, - 0.16834852, - 0.131699, - 1.01038427, - 0.50971773, - 0.37668987, - 0.27438032 + 0.10679559, + 0.10547293, + 0.10494688, + 0.10461475, + 0.31651284, + 0.31306477, + 0.31189373, + 0.31149969, + 1.04054557, + 1.032719, + 1.03181024, + 1.03275386, + 2.65368672, + 2.61416605, + 2.5994033, + 2.5902975 ], "drift": [ 0.0, - 0.00046, - 0.00194, - 0.0039, - 0.006, - 0.00136, - 0.00491, - 0.0109, - 0.01666, - 0.00421, - 0.01525, - 0.02819, - 0.0435, - 0.01111, - 0.02828, - 0.05228, - 0.08124 + 0.00863, + 0.02274, + 0.0322, + 0.04047, + 0.01862, + 0.04209, + 0.05783, + 0.07152, + 0.03753, + 0.07301, + 0.09505, + 0.11354, + 0.06027, + 0.10335, + 0.1309, + 0.156 ], "capped": [ 0, @@ -1381,90 +1359,90 @@ 0, 0, 0, - 11, - 6, - 2, - 1 + 168, + 167, + 166, + 162 ], "mean_delta": [ - 0.013699, - 0.013807, - 0.015059, - 0.016398, - 0.017741, - 0.014045, - 0.017069, - 0.021082, - 0.02475, - 0.013845, - 0.023644, - 0.032565, - 0.042299, - 0.013797, - 0.02944, - 0.045704, - 0.063913 + -0.040432, + -0.02836, + -0.01347, + -0.004831, + 0.002238, + -0.016139, + 0.005467, + 0.01782, + 0.027606, + -0.001613, + 0.023576, + 0.035519, + 0.044047, + 0.003194, + 0.023628, + 0.032835, + 0.039144 ], "mean_abs_delta": [ - 0.183171, - 0.183126, - 0.183024, - 0.182942, - 0.18287, - 0.183048, - 0.182837, - 0.182735, - 0.182695, - 0.182865, - 0.182626, - 0.182803, - 0.1836, - 0.182576, - 0.182491, - 0.184005, - 0.187536 + 0.243221, + 0.240296, + 0.23729, + 0.235926, + 0.234916, + 0.237417, + 0.234076, + 0.232521, + 0.231564, + 0.233647, + 0.230187, + 0.229127, + 0.228694, + 0.2304, + 0.227297, + 0.226267, + 0.225737 ] }, { - "n": 1128, - "day": "2026-08-05", + "n": 1103, + "day": "2026-09-18", "step": [ 0.0, - 0.05903479, - 0.05507833, - 0.05134258, - 0.04764679, - 0.17682159, - 0.15135335, - 0.12787613, - 0.11643432, - 0.6171573, - 0.40581984, - 0.34735324, - 0.30416585, - 1.83372795, - 1.00153648, - 0.85559196, - 0.64182627 + 0.10171894, + 0.1004097, + 0.09985218, + 0.09957259, + 0.30264728, + 0.29896345, + 0.2982446, + 0.29861928, + 1.00028158, + 0.9938767, + 0.9933884, + 0.99473531, + 2.53034597, + 2.5091195, + 2.50166478, + 2.49738774 ], "drift": [ 0.0, - 0.00107, - 0.00083, - 0.00233, - 0.00462, - 0.00313, - 0.00239, - 0.00694, - 0.01331, - 0.01023, - 0.0079, - 0.01759, - 0.03386, - 0.01932, - 0.01897, - 0.03149, - 0.05838 + 0.00482, + 0.0178, + 0.02829, + 0.0381, + 0.01, + 0.03078, + 0.04755, + 0.06328, + 0.01965, + 0.0503, + 0.07451, + 0.09663, + 0.03529, + 0.07388, + 0.10473, + 0.13561 ], "capped": [ 0, @@ -1480,90 +1458,90 @@ 0, 0, 0, - 63, - 44, - 39, - 31 + 156, + 157, + 153, + 150 ], "mean_delta": [ - 0.045258, - 0.044848, - 0.04591, - 0.047412, - 0.04904, - 0.044154, - 0.046767, - 0.051566, - 0.05613, - 0.041694, - 0.051311, - 0.060843, - 0.072213, - 0.041475, - 0.054393, - 0.067997, - 0.08784 + -0.011827, + -0.004222, + 0.01002, + 0.019707, + 0.027927, + 0.003769, + 0.024129, + 0.037144, + 0.047967, + 0.013337, + 0.032898, + 0.042796, + 0.050177, + 0.01812, + 0.033057, + 0.039623, + 0.043918 ], "mean_abs_delta": [ - 0.173116, - 0.173179, - 0.173065, - 0.172906, - 0.172747, - 0.173298, - 0.173014, - 0.172589, - 0.172375, - 0.173776, - 0.17276, - 0.172651, - 0.173458, - 0.173557, - 0.173238, - 0.173589, - 0.176942 + 0.225247, + 0.223176, + 0.219988, + 0.218332, + 0.217242, + 0.221218, + 0.217691, + 0.216313, + 0.215799, + 0.2191, + 0.21651, + 0.215622, + 0.21536, + 0.21802, + 0.216047, + 0.215298, + 0.214902 ] }, { - "n": 1058, - "day": "2026-08-06", + "n": 1036, + "day": "2026-09-19", "step": [ 0.0, - 0.03945935, - 0.03698631, - 0.03220626, - 0.02746376, - 0.12985755, - 0.11435575, - 0.07488091, - 0.05460495, - 0.63672671, - 0.3500779, - 0.1885276, - 0.11419593, - 1.80572728, - 1.57435511, - 0.62120528, - 0.28846189 + 0.11678434, + 0.11557956, + 0.11478874, + 0.11426845, + 0.34844892, + 0.34460238, + 0.34276291, + 0.34191144, + 1.14770467, + 1.14334365, + 1.13901164, + 1.13590429, + 2.75977484, + 2.761065, + 2.75854016, + 2.75362164 ], "drift": [ 0.0, - 0.00064, - 0.00098, - 0.00238, - 0.00474, - 0.00196, - 0.00278, - 0.00677, - 0.01397, - 0.00734, - 0.00897, - 0.01889, - 0.03563, - 0.01789, - 0.0232, - 0.03536, - 0.06194 + 0.00438, + 0.01608, + 0.02714, + 0.03826, + 0.01012, + 0.02845, + 0.04542, + 0.06262, + 0.02201, + 0.04827, + 0.07261, + 0.09706, + 0.03463, + 0.06915, + 0.10194, + 0.13677 ], "capped": [ 0, @@ -1575,94 +1553,94 @@ 0, 0, 0, - 0, - 0, - 0, - 0, - 46, - 40, - 17, - 6 + 3, + 2, + 2, + 2, + 192, + 193, + 190, + 191 ], "mean_delta": [ - 0.003393, - 0.002579, - 0.003265, - 0.004759, - 0.006634, - 0.00097, - 0.002713, - 0.007733, - 0.01337, - -0.00438, - 0.003119, - 0.014892, - 0.02849, - -0.00547, - -0.003837, - 0.016483, - 0.041025 + -0.054567, + -0.049741, + -0.037411, + -0.027108, + -0.017606, + -0.044421, + -0.027776, + -0.01496, + -0.003504, + -0.037136, + -0.025033, + -0.016661, + -0.009471, + -0.032599, + -0.024503, + -0.019776, + -0.015409 ], "mean_abs_delta": [ - 0.168503, - 0.168553, - 0.168513, - 0.168429, - 0.168352, - 0.168686, - 0.168561, - 0.168344, - 0.16832, - 0.169263, - 0.168633, - 0.168449, - 0.169828, - 0.169664, - 0.169537, - 0.168916, - 0.172572 + 0.245887, + 0.245196, + 0.243845, + 0.242908, + 0.242319, + 0.244383, + 0.242828, + 0.242088, + 0.241955, + 0.242962, + 0.242128, + 0.241527, + 0.241323, + 0.241465, + 0.241207, + 0.240942, + 0.240554 ] }, { - "n": 892, - "day": "2026-08-07", + "n": 1187, + "day": "2026-09-20", "step": [ 0.0, - 0.04321418, - 0.04232384, - 0.03822837, - 0.03495312, - 0.13560243, - 0.12616215, - 0.09834065, - 0.08321215, - 0.49905728, - 0.35899078, - 0.26573406, - 0.22310838, - 0.96535605, - 1.01828484, - 0.73811684, - 0.58092871 + 0.13266736, + 0.13004195, + 0.12775291, + 0.12577976, + 0.39207659, + 0.38208249, + 0.37475607, + 0.36877949, + 1.26922513, + 1.24006389, + 1.22049149, + 1.2042866, + 3.04776096, + 3.01592225, + 2.99810015, + 2.98511973 ], "drift": [ 0.0, - 0.0006, - 0.00106, - 0.00252, - 0.00501, - 0.00186, - 0.00299, - 0.00707, - 0.01449, - 0.00655, - 0.01045, - 0.01962, - 0.03622, - 0.01756, + 0.00569, + 0.01606, 0.0272, - 0.03745, - 0.06359 + 0.03925, + 0.01458, + 0.03117, + 0.04756, + 0.06545, + 0.03589, + 0.06281, + 0.08566, + 0.10953, + 0.06189, + 0.09764, + 0.12626, + 0.15856 ], "capped": [ 0, @@ -1674,94 +1652,94 @@ 0, 0, 0, + 1, 0, 0, 0, - 0, - 12, - 12, - 12, - 11 + 192, + 180, + 181, + 182 ], "mean_delta": [ - -0.026824, - -0.026974, - -0.026615, - -0.025306, - -0.023529, - -0.027278, - -0.026385, - -0.022427, - -0.017015, - -0.028057, - -0.023258, - -0.014869, - -0.0028, - -0.020316, - -0.019829, - -0.007844, - 0.009888 + -0.056364, + -0.052038, + -0.040912, + -0.030003, + -0.019167, + -0.046364, + -0.031376, + -0.018432, + -0.006145, + -0.036509, + -0.02429, + -0.016045, + -0.008912, + -0.028272, + -0.018874, + -0.014138, + -0.010484 ], "mean_abs_delta": [ - 0.165735, - 0.165729, - 0.165702, - 0.16562, - 0.165527, - 0.165724, - 0.165654, - 0.165483, - 0.165431, - 0.165654, - 0.165466, - 0.165447, - 0.166151, - 0.165184, - 0.165209, - 0.165848, - 0.168385 + 0.245185, + 0.24333, + 0.239708, + 0.236471, + 0.233578, + 0.240728, + 0.235891, + 0.232144, + 0.229056, + 0.23563, + 0.230898, + 0.227667, + 0.225108, + 0.229873, + 0.225418, + 0.223095, + 0.221617 ] }, { - "n": 1076, - "day": "2026-08-08", + "n": 1206, + "day": "2026-09-21", "step": [ 0.0, - 0.07095081, - 0.07041009, - 0.06571411, - 0.05991169, - 0.21319146, - 0.20814377, - 0.1705785, - 0.13642696, - 0.70387371, - 0.57735057, - 0.42904051, - 0.32422108, - 1.63103617, - 1.54005343, - 1.00802515, - 0.76446274 + 0.11644781, + 0.11481612, + 0.11338398, + 0.11221946, + 0.34545412, + 0.34004148, + 0.33650485, + 0.33410854, + 1.12699008, + 1.11441112, + 1.10691122, + 1.10130664, + 2.91439664, + 2.88506963, + 2.86526596, + 2.84790216 ], "drift": [ 0.0, - 0.00139, - 0.00201, - 0.00348, - 0.00597, - 0.00421, - 0.00595, - 0.00945, - 0.01669, - 0.01451, - 0.01815, - 0.02519, - 0.04134, - 0.03226, - 0.04307, - 0.04922, - 0.07187 + 0.00532, + 0.01534, + 0.02674, + 0.03978, + 0.0137, + 0.03031, + 0.04697, + 0.06614, + 0.03342, + 0.06086, + 0.08544, + 0.11219, + 0.06609, + 0.10559, + 0.13678, + 0.17212 ], "capped": [ 0, @@ -1773,94 +1751,94 @@ 0, 0, 0, - 0, - 0, - 0, - 0, - 51, - 50, - 29, - 15 + 4, + 4, + 5, + 5, + 136, + 133, + 128, + 128 ], "mean_delta": [ - -0.031996, - -0.031957, - -0.031714, - -0.030445, - -0.028644, - -0.031818, - -0.031171, - -0.027531, - -0.022065, - -0.03076, - -0.027356, - -0.019525, - -0.007765, - -0.026253, - -0.024635, - -0.010153, - 0.006282 + -0.085746, + -0.08049, + -0.07024, + -0.059297, + -0.047764, + -0.073021, + -0.059314, + -0.047263, + -0.035521, + -0.060101, + -0.048115, + -0.040621, + -0.034493, + -0.045142, + -0.034915, + -0.029471, + -0.025509 ], "mean_abs_delta": [ - 0.199058, - 0.199035, - 0.198997, - 0.198821, - 0.198581, - 0.19898, - 0.198881, - 0.198421, - 0.197903, - 0.198693, - 0.198275, - 0.19768, - 0.197643, - 0.197837, - 0.197494, - 0.197324, - 0.198843 + 0.233576, + 0.232077, + 0.229566, + 0.227273, + 0.225168, + 0.230144, + 0.227265, + 0.224994, + 0.223114, + 0.227143, + 0.225112, + 0.223683, + 0.222413, + 0.223257, + 0.221995, + 0.221008, + 0.220042 ] }, { - "n": 868, - "day": "2026-08-09", + "n": 1079, + "day": "2026-09-22", "step": [ 0.0, - 0.06866672, - 0.0674683, - 0.06261659, - 0.0572501, - 0.19572992, - 0.18850666, - 0.16446996, - 0.13726034, - 0.57460789, - 0.53087509, - 0.43740982, - 0.34744486, - 1.49970195, - 1.28868771, - 1.09052244, - 0.83394681 + 0.09756422, + 0.09530408, + 0.093099, + 0.09107224, + 0.28622661, + 0.278017, + 0.27232901, + 0.26792277, + 0.92025189, + 0.89941675, + 0.8904902, + 0.88611577, + 2.42137875, + 2.400505, + 2.39369573, + 2.38974493 ], "drift": [ 0.0, - 0.00092, - 0.00141, - 0.00246, - 0.00496, - 0.00263, - 0.00422, - 0.00675, - 0.01374, - 0.00836, - 0.01363, - 0.01823, - 0.03352, - 0.0207, - 0.03373, - 0.04033, - 0.06014 + 0.00578, + 0.01565, + 0.02711, + 0.04076, + 0.01461, + 0.0311, + 0.04756, + 0.06751, + 0.0314, + 0.05874, + 0.08347, + 0.11224, + 0.05136, + 0.09116, + 0.12454, + 0.16416 ], "capped": [ 0, @@ -1876,90 +1854,90 @@ 0, 0, 0, - 50, - 48, - 38, - 26 + 109, + 102, + 100, + 96 ], "mean_delta": [ - 0.045081, - 0.045225, - 0.045497, - 0.046645, - 0.048417, - 0.045572, - 0.046402, - 0.049269, - 0.054513, - 0.047274, - 0.049228, - 0.05597, - 0.068074, - 0.049162, - 0.053368, - 0.062153, - 0.07952 + -0.077488, + -0.070788, + -0.060436, + -0.048752, + -0.035794, + -0.060456, + -0.045885, + -0.033158, + -0.020082, + -0.04091, + -0.025862, + -0.016616, + -0.009334, + -0.022349, + -0.006128, + 0.001317, + 0.006364 ], "mean_abs_delta": [ - 0.223275, - 0.223163, - 0.223108, - 0.222843, - 0.222452, - 0.222937, - 0.222776, - 0.222128, - 0.221108, - 0.22213, - 0.221745, - 0.220548, - 0.219228, - 0.220893, - 0.220428, - 0.219255, - 0.218497 + 0.216724, + 0.214793, + 0.211789, + 0.208763, + 0.205896, + 0.21193, + 0.208067, + 0.20529, + 0.203191, + 0.207176, + 0.203862, + 0.20237, + 0.201428, + 0.20337, + 0.200791, + 0.199887, + 0.199492 ] }, { - "n": 750, - "day": "2026-08-10", + "n": 976, + "day": "2026-09-23", "step": [ 0.0, - 0.05472073, - 0.05364319, - 0.04933666, - 0.04520726, - 0.16364656, - 0.1575568, - 0.13419577, - 0.10970383, - 0.5261399, - 0.49561711, - 0.3807543, - 0.30355334, - 1.53532154, - 1.38939052, - 1.08685966, - 0.78345868 + 0.09426381, + 0.09374157, + 0.09338934, + 0.09337377, + 0.28257084, + 0.28077042, + 0.28023092, + 0.28056747, + 0.94434258, + 0.93987403, + 0.93681763, + 0.93477549, + 2.2962897, + 2.29777617, + 2.29499464, + 2.28755804 ], "drift": [ 0.0, - 0.0007, - 0.00148, - 0.00293, - 0.00562, - 0.00219, - 0.00443, - 0.00785, - 0.01565, - 0.00741, - 0.01389, - 0.02234, - 0.03848, - 0.01995, - 0.03779, - 0.05147, - 0.07327 + 0.00244, + 0.01117, + 0.02258, + 0.03706, + 0.00582, + 0.02066, + 0.03749, + 0.05902, + 0.01409, + 0.04003, + 0.06714, + 0.09998, + 0.032, + 0.07254, + 0.10976, + 0.1545 ], "capped": [ 0, @@ -1971,94 +1949,94 @@ 0, 0, 0, - 0, - 0, - 0, - 0, - 50, - 44, - 35, - 20 + 2, + 2, + 2, + 2, + 158, + 158, + 147, + 149 ], "mean_delta": [ - -0.038261, - -0.038247, - -0.038023, - -0.036867, - -0.034865, - -0.038081, - -0.037483, - -0.034662, - -0.02846, - -0.036775, - -0.035351, - -0.027956, - -0.015405, - -0.03419, - -0.029574, - -0.020264, - -0.004209 + 0.014295, + 0.01724, + 0.026804, + 0.038089, + 0.051245, + 0.020594, + 0.034768, + 0.046982, + 0.05994, + 0.020319, + 0.034759, + 0.043281, + 0.050589, + 0.014688, + 0.026629, + 0.032576, + 0.037127 ], "mean_abs_delta": [ - 0.197346, - 0.19733, - 0.197299, - 0.197197, - 0.197041, - 0.197289, - 0.197205, - 0.197002, - 0.19669, - 0.197076, - 0.196911, - 0.196633, - 0.196898, - 0.196582, - 0.195933, - 0.196452, - 0.197811 + 0.22437, + 0.224002, + 0.222885, + 0.222081, + 0.221847, + 0.2237, + 0.222278, + 0.221525, + 0.221327, + 0.22391, + 0.222332, + 0.221141, + 0.220453, + 0.223597, + 0.221879, + 0.220773, + 0.219848 ] }, { - "n": 783, - "day": "2026-08-11", + "n": 903, + "day": "2026-09-24", "step": [ 0.0, - 0.05639539, - 0.05519085, - 0.05250994, - 0.0479547, - 0.16113087, - 0.153371, - 0.13901212, - 0.11707776, - 0.46874315, - 0.42715407, - 0.35583433, - 0.32026198, - 1.04951349, - 0.93101532, - 0.88466587, - 0.79205738 + 0.08876457, + 0.08874128, + 0.08892377, + 0.08963874, + 0.2664914, + 0.26678571, + 0.26791933, + 0.27041107, + 0.88996891, + 0.89197688, + 0.89317843, + 0.8949892, + 2.18884247, + 2.18298296, + 2.17738741, + 2.16912733 ], "drift": [ 0.0, - 0.00075, - 0.00164, - 0.00333, - 0.00616, - 0.00249, - 0.00511, - 0.00897, - 0.01773, - 0.00797, - 0.0169, - 0.02615, - 0.0421, - 0.02123, - 0.03919, - 0.05574, - 0.07721 + 0.00084, + 0.00699, + 0.01785, + 0.033, + 0.00277, + 0.01211, + 0.02822, + 0.05099, + 0.01055, + 0.02981, + 0.05568, + 0.09072, + 0.01841, + 0.05459, + 0.09394, + 0.14281 ], "capped": [ 0, @@ -2070,94 +2048,94 @@ 0, 0, 0, - 0, - 0, - 0, - 0, - 23, - 18, - 13, - 10 + 2, + 2, + 3, + 4, + 132, + 133, + 136, + 133 ], "mean_delta": [ - -0.017449, - -0.016968, - -0.01666, - -0.015667, - -0.013979, - -0.015814, - -0.01491, - -0.012715, - -0.007314, - -0.011976, - -0.008872, - -0.003129, - 0.00642, - -0.002249, - 0.004524, - 0.012359, - 0.023258 + 0.027675, + 0.02845, + 0.037452, + 0.049463, + 0.06424, + 0.028344, + 0.040858, + 0.053426, + 0.067364, + 0.022923, + 0.032271, + 0.040672, + 0.049039, + 0.026384, + 0.034044, + 0.039349, + 0.045038 ], "mean_abs_delta": [ - 0.223001, - 0.222889, - 0.222832, - 0.222668, - 0.2224, - 0.222629, - 0.222472, - 0.22216, - 0.221525, - 0.221802, - 0.221368, - 0.220954, - 0.220967, - 0.220241, - 0.21992, - 0.220531, - 0.221761 + 0.21318, + 0.213152, + 0.212808, + 0.212744, + 0.213551, + 0.213189, + 0.212854, + 0.213044, + 0.213903, + 0.213385, + 0.213147, + 0.21294, + 0.212884, + 0.212399, + 0.212564, + 0.212668, + 0.212694 ] }, { - "n": 737, - "day": "2026-08-12", + "n": 954, + "day": "2026-09-25", "step": [ 0.0, - 0.05897562, - 0.05788302, - 0.05464898, - 0.0493692, - 0.16978297, - 0.15688009, - 0.13920504, - 0.112585, - 0.47220465, - 0.39037227, - 0.34833647, - 0.29217996, - 1.05956363, - 0.93436273, - 0.77778303, - 0.59210142 + 0.09233516, + 0.09085702, + 0.0886607, + 0.08625427, + 0.27524403, + 0.27045533, + 0.26491697, + 0.25959592, + 0.90289918, + 0.90081693, + 0.89529136, + 0.88955279, + 2.20147544, + 2.19533407, + 2.19098079, + 2.18440822 ], "drift": [ 0.0, - 0.00193, - 0.00289, - 0.00468, - 0.00769, - 0.00563, - 0.00849, - 0.01268, - 0.02191, - 0.01657, - 0.02624, - 0.0356, - 0.05084, - 0.0402, - 0.05596, - 0.07187, - 0.09195 + 0.00129, + 0.00665, + 0.01669, + 0.03189, + 0.00318, + 0.01136, + 0.02592, + 0.04865, + 0.00796, + 0.0232, + 0.04857, + 0.08559, + 0.01868, + 0.04431, + 0.08187, + 0.13334 ], "capped": [ 0, @@ -2169,94 +2147,94 @@ 0, 0, 0, - 0, - 0, - 0, - 0, - 21, - 21, - 14, - 13 + 2, + 2, + 2, + 2, + 133, + 135, + 132, + 134 ], "mean_delta": [ - -0.070931, - -0.070332, - -0.069889, - -0.068616, - -0.066434, - -0.069028, - -0.067488, - -0.064536, - -0.057181, - -0.064527, - -0.058243, - -0.05118, - -0.039822, - -0.0522, - -0.04429, - -0.032432, - -0.017798 + -0.033727, + -0.031701, + -0.024465, + -0.012349, + 0.004022, + -0.028724, + -0.020267, + -0.008991, + 0.004833, + -0.022034, + -0.018706, + -0.013501, + -0.007184, + -0.011848, + -0.008334, + -0.005493, + -0.001478 ], "mean_abs_delta": [ - 0.21199, - 0.211742, - 0.211594, - 0.211213, - 0.210562, - 0.211234, - 0.210759, - 0.209887, - 0.207963, - 0.209641, - 0.207901, - 0.206368, - 0.204237, - 0.206052, - 0.204262, - 0.202601, - 0.201103 + 0.202285, + 0.201667, + 0.199375, + 0.195928, + 0.191963, + 0.200805, + 0.198149, + 0.194988, + 0.19166, + 0.198704, + 0.197768, + 0.196351, + 0.194949, + 0.19472, + 0.194025, + 0.193564, + 0.192967 ] }, { - "n": 911, - "day": "2026-08-13", + "n": 1402, + "day": "2026-09-26", "step": [ 0.0, - 0.07731382, - 0.07438362, - 0.06862943, - 0.06052568, - 0.20060044, - 0.17363393, - 0.15290134, - 0.12778348, - 0.47968175, - 0.39746136, - 0.34151492, - 0.28810784, - 0.95362768, - 0.76048055, - 0.63285901, - 0.47991281 + 0.12922601, + 0.12729616, + 0.12402046, + 0.11979357, + 0.37791886, + 0.37161283, + 0.36364022, + 0.35448112, + 1.18658264, + 1.17125415, + 1.15828728, + 1.14850897, + 2.7113238, + 2.68743956, + 2.66943223, + 2.65527476 ], "drift": [ 0.0, - 0.00205, - 0.00341, - 0.00526, - 0.00838, - 0.00603, - 0.00985, - 0.01463, - 0.02425, - 0.01872, - 0.02971, - 0.04066, - 0.05536, - 0.04117, - 0.06156, - 0.07875, - 0.09625 + 0.00618, + 0.01214, + 0.0215, + 0.03642, + 0.01599, + 0.02603, + 0.03831, + 0.05891, + 0.0361, + 0.05437, + 0.07398, + 0.10518, + 0.05695, + 0.08522, + 0.11411, + 0.15866 ], "capped": [ 0, @@ -2268,94 +2246,94 @@ 0, 0, 0, - 0, - 0, - 0, - 0, - 36, - 24, - 22, - 19 + 8, + 8, + 8, + 8, + 170, + 161, + 154, + 150 ], "mean_delta": [ - -0.009271, - -0.008132, - -0.007288, - -0.005889, - -0.00358, - -0.005773, - -0.003224, - 0.000174, - 0.007648, - 0.001879, - 0.009824, - 0.017645, - 0.028744, - 0.017, - 0.03027, - 0.041444, - 0.053666 + -0.091814, + -0.086608, + -0.081194, + -0.071274, + -0.056776, + -0.078095, + -0.071978, + -0.063704, + -0.053213, + -0.061036, + -0.056415, + -0.05259, + -0.049381, + -0.049961, + -0.045256, + -0.04266, + -0.040918 ], "mean_abs_delta": [ - 0.213292, - 0.21315, - 0.213064, - 0.212925, - 0.21271, - 0.212861, - 0.212615, - 0.212311, - 0.211911, - 0.211981, - 0.211575, - 0.211535, - 0.212126, - 0.21108, - 0.211846, - 0.213461, - 0.21583 + 0.215423, + 0.213271, + 0.21127, + 0.207819, + 0.203319, + 0.209965, + 0.2078, + 0.204945, + 0.201565, + 0.203618, + 0.201919, + 0.200678, + 0.199856, + 0.199458, + 0.197915, + 0.197247, + 0.196956 ] }, { - "n": 1017, - "day": "2026-08-14", + "n": 1387, + "day": "2026-09-27", "step": [ 0.0, - 0.07309305, - 0.0708886, - 0.06746373, - 0.06293516, - 0.20178701, - 0.18496513, - 0.16956349, - 0.14622314, - 0.54395029, - 0.47835179, - 0.42497278, - 0.3933873, - 1.29079118, - 1.11832579, - 1.00047263, - 0.93534584 + 0.13277694, + 0.13062171, + 0.12782822, + 0.12422161, + 0.38694882, + 0.37886373, + 0.37264205, + 0.36680423, + 1.24036559, + 1.21573718, + 1.20314916, + 1.19559413, + 2.91378065, + 2.88427031, + 2.8693509, + 2.85729445 ], "drift": [ 0.0, - 0.00171, - 0.00352, - 0.00573, - 0.00941, - 0.00537, - 0.01105, - 0.01672, - 0.0267, - 0.01796, - 0.03158, - 0.04416, - 0.06072, - 0.03368, - 0.06324, - 0.08527, - 0.10681 + 0.00684, + 0.01369, + 0.02285, + 0.03773, + 0.01661, + 0.02852, + 0.04057, + 0.06063, + 0.03357, + 0.05326, + 0.07187, + 0.10318, + 0.05215, + 0.08074, + 0.10847, + 0.1537 ], "capped": [ 0, @@ -2367,94 +2345,94 @@ 0, 0, 0, - 0, - 0, - 0, - 0, - 35, - 28, - 27, - 24 + 1, + 1, + 1, + 1, + 197, + 189, + 183, + 178 ], "mean_delta": [ - -0.033997, - -0.033083, - -0.032061, - -0.030691, - -0.028448, - -0.030913, - -0.027661, - -0.024266, - -0.017608, - -0.022265, - -0.014321, - -0.006806, - 0.002499, - -0.008865, - 0.006105, - 0.01656, - 0.025832 + -0.072977, + -0.064596, + -0.057662, + -0.047756, + -0.03277, + -0.052273, + -0.042247, + -0.033045, + -0.022007, + -0.034081, + -0.022778, + -0.015348, + -0.008754, + -0.020496, + -0.009582, + -0.004055, + 0.000154 ], "mean_abs_delta": [ - 0.237865, - 0.237737, - 0.237584, - 0.237392, - 0.237101, - 0.23747, - 0.237036, - 0.236725, - 0.236332, - 0.23671, - 0.236348, - 0.236374, - 0.237103, - 0.236494, - 0.23732, - 0.238586, - 0.239962 + 0.234153, + 0.231081, + 0.228776, + 0.225892, + 0.222164, + 0.227098, + 0.224311, + 0.222096, + 0.219981, + 0.222008, + 0.21942, + 0.217972, + 0.216887, + 0.218708, + 0.21647, + 0.21535, + 0.214617 ] }, { - "n": 782, - "day": "2026-08-15", + "n": 989, + "day": "2026-09-28", "step": [ 0.0, - 0.05105143, - 0.04709918, - 0.04199555, - 0.03510073, - 0.13004292, - 0.1017861, - 0.08234778, - 0.0686654, - 0.28074686, - 0.22478825, - 0.18507749, - 0.14782099, - 0.65555293, - 0.46043469, - 0.34151427, - 0.23708264 + 0.13334907, + 0.13084444, + 0.12786871, + 0.12374961, + 0.38378832, + 0.37389561, + 0.36598754, + 0.35700667, + 1.19671118, + 1.16303214, + 1.14210216, + 1.12409864, + 2.72350092, + 2.68801324, + 2.67189312, + 2.66135791 ], "drift": [ 0.0, - 0.00118, - 0.00309, - 0.0054, - 0.00937, - 0.00334, - 0.0091, - 0.01512, - 0.02533, - 0.00953, - 0.02327, - 0.03684, - 0.05429, - 0.01554, - 0.04102, - 0.06802, - 0.09635 + 0.01038, + 0.01869, + 0.02813, + 0.04271, + 0.02451, + 0.03858, + 0.05158, + 0.07216, + 0.04929, + 0.07503, + 0.09402, + 0.123, + 0.07038, + 0.10352, + 0.12949, + 0.17071 ], "capped": [ 0, @@ -2466,292 +2444,94 @@ 0, 0, 0, - 0, - 0, - 0, - 0, - 13, - 7, 4, - 2 - ], - "mean_delta": [ - -0.021645, - -0.02061, - -0.019244, - -0.017492, - -0.014532, - -0.01839, - -0.014151, - -0.009748, - -0.001841, - -0.011884, - -0.001369, - 0.008715, - 0.021385, - -0.005379, - 0.015963, - 0.033387, - 0.049669 - ], - "mean_abs_delta": [ - 0.181345, - 0.181145, - 0.18091, - 0.180626, - 0.18017, - 0.180769, - 0.180115, - 0.179624, - 0.179017, - 0.179797, - 0.178949, - 0.178738, - 0.17941, - 0.179029, - 0.178991, - 0.180583, - 0.18319 - ] - }, - { - "n": 756, - "day": "2026-08-16", - "step": [ - 0.0, - 0.04780883, - 0.04495553, - 0.03995133, - 0.03440757, - 0.13295923, - 0.10577023, - 0.08685807, - 0.0707289, - 0.34714333, - 0.25140565, - 0.19970523, - 0.15146923, - 0.91328464, - 0.58876806, - 0.40649874, - 0.28196843 - ], - "drift": [ - 0.0, - 0.00237, - 0.00446, - 0.00687, - 0.01082, - 0.0068, - 0.01227, - 0.01787, - 0.02777, - 0.01864, - 0.02973, - 0.04149, - 0.05804, - 0.04125, - 0.05632, - 0.07413, - 0.10157 - ], - "capped": [ - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 11, - 3, - 3, - 0 - ], - "mean_delta": [ - -0.078631, - -0.077277, - -0.075808, - -0.073971, - -0.070846, - -0.074792, - -0.070724, - -0.066101, - -0.057693, - -0.068506, - -0.059447, - -0.048732, - -0.035315, - -0.060991, - -0.046701, - -0.028103, - -0.007035 - ], - "mean_abs_delta": [ - 0.185639, - 0.185132, - 0.184619, - 0.183987, - 0.182947, - 0.184228, - 0.182886, - 0.181397, - 0.179011, - 0.182152, - 0.17957, - 0.17682, - 0.17409, - 0.180342, - 0.176742, - 0.173368, - 0.171709 - ] - }, - { - "n": 848, - "day": "2026-08-17", - "step": [ - 0.0, - 0.0558308, - 0.05080038, - 0.04544581, - 0.03801651, - 0.14489821, - 0.11096367, - 0.09344246, - 0.07635285, - 0.33652354, - 0.2530741, - 0.21792884, - 0.1736102, - 0.68794068, - 0.54871009, - 0.45554741, - 0.37407675 - ], - "drift": [ - 0.0, - 0.00201, - 0.00461, - 0.00733, - 0.01173, - 0.00597, - 0.01257, - 0.01913, - 0.02969, - 0.01726, - 0.0309, - 0.04521, - 0.06438, - 0.03726, - 0.06387, - 0.08498, - 0.11341 - ], - "capped": [ - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 13, - 6, - 3, - 1 + 4, + 2, + 2, + 182, + 175, + 173, + 170 ], "mean_delta": [ - -0.03783, - -0.03619, - -0.034363, - -0.032334, - -0.028952, - -0.033246, - -0.028327, - -0.023285, - -0.014807, - -0.024236, - -0.013981, - -0.003355, - 0.010547, - -0.006998, - 0.010123, - 0.023707, - 0.04196 + -0.143762, + -0.132013, + -0.12422, + -0.11445, + -0.099898, + -0.115189, + -0.103814, + -0.093735, + -0.08035, + -0.08774, + -0.072903, + -0.062548, + -0.051449, + -0.066522, + -0.05072, + -0.042016, + -0.034382 ], "mean_abs_delta": [ - 0.190818, - 0.190362, - 0.189849, - 0.189308, - 0.188485, - 0.189594, - 0.188379, - 0.187249, - 0.185712, - 0.187543, - 0.185483, - 0.184258, - 0.18398, - 0.184215, - 0.183429, - 0.184094, - 0.186066 + 0.251834, + 0.247031, + 0.244054, + 0.240572, + 0.236161, + 0.240595, + 0.236981, + 0.23414, + 0.230845, + 0.232134, + 0.228633, + 0.226308, + 0.224249, + 0.226687, + 0.223205, + 0.221473, + 0.220168 ] }, { - "n": 454, - "day": "2026-08-18", + "n": 1002, + "day": "2026-09-29", "step": [ 0.0, - 0.04264899, - 0.04017388, - 0.03644714, - 0.03388629, - 0.11645878, - 0.10072397, - 0.09010919, - 0.07385865, - 0.31588276, - 0.24178546, - 0.1966936, - 0.15891182, - 0.66131289, - 0.47835376, - 0.35054658, - 0.22946556 + 0.09413893, + 0.09190975, + 0.08997826, + 0.08781625, + 0.27292635, + 0.26521267, + 0.26103082, + 0.25761413, + 0.87567712, + 0.85592157, + 0.84993359, + 0.84860367, + 2.21190045, + 2.18811738, + 2.182877, + 2.18202282 ], "drift": [ 0.0, - 0.00171, - 0.00439, - 0.00743, - 0.01223, - 0.00493, - 0.01222, - 0.01965, - 0.03117, - 0.01535, - 0.03179, - 0.0463, - 0.06717, - 0.03504, - 0.06446, - 0.08716, - 0.11527 + 0.00814, + 0.01823, + 0.02853, + 0.04391, + 0.0192, + 0.03701, + 0.05178, + 0.07392, + 0.0379, + 0.06854, + 0.09012, + 0.12078, + 0.05713, + 0.09532, + 0.12383, + 0.16785 ], "capped": [ 0, @@ -2767,90 +2547,90 @@ 0, 0, 0, - 8, - 1, - 1, - 0 + 140, + 138, + 134, + 135 ], "mean_delta": [ - -0.065056, - -0.063888, - -0.062266, - -0.060534, - -0.057736, - -0.061668, - -0.057426, - -0.053271, - -0.046597, - -0.055009, - -0.046043, - -0.037613, - -0.026444, - -0.04393, - -0.027723, - -0.015299, - -0.000573 + -0.09698, + -0.086293, + -0.076804, + -0.066992, + -0.053293, + -0.071843, + -0.057373, + -0.046638, + -0.032949, + -0.050082, + -0.031224, + -0.020706, + -0.010655, + -0.033963, + -0.017253, + -0.009769, + -0.004676 ], "mean_abs_delta": [ - 0.263289, - 0.262836, - 0.262254, - 0.261622, - 0.260693, - 0.262006, - 0.260626, - 0.259372, - 0.25756, - 0.25982, - 0.257263, - 0.25544, - 0.253375, - 0.256353, - 0.253108, - 0.251597, - 0.250156 + 0.22659, + 0.2224, + 0.218908, + 0.215701, + 0.211942, + 0.217415, + 0.212997, + 0.21039, + 0.208019, + 0.211532, + 0.207496, + 0.206094, + 0.205588, + 0.208068, + 0.205317, + 0.204714, + 0.204999 ] }, { - "n": 517, - "day": "2026-08-19", + "n": 1071, + "day": "2026-09-30", "step": [ 0.0, - 0.0474315, - 0.04435627, - 0.04203624, - 0.03866324, - 0.13560452, - 0.1174881, - 0.11074909, - 0.09387226, - 0.3948683, - 0.32294493, - 0.26289569, - 0.19850024, - 0.97131711, - 0.63472109, - 0.49923458, - 0.3656384 + 0.10194504, + 0.10050518, + 0.09937599, + 0.09825005, + 0.30133916, + 0.29625166, + 0.29364143, + 0.29202814, + 0.98718286, + 0.97197545, + 0.96722052, + 0.96552225, + 2.5882594, + 2.56420826, + 2.55513859, + 2.5457696 ], "drift": [ 0.0, - 0.00078, - 0.00324, - 0.00651, - 0.01163, - 0.00211, - 0.00898, - 0.0168, - 0.02923, - 0.00666, - 0.02246, - 0.03943, - 0.06472, - 0.01641, - 0.0475, - 0.07743, - 0.10934 + 0.00692, + 0.01731, + 0.02815, + 0.04419, + 0.01692, + 0.03471, + 0.0504, + 0.07369, + 0.0354, + 0.06334, + 0.08641, + 0.11969, + 0.06149, + 0.09644, + 0.12773, + 0.17537 ], "capped": [ 0, @@ -2866,90 +2646,90 @@ 0, 0, 0, - 35, - 9, - 4, - 3 + 131, + 129, + 125, + 126 ], "mean_delta": [ - 0.008636, - 0.009295, - 0.010871, - 0.012725, - 0.015575, - 0.010438, - 0.014863, - 0.019304, - 0.026056, - 0.014393, - 0.024352, - 0.033481, - 0.045581, - 0.021344, - 0.040674, - 0.054498, - 0.068201 + -0.067915, + -0.060281, + -0.050204, + -0.040243, + -0.02649, + -0.050151, + -0.034241, + -0.022738, + -0.008726, + -0.036083, + -0.017108, + -0.006789, + 0.002434, + -0.026464, + -0.011705, + -0.005099, + -0.000143 ], "mean_abs_delta": [ - 0.254107, - 0.253952, - 0.253632, - 0.253282, - 0.252768, - 0.253692, - 0.252861, - 0.252089, - 0.251132, - 0.252837, - 0.251192, - 0.250173, - 0.249123, - 0.251433, - 0.24925, - 0.248184, - 0.247839 + 0.205825, + 0.203874, + 0.201727, + 0.200135, + 0.198375, + 0.201546, + 0.199148, + 0.197769, + 0.196989, + 0.198831, + 0.196391, + 0.195852, + 0.195874, + 0.196059, + 0.194181, + 0.193739, + 0.193529 ] }, { - "n": 501, - "day": "2026-08-20", + "n": 609, + "day": "2026-10-01", "step": [ 0.0, - 0.0512835, - 0.04814666, - 0.04310128, - 0.03719705, - 0.14805485, - 0.12118479, - 0.09997414, - 0.0662997, - 0.4322925, - 0.30496283, - 0.18570551, - 0.11847896, - 1.07113278, - 0.52280753, - 0.32041808, - 0.22553451 + 0.05926515, + 0.05865917, + 0.05822444, + 0.05794076, + 0.17634914, + 0.17416903, + 0.17318638, + 0.17284884, + 0.58035974, + 0.57446971, + 0.57139737, + 0.56842638, + 1.51804069, + 1.51206034, + 1.50497408, + 1.49569031 ], "drift": [ 0.0, - 0.00121, - 0.00347, - 0.00674, - 0.01217, - 0.00332, - 0.00938, - 0.01713, - 0.02928, - 0.00984, - 0.0231, - 0.03693, - 0.06333, - 0.02102, - 0.04091, - 0.0694, - 0.10466 + 0.0044, + 0.01461, + 0.02607, + 0.04328, + 0.01084, + 0.02906, + 0.04623, + 0.07156, + 0.02358, + 0.05323, + 0.0794, + 0.11688, + 0.04212, + 0.0843, + 0.12078, + 0.17435 ], "capped": [ 0, @@ -2965,90 +2745,90 @@ 0, 0, 0, - 47, - 9, - 3, - 3 + 97, + 100, + 95, + 95 ], "mean_delta": [ - -0.074604, - -0.073799, - -0.072062, - -0.06974, - -0.066028, - -0.072373, - -0.067635, - -0.062199, - -0.053684, - -0.067778, - -0.057609, - -0.047488, - -0.030146, - -0.060298, - -0.044217, - -0.025152, - -0.005762 + -0.028565, + -0.023441, + -0.013502, + -0.003131, + 0.011305, + -0.016516, + -0.000566, + 0.011606, + 0.026104, + -0.007081, + 0.010331, + 0.020195, + 0.027956, + -0.003297, + 0.009381, + 0.015558, + 0.019243 ], "mean_abs_delta": [ - 0.248258, - 0.248032, - 0.247679, - 0.247237, - 0.246558, - 0.247613, - 0.246757, - 0.245898, - 0.244698, - 0.246351, - 0.24496, - 0.243951, - 0.242646, - 0.244344, - 0.243456, - 0.242858, - 0.242606 + 0.191003, + 0.190211, + 0.189107, + 0.188513, + 0.188399, + 0.189233, + 0.188107, + 0.187631, + 0.187441, + 0.187589, + 0.18638, + 0.185337, + 0.184084, + 0.185555, + 0.183718, + 0.182369, + 0.180954 ] }, { - "n": 516, - "day": "2026-08-21", + "n": 696, + "day": "2026-10-02", "step": [ 0.0, - 0.06285724, - 0.06112776, - 0.05891028, - 0.0567097, - 0.18427345, - 0.17233692, - 0.16541138, - 0.14897121, - 0.57162594, - 0.50935302, - 0.43821617, - 0.34439457, - 1.37686233, - 1.17150963, - 0.89756371, - 0.65063914 + 0.07735933, + 0.07594488, + 0.07460486, + 0.07319296, + 0.22883225, + 0.22280266, + 0.21882868, + 0.21551559, + 0.74457603, + 0.7255056, + 0.71500672, + 0.70750136, + 1.94697659, + 1.92121591, + 1.90359487, + 1.88871225 ], "drift": [ 0.0, - 0.00261, - 0.00523, - 0.00853, - 0.0142, - 0.00755, - 0.01451, - 0.02251, - 0.03492, - 0.02328, - 0.04082, - 0.05336, - 0.07391, - 0.05428, - 0.07838, - 0.0956, - 0.12411 + 0.00394, + 0.01349, + 0.02511, + 0.04299, + 0.00992, + 0.02694, + 0.04418, + 0.07025, + 0.02178, + 0.04877, + 0.07491, + 0.11393, + 0.03718, + 0.07718, + 0.11427, + 0.17033 ], "capped": [ 0, @@ -3064,90 +2844,90 @@ 0, 0, 0, - 48, - 37, - 19, - 9 + 105, + 93, + 89, + 88 ], "mean_delta": [ - -0.147489, - -0.146347, - -0.145006, - -0.143132, - -0.140017, - -0.144252, - -0.14068, - -0.136245, - -0.129341, - -0.137658, - -0.1296, - -0.122658, - -0.109237, - -0.125454, - -0.115666, - -0.103236, - -0.087733 + -0.055935, + -0.050386, + -0.039228, + -0.026658, + -0.009126, + -0.042142, + -0.024303, + -0.009323, + 0.008578, + -0.027495, + -0.009075, + 0.003315, + 0.014152, + -0.017166, + -0.00323, + 0.004943, + 0.011467 ], "mean_abs_delta": [ - 0.287041, - 0.286704, - 0.286389, - 0.285947, - 0.285183, - 0.286079, - 0.285216, - 0.284184, - 0.282691, - 0.284046, - 0.282292, - 0.281063, - 0.278676, - 0.280197, - 0.278503, - 0.276538, - 0.275205 + 0.227442, + 0.226246, + 0.224205, + 0.22251, + 0.220733, + 0.224583, + 0.221964, + 0.220058, + 0.217969, + 0.222115, + 0.21898, + 0.216752, + 0.214464, + 0.219752, + 0.216397, + 0.214184, + 0.212456 ] }, { - "n": 984, - "day": "2026-08-22", + "n": 558, + "day": "2026-10-03", "step": [ 0.0, - 0.07079557, - 0.06621022, - 0.06018513, - 0.05257794, - 0.19193424, - 0.16152149, - 0.13697887, - 0.11283919, - 0.48820673, - 0.35875452, - 0.31211081, - 0.24346503, - 1.00120737, - 0.7723155, - 0.54850571, - 0.24029123 + 0.06263698, + 0.0621514, + 0.06166871, + 0.06117463, + 0.18702926, + 0.18505066, + 0.18377459, + 0.18297868, + 0.61843144, + 0.61397421, + 0.61255781, + 0.61330184, + 1.63537717, + 1.63550191, + 1.63811187, + 1.64384677 ], "drift": [ 0.0, - 0.00287, - 0.00597, - 0.00952, - 0.01553, - 0.00788, - 0.01613, - 0.02413, - 0.03714, - 0.02219, - 0.04046, - 0.05341, - 0.07458, - 0.04108, - 0.06905, - 0.08551, - 0.1208 + 0.00309, + 0.01177, + 0.02352, + 0.04229, + 0.00805, + 0.02373, + 0.04129, + 0.06861, + 0.01967, + 0.04418, + 0.07033, + 0.11117, + 0.03786, + 0.07383, + 0.10988, + 0.16699 ], "capped": [ 0, @@ -3163,90 +2943,90 @@ 0, 0, 0, - 29, - 24, - 18, - 5 + 84, + 86, + 89, + 88 ], "mean_delta": [ - -0.062656, - -0.060646, - -0.058451, - -0.055591, - -0.050791, - -0.057183, - -0.051396, - -0.044593, - -0.033894, - -0.046839, - -0.033256, - -0.023211, - -0.005956, - -0.029865, - -0.016195, - -0.004611, - 0.020509 + -0.039456, + -0.036591, + -0.028316, + -0.017888, + -0.002646, + -0.032502, + -0.019147, + -0.006858, + 0.008475, + -0.026261, + -0.013511, + -0.004735, + 0.003525, + -0.022147, + -0.014655, + -0.01061, + -0.006348 ], "mean_abs_delta": [ - 0.184449, - 0.183733, - 0.183, - 0.182128, - 0.180773, - 0.182562, - 0.180852, - 0.179072, - 0.17683, - 0.179495, - 0.17625, - 0.174865, - 0.174033, - 0.17545, - 0.173488, - 0.172961, - 0.174931 + 0.226071, + 0.225439, + 0.22409, + 0.222595, + 0.221284, + 0.22461, + 0.222586, + 0.221257, + 0.22087, + 0.223118, + 0.221433, + 0.220834, + 0.221022, + 0.221351, + 0.22072, + 0.221111, + 0.222415 ] }, { - "n": 738, - "day": "2026-08-23", + "n": 826, + "day": "2026-10-04", "step": [ 0.0, - 0.04704602, - 0.04141123, - 0.03620002, - 0.03101032, - 0.12249323, - 0.09463202, - 0.07637467, - 0.06118426, - 0.28513528, - 0.19911735, - 0.15631127, - 0.11387271, - 0.59940241, - 0.38841791, - 0.33015157, - 0.22280503 + 0.09874968, + 0.09753729, + 0.09601532, + 0.09398026, + 0.29359792, + 0.2878481, + 0.28260711, + 0.27691978, + 0.95724335, + 0.93780841, + 0.92390935, + 0.91154243, + 2.32348453, + 2.28884165, + 2.26555067, + 2.24703477 ], "drift": [ 0.0, - 0.00231, - 0.00581, - 0.00983, - 0.01631, - 0.00643, - 0.01581, - 0.02458, - 0.03834, - 0.01784, - 0.03734, - 0.05325, - 0.07589, - 0.03373, - 0.06374, - 0.08371, - 0.12176 + 0.00379, + 0.01166, + 0.02308, + 0.04205, + 0.01006, + 0.02409, + 0.04065, + 0.06786, + 0.0238, + 0.04618, + 0.07072, + 0.1107, + 0.04384, + 0.07735, + 0.11025, + 0.16542 ], "capped": [ 0, @@ -3258,94 +3038,94 @@ 0, 0, 0, - 0, - 0, - 0, - 0, - 19, - 7, + 2, + 2, + 2, 3, - 1 + 157, + 148, + 149, + 150 ], "mean_delta": [ - -0.04995, - -0.047962, - -0.04565, - -0.042778, - -0.037946, - -0.044612, - -0.038553, - -0.032108, - -0.021452, - -0.035592, - -0.023722, - -0.013256, - 0.004438, - -0.024243, - -0.010924, - 0.000147, - 0.030133 + -0.049581, + -0.04598, + -0.037616, + -0.026184, + -0.008877, + -0.040239, + -0.026648, + -0.012682, + 0.005256, + -0.029238, + -0.014891, + -0.003499, + 0.007922, + -0.019355, + -0.005969, + 0.002579, + 0.010667 ], "mean_abs_delta": [ - 0.185658, - 0.185118, - 0.184506, - 0.183816, - 0.182837, - 0.184212, - 0.182902, - 0.181814, - 0.180355, - 0.182238, - 0.180339, - 0.179151, - 0.179191, - 0.180085, - 0.178329, - 0.177767, - 0.181409 + 0.252912, + 0.251617, + 0.248853, + 0.245207, + 0.240312, + 0.249592, + 0.245098, + 0.240822, + 0.236063, + 0.245219, + 0.240462, + 0.236945, + 0.233666, + 0.240948, + 0.236375, + 0.233339, + 0.230816 ] }, { - "n": 556, - "day": "2026-08-24", + "n": 888, + "day": "2026-10-05", "step": [ 0.0, - 0.05199018, - 0.04871911, - 0.04527927, - 0.040009, - 0.14488231, - 0.12315972, - 0.10906305, - 0.09013911, - 0.40353722, - 0.30657548, - 0.24763644, - 0.19324898, - 0.99032098, - 0.67078023, - 0.56913058, - 0.32789244 + 0.09409228, + 0.09432181, + 0.09478614, + 0.09578325, + 0.28232353, + 0.28357013, + 0.28556016, + 0.28895636, + 0.93958214, + 0.94359298, + 0.94862612, + 0.9555521, + 2.33173234, + 2.33359715, + 2.34122151, + 2.35177652 ], "drift": [ 0.0, - 0.00217, - 0.00579, - 0.01006, + 0.00241, + 0.00845, + 0.01926, + 0.03843, + 0.00687, 0.01708, - 0.00624, - 0.0155, - 0.02514, - 0.03965, - 0.01756, - 0.03639, - 0.05457, - 0.07878, - 0.03532, - 0.06671, - 0.08866, - 0.12579 + 0.0324, + 0.06006, + 0.02122, + 0.03837, + 0.06042, + 0.10108, + 0.0435, + 0.06998, + 0.09878, + 0.1533 ], "capped": [ 0, @@ -3359,290 +3139,92 @@ 0, 0, 0, - 0, - 0, - 22, - 11, - 10, - 8 + 1, + 1, + 164, + 160, + 159, + 159 ], "mean_delta": [ - -0.072144, - -0.070691, - -0.068244, - -0.065315, - -0.06047, - -0.068015, - -0.061636, - -0.055142, - -0.045122, - -0.06035, - -0.048188, - -0.036675, - -0.020267, - -0.049637, - -0.031801, - -0.020553, - 0.004401 + 0.039845, + 0.040729, + 0.047845, + 0.058335, + 0.074857, + 0.041355, + 0.05247, + 0.064246, + 0.0796, + 0.037206, + 0.04687, + 0.054502, + 0.06201, + 0.033405, + 0.041253, + 0.045839, + 0.050753 ], "mean_abs_delta": [ - 0.236508, - 0.236001, - 0.235204, - 0.234242, - 0.232657, - 0.235065, - 0.233036, - 0.231087, - 0.228547, - 0.232386, - 0.229153, - 0.226289, - 0.223166, - 0.228804, - 0.224558, - 0.222339, - 0.220227 + 0.227335, + 0.227219, + 0.227025, + 0.226991, + 0.227596, + 0.227035, + 0.226736, + 0.226895, + 0.227964, + 0.2267, + 0.226228, + 0.226488, + 0.227721, + 0.226258, + 0.226056, + 0.226888, + 0.22869 ] }, { - "n": 644, - "day": "2026-08-25", + "n": 720, + "day": "2026-10-06", "step": [ 0.0, - 0.04470174, - 0.0406165, - 0.03516736, - 0.02863363, - 0.12024254, - 0.0903945, - 0.07167589, - 0.05667827, - 0.29734376, - 0.2064007, - 0.15222247, - 0.12518414, - 0.65988753, - 0.42234057, - 0.33745672, - 0.25093043 + 0.07442262, + 0.07390338, + 0.07311527, + 0.07213194, + 0.22239294, + 0.2203837, + 0.21838889, + 0.21678979, + 0.73532286, + 0.73350849, + 0.73325055, + 0.73713676, + 1.92132994, + 1.92265451, + 1.92718839, + 1.93436573 ], "drift": [ 0.0, - 0.00154, - 0.00507, - 0.00953, - 0.01706, - 0.00428, - 0.01341, - 0.02367, - 0.03932, + 0.00177, + 0.00742, + 0.01781, + 0.03736, + 0.00472, + 0.01473, + 0.02953, + 0.0576, 0.01205, - 0.03114, - 0.05043, - 0.07709, - 0.02609, - 0.05574, - 0.08201, - 0.12521 - ], - "capped": [ - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 15, - 9, - 7, - 3 - ], - "mean_delta": [ - -0.040169, - -0.038763, - -0.036122, - -0.032954, - -0.027635, - -0.036294, - -0.029576, - -0.022216, - -0.011144, - -0.029791, - -0.015745, - -0.002789, - 0.014981, - -0.019097, - 0.001565, - 0.014984, - 0.040777 - ], - "mean_abs_delta": [ - 0.196488, - 0.196209, - 0.195725, - 0.195222, - 0.1946, - 0.19574, - 0.194742, - 0.194119, - 0.194085, - 0.194747, - 0.19374, - 0.193818, - 0.1968, - 0.19402, - 0.194103, - 0.195771, - 0.203331 - ] - }, - { - "n": 1525, - "day": "2026-08-26", - "step": [ - 0.0, - 0.09167986, - 0.08027399, - 0.07324188, - 0.06339315, - 0.2489593, - 0.20470223, - 0.17083082, - 0.13164434, - 0.70956551, - 0.52298793, - 0.3795703, - 0.24847017, - 1.64649651, - 1.0648022, - 0.72603168, - 0.38225164 - ], - "drift": [ - 0.0, - 0.0021, - 0.00533, - 0.00984, - 0.01784, - 0.00635, - 0.01422, - 0.02464, - 0.04067, - 0.01809, - 0.0348, - 0.05169, - 0.07962, - 0.04288, - 0.06292, - 0.08695, - 0.126 - ], - "capped": [ - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 46, - 27, - 17, - 8 - ], - "mean_delta": [ - -0.0269, - -0.026158, - -0.023768, - -0.02057, - -0.014999, - -0.024668, - -0.018612, - -0.011086, - 0.000449, - -0.020785, - -0.00779, - 0.005347, - 0.025772, - -0.011685, - 0.00583, - 0.024162, - 0.051822 - ], - "mean_abs_delta": [ - 0.191201, - 0.190981, - 0.190542, - 0.190028, - 0.189271, - 0.190537, - 0.189615, - 0.188747, - 0.187952, - 0.189384, - 0.188105, - 0.187555, - 0.188566, - 0.187303, - 0.187031, - 0.187802, - 0.193864 - ] - }, - { - "n": 1375, - "day": "2026-08-27", - "step": [ - 0.0, - 0.08118152, - 0.06985468, - 0.05589185, - 0.03914406, - 0.1985926, - 0.13376636, - 0.09940184, - 0.07505503, - 0.42244928, - 0.28212476, - 0.21233934, - 0.16099011, - 0.82615748, - 0.60867617, - 0.46572436, - 0.33367715 - ], - "drift": [ - 0.0, - 0.00291, - 0.00632, - 0.01077, - 0.01872, - 0.00782, - 0.01525, - 0.02565, - 0.04233, - 0.01814, - 0.03675, - 0.05328, - 0.08136, - 0.04126, - 0.06447, - 0.08797, - 0.12769 + 0.03007, + 0.05286, + 0.09557, + 0.02289, + 0.05089, + 0.08233, + 0.14169 ], "capped": [ 0, @@ -3658,90 +3240,90 @@ 0, 0, 0, - 5, - 3, - 3, - 1 + 98, + 100, + 104, + 108 ], "mean_delta": [ - -0.048981, - -0.047106, - -0.044455, - -0.040856, - -0.034472, - -0.043466, - -0.03739, - -0.028902, - -0.015966, - -0.034957, - -0.020119, - -0.008064, - 0.012922, - -0.015645, - 0.000754, - 0.015879, - 0.041156 + -0.03252, + -0.030238, + -0.024112, + -0.01339, + 0.004655, + -0.026526, + -0.017717, + -0.006543, + 0.008825, + -0.017932, + -0.012552, + -0.007655, + -0.002437, + -0.005788, + -0.001286, + 0.000941, + 0.002738 ], "mean_abs_delta": [ - 0.173538, - 0.173107, - 0.172563, - 0.171906, - 0.170879, - 0.172345, - 0.171306, - 0.170143, - 0.169173, - 0.170896, - 0.169355, - 0.169006, - 0.17, - 0.169029, - 0.169379, - 0.170392, - 0.174663 + 0.221115, + 0.220763, + 0.219754, + 0.21816, + 0.215927, + 0.220203, + 0.218961, + 0.217552, + 0.216113, + 0.219026, + 0.218868, + 0.21881, + 0.219318, + 0.216911, + 0.217283, + 0.217852, + 0.218983 ] }, { - "n": 467, - "day": "2026-08-28", + "n": 656, + "day": "2026-10-07", "step": [ 0.0, - 0.04375223, - 0.03980578, - 0.03540602, - 0.03030321, - 0.11834353, - 0.09898567, - 0.08402507, - 0.06656402, - 0.33475573, - 0.26345799, - 0.20212572, - 0.14414459, - 0.72334079, - 0.5348303, - 0.38463634, - 0.21109113 + 0.06619672, + 0.06549448, + 0.06432556, + 0.06277903, + 0.19694437, + 0.19383499, + 0.19051167, + 0.18687819, + 0.64482673, + 0.63616278, + 0.63036907, + 0.62679891, + 1.75893035, + 1.74544039, + 1.73679071, + 1.72847138 ], "drift": [ 0.0, - 0.00135, - 0.0048, - 0.0096, - 0.01825, - 0.00371, - 0.01181, - 0.02316, - 0.04156, - 0.00935, - 0.02977, - 0.04957, - 0.07977, - 0.02628, - 0.05585, - 0.0805, - 0.12531 + 0.00281, + 0.00805, + 0.01799, + 0.03765, + 0.00777, + 0.01711, + 0.03098, + 0.05863, + 0.02012, + 0.03597, + 0.05669, + 0.09815, + 0.03921, + 0.06338, + 0.09101, + 0.14744 ], "capped": [ 0, @@ -3757,90 +3339,90 @@ 0, 0, 0, - 30, - 20, - 14, - 9 + 79, + 82, + 80, + 79 ], "mean_delta": [ - 0.001875, - 0.003347, - 0.005717, - 0.008744, - 0.014131, - 0.005893, - 0.010976, - 0.018103, - 0.029636, - 0.010507, - 0.023759, - 0.035278, - 0.053415, - 0.022608, - 0.040238, - 0.053798, - 0.07677 + -0.065151, + -0.06304, + -0.057941, + -0.04813, + -0.030583, + -0.059579, + -0.05238, + -0.042746, + -0.029114, + -0.052174, + -0.0476, + -0.044104, + -0.040995, + -0.046854, + -0.044913, + -0.043829, + -0.042953 ], "mean_abs_delta": [ - 0.231976, - 0.23186, - 0.231583, - 0.231264, - 0.230809, - 0.231702, - 0.231147, - 0.230689, - 0.23049, - 0.231478, - 0.230825, - 0.230688, - 0.231431, - 0.231387, - 0.231255, - 0.231692, - 0.235412 + 0.219186, + 0.218356, + 0.216596, + 0.213813, + 0.210038, + 0.21696, + 0.214365, + 0.21149, + 0.207855, + 0.214052, + 0.211439, + 0.209347, + 0.206999, + 0.211129, + 0.208821, + 0.20721, + 0.205786 ] }, { - "n": 1316, - "day": "2026-08-29", + "n": 489, + "day": "2026-10-08", "step": [ 0.0, - 0.08629961, - 0.074102, - 0.0617587, - 0.04749534, - 0.2378084, - 0.17612329, - 0.13548512, - 0.10750243, - 0.68545477, - 0.43172701, - 0.33850964, - 0.21798338, - 1.37938537, - 0.94471312, - 0.66335817, - 0.37055486 + 0.04758729, + 0.04745913, + 0.04733163, + 0.04734279, + 0.1423879, + 0.14187612, + 0.14172263, + 0.14244212, + 0.47239631, + 0.47090257, + 0.47129491, + 0.47589861, + 1.30655058, + 1.30274417, + 1.30670432, + 1.31761537 ], "drift": [ 0.0, - 0.00133, - 0.00434, - 0.00902, - 0.01829, - 0.00381, - 0.01053, - 0.02242, - 0.04222, - 0.0109, - 0.02924, - 0.04937, - 0.08057, - 0.02976, - 0.05662, - 0.08318, - 0.12906 + 0.0016, + 0.00654, + 0.01618, + 0.03608, + 0.0042, + 0.01346, + 0.02722, + 0.05559, + 0.00989, + 0.02704, + 0.04911, + 0.09313, + 0.02027, + 0.04688, + 0.07734, + 0.13801 ], "capped": [ 0, @@ -3856,90 +3438,90 @@ 0, 0, 0, - 40, - 23, - 16, - 5 + 63, + 61, + 58, + 58 ], "mean_delta": [ - -0.014835, - -0.014124, - -0.011384, - -0.007726, - -0.000931, - -0.012717, - -0.006712, - 0.00231, - 0.016959, - -0.010527, - 0.005986, - 0.021256, - 0.043403, - -0.000581, - 0.02187, - 0.040534, - 0.07154 + 0.001017, + 0.003374, + 0.008615, + 0.01847, + 0.036567, + 0.007282, + 0.015568, + 0.025251, + 0.038331, + 0.015093, + 0.023941, + 0.028631, + 0.031492, + 0.020845, + 0.0295, + 0.032877, + 0.034275 ], "mean_abs_delta": [ - 0.194105, - 0.193981, - 0.193731, - 0.19348, - 0.193273, - 0.19378, - 0.193352, - 0.193213, - 0.194318, - 0.193394, - 0.193076, - 0.1947, - 0.199462, - 0.192366, - 0.194274, - 0.198236, - 0.208934 + 0.204622, + 0.204357, + 0.203914, + 0.203279, + 0.202777, + 0.203964, + 0.203343, + 0.202986, + 0.203333, + 0.203087, + 0.202589, + 0.202821, + 0.204273, + 0.202197, + 0.201757, + 0.202304, + 0.203865 ] }, { - "n": 345, - "day": "2026-08-30", + "n": 502, + "day": "2026-10-09", "step": [ 0.0, - 0.02939512, - 0.02634118, - 0.02106314, - 0.01576835, - 0.08304256, - 0.06171494, - 0.0449447, - 0.031791, - 0.23938493, - 0.14502543, - 0.09164533, - 0.07063859, - 0.49512937, - 0.26300237, - 0.20475419, - 0.13706382 + 0.04210724, + 0.04156587, + 0.04061964, + 0.03920821, + 0.12507674, + 0.12264298, + 0.12008081, + 0.11740424, + 0.40925542, + 0.40202745, + 0.39771535, + 0.39591208, + 1.13048631, + 1.12367482, + 1.12057501, + 1.11943064 ], "drift": [ 0.0, - 0.00112, - 0.00389, - 0.00859, - 0.01832, - 0.00312, - 0.00955, - 0.02165, - 0.0425, - 0.009, - 0.02738, - 0.04814, - 0.08162, - 0.02397, - 0.05393, - 0.08333, - 0.12813 + 0.00203, + 0.00642, + 0.01558, + 0.03573, + 0.00551, + 0.01372, + 0.02658, + 0.05493, + 0.0136, + 0.02719, + 0.04723, + 0.09155, + 0.02411, + 0.04607, + 0.07484, + 0.13661 ], "capped": [ 0, @@ -3955,2569 +3537,2608 @@ 0, 0, 0, - 7, - 3, - 3, - 2 + 36, + 33, + 35, + 33 ], "mean_delta": [ - -0.05648, - -0.055857, - -0.053708, - -0.050188, - -0.043223, - -0.054794, - -0.049888, - -0.040677, - -0.026103, - -0.052482, - -0.037555, - -0.022475, - -0.000264, - -0.042398, - -0.021649, - -0.001424, - 0.02677 + -0.073759, + -0.071118, + -0.065983, + -0.055813, + -0.036347, + -0.066595, + -0.058104, + -0.047477, + -0.032392, + -0.056242, + -0.047095, + -0.040388, + -0.034335, + -0.044545, + -0.037442, + -0.033248, + -0.029847 ], "mean_abs_delta": [ - 0.210476, - 0.210248, - 0.209698, - 0.208814, - 0.207283, - 0.209851, - 0.208584, - 0.20673, - 0.204709, - 0.208838, - 0.206018, - 0.204099, - 0.202998, - 0.206186, - 0.203376, - 0.202384, - 0.20401 + 0.174975, + 0.174242, + 0.172748, + 0.170111, + 0.166815, + 0.173077, + 0.170829, + 0.168616, + 0.166488, + 0.170928, + 0.169057, + 0.167969, + 0.167479, + 0.169159, + 0.168292, + 0.167735, + 0.167392 ] + } + ], + "bootstrap": { + "lines": { + "capitulation": 0.016041791066527367 }, - { - "n": 286, - "day": "2026-08-31", - "step": [ - 0.0, - 0.03179052, - 0.02905972, - 0.02426906, - 0.01884788, - 0.09156561, - 0.07276787, - 0.05196809, - 0.03524788, - 0.25354667, - 0.15317311, - 0.10622756, - 0.06961744, - 0.50010213, - 0.29178164, - 0.18970141, - 0.07194889 - ], - "drift": [ - 0.0, - 0.00087, - 0.00337, - 0.00793, - 0.01804, - 0.00248, - 0.00809, - 0.02018, - 0.04236, - 0.00704, - 0.02412, - 0.04619, - 0.08153, - 0.01897, - 0.05054, - 0.08183, - 0.12697 - ], - "capped": [ - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 19, - 11, - 4, - 1 - ], - "mean_delta": [ - -0.019745, - -0.019099, - -0.017146, - -0.013692, - -0.006285, - -0.018009, - -0.013746, - -0.004426, - 0.011453, - -0.015654, - -0.001914, - 0.014086, - 0.03815, - -0.007841, - 0.015306, - 0.037138, - 0.063849 - ], - "mean_abs_delta": [ - 0.230497, - 0.230357, - 0.230005, - 0.229447, - 0.228391, - 0.230114, - 0.229333, - 0.228066, - 0.22686, - 0.229423, - 0.227543, - 0.226424, - 0.2261, - 0.227506, - 0.22569, - 0.224916, - 0.228186 - ] + "gates_ok": true, + "calibration": { + "mean": 0.07089309626579844, + "total": 158.8714287316543, + "trades": 2241, + "per_strategy": { + "capitulation": { + "line": 0.016041791066527367, + "mean": 0.11632242810015601, + "rule": "match", + "chain": "sol", + "total": 18.844233352225274, + "trades": 162, + "hold_min": 240, + "teacher_trades": 162 + }, + "capitulation@rh": { + "line": 0.07225678861141205, + "mean": 0.0673531483306537, + "rule": "match", + "chain": "rh", + "total": 140.02719537942903, + "trades": 2079, + "hold_min": 240, + "teacher_trades": 2079 + } + } }, - { - "n": 262, - "day": "2026-09-01", - "step": [ - 0.0, - 0.02220945, - 0.01957663, - 0.01622015, - 0.01220125, - 0.06103343, - 0.04962636, - 0.03571233, - 0.02316173, - 0.17014137, - 0.11125148, - 0.06986754, - 0.02751291, - 0.38155811, - 0.19625071, - 0.08645905, - 0.04951678 - ], - "drift": [ - 0.0, - 0.0009, - 0.00329, - 0.00772, - 0.01794, - 0.00279, - 0.00799, - 0.01925, - 0.04183, - 0.00778, - 0.02225, - 0.04328, - 0.0806, - 0.01858, - 0.04371, - 0.07682, - 0.12559 - ], - "capped": [ - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 5, - 1, - 0, - 0 - ], - "mean_delta": [ - -0.050607, - -0.049948, - -0.048432, - -0.045629, - -0.039372, - -0.048666, - -0.045356, - -0.038031, - -0.024472, - -0.045246, - -0.035568, - -0.022469, - -0.001467, - -0.037581, - -0.021128, - -0.002308, - 0.020669 - ], - "mean_abs_delta": [ - 0.209404, - 0.209058, - 0.208367, - 0.207159, - 0.204585, - 0.208395, - 0.206905, - 0.203939, - 0.1999, - 0.206587, - 0.202618, - 0.199088, - 0.194615, - 0.20279, - 0.197804, - 0.194306, - 0.193425 - ] + "diagnostics": { + "rows": 15402, + "slope": 1.0249917184344512, + "kc_active": 0.10003621876239777, + "rank_corr": 0.9010552624530308, + "mbon_share": 0.340462761402956, + "mbon_saturated": 0.02083333395421505 }, - { - "n": 215, - "day": "2026-09-02", - "step": [ - 0.0, - 0.01950953, - 0.01855094, - 0.01688687, - 0.01535418, - 0.05658671, - 0.05216318, - 0.04597486, - 0.03700255, - 0.17603072, - 0.1492127, - 0.11986204, - 0.08855332, - 0.43408566, - 0.33790854, - 0.25116014, - 0.12548558 - ], - "drift": [ - 0.0, - 0.00085, - 0.00301, - 0.00738, - 0.01788, - 0.00263, - 0.00749, - 0.01871, - 0.04187, - 0.00813, - 0.02176, - 0.04291, - 0.08015, - 0.02065, - 0.04383, - 0.07437, - 0.12557 - ], - "capped": [ - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 15, - 11, - 6, - 0 - ], - "mean_delta": [ - -0.01788, - -0.017634, - -0.016312, - -0.013712, - -0.007847, - -0.017064, - -0.014105, - -0.007431, - 0.005483, - -0.015539, - -0.007046, - 0.005721, - 0.026103, - -0.012387, - 0.002943, - 0.022069, - 0.04647 - ], - "mean_abs_delta": [ - 0.230125, - 0.230057, - 0.229883, - 0.229648, - 0.22928, - 0.229913, - 0.229551, - 0.229102, - 0.228939, - 0.229479, - 0.228688, - 0.228447, - 0.229273, - 0.228839, - 0.227682, - 0.228334, - 0.231888 - ] - }, - { - "n": 195, - "day": "2026-09-03", - "step": [ - 0.0, - 0.01902172, - 0.01621971, - 0.01246771, - 0.00844851, - 0.05370394, - 0.03817254, - 0.02533463, - 0.01299596, - 0.14095247, - 0.08077155, - 0.04582197, - 0.02751261, - 0.26999788, - 0.14341474, - 0.08500843, - 0.0433968 - ], - "drift": [ - 0.0, - 0.00066, - 0.00252, - 0.00666, - 0.01735, - 0.00195, - 0.00609, - 0.01648, - 0.04075, - 0.00533, - 0.01631, - 0.03798, - 0.07851, - 0.01073, - 0.03308, - 0.06679, - 0.12382 - ], - "capped": [ - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 7, - 2, - 2, - 2 - ], - "mean_delta": [ - -0.074095, - -0.073693, - -0.07247, - -0.069856, - -0.063262, - -0.072899, - -0.070276, - -0.063475, - -0.048718, - -0.070584, - -0.063795, - -0.050275, - -0.026565, - -0.068727, - -0.055076, - -0.034139, - -0.003212 - ], - "mean_abs_delta": [ - 0.235281, - 0.235175, - 0.234946, - 0.234486, - 0.233372, - 0.234967, - 0.234529, - 0.233359, - 0.230828, - 0.234402, - 0.23328, - 0.231155, - 0.227299, - 0.233989, - 0.23181, - 0.228659, - 0.225279 - ] - }, - { - "n": 301, - "day": "2026-09-04", - "step": [ - 0.0, - 0.02933012, - 0.02813112, - 0.02632646, - 0.02038493, - 0.0870515, - 0.07975674, - 0.06365537, - 0.03483518, - 0.27816772, - 0.23210254, - 0.1298645, - 0.05609047, - 0.80538895, - 0.49541149, - 0.23193107, - 0.08924744 - ], - "drift": [ - 0.0, - 0.0007, - 0.00222, - 0.00616, - 0.01692, - 0.00198, - 0.00539, - 0.01488, - 0.03981, - 0.006, - 0.01373, - 0.03476, - 0.07505, - 0.01537, - 0.0292, - 0.05698, - 0.11641 - ], - "capped": [ - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 26, - 9, - 2, - 3 - ], - "mean_delta": [ - -0.008926, - -0.008796, - -0.007617, - -0.004779, - 0.002813, - -0.008594, - -0.005971, - 0.001216, - 0.01878, - -0.008111, - -0.001724, - 0.014758, - 0.041606, - -0.008391, - 0.00581, - 0.029073, - 0.065003 - ], - "mean_abs_delta": [ - 0.231664, - 0.231597, - 0.231179, - 0.23021, - 0.227716, - 0.23149, - 0.230585, - 0.228231, - 0.223005, - 0.231199, - 0.229116, - 0.22411, - 0.219352, - 0.230975, - 0.226723, - 0.221188, - 0.218476 - ] - }, - { - "n": 133, - "day": "2026-09-05", - "step": [ - 0.0, - 0.00975421, - 0.00895728, - 0.00811434, - 0.0072122, - 0.02755716, - 0.02465262, - 0.0218246, - 0.01928112, - 0.08333245, - 0.07514489, - 0.06559157, - 0.05174995, - 0.2404971, - 0.21020005, - 0.16684518, - 0.12141792 - ], - "drift": [ - 0.0, - 0.00029, - 0.00168, - 0.00549, - 0.01652, - 0.00087, - 0.00411, - 0.01339, - 0.0387, - 0.00284, - 0.01077, - 0.03097, - 0.07288, - 0.00765, - 0.02306, - 0.05126, - 0.11268 - ], - "capped": [ - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 0, - 0, - 0 - ], - "mean_delta": [ - -0.017264, - -0.017125, - -0.016208, - -0.013737, - -0.006923, - -0.016843, - -0.014788, - -0.008722, - 0.006836, - -0.01581, - -0.011096, - 0.002354, - 0.026787, - -0.013528, - -0.004682, - 0.013665, - 0.046423 - ], - "mean_abs_delta": [ - 0.204211, - 0.204155, - 0.203775, - 0.202752, - 0.200191, - 0.204046, - 0.203193, - 0.200857, - 0.194883, - 0.203664, - 0.201818, - 0.1965, - 0.190329, - 0.202774, - 0.199263, - 0.19261, - 0.188486 - ] - }, - { - "n": 246, - "day": "2026-09-06", - "step": [ - 0.0, - 0.02326191, - 0.02235886, - 0.01986652, - 0.01598883, - 0.0693392, - 0.06285499, - 0.05081196, - 0.03662, - 0.22246512, - 0.19159761, - 0.14067066, - 0.09014761, - 0.66139445, - 0.5137931, - 0.35138259, - 0.13267747 - ], - "drift": [ - 0.0, - 0.00035, - 0.00113, - 0.00457, - 0.01564, - 0.00104, - 0.00284, - 0.01097, - 0.03593, - 0.00366, - 0.0077, - 0.02399, - 0.06572, - 0.00903, - 0.01867, - 0.03506, - 0.10298 - ], - "capped": [ - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 21, - 17, - 11, - 6 - ], - "mean_delta": [ - 0.040899, - 0.040892, - 0.041702, - 0.044247, - 0.051908, - 0.040874, - 0.042741, - 0.04899, - 0.066231, - 0.040774, - 0.045132, - 0.058354, - 0.085991, - 0.040447, - 0.048248, - 0.064342, - 0.106088 - ], - "mean_abs_delta": [ - 0.235625, - 0.235633, - 0.235618, - 0.23554, - 0.235368, - 0.23566, - 0.235609, - 0.235449, - 0.236186, - 0.23576, - 0.235638, - 0.235688, - 0.239191, - 0.235951, - 0.235605, - 0.236632, - 0.243836 - ] - }, - { - "n": 137, - "day": "2026-09-07", - "step": [ - 0.0, - 0.01009121, - 0.00953655, - 0.00757871, - 0.0049945, - 0.03115872, - 0.0281916, - 0.01804591, - 0.00879754, - 0.11144284, - 0.08926339, - 0.04007388, - 0.01630277, - 0.33589161, - 0.25725566, - 0.12486059, - 0.01063137 - ], - "drift": [ - 0.0, - 0.00026, - 0.00095, - 0.00418, - 0.0153, - 0.0008, - 0.00245, - 0.01007, - 0.03514, - 0.00269, - 0.00692, - 0.02212, - 0.06434, - 0.00599, - 0.01722, - 0.03329, - 0.1007 - ], - "capped": [ - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 1, - 1, - 0 - ], - "mean_delta": [ - -0.016121, - -0.016262, - -0.015603, - -0.012912, - -0.003762, - -0.016562, - -0.015234, - -0.008608, - 0.011637, - -0.017701, - -0.01511, - -0.001719, - 0.031751, - -0.016158, - -0.014891, - -0.003135, - 0.05433 - ], - "mean_abs_delta": [ - 0.193758, - 0.193774, - 0.193658, - 0.193206, - 0.191866, - 0.193804, - 0.193558, - 0.192493, - 0.190242, - 0.193997, - 0.193409, - 0.191422, - 0.189044, - 0.193852, - 0.193063, - 0.191096, - 0.190321 - ] - }, - { - "n": 167, - "day": "2026-09-08", - "step": [ - 0.0, - 0.01312416, - 0.01311906, - 0.01243212, - 0.01101319, - 0.04037402, - 0.03838226, - 0.03498283, - 0.0288442, - 0.13334417, - 0.12410463, - 0.10502601, - 0.0544116, - 0.36169737, - 0.34682685, - 0.29105325, - 0.09765011 - ], - "drift": [ - 0.0, - 0.00039, - 0.00108, - 0.00412, - 0.01519, - 0.00114, - 0.00285, - 0.00995, - 0.0349, - 0.00361, - 0.00798, - 0.02222, - 0.06319, - 0.0101, - 0.02004, - 0.0353, - 0.09691 - ], - "capped": [ - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 10, - 9, - 8, - 0 - ], - "mean_delta": [ - -0.068602, - -0.068583, - -0.068238, - -0.066254, - -0.059141, - -0.068598, - -0.067828, - -0.063003, - -0.046731, - -0.068612, - -0.067087, - -0.056897, - -0.029587, - -0.066652, - -0.064929, - -0.054877, - -0.011713 - ], - "mean_abs_delta": [ - 0.195573, - 0.195531, - 0.195511, - 0.195489, - 0.195772, - 0.195447, - 0.195369, - 0.195454, - 0.196446, - 0.195167, - 0.194961, - 0.195327, - 0.197649, - 0.194516, - 0.19387, - 0.194313, - 0.20038 - ] - }, - { - "n": 65, - "day": "2026-09-09", - "step": [ - 0.0, - 0.00665872, - 0.00665723, - 0.00619565, - 0.00542629, - 0.01997365, - 0.01974988, - 0.01710868, - 0.01312009, - 0.06583992, - 0.05978317, - 0.04799426, - 0.02451812, - 0.18086942, - 0.14875548, - 0.13515674, - 0.04138675 - ], - "drift": [ - 0.0, - 0.00026, - 0.00086, - 0.00384, - 0.0149, - 0.00074, - 0.00221, - 0.00914, - 0.03426, - 0.00235, - 0.00598, - 0.02019, - 0.06202, - 0.00729, - 0.01507, - 0.03144, - 0.09584 - ], - "capped": [ - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 3, - 2, - 2, - 2 - ], - "mean_delta": [ - -0.016557, - -0.016481, - -0.01612, - -0.014263, - -0.007485, - -0.016365, - -0.015528, - -0.01113, - 0.004215, - -0.016017, - -0.014099, - -0.004853, - 0.019949, - -0.014279, - -0.010224, - -0.001068, - 0.034953 - ], - "mean_abs_delta": [ - 0.244374, - 0.244347, - 0.244295, - 0.243738, - 0.241291, - 0.24431, - 0.244242, - 0.242924, - 0.237317, - 0.24418, - 0.244277, - 0.241599, - 0.231702, - 0.243548, - 0.244356, - 0.241885, - 0.224518 - ] - }, - { - "n": 77, - "day": "2026-09-10", - "step": [ - 0.0, - 0.01405659, - 0.01380972, - 0.01371657, - 0.01313055, - 0.04142935, - 0.04116493, - 0.04044309, - 0.03408544, - 0.13658654, - 0.13487526, - 0.12856365, - 0.06083363, - 0.37658455, - 0.37280872, - 0.36325691, - 0.05562118 - ], - "drift": [ - 0.0, - 0.00097, - 0.00158, - 0.00436, - 0.01542, - 0.00292, - 0.00442, - 0.01091, - 0.03572, - 0.0098, - 0.01342, - 0.02694, - 0.06419, - 0.0283, - 0.03663, - 0.05147, - 0.09453 - ], - "capped": [ - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 18, - 18, - 18, - 4 - ], - "mean_delta": [ - -0.325405, - -0.325309, - -0.325191, - -0.324547, - -0.321863, - -0.325126, - -0.324856, - -0.323358, - -0.3172, - -0.324491, - -0.323904, - -0.320783, - -0.311176, - -0.322451, - -0.321274, - -0.318371, - -0.305 - ], - "mean_abs_delta": [ - 0.412495, - 0.412479, - 0.412434, - 0.412177, - 0.411142, - 0.41245, - 0.412349, - 0.411746, - 0.409522, - 0.412352, - 0.412126, - 0.410878, - 0.407852, - 0.411813, - 0.411371, - 0.410266, - 0.405797 - ] - }, - { - "n": 79, - "day": "2026-09-11", - "step": [ - 0.0, - 0.00534917, - 0.00528184, - 0.0050939, - 0.00442459, - 0.01583146, - 0.0152748, - 0.01371148, - 0.01162795, - 0.0457517, - 0.04556668, - 0.04149637, - 0.02252047, - 0.12201497, - 0.11605729, - 0.09059171, - 0.04589043 - ], - "drift": [ - 0.0, - 0.00087, - 0.00159, - 0.00429, - 0.01537, - 0.00259, - 0.00446, - 0.01081, - 0.03569, - 0.00847, - 0.01355, - 0.0268, - 0.06415, - 0.02385, - 0.0366, - 0.05132, - 0.09513 - ], - "capped": [ - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0 - ], - "mean_delta": [ - -0.12375, - -0.123222, - -0.12281, - -0.121171, - -0.113884, - -0.122179, - -0.121137, - -0.117325, - -0.101147, - -0.118601, - -0.115969, - -0.108226, - -0.084206, - -0.109043, - -0.102222, - -0.094814, - -0.067413 - ], - "mean_abs_delta": [ - 0.192215, - 0.192039, - 0.191933, - 0.191517, - 0.189605, - 0.191704, - 0.191437, - 0.190475, - 0.186239, - 0.190558, - 0.189884, - 0.187978, - 0.181836, - 0.187596, - 0.18583, - 0.18412, - 0.176645 - ] - }, - { - "n": 493, - "day": "2026-09-12", - "step": [ - 0.0, - 0.0370898, - 0.03507038, - 0.03225774, - 0.01932832, - 0.10301545, - 0.09570568, - 0.0697418, - 0.03207083, - 0.29446763, - 0.21732125, - 0.14732182, - 0.06674761, - 0.54893624, - 0.33661904, - 0.25043254, - 0.12497538 - ], - "drift": [ - 0.0, - 0.00097, - 0.00184, - 0.00437, - 0.01509, - 0.00268, - 0.00496, - 0.01069, - 0.03383, - 0.00712, - 0.01286, - 0.02324, - 0.0603, - 0.01572, - 0.02459, - 0.04117, - 0.08817 - ], - "capped": [ - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 20, - 17, - 12, - 4 - ], - "mean_delta": [ - -0.052176, - -0.05147, - -0.050841, - -0.048837, - -0.040248, - -0.050155, - -0.048558, - -0.043957, - -0.025105, - -0.046425, - -0.042587, - -0.033848, - -0.006406, - -0.040447, - -0.03061, - -0.019606, - 0.010027 - ], - "mean_abs_delta": [ - 0.172821, - 0.172692, - 0.17259, - 0.172277, - 0.171538, - 0.172457, - 0.172205, - 0.171744, - 0.171229, - 0.171891, - 0.171546, - 0.171197, - 0.172319, - 0.171372, - 0.171058, - 0.171113, - 0.175126 - ] - }, - { - "n": 84, - "day": "2026-09-13", - "step": [ - 0.0, - 0.00483925, - 0.00460524, - 0.00435357, - 0.0029117, - 0.01376317, - 0.01295667, - 0.00991481, - 0.00450319, - 0.04041816, - 0.03158356, - 0.02217941, - 0.01329835, - 0.09713418, - 0.06785647, - 0.04497612, - 0.03795227 - ], - "drift": [ - 0.0, - 0.00056, - 0.00154, - 0.00403, - 0.01479, - 0.00151, - 0.00413, - 0.00978, - 0.03299, - 0.00389, - 0.0104, - 0.02135, - 0.05848, - 0.00808, - 0.02009, - 0.03642, - 0.0844 - ], - "capped": [ - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 1, - 0, - 0 - ], - "mean_delta": [ - -0.003795, - -0.003168, - -0.002474, - -0.000561, - 0.007693, - -0.00209, - -0.000259, - 0.004098, - 0.021821, - 0.000539, - 0.005203, - 0.012976, - 0.039871, - 0.005103, - 0.012472, - 0.023773, - 0.053932 - ], - "mean_abs_delta": [ - 0.129046, - 0.129024, - 0.128986, - 0.128928, - 0.129218, - 0.128974, - 0.128919, - 0.128951, - 0.131391, - 0.128947, - 0.129011, - 0.129546, - 0.136366, - 0.129146, - 0.129562, - 0.131503, - 0.141918 - ] - }, - { - "n": 143, - "day": "2026-09-14", - "step": [ - 0.0, - 0.01395576, - 0.01309729, - 0.01124796, - 0.00897606, - 0.04010188, - 0.03377439, - 0.02885052, - 0.01906088, - 0.12376363, - 0.09607635, - 0.07550813, - 0.05758084, - 0.31571325, - 0.24308048, - 0.18792083, - 0.14813834 - ], - "drift": [ - 0.0, - 0.00038, - 0.00114, - 0.00339, - 0.01425, - 0.00111, - 0.00276, - 0.00818, - 0.03209, - 0.00348, - 0.0069, - 0.01847, - 0.05609, - 0.01049, - 0.01591, - 0.03238, - 0.08018 - ], - "capped": [ - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 2, - 2, - 0, - 0 - ], - "mean_delta": [ - 0.042167, - 0.042324, - 0.042822, - 0.04409, - 0.04979, - 0.042599, - 0.043851, - 0.046939, - 0.059201, - 0.043063, - 0.046646, - 0.052594, - 0.071165, - 0.043394, - 0.050826, - 0.059147, - 0.080106 - ], - "mean_abs_delta": [ - 0.28548, - 0.285509, - 0.285652, - 0.286063, - 0.28775, - 0.285551, - 0.285925, - 0.286922, - 0.290437, - 0.285563, - 0.28668, - 0.288459, - 0.293552, - 0.285445, - 0.28746, - 0.289817, - 0.295308 - ] - }, - { - "n": 87, - "day": "2026-09-15", - "step": [ - 0.0, - 0.00589553, - 0.00534711, - 0.00437177, - 0.0023167, - 0.0176887, - 0.01574199, - 0.0092213, - 0.00417802, - 0.06091992, - 0.03922848, - 0.02030019, - 0.00725556, - 0.24855052, - 0.07829639, - 0.04196794, - 0.01774509 - ], - "drift": [ - 0.0, - 0.00018, - 0.00085, - 0.00303, - 0.01393, - 0.00052, - 0.00207, - 0.00752, - 0.03138, - 0.00165, - 0.00588, - 0.01693, - 0.05464, - 0.00508, - 0.0136, - 0.03038, - 0.07811 - ], - "capped": [ - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 7, - 3, - 0, - 0 - ], - "mean_delta": [ - -0.029959, - -0.029943, - -0.029289, - -0.027475, - -0.018695, - -0.029969, - -0.028517, - -0.023939, - -0.004823, - -0.030586, - -0.026434, - -0.016761, - 0.011357, - -0.03272, - -0.023351, - -0.00884, - 0.022281 - ], - "mean_abs_delta": [ - 0.204738, - 0.204741, - 0.204547, - 0.204045, - 0.202169, - 0.204765, - 0.204331, - 0.203264, - 0.199482, - 0.204999, - 0.203755, - 0.201677, - 0.197665, - 0.205918, - 0.202919, - 0.199844, - 0.196499 - ] - }, - { - "n": 124, - "day": "2026-09-16", - "step": [ - 0.0, - 0.01832044, - 0.01804405, - 0.0175943, - 0.01563341, - 0.05491715, - 0.05347187, - 0.04937034, - 0.04400562, - 0.18374212, - 0.17280494, - 0.15474949, - 0.11915406, - 0.47576329, - 0.42201889, - 0.38254163, - 0.2429094 - ], - "drift": [ - 0.0, - 0.00023, - 0.00054, - 0.00263, - 0.01353, - 0.00072, - 0.00129, - 0.00655, - 0.03016, - 0.0027, - 0.00378, - 0.01431, - 0.05184, - 0.0057, - 0.00948, - 0.02491, - 0.075 - ], - "capped": [ - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 30, - 29, - 28, - 18 - ], - "mean_delta": [ - 0.022279, - 0.022561, - 0.023203, - 0.025178, - 0.035284, - 0.023082, - 0.024587, - 0.029544, - 0.051251, - 0.024512, - 0.029217, - 0.038664, - 0.069699, - 0.028913, - 0.036882, - 0.049392, - 0.085509 - ], - "mean_abs_delta": [ - 0.230778, - 0.230706, - 0.230617, - 0.230304, - 0.229062, - 0.230567, - 0.23037, - 0.229659, - 0.227468, - 0.230131, - 0.229473, - 0.228431, - 0.225807, - 0.229013, - 0.228208, - 0.226842, - 0.225972 - ] - }, - { - "n": 191, - "day": "2026-09-17", - "step": [ - 0.0, - 0.01182056, - 0.01127616, - 0.01076521, - 0.00703857, - 0.0360324, - 0.0341499, - 0.02775102, - 0.01566044, - 0.15178881, - 0.11024774, - 0.07863011, - 0.04397652, - 0.56895505, - 0.39013295, - 0.18317343, - 0.09496041 - ], - "drift": [ - 0.0, - 0.00035, - 0.00027, - 0.002, - 0.01281, - 0.00103, - 0.0008, - 0.00492, - 0.02848, - 0.003, - 0.00262, - 0.00954, - 0.048, - 0.01015, - 0.00849, - 0.01595, - 0.06834 - ], - "capped": [ - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 12, - 8, - 1, - 0 - ], - "mean_delta": [ - 0.06222, - 0.061848, - 0.062237, - 0.063944, - 0.073174, - 0.061108, - 0.061989, - 0.066574, - 0.08627, - 0.058697, - 0.061237, - 0.070202, - 0.100012, - 0.054439, - 0.060366, - 0.072398, - 0.110221 - ], - "mean_abs_delta": [ - 0.211133, - 0.211242, - 0.211198, - 0.210932, - 0.209298, - 0.211448, - 0.211366, - 0.210619, - 0.208113, - 0.211837, - 0.211904, - 0.21059, - 0.208378, - 0.211711, - 0.211887, - 0.211899, - 0.208514 - ] - }, - { - "n": 319, - "day": "2026-09-18", - "step": [ - 0.0, - 0.02242248, - 0.02217253, - 0.02045203, - 0.01633869, - 0.07067607, - 0.06859963, - 0.05671508, - 0.04174889, - 0.25822656, - 0.24802058, - 0.18150416, - 0.11894941, - 0.78144932, - 0.68174072, - 0.54618854, - 0.29896796 - ], - "drift": [ - 0.0, - 0.00047, - 0.00052, - 0.00155, - 0.01229, - 0.00136, - 0.00153, - 0.00396, - 0.02727, - 0.00401, - 0.00485, - 0.00799, - 0.0462, - 0.01092, - 0.01303, - 0.01686, - 0.06539 - ], - "capped": [ - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 25, - 24, - 20, - 8 - ], - "mean_delta": [ - 0.009872, - 0.009533, - 0.009665, - 0.010969, - 0.018562, - 0.008852, - 0.009137, - 0.012604, - 0.028778, - 0.006949, - 0.007383, - 0.014042, - 0.039294, - 0.004102, - 0.006137, - 0.013019, - 0.045467 - ], - "mean_abs_delta": [ - 0.207011, - 0.207039, - 0.207031, - 0.206975, - 0.206794, - 0.207103, - 0.207089, - 0.20695, - 0.207296, - 0.207315, - 0.207282, - 0.206951, - 0.208302, - 0.207783, - 0.207602, - 0.207094, - 0.208569 - ] - }, - { - "n": 192, - "day": "2026-09-19", - "step": [ - 0.0, - 0.02022867, - 0.02022888, - 0.01973887, - 0.01385753, - 0.06113315, - 0.06201499, - 0.05496726, - 0.02287643, - 0.20289687, - 0.20613353, - 0.17609878, - 0.05229677, - 0.56105344, - 0.54359421, - 0.49684691, - 0.1352664 - ], - "drift": [ - 0.0, - 0.00044, - 0.00063, - 0.00197, - 0.01252, - 0.00137, - 0.00194, - 0.00538, - 0.02754, - 0.00466, - 0.0069, - 0.01263, - 0.04721, - 0.01402, - 0.02082, - 0.02768, - 0.06799 - ], - "capped": [ - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 21, - 20, - 18, - 1 - ], - "mean_delta": [ - -0.105825, - -0.105675, - -0.105674, - -0.10471, - -0.098195, - -0.105318, - -0.105303, - -0.102805, - -0.089373, - -0.103792, - -0.103744, - -0.099554, - -0.079574, - -0.098469, - -0.097592, - -0.094821, - -0.072352 - ], - "mean_abs_delta": [ - 0.235298, - 0.235257, - 0.235256, - 0.234786, - 0.231752, - 0.235149, - 0.235154, - 0.233942, - 0.227892, - 0.234663, - 0.234644, - 0.232686, - 0.2241, - 0.232779, - 0.232509, - 0.230973, - 0.221783 - ] - }, - { - "n": 285, - "day": "2026-09-20", - "step": [ - 0.0, - 0.03098814, - 0.03096187, - 0.02952989, - 0.01886986, - 0.09253663, - 0.09229132, - 0.07469206, - 0.05010612, - 0.29653753, - 0.28435932, - 0.19785869, - 0.15575038, - 0.63008355, - 0.53885392, - 0.50478423, - 0.42051997 - ], - "drift": [ - 0.0, - 0.00061, - 0.00083, - 0.00211, - 0.01247, - 0.00185, - 0.00245, - 0.00576, - 0.02776, - 0.00626, - 0.0084, - 0.01378, - 0.0487, - 0.01765, - 0.02334, - 0.0337, - 0.07197 - ], - "capped": [ - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 21, - 19, - 19, - 17 - ], - "mean_delta": [ - -0.036569, - -0.036506, - -0.036457, - -0.0355, - -0.028211, - -0.036347, - -0.036182, - -0.033628, - -0.018538, - -0.035528, - -0.034718, - -0.030588, - -0.008546, - -0.031474, - -0.028088, - -0.024574, - -0.001976 - ], - "mean_abs_delta": [ - 0.254601, - 0.254568, - 0.25457, - 0.254423, - 0.253539, - 0.254498, - 0.254512, - 0.25415, - 0.252636, - 0.254219, - 0.254275, - 0.253819, - 0.251753, - 0.253477, - 0.253899, - 0.253214, - 0.251228 - ] - }, - { - "n": 242, - "day": "2026-09-21", - "step": [ - 0.0, - 0.02376249, - 0.02343056, - 0.02027723, - 0.00928573, - 0.06476467, - 0.06307917, - 0.0473858, - 0.01710585, - 0.16063197, - 0.14352258, - 0.10422501, - 0.03475859, - 0.23264679, - 0.18126442, - 0.14457705, - 0.06711467 - ], - "drift": [ - 0.0, - 0.00075, - 0.00104, - 0.00224, - 0.01247, - 0.00227, - 0.00311, - 0.00651, - 0.02785, - 0.0074, - 0.01064, - 0.01614, - 0.04934, - 0.01628, - 0.02541, - 0.03641, - 0.07323 - ], - "capped": [ - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 15, - 13, - 12, - 8 - ], - "mean_delta": [ - -0.048613, - -0.04834, - -0.04822, - -0.047225, - -0.038763, - -0.047612, - -0.047224, - -0.044301, - -0.027413, - -0.044167, - -0.042611, - -0.03894, - -0.015979, - -0.035038, - -0.031349, - -0.027927, - -0.007488 + "gate_failures": [] + }, + "evaluation": { + "n": 1785, + "pf": 1.2904917167018137, + "win": 0.47619047619047616, + "days": 16, + "mean": 0.028513016179203987, + "median": -0.009349760599434376, + "days_positive": 12 + }, + "strategies": [ + "capitulation" + ], + "finished_at": "2026-10-10T23:37:21.188648+00:00", + "trades_file": "/Users/bryce/Documents/code/fly-trader/data/brain/replay/trades_20261010T233728Z.json", + "per_strategy": { + "capitulation": { + "alpha": 0.0001, + "chosen": { + "mean": 0.030173424631357193, + "alpha": 0.0001, + "total": 89.58489990234375, + "config": 1, + "trades": 2969, + "governance": { + "ic": 35, + "any": 35, + "drift": 0, + "checks": 1482, + "shadow": 0 + }, + "half_life_days": 1.0 + }, + "configs": [ + { + "mean": 0.03036709874868393, + "alpha": 0.0, + "total": 91.28350067138672, + "config": 0, + "trades": 3006, + "governance": { + "ic": 0, + "any": 0, + "drift": 0, + "checks": 0, + "shadow": 0 + }, + "half_life_days": null + }, + { + "mean": 0.030173424631357193, + "alpha": 0.0001, + "total": 89.58489990234375, + "config": 1, + "trades": 2969, + "governance": { + "ic": 35, + "any": 35, + "drift": 0, + "checks": 1482, + "shadow": 0 + }, + "half_life_days": 1.0 + }, + { + "mean": 0.032286640256643295, + "alpha": 0.0001, + "total": 67.99566650390625, + "config": 2, + "trades": 2106, + "governance": { + "ic": 28, + "any": 29, + "drift": 0, + "checks": 1482, + "shadow": 1 + }, + "half_life_days": 3.0 + }, + { + "mean": 0.03781268000602722, + "alpha": 0.0001, + "total": 55.92495346069336, + "config": 3, + "trades": 1479, + "governance": { + "ic": 17, + "any": 69, + "drift": 0, + "checks": 1482, + "shadow": 52 + }, + "half_life_days": 7.0 + }, + { + "mean": 0.04063548520207405, + "alpha": 0.0001, + "total": 48.437496185302734, + "config": 4, + "trades": 1192, + "governance": { + "ic": 7, + "any": 40, + "drift": 0, + "checks": 1482, + "shadow": 33 + }, + "half_life_days": 30.0 + }, + { + "mean": 0.03773646801710129, + "alpha": 0.0003, + "total": 79.32205200195312, + "config": 5, + "trades": 2102, + "governance": { + "ic": 30, + "any": 31, + "drift": 0, + "checks": 1482, + "shadow": 1 + }, + "half_life_days": 1.0 + }, + { + "mean": 0.04369469732046127, + "alpha": 0.0003, + "total": 42.51494216918945, + "config": 6, + "trades": 973, + "governance": { + "ic": 16, + "any": 53, + "drift": 0, + "checks": 1482, + "shadow": 37 + }, + "half_life_days": 3.0 + }, + { + "mean": 0.05426740273833275, + "alpha": 0.0003, + "total": 37.17317199707031, + "config": 7, + "trades": 685, + "governance": { + "ic": 8, + "any": 62, + "drift": 0, + "checks": 1482, + "shadow": 54 + }, + "half_life_days": 7.0 + }, + { + "mean": 0.034157898277044296, + "alpha": 0.0003, + "total": 20.494739532470703, + "config": 8, + "trades": 600, + "governance": { + "ic": 0, + "any": 58, + "drift": 0, + "checks": 1482, + "shadow": 58 + }, + "half_life_days": 30.0 + }, + { + "mean": 0.04557060822844505, + "alpha": 0.001, + "total": 51.90492248535156, + "config": 9, + "trades": 1139, + "governance": { + "ic": 20, + "any": 20, + "drift": 0, + "checks": 1482, + "shadow": 0 + }, + "half_life_days": 1.0 + }, + { + "mean": 0.03615965694189072, + "alpha": 0.001, + "total": 22.129711151123047, + "config": 10, + "trades": 612, + "governance": { + "ic": 5, + "any": 53, + "drift": 0, + "checks": 1482, + "shadow": 48 + }, + "half_life_days": 3.0 + }, + { + "mean": 0.04262426868081093, + "alpha": 0.001, + "total": 21.3121337890625, + "config": 11, + "trades": 500, + "governance": { + "ic": 0, + "any": 78, + "drift": 0, + "checks": 1482, + "shadow": 78 + }, + "half_life_days": 7.0 + }, + { + "mean": 0.023908229544758797, + "alpha": 0.001, + "total": 10.639162063598633, + "config": 12, + "trades": 445, + "governance": { + "ic": 0, + "any": 66, + "drift": 0, + "checks": 1482, + "shadow": 66 + }, + "half_life_days": 30.0 + }, + { + "mean": 0.04726389795541763, + "alpha": 0.003, + "total": 46.224090576171875, + "config": 13, + "trades": 978, + "governance": { + "ic": 6, + "any": 58, + "drift": 0, + "checks": 1482, + "shadow": 52 + }, + "half_life_days": 1.0 + }, + { + "mean": 0.03016720339655876, + "alpha": 0.003, + "total": 20.634366989135742, + "config": 14, + "trades": 684, + "governance": { + "ic": 0, + "any": 137, + "drift": 0, + "checks": 1482, + "shadow": 137 + }, + "half_life_days": 3.0 + }, + { + "mean": 0.034875430166721344, + "alpha": 0.003, + "total": 21.832019805908203, + "config": 15, + "trades": 626, + "governance": { + "ic": 0, + "any": 136, + "drift": 0, + "checks": 1482, + "shadow": 136 + }, + "half_life_days": 7.0 + }, + { + "mean": 0.03963538631796837, + "alpha": 0.003, + "total": 23.345243453979492, + "config": 16, + "trades": 589, + "governance": { + "ic": 0, + "any": 199, + "drift": 0, + "checks": 1482, + "shadow": 199 + }, + "half_life_days": 30.0 + } ], - "mean_abs_delta": [ - 0.189753, - 0.18963, - 0.189622, - 0.18943, - 0.188027, - 0.189376, - 0.189349, - 0.188778, - 0.187194, - 0.188566, - 0.188461, - 0.188063, - 0.187236, - 0.187389, - 0.187586, - 0.18761, - 0.188787 - ] + "plastic": true, + "hold_min": 240, + "half_life_days": 1.0 } + }, + "half_life_days": 1.0, + "selection_days": [ + "2026-09-09", + "2026-09-23" ], - "bootstrap": { - "lines": { - "capitulation": 0.0025 - }, - "gates_ok": true, - "calibration": { - "mean": 0.015530285738707813, - "total": 19.59922060224926, - "trades": 1262, - "per_strategy": { - "capitulation": { - "line": 0.0025, - "mean": 0.015530285738707813, - "total": 19.59922060224926, - "trades": 1262, - "hold_min": 240 - } + "evaluation_days": [ + "2026-09-24", + "2026-10-09" + ], + "plastic_beats_frozen": true + }, + "pinned_selector_sha256": "b38f0633ee4a9b693bfb9a01a5e5de044e2ce317e08fb9726e77b48dfbb603fe", + "autostart": { + "pumpstream": true, + "runner": true, + "train": false, + "replay": false, + "discover": false, + "vault": true, + "rh_stream": true + }, + "rh_assets": [ + { + "asset": "0x0000000000000000000000000000000000000000", + "symbol": "ETH", + "class": "eth", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "4200000000000000000", + "ref_pool": null, + "ref_kind": null, + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": null + }, + { + "asset": "0x03dfbbe0ac4e7bcdafd08ed41a400326b77d8c80", + "symbol": "JNJ", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "36524326877657062089", + "ref_pool": "0x7f0ace18d1dced47063cf26e649bc8ab14d09e67", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 18, + "dec1": 6, + "pool": "0x7f0ace18d1dced47063cf26e649bc8ab14d09e67", + "token0": "0x03dfbbe0ac4e7bcdafd08ed41a400326b77d8c80", + "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "emitter": "0x7f0ace18d1dced47063cf26e649bc8ab14d09e67", + "token_in": "0x03dfbbe0ac4e7bcdafd08ed41a400326b77d8c80" + }, + { + "dec0": 18, + "dec1": 6, + "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168" + } + ] + }, + { + "asset": "0x05a3d1cd21d0c88145e82600e62e7e496e0f222b", + "symbol": "AMC", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "3840064684014871241292", + "ref_pool": "0xaa34fea710a1a737840329051d81d3b0b7c564d5", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 18, + "dec1": 6, + "pool": "0xaa34fea710a1a737840329051d81d3b0b7c564d5", + "token0": "0x05a3d1cd21d0c88145e82600e62e7e496e0f222b", + "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "emitter": "0xaa34fea710a1a737840329051d81d3b0b7c564d5", + "token_in": "0x05a3d1cd21d0c88145e82600e62e7e496e0f222b" + }, + { + "dec0": 18, + "dec1": 6, + "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168" + } + ] + }, + { + "asset": "0x05b37fb53a299a1b874a619e1c4c404d52c36f4c", + "symbol": "RDDT", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "42347152428810721502", + "ref_pool": "0xa541143f20d7b0643123064abf25f423e375b531", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 18, + "dec1": 18, + "pool": "0xa541143f20d7b0643123064abf25f423e375b531", + "token0": "0x05b37fb53a299a1b874a619e1c4c404d52c36f4c", + "token1": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "emitter": "0xa541143f20d7b0643123064abf25f423e375b531", + "token_in": "0x05b37fb53a299a1b874a619e1c4c404d52c36f4c" + } + ] + }, + { + "asset": "0x07ebb29a38fbcb41563817e5e19f2cec619c90d2", + "symbol": "BUN", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "191347838750403524965795", + "ref_pool": "0x4f22b1a543711903604d5e84333a9fdd4a63166d", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 18, + "dec1": 18, + "pool": "0x4f22b1a543711903604d5e84333a9fdd4a63166d", + "token0": "0x07ebb29a38fbcb41563817e5e19f2cec619c90d2", + "token1": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "emitter": "0x4f22b1a543711903604d5e84333a9fdd4a63166d", + "token_in": "0x07ebb29a38fbcb41563817e5e19f2cec619c90d2" + } + ] + }, + { + "asset": "0x116f00968269b7bfbad4109ce591d6e74c0601d4", + "symbol": "NET", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "33075489282385834154", + "ref_pool": "0xa9caebb1b2fb2b954f192572b309aef1caf6a330", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 18, + "dec1": 6, + "pool": "0xa9caebb1b2fb2b954f192572b309aef1caf6a330", + "token0": "0x116f00968269b7bfbad4109ce591d6e74c0601d4", + "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "emitter": "0xa9caebb1b2fb2b954f192572b309aef1caf6a330", + "token_in": "0x116f00968269b7bfbad4109ce591d6e74c0601d4" + }, + { + "dec0": 18, + "dec1": 6, + "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168" + } + ] + }, + { + "asset": "0x117cc2133c37b721f49de2a7a74833232b3b4c0c", + "symbol": "SPY", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "10900000000000000000", + "ref_pool": "0xa7bb1ac63bbab0c44316e6c8c455213441689167", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 18, + "dec1": 6, + "pool": "0xa7bb1ac63bbab0c44316e6c8c455213441689167", + "token0": "0x117cc2133c37b721f49de2a7a74833232b3b4c0c", + "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "emitter": "0xa7bb1ac63bbab0c44316e6c8c455213441689167", + "token_in": "0x117cc2133c37b721f49de2a7a74833232b3b4c0c" + }, + { + "dec0": 18, + "dec1": 6, + "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168" + } + ] + }, + { + "asset": "0x12f190a9f9d7d37a250758b26824b97ce941bf54", + "symbol": "AMZN", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "29330291092142702509", + "ref_pool": "0x8ac92da74ab5f3b1d024dc1943ad7e15dc4179ef", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 18, + "dec1": 6, + "pool": "0x8ac92da74ab5f3b1d024dc1943ad7e15dc4179ef", + "token0": "0x12f190a9f9d7d37a250758b26824b97ce941bf54", + "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "emitter": "0x8ac92da74ab5f3b1d024dc1943ad7e15dc4179ef", + "token_in": "0x12f190a9f9d7d37a250758b26824b97ce941bf54" + }, + { + "dec0": 18, + "dec1": 6, + "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168" + } + ] + }, + { + "asset": "0x1b0e319c6a659f002271b69db8a7df2f911c153e", + "symbol": "GME", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "369000000000000000000", + "ref_pool": "0xc6bcc95043dc48c204bb2d57fb264a10efe0a607", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 18, + "dec1": 18, + "pool": "0xc6bcc95043dc48c204bb2d57fb264a10efe0a607", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0x1b0e319c6a659f002271b69db8a7df2f911c153e", + "emitter": "0xc6bcc95043dc48c204bb2d57fb264a10efe0a607", + "token_in": "0x1b0e319c6a659f002271b69db8a7df2f911c153e" + } + ] + }, + { + "asset": "0x1cdad396db64bda184d5182a97dd9b3c62100b7d", + "symbol": "P", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "66700197238658777370", + "ref_pool": null, + "ref_kind": null, + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": null + }, + { + "asset": "0x1d11f0496982706c5e14a514d4e79f2e6bde4516", + "symbol": "DJT", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "917162201643832986520", + "ref_pool": "0x95def4ea143630d64caa8f55f7570d8023f20265", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 18, + "dec1": 18, + "pool": "0x95def4ea143630d64caa8f55f7570d8023f20265", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0x1d11f0496982706c5e14a514d4e79f2e6bde4516", + "emitter": "0x95def4ea143630d64caa8f55f7570d8023f20265", + "token_in": "0x1d11f0496982706c5e14a514d4e79f2e6bde4516" + } + ] + }, + { + "asset": "0x25c288e6d899b9bc30160965ad9644c67e73be0c", + "symbol": "F", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "711416942148760410079", + "ref_pool": "0x4dbac19e895322ac5b93abad9008691632bffc05", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 18, + "dec1": 6, + "pool": "0x4dbac19e895322ac5b93abad9008691632bffc05", + "token0": "0x25c288e6d899b9bc30160965ad9644c67e73be0c", + "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "emitter": "0x4dbac19e895322ac5b93abad9008691632bffc05", + "token_in": "0x25c288e6d899b9bc30160965ad9644c67e73be0c" + }, + { + "dec0": 18, + "dec1": 6, + "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168" + } + ] + }, + { + "asset": "0x27c99fbde9d0d2aa4f4bfb4943f237843ddf6958", + "symbol": "DDOG", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "37256385303244523662", + "ref_pool": null, + "ref_kind": null, + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": null + }, + { + "asset": "0x2e0847e8910a9732eb3fb1bb4b70a580adad4fe3", + "symbol": "GOOGL", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "24200000000000000000", + "ref_pool": "0x34d0dc122cf9a8eb296fc5e0d3a233625d7d19b7", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 18, + "dec1": 6, + "pool": "0x34d0dc122cf9a8eb296fc5e0d3a233625d7d19b7", + "token0": "0x2e0847e8910a9732eb3fb1bb4b70a580adad4fe3", + "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "emitter": "0x34d0dc122cf9a8eb296fc5e0d3a233625d7d19b7", + "token_in": "0x2e0847e8910a9732eb3fb1bb4b70a580adad4fe3" + }, + { + "dec0": 18, + "dec1": 6, + "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168" + } + ] + }, + { + "asset": "0x322f0929c4625ed5bad873c95208d54e1c003b2d", + "symbol": "TSLA", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "26000000000000000000", + "ref_pool": "0xc4f0172d6ac8dd294dd1137d047d5e1893760236", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 18, + "dec1": 6, + "pool": "0xc4f0172d6ac8dd294dd1137d047d5e1893760236", + "token0": "0x322f0929c4625ed5bad873c95208d54e1c003b2d", + "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "emitter": "0xc4f0172d6ac8dd294dd1137d047d5e1893760236", + "token_in": "0x322f0929c4625ed5bad873c95208d54e1c003b2d" + } + ] + }, + { + "asset": "0x329fcaceb9ad6f9580dd5f643fed0646900d043c", + "symbol": "LMT", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "19417850658193372681", + "ref_pool": "0x54f160bea60ec5207918c539d86e0b906b4da297", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 18, + "dec1": 6, + "pool": "0x54f160bea60ec5207918c539d86e0b906b4da297", + "token0": "0x329fcaceb9ad6f9580dd5f643fed0646900d043c", + "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "emitter": "0x54f160bea60ec5207918c539d86e0b906b4da297", + "token_in": "0x329fcaceb9ad6f9580dd5f643fed0646900d043c" + }, + { + "dec0": 18, + "dec1": 6, + "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168" + } + ] + }, + { + "asset": "0x3b14c39e89d60d627b42a1a4ca45b5bb45fc12e2", + "symbol": "RKLB", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "162685692553916369167", + "ref_pool": "0xbbc15d66fef8391f53569e19016e86545be9de3f302b82db93ed6165236d22f3", + "ref_kind": "uniswap-v4", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 18, + "dec1": 6, + "pool": "0xbbc15d66fef8391f53569e19016e86545be9de3f302b82db93ed6165236d22f3", + "token0": "0x3b14c39e89d60d627b42a1a4ca45b5bb45fc12e2", + "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "emitter": "0x8366a39cc670b4001a1121b8f6a443a643e40951", + "token_in": "0x3b14c39e89d60d627b42a1a4ca45b5bb45fc12e2" + }, + { + "dec0": 18, + "dec1": 6, + "pool": "0x84bd4e2d8be11aeb0afc1195b38f587b61e90068548f1063fdbe448fb8cad0b6", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "emitter": "0x8366a39cc670b4001a1121b8f6a443a643e40951", + "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168" + } + ] + }, + { + "asset": "0x408c14038a04f7bd235329e26d2bf569ee20e250", + "symbol": "NU", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "667513667205169680597", + "ref_pool": "0x0e3faed512e7909758eb924e6919e0057bd6b45e", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 18, + "dec1": 6, + "pool": "0x0e3faed512e7909758eb924e6919e0057bd6b45e", + "token0": "0x408c14038a04f7bd235329e26d2bf569ee20e250", + "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "emitter": "0x0e3faed512e7909758eb924e6919e0057bd6b45e", + "token_in": "0x408c14038a04f7bd235329e26d2bf569ee20e250" + }, + { + "dec0": 18, + "dec1": 6, + "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168" + } + ] + }, + { + "asset": "0x411efb0e7f985935daec3d4c3ebaea0d0ad7d89f", + "symbol": "SLV", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "172796487119437943163", + "ref_pool": "0x48abc7052509caf820c4c22559f08381f9afd823", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 18, + "dec1": 18, + "pool": "0x48abc7052509caf820c4c22559f08381f9afd823", + "token0": "0x411efb0e7f985935daec3d4c3ebaea0d0ad7d89f", + "token1": "0xc9a981fee1f9dec688bb123ccdecc63d0debfc4e", + "emitter": "0x48abc7052509caf820c4c22559f08381f9afd823", + "token_in": "0x411efb0e7f985935daec3d4c3ebaea0d0ad7d89f" + }, + { + "dec0": 18, + "dec1": 18, + "pool": "0x26250ba84465454bc731f710e46f1f32b167d66b", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0xc9a981fee1f9dec688bb123ccdecc63d0debfc4e", + "emitter": "0x26250ba84465454bc731f710e46f1f32b167d66b", + "token_in": "0xc9a981fee1f9dec688bb123ccdecc63d0debfc4e" + } + ] + }, + { + "asset": "0x41f4267525a8aff329540ef24fd83d9044758b33", + "symbol": "FIG", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "390666925767586478597", + "ref_pool": "0xca5904c0a9d42f0ec1bf760fdf779907877144fd", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 18, + "dec1": 6, + "pool": "0xca5904c0a9d42f0ec1bf760fdf779907877144fd", + "token0": "0x41f4267525a8aff329540ef24fd83d9044758b33", + "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "emitter": "0xca5904c0a9d42f0ec1bf760fdf779907877144fd", + "token_in": "0x41f4267525a8aff329540ef24fd83d9044758b33" + }, + { + "dec0": 18, + "dec1": 6, + "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168" + } + ] + }, + { + "asset": "0x43b07d15ce533bec5476d70c22a78a1b2b662155", + "symbol": "MRNA", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "66652476626218421319", + "ref_pool": "0xae6d85e96bee054417d8d627b310a945484415be", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 18, + "dec1": 18, + "pool": "0xae6d85e96bee054417d8d627b310a945484415be", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0x43b07d15ce533bec5476d70c22a78a1b2b662155", + "emitter": "0xae6d85e96bee054417d8d627b310a945484415be", + "token_in": "0x43b07d15ce533bec5476d70c22a78a1b2b662155" + } + ] + }, + { + "asset": "0x48e39e56acdba37b09020c0b734a613c9a2f100a", + "symbol": "BB", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "1259561425061425366930", + "ref_pool": "0x183304567485e97e68835708f572aaa0e0e71d08", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 18, + "dec1": 18, + "pool": "0x183304567485e97e68835708f572aaa0e0e71d08", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0x48e39e56acdba37b09020c0b734a613c9a2f100a", + "emitter": "0x183304567485e97e68835708f572aaa0e0e71d08", + "token_in": "0x48e39e56acdba37b09020c0b734a613c9a2f100a" + } + ] + }, + { + "asset": "0x4a0e65a3eccec6dbe60ae065f2e7bb85fae35eea", + "symbol": "SPCX", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "72200000000000000000", + "ref_pool": "0xc61284332117c3fb23a2a56cceffd07f7af60029", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 18, + "dec1": 6, + "pool": "0xc61284332117c3fb23a2a56cceffd07f7af60029", + "token0": "0x4a0e65a3eccec6dbe60ae065f2e7bb85fae35eea", + "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "emitter": "0xc61284332117c3fb23a2a56cceffd07f7af60029", + "token_in": "0x4a0e65a3eccec6dbe60ae065f2e7bb85fae35eea" + }, + { + "dec0": 18, + "dec1": 6, + "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168" + } + ] + }, + { + "asset": "0x4d21483a44bf67a86b77e3da301411880797d452", + "symbol": "BA", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "49724925896506964807", + "ref_pool": null, + "ref_kind": null, + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": null + }, + { + "asset": "0x4e62068525ab11fe768e29dfd00ef909b9803016", + "symbol": "LULU", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "83716665278587490919", + "ref_pool": "0x0f4227d27082b3bca6818381b9ea6460275e49f4", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 18, + "dec1": 6, + "pool": "0x0f4227d27082b3bca6818381b9ea6460275e49f4", + "token0": "0x4e62068525ab11fe768e29dfd00ef909b9803016", + "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "emitter": "0x0f4227d27082b3bca6818381b9ea6460275e49f4", + "token_in": "0x4e62068525ab11fe768e29dfd00ef909b9803016" + }, + { + "dec0": 18, + "dec1": 6, + "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168" + } + ] + }, + { + "asset": "0x4ea005168d7f09a7a0ba9d1def21a479950e44c2", + "symbol": "COST", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "8470405930331172950", + "ref_pool": "0x0a2121a50a09ed0796ae81f9c53ff9398355a398", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 18, + "dec1": 6, + "pool": "0x0a2121a50a09ed0796ae81f9c53ff9398355a398", + "token0": "0x4ea005168d7f09a7a0ba9d1def21a479950e44c2", + "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "emitter": "0x0a2121a50a09ed0796ae81f9c53ff9398355a398", + "token_in": "0x4ea005168d7f09a7a0ba9d1def21a479950e44c2" + }, + { + "dec0": 18, + "dec1": 6, + "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168" + } + ] + }, + { + "asset": "0x56910d4409f3a0c78c64dd8d0545ff0705389870", + "symbol": "Index", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "620133096008220802271782", + "ref_pool": "0xd29893ffac8b29ec4db2cfe0cdb3fe1377c028ff", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 18, + "dec1": 18, + "pool": "0xd29893ffac8b29ec4db2cfe0cdb3fe1377c028ff", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0x56910d4409f3a0c78c64dd8d0545ff0705389870", + "emitter": "0xd29893ffac8b29ec4db2cfe0cdb3fe1377c028ff", + "token_in": "0x56910d4409f3a0c78c64dd8d0545ff0705389870" + } + ] + }, + { + "asset": "0x58ffe4a942d3885baa22d7520691f611ef09e7aa", + "symbol": "TSM", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "24453231884057971122", + "ref_pool": "0x91280db3392ea92c08d8134b5760fb4798b69547", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 18, + "dec1": 18, + "pool": "0x91280db3392ea92c08d8134b5760fb4798b69547", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0x58ffe4a942d3885baa22d7520691f611ef09e7aa", + "emitter": "0x91280db3392ea92c08d8134b5760fb4798b69547", + "token_in": "0x58ffe4a942d3885baa22d7520691f611ef09e7aa" + } + ] + }, + { + "asset": "0x59818904ab4ce163b3ce4ffb64f2d6ca02c434b4", + "symbol": "QUBT", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "1294653972257250949347", + "ref_pool": "0x227bbce9a81b3694b01754298a983be3f9e44a93", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 18, + "dec1": 6, + "pool": "0x227bbce9a81b3694b01754298a983be3f9e44a93", + "token0": "0x59818904ab4ce163b3ce4ffb64f2d6ca02c434b4", + "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "emitter": "0x227bbce9a81b3694b01754298a983be3f9e44a93", + "token_in": "0x59818904ab4ce163b3ce4ffb64f2d6ca02c434b4" + }, + { + "dec0": 18, + "dec1": 6, + "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168" + } + ] + }, + { + "asset": "0x5e81213613b6b86eab4c6c50d718d34359459786", + "symbol": "TTWO", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "26639006882017848277", + "ref_pool": "0xe69fe23b708362ef5817048f16e4171e0831341c", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 18, + "dec1": 18, + "pool": "0xe69fe23b708362ef5817048f16e4171e0831341c", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0x5e81213613b6b86eab4c6c50d718d34359459786", + "emitter": "0xe69fe23b708362ef5817048f16e4171e0831341c", + "token_in": "0x5e81213613b6b86eab4c6c50d718d34359459786" + } + ] + }, + { + "asset": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "symbol": "USDG", + "class": "stable", + "decimals": 6, + "approved": true, + "graduation_threshold_raw": "8090000000", + "ref_pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 18, + "dec1": 6, + "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168" + } + ] + }, + { + "asset": "0x62fd0668e10d8b72339be2dcf7643001688ff13b", + "symbol": "MRVL", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "48682003099573809148", + "ref_pool": "0x5201ea77c950abb8d3706a20bf3b83d872e4a98b", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 18, + "dec1": 18, + "pool": "0x5201ea77c950abb8d3706a20bf3b83d872e4a98b", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0x62fd0668e10d8b72339be2dcf7643001688ff13b", + "emitter": "0x5201ea77c950abb8d3706a20bf3b83d872e4a98b", + "token_in": "0x62fd0668e10d8b72339be2dcf7643001688ff13b" + } + ] + }, + { + "asset": "0x6330d8c3178a418788df01a47479c0ce7ccf450b", + "symbol": "COIN", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "52470673453406424989", + "ref_pool": "0x6707aeac7d0e519b083219d27bb427364363183a", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 18, + "dec1": 18, + "pool": "0x6707aeac7d0e519b083219d27bb427364363183a", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0x6330d8c3178a418788df01a47479c0ce7ccf450b", + "emitter": "0x6707aeac7d0e519b083219d27bb427364363183a", + "token_in": "0x6330d8c3178a418788df01a47479c0ce7ccf450b" + } + ] + }, + { + "asset": "0x63c12667638f2ae6fc6ae09b43d98ec84a8586ea", + "symbol": "wsNET", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "10063226872967693437", + "ref_pool": null, + "ref_kind": null, + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": null + }, + { + "asset": "0x7066a64c24e4206cd62e83bf198c1e7eb361f51e", + "symbol": "PFE", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "346376981392143376566", + "ref_pool": "0xc7d573fcda6d2107c97fb582ae18411f9db32e7f", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 6, + "dec1": 18, + "pool": "0xc7d573fcda6d2107c97fb582ae18411f9db32e7f", + "token0": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "token1": "0x7066a64c24e4206cd62e83bf198c1e7eb361f51e", + "emitter": "0xc7d573fcda6d2107c97fb582ae18411f9db32e7f", + "token_in": "0x7066a64c24e4206cd62e83bf198c1e7eb361f51e" + }, + { + "dec0": 18, + "dec1": 6, + "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168" + } + ] + }, + { + "asset": "0x75742c18bc1f1c5c5f448f4c9d9c6f66dafaaa38", + "symbol": "SOXX", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "18167833670600454869", + "ref_pool": null, + "ref_kind": null, + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": null + }, + { + "asset": "0x7f0abef0c07280f82c6a08ead09ded6bae2c13fc", + "symbol": "EWY", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "55805870522367777811", + "ref_pool": "0x3cba52145fb271e4a76f44df3d8b86f460a04e767841bcb8288a02729047dbbb", + "ref_kind": "uniswap-v4", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 6, + "dec1": 18, + "pool": "0x3cba52145fb271e4a76f44df3d8b86f460a04e767841bcb8288a02729047dbbb", + "token0": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "token1": "0x7f0abef0c07280f82c6a08ead09ded6bae2c13fc", + "emitter": "0x8366a39cc670b4001a1121b8f6a443a643e40951", + "token_in": "0x7f0abef0c07280f82c6a08ead09ded6bae2c13fc" + }, + { + "dec0": 18, + "dec1": 6, + "pool": "0x84bd4e2d8be11aeb0afc1195b38f587b61e90068548f1063fdbe448fb8cad0b6", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "emitter": "0x8366a39cc670b4001a1121b8f6a443a643e40951", + "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168" + } + ] + }, + { + "asset": "0x8005d266423c7ea827372c9c864491e5786600ea", + "symbol": "LLY", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "8727274137931034486", + "ref_pool": "0xf212d02146a897f5f686e9d629f6a73da534324a", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 6, + "dec1": 18, + "pool": "0xf212d02146a897f5f686e9d629f6a73da534324a", + "token0": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "token1": "0x8005d266423c7ea827372c9c864491e5786600ea", + "emitter": "0xf212d02146a897f5f686e9d629f6a73da534324a", + "token_in": "0x8005d266423c7ea827372c9c864491e5786600ea" + }, + { + "dec0": 18, + "dec1": 6, + "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168" + } + ] + }, + { + "asset": "0x822cc93ffd030293e9842c30bbd678f530701867", + "symbol": "BE", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "42050779564420924105", + "ref_pool": "0x1bad145c8f06444e0df81c28257cd20231bd1f16", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 6, + "dec1": 18, + "pool": "0x1bad145c8f06444e0df81c28257cd20231bd1f16", + "token0": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "token1": "0x822cc93ffd030293e9842c30bbd678f530701867", + "emitter": "0x1bad145c8f06444e0df81c28257cd20231bd1f16", + "token_in": "0x822cc93ffd030293e9842c30bbd678f530701867" + }, + { + "dec0": 18, + "dec1": 6, + "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168" + } + ] + }, + { + "asset": "0x84cab63bc87912e71ad199ff14a0ba45de68fef8", + "symbol": "SKHY", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "62965779325786790016", + "ref_pool": "0x5f2a5025feb93a4c44da4e3b37e4fd3fb0ab5171", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 6, + "dec1": 18, + "pool": "0x5f2a5025feb93a4c44da4e3b37e4fd3fb0ab5171", + "token0": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "token1": "0x84cab63bc87912e71ad199ff14a0ba45de68fef8", + "emitter": "0x5f2a5025feb93a4c44da4e3b37e4fd3fb0ab5171", + "token_in": "0x84cab63bc87912e71ad199ff14a0ba45de68fef8" + }, + { + "dec0": 18, + "dec1": 6, + "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168" + } + ] + }, + { + "asset": "0x86923f96303d656e4aa86d9d42d1e57ad2023fdc", + "symbol": "AMD", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "16665504590959022517", + "ref_pool": "0x48d284a2a4d3dc1b3da08231fe44317e7e7aa51f", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 6, + "dec1": 18, + "pool": "0x48d284a2a4d3dc1b3da08231fe44317e7e7aa51f", + "token0": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "token1": "0x86923f96303d656e4aa86d9d42d1e57ad2023fdc", + "emitter": "0x48d284a2a4d3dc1b3da08231fe44317e7e7aa51f", + "token_in": "0x86923f96303d656e4aa86d9d42d1e57ad2023fdc" + }, + { + "dec0": 18, + "dec1": 6, + "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168" + } + ] + }, + { + "asset": "0x894e1ec2d74ffe5aef8dc8a9e84686accb964f2a", + "symbol": "PLTR", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "47065125263404356876", + "ref_pool": "0x61be5bfbaf17ae28bf68006103b2e78fe6112638", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 18, + "dec1": 18, + "pool": "0x61be5bfbaf17ae28bf68006103b2e78fe6112638", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0x894e1ec2d74ffe5aef8dc8a9e84686accb964f2a", + "emitter": "0x61be5bfbaf17ae28bf68006103b2e78fe6112638", + "token_in": "0x894e1ec2d74ffe5aef8dc8a9e84686accb964f2a" + } + ] + }, + { + "asset": "0x92fd66527192e3e61d4ddd13322aa222de86f9b5", + "symbol": "SGOV", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "102816558506191026323", + "ref_pool": "0x6ba50150b17ffd0972915aaf04ffd5e8f4fa49b4", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 6, + "dec1": 18, + "pool": "0x6ba50150b17ffd0972915aaf04ffd5e8f4fa49b4", + "token0": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "token1": "0x92fd66527192e3e61d4ddd13322aa222de86f9b5", + "emitter": "0x6ba50150b17ffd0972915aaf04ffd5e8f4fa49b4", + "token_in": "0x92fd66527192e3e61d4ddd13322aa222de86f9b5" + }, + { + "dec0": 18, + "dec1": 6, + "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168" + } + ] + }, + { + "asset": "0x941ae714ec6d8130c7b75d67160ca08f1e7d11dd", + "symbol": "DELL", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "23820324705882352994", + "ref_pool": "0xc30c89cb7815a1488b7998d15eec73961707fc5a", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 6, + "dec1": 18, + "pool": "0xc30c89cb7815a1488b7998d15eec73961707fc5a", + "token0": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "token1": "0x941ae714ec6d8130c7b75d67160ca08f1e7d11dd", + "emitter": "0xc30c89cb7815a1488b7998d15eec73961707fc5a", + "token_in": "0x941ae714ec6d8130c7b75d67160ca08f1e7d11dd" + }, + { + "dec0": 18, + "dec1": 6, + "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168" + } + ] + }, + { + "asset": "0x980dcf6766fa79f5cf0c4aadb3ab477ff15a9619", + "symbol": "IBM", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "43891115360101975839", + "ref_pool": "0x7502f81aef932e82ccf240f023dd989689a3ef65", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 18, + "dec1": 18, + "pool": "0x7502f81aef932e82ccf240f023dd989689a3ef65", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0x980dcf6766fa79f5cf0c4aadb3ab477ff15a9619", + "emitter": "0x7502f81aef932e82ccf240f023dd989689a3ef65", + "token_in": "0x980dcf6766fa79f5cf0c4aadb3ab477ff15a9619" + } + ] + }, + { + "asset": "0x9b23573b156b52565012f5ce02cdf60afbaa70be", + "symbol": "PENG", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "143586441157738307921", + "ref_pool": null, + "ref_kind": null, + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": null + }, + { + "asset": "0x9d9c6684f596f66a64c030b93a886d51fd4d7931", + "symbol": "NBIS", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "46093139482053612464", + "ref_pool": null, + "ref_kind": null, + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": null + }, + { + "asset": "0x9e7abd3c9139d14e4c86dce0e455aab7a0c2fb3e", + "symbol": "WYFI", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "578493600000000022375", + "ref_pool": "0x2a3063e34c60253abb23c2442f4cdfbc5cbd02c2", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 6, + "dec1": 18, + "pool": "0x2a3063e34c60253abb23c2442f4cdfbc5cbd02c2", + "token0": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "token1": "0x9e7abd3c9139d14e4c86dce0e455aab7a0c2fb3e", + "emitter": "0x2a3063e34c60253abb23c2442f4cdfbc5cbd02c2", + "token_in": "0x9e7abd3c9139d14e4c86dce0e455aab7a0c2fb3e" + }, + { + "dec0": 18, + "dec1": 6, + "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168" + } + ] + }, + { + "asset": "0xa30fa36db767ad9ed3f7a60fc79526fb4d56d344", + "symbol": "USO", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "71798851063829787942", + "ref_pool": "0x02175608f1b5e6b5ed221ccfdc7be197d111d915", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 6, + "dec1": 18, + "pool": "0x02175608f1b5e6b5ed221ccfdc7be197d111d915", + "token0": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "token1": "0xa30fa36db767ad9ed3f7a60fc79526fb4d56d344", + "emitter": "0x02175608f1b5e6b5ed221ccfdc7be197d111d915", + "token_in": "0xa30fa36db767ad9ed3f7a60fc79526fb4d56d344" + }, + { + "dec0": 18, + "dec1": 6, + "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168" + } + ] + }, + { + "asset": "0xaa07a0e9209e16ac99708c3ec70159c6ef3128a3", + "symbol": "ORBIO", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "193502872457166537181000", + "ref_pool": "0x34f73f488309208b8cb6012eb47ffeb086ca1c2d", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 18, + "dec1": 18, + "pool": "0x34f73f488309208b8cb6012eb47ffeb086ca1c2d", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0xaa07a0e9209e16ac99708c3ec70159c6ef3128a3", + "emitter": "0x34f73f488309208b8cb6012eb47ffeb086ca1c2d", + "token_in": "0xaa07a0e9209e16ac99708c3ec70159c6ef3128a3" + } + ] + }, + { + "asset": "0xab093def657f15df31b33922a95e047add645b29", + "symbol": "SHROOM", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "1393388248231318945176878", + "ref_pool": "0xc641a0dc848e7aadd7c69d800bae2fea9b258610", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 18, + "dec1": 18, + "pool": "0xc641a0dc848e7aadd7c69d800bae2fea9b258610", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0xab093def657f15df31b33922a95e047add645b29", + "emitter": "0xc641a0dc848e7aadd7c69d800bae2fea9b258610", + "token_in": "0xab093def657f15df31b33922a95e047add645b29" + } + ] + }, + { + "asset": "0xacef2e09adb47ad6abebad9ff06689e60615c2b6", + "symbol": "INDA", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "206740198138697096367", + "ref_pool": "0xf5b37a305e7304a70067be356ee611ac29f706eb", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 18, + "dec1": 18, + "pool": "0xf5b37a305e7304a70067be356ee611ac29f706eb", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0xacef2e09adb47ad6abebad9ff06689e60615c2b6", + "emitter": "0xf5b37a305e7304a70067be356ee611ac29f706eb", + "token_in": "0xacef2e09adb47ad6abebad9ff06689e60615c2b6" + } + ] + }, + { + "asset": "0xad25ac6c84d497db898fa1e8387bf6af3532a1c4", + "symbol": "BABA", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "91284676564156947526", + "ref_pool": "0xa57ab582b310dd6f9e934ea1eeea152741545e6a", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 6, + "dec1": 18, + "pool": "0xa57ab582b310dd6f9e934ea1eeea152741545e6a", + "token0": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "token1": "0xad25ac6c84d497db898fa1e8387bf6af3532a1c4", + "emitter": "0xa57ab582b310dd6f9e934ea1eeea152741545e6a", + "token_in": "0xad25ac6c84d497db898fa1e8387bf6af3532a1c4" + }, + { + "dec0": 18, + "dec1": 6, + "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168" + } + ] + }, + { + "asset": "0xae517a2903e68bd929dfd15be875f8369d53e94a", + "symbol": "CEG", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "35795219915276397737", + "ref_pool": null, + "ref_kind": null, + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": null + }, + { + "asset": "0xaf3d76f1834a1d425780943c99ea8a608f8a93f9", + "symbol": "AAPL", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "24200000000000000000", + "ref_pool": "0xaae0d815ee56e4092a5e5c2911e676fea50b2d6d", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 6, + "dec1": 18, + "pool": "0xaae0d815ee56e4092a5e5c2911e676fea50b2d6d", + "token0": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "token1": "0xaf3d76f1834a1d425780943c99ea8a608f8a93f9", + "emitter": "0xaae0d815ee56e4092a5e5c2911e676fea50b2d6d", + "token_in": "0xaf3d76f1834a1d425780943c99ea8a608f8a93f9" + }, + { + "dec0": 18, + "dec1": 6, + "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168" + } + ] + }, + { + "asset": "0xb0992820e760d836549ba69bc7598b4af75dee03", + "symbol": "ORCL", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "74826304376687171137", + "ref_pool": null, + "ref_kind": null, + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": null + }, + { + "asset": "0xb1bf26c1d20ff267a4f93550d1e0d06ac40a114b", + "symbol": "RIVN", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "643937219730941726039", + "ref_pool": "0xf9e71558eef115b6c8fbfc9ad5fda00033e9d69c", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 6, + "dec1": 18, + "pool": "0xf9e71558eef115b6c8fbfc9ad5fda00033e9d69c", + "token0": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "token1": "0xb1bf26c1d20ff267a4f93550d1e0d06ac40a114b", + "emitter": "0xf9e71558eef115b6c8fbfc9ad5fda00033e9d69c", + "token_in": "0xb1bf26c1d20ff267a4f93550d1e0d06ac40a114b" + }, + { + "dec0": 18, + "dec1": 6, + "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168" + } + ] + }, + { + "asset": "0xb90a19ff0af67f7779aff50a882a9cff42446400", + "symbol": "SNDK", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "6630443315631494151", + "ref_pool": "0x995c1ad5eb998b1bdd89f515c4bb64760c411b62", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 18, + "dec1": 18, + "pool": "0x995c1ad5eb998b1bdd89f515c4bb64760c411b62", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0xb90a19ff0af67f7779aff50a882a9cff42446400", + "emitter": "0x995c1ad5eb998b1bdd89f515c4bb64760c411b62", + "token_in": "0xb90a19ff0af67f7779aff50a882a9cff42446400" + } + ] + }, + { + "asset": "0xba0cab75495255d0cb58e22b648bfed4ecd1f47e", + "symbol": "SNOW", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "30736040715515305748", + "ref_pool": "0xe0ca1dfc1161500fb410c8181f7cc33860ffb16b", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 6, + "dec1": 18, + "pool": "0xe0ca1dfc1161500fb410c8181f7cc33860ffb16b", + "token0": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "token1": "0xba0cab75495255d0cb58e22b648bfed4ecd1f47e", + "emitter": "0xe0ca1dfc1161500fb410c8181f7cc33860ffb16b", + "token_in": "0xba0cab75495255d0cb58e22b648bfed4ecd1f47e" + }, + { + "dec0": 18, + "dec1": 6, + "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168" + } + ] + }, + { + "asset": "0xc0d6457c16cc70d6790dd43521c899c87ce02f35", + "symbol": "META", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "13567895109851171552", + "ref_pool": "0xa4bdb396a69617eb7f70e2cc1ef526f7340b1b0d", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 18, + "dec1": 18, + "pool": "0xa4bdb396a69617eb7f70e2cc1ef526f7340b1b0d", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0xc0d6457c16cc70d6790dd43521c899c87ce02f35", + "emitter": "0xa4bdb396a69617eb7f70e2cc1ef526f7340b1b0d", + "token_in": "0xc0d6457c16cc70d6790dd43521c899c87ce02f35" + } + ] + }, + { + "asset": "0xc72b96e0e48ecd4dc75e1e45396e26300bc39681", + "symbol": "INTC", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "101528935917721519434", + "ref_pool": "0x2e5a92f5013a64661a49312111be2e8abd33f56a", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 6, + "dec1": 18, + "pool": "0x2e5a92f5013a64661a49312111be2e8abd33f56a", + "token0": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "token1": "0xc72b96e0e48ecd4dc75e1e45396e26300bc39681", + "emitter": "0x2e5a92f5013a64661a49312111be2e8abd33f56a", + "token_in": "0xc72b96e0e48ecd4dc75e1e45396e26300bc39681" + }, + { + "dec0": 18, + "dec1": 6, + "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168" } - }, - "diagnostics": { - "rows": 5648, - "slope": 0.9636524869032652, - "kc_active": 0.09993186593055725, - "rank_corr": 0.8975306157777404, - "mbon_share": 0.48021747339581045, - "mbon_saturated": 0.010416666977107525 - }, - "gate_failures": [] + ] }, - "evaluation": { - "n": 1651, - "pf": 1.1344105930713058, - "win": 0.4476075105996366, - "days": 29, - "mean": 0.015233186073601246, - "median": -0.02210576832294464, - "days_positive": 19 + { + "asset": "0xc9a981fee1f9dec688bb123ccdecc63d0debfc4e", + "symbol": "GLD", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "24940596949573086306", + "ref_pool": "0x26250ba84465454bc731f710e46f1f32b167d66b", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 18, + "dec1": 18, + "pool": "0x26250ba84465454bc731f710e46f1f32b167d66b", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0xc9a981fee1f9dec688bb123ccdecc63d0debfc4e", + "emitter": "0x26250ba84465454bc731f710e46f1f32b167d66b", + "token_in": "0xc9a981fee1f9dec688bb123ccdecc63d0debfc4e" + } + ] }, - "strategies": [ - "capitulation" - ], - "finished_at": "2026-09-22T07:27:02.794988+00:00", - "per_strategy": { - "capitulation": { - "alpha": 0.0003, - "chosen": { - "mean": 0.023196974769234657, - "alpha": 0.0003, - "total": 75.43656158447266, - "config": 5, - "trades": 3252, - "governance": { - "ic": 0, - "any": 0, - "drift": 0, - "checks": 1365, - "shadow": 0 - }, - "half_life_days": 1.0 + { + "asset": "0xccee82fe024c36fa15e1005ede3e9e4787e23d09", + "symbol": "HIMS", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "354892004153686420827", + "ref_pool": "0xeb576c467d69e084a0fdc6ddf744467804634650", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 18, + "dec1": 18, + "pool": "0xeb576c467d69e084a0fdc6ddf744467804634650", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0xccee82fe024c36fa15e1005ede3e9e4787e23d09", + "emitter": "0xeb576c467d69e084a0fdc6ddf744467804634650", + "token_in": "0xccee82fe024c36fa15e1005ede3e9e4787e23d09" + } + ] + }, + { + "asset": "0xcec185eb182c47d1ba1efc84e6959e18cd620be4", + "symbol": "cbBTC", + "class": "btc", + "decimals": 8, + "approved": true, + "graduation_threshold_raw": "13184752", + "ref_pool": "0xd30e44aae604b42a63f6f9a8109fd0408f35b9fb", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 18, + "dec1": 8, + "pool": "0xd30e44aae604b42a63f6f9a8109fd0408f35b9fb", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0xcec185eb182c47d1ba1efc84e6959e18cd620be4", + "emitter": "0xd30e44aae604b42a63f6f9a8109fd0408f35b9fb", + "token_in": "0xcec185eb182c47d1ba1efc84e6959e18cd620be4" + } + ] + }, + { + "asset": "0xcef9027c7d6985b85f0ba431125073529a947a68", + "symbol": "BULL", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "1054596630934150255151", + "ref_pool": "0x0a393801fea32a50be77dab6c8bbb503db3a4663", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 18, + "dec1": 18, + "pool": "0x0a393801fea32a50be77dab6c8bbb503db3a4663", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0xcef9027c7d6985b85f0ba431125073529a947a68", + "emitter": "0x0a393801fea32a50be77dab6c8bbb503db3a4663", + "token_in": "0xcef9027c7d6985b85f0ba431125073529a947a68" + } + ] + }, + { + "asset": "0xd0601ce157db5bdc3162bbac2a2c8af5320d9eec", + "symbol": "NVDA", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "41600000000000000000", + "ref_pool": "0x62ab521f71431f78ac374cdbadc6cda3c8916b6c", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 18, + "dec1": 18, + "pool": "0x62ab521f71431f78ac374cdbadc6cda3c8916b6c", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0xd0601ce157db5bdc3162bbac2a2c8af5320d9eec", + "emitter": "0x62ab521f71431f78ac374cdbadc6cda3c8916b6c", + "token_in": "0xd0601ce157db5bdc3162bbac2a2c8af5320d9eec" + } + ] + }, + { + "asset": "0xd5f3879160bc7c32ebb4dc785f8a4f505888de68", + "symbol": "QQQ", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "11146474875652736535", + "ref_pool": "0xd60a5d14db690b7afad71f76b108071d7175597d", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 6, + "dec1": 18, + "pool": "0xd60a5d14db690b7afad71f76b108071d7175597d", + "token0": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "token1": "0xd5f3879160bc7c32ebb4dc785f8a4f505888de68", + "emitter": "0xd60a5d14db690b7afad71f76b108071d7175597d", + "token_in": "0xd5f3879160bc7c32ebb4dc785f8a4f505888de68" }, - "configs": [ - { - "mean": 0.016807934269309044, - "alpha": 0.0, - "total": 65.02989959716797, - "config": 0, - "trades": 3869, - "governance": { - "ic": 0, - "any": 0, - "drift": 0, - "checks": 0, - "shadow": 0 - }, - "half_life_days": null - }, - { - "mean": 0.01756637543439865, - "alpha": 0.0001, - "total": 63.76594543457031, - "config": 1, - "trades": 3630, - "governance": { - "ic": 0, - "any": 0, - "drift": 0, - "checks": 1365, - "shadow": 0 - }, - "half_life_days": 1.0 - }, - { - "mean": 0.020793743431568146, - "alpha": 0.0001, - "total": 68.8896713256836, - "config": 2, - "trades": 3313, - "governance": { - "ic": 0, - "any": 0, - "drift": 0, - "checks": 1365, - "shadow": 0 - }, - "half_life_days": 3.0 - }, - { - "mean": 0.02111992798745632, - "alpha": 0.0001, - "total": 61.73354721069336, - "config": 3, - "trades": 2923, - "governance": { - "ic": 0, - "any": 0, - "drift": 0, - "checks": 1365, - "shadow": 0 - }, - "half_life_days": 7.0 - }, - { - "mean": 0.020111948251724243, - "alpha": 0.0001, - "total": 50.62177276611328, - "config": 4, - "trades": 2517, - "governance": { - "ic": 0, - "any": 68, - "drift": 0, - "checks": 1365, - "shadow": 68 - }, - "half_life_days": 30.0 - }, - { - "mean": 0.023196974769234657, - "alpha": 0.0003, - "total": 75.43656158447266, - "config": 5, - "trades": 3252, - "governance": { - "ic": 0, - "any": 0, - "drift": 0, - "checks": 1365, - "shadow": 0 - }, - "half_life_days": 1.0 - }, - { - "mean": 0.020104866474866867, - "alpha": 0.0003, - "total": 52.694854736328125, - "config": 6, - "trades": 2621, - "governance": { - "ic": 0, - "any": 34, - "drift": 0, - "checks": 1365, - "shadow": 34 - }, - "half_life_days": 3.0 - }, - { - "mean": 0.015242534689605236, - "alpha": 0.0003, - "total": 31.61301612854004, - "config": 7, - "trades": 2074, - "governance": { - "ic": 0, - "any": 93, - "drift": 0, - "checks": 1365, - "shadow": 93 - }, - "half_life_days": 7.0 - }, - { - "mean": 0.014115910045802593, - "alpha": 0.0003, - "total": 23.277135848999023, - "config": 8, - "trades": 1649, - "governance": { - "ic": 0, - "any": 111, - "drift": 0, - "checks": 1365, - "shadow": 111 - }, - "half_life_days": 30.0 - }, - { - "mean": 0.014811468310654163, - "alpha": 0.001, - "total": 39.4577522277832, - "config": 9, - "trades": 2664, - "governance": { - "ic": 0, - "any": 74, - "drift": 0, - "checks": 1365, - "shadow": 74 - }, - "half_life_days": 1.0 - }, - { - "mean": 0.011060998775064945, - "alpha": 0.001, - "total": 18.847942352294922, - "config": 10, - "trades": 1704, - "governance": { - "ic": 0, - "any": 139, - "drift": 0, - "checks": 1365, - "shadow": 139 - }, - "half_life_days": 3.0 - }, - { - "mean": 0.002477346919476986, - "alpha": 0.001, - "total": 3.146230697631836, - "config": 11, - "trades": 1270, - "governance": { - "ic": 0, - "any": 184, - "drift": 0, - "checks": 1365, - "shadow": 184 - }, - "half_life_days": 7.0 - }, - { - "mean": 0.018671352416276932, - "alpha": 0.001, - "total": 19.77296257019043, - "config": 12, - "trades": 1059, - "governance": { - "ic": 0, - "any": 144, - "drift": 0, - "checks": 1365, - "shadow": 144 - }, - "half_life_days": 30.0 - }, - { - "mean": 0.01424670871347189, - "alpha": 0.003, - "total": 28.10875701904297, - "config": 13, - "trades": 1973, - "governance": { - "ic": 0, - "any": 203, - "drift": 0, - "checks": 1365, - "shadow": 203 - }, - "half_life_days": 1.0 - }, - { - "mean": 0.027894049882888794, - "alpha": 0.003, - "total": 38.716941833496094, - "config": 14, - "trades": 1388, - "governance": { - "ic": 0, - "any": 71, - "drift": 0, - "checks": 1365, - "shadow": 71 - }, - "half_life_days": 3.0 - }, - { - "mean": 0.01914161443710327, - "alpha": 0.003, - "total": 18.816207885742188, - "config": 15, - "trades": 983, - "governance": { - "ic": 0, - "any": 112, - "drift": 0, - "checks": 1365, - "shadow": 112 - }, - "half_life_days": 7.0 - }, - { - "mean": 0.028120551258325577, - "alpha": 0.003, - "total": 22.524560928344727, - "config": 16, - "trades": 801, - "governance": { - "ic": 0, - "any": 196, - "drift": 0, - "checks": 1365, - "shadow": 196 - }, - "half_life_days": 30.0 - } - ], - "plastic": true, - "hold_min": 240, - "half_life_days": 1.0 - } + { + "dec0": 18, + "dec1": 6, + "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168" + } + ] }, - "half_life_days": 1.0, - "selection_days": [ - "2026-07-27", - "2026-08-23" - ], - "evaluation_days": [ - "2026-08-24", - "2026-09-21" - ], - "plastic_beats_frozen": true - }, - "pinned_selector_sha256": "3997c13242e1ef3abcd9a4105cdb58c39a29a72f55c55d2585166d522d948ac1", - "autostart": { - "pumpstream": true, - "runner": true, - "train": false, - "replay": false, - "discover": false, - "vault": true - } + { + "asset": "0xdf0992e440dd0be65bd8439b609d6d4366bf1cb5", + "symbol": "CRCL", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "119901670146137782822", + "ref_pool": "0x654e4143e82a5824445ade0824351c2a9acd95a8", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 6, + "dec1": 18, + "pool": "0x654e4143e82a5824445ade0824351c2a9acd95a8", + "token0": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "token1": "0xdf0992e440dd0be65bd8439b609d6d4366bf1cb5", + "emitter": "0x654e4143e82a5824445ade0824351c2a9acd95a8", + "token_in": "0xdf0992e440dd0be65bd8439b609d6d4366bf1cb5" + }, + { + "dec0": 18, + "dec1": 6, + "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168" + } + ] + }, + { + "asset": "0xe0444ef8bf4ed74f74fd73686e2ddf4c1c5591e8", + "symbol": "NFLX", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "130303046357615894393", + "ref_pool": "0x59895c0302f41aeaa129d2fa2442cec01e7ef45e", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 6, + "dec1": 18, + "pool": "0x59895c0302f41aeaa129d2fa2442cec01e7ef45e", + "token0": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "token1": "0xe0444ef8bf4ed74f74fd73686e2ddf4c1c5591e8", + "emitter": "0x59895c0302f41aeaa129d2fa2442cec01e7ef45e", + "token_in": "0xe0444ef8bf4ed74f74fd73686e2ddf4c1c5591e8" + }, + { + "dec0": 18, + "dec1": 6, + "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168" + } + ] + }, + { + "asset": "0xe93237c50d904957cf27e7b1133b510c669c2e74", + "symbol": "MSFT", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "16078639417693171399", + "ref_pool": "0xeb60bcd1d920ad6e102690ccfc6fb488899e1510", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 6, + "dec1": 18, + "pool": "0xeb60bcd1d920ad6e102690ccfc6fb488899e1510", + "token0": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "token1": "0xe93237c50d904957cf27e7b1133b510c669c2e74", + "emitter": "0xeb60bcd1d920ad6e102690ccfc6fb488899e1510", + "token_in": "0xe93237c50d904957cf27e7b1133b510c669c2e74" + }, + { + "dec0": 18, + "dec1": 6, + "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168" + } + ] + }, + { + "asset": "0xec262a75e413fafd0df80480274532c79d42da09", + "symbol": "MSTR", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "79980227385071671620", + "ref_pool": "0x6f8dc7122b56017c46892f91d38d871acf787f9b", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 8, + "dec1": 18, + "pool": "0x6f8dc7122b56017c46892f91d38d871acf787f9b", + "token0": "0xcec185eb182c47d1ba1efc84e6959e18cd620be4", + "token1": "0xec262a75e413fafd0df80480274532c79d42da09", + "emitter": "0x6f8dc7122b56017c46892f91d38d871acf787f9b", + "token_in": "0xec262a75e413fafd0df80480274532c79d42da09" + }, + { + "dec0": 6, + "dec1": 8, + "pool": "0x9664d869540e9d0a76f12c6623946c6d5d201e09", + "token0": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "token1": "0xcec185eb182c47d1ba1efc84e6959e18cd620be4", + "emitter": "0x9664d869540e9d0a76f12c6623946c6d5d201e09", + "token_in": "0xcec185eb182c47d1ba1efc84e6959e18cd620be4" + }, + { + "dec0": 18, + "dec1": 6, + "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168" + } + ] + }, + { + "asset": "0xf0c4bf4c582cb3836e98394b1d4e7b7281101be8", + "symbol": "RBLX", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "248921514629948368496", + "ref_pool": "0x6d25417718a8d6c529130a8ccc4bfbf0a18219d3", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 18, + "dec1": 18, + "pool": "0x6d25417718a8d6c529130a8ccc4bfbf0a18219d3", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0xf0c4bf4c582cb3836e98394b1d4e7b7281101be8", + "emitter": "0x6d25417718a8d6c529130a8ccc4bfbf0a18219d3", + "token_in": "0xf0c4bf4c582cb3836e98394b1d4e7b7281101be8" + } + ] + }, + { + "asset": "0xf23250dac154d05bb671cb0d0ebef3c635c79ce2", + "symbol": "UPS", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "100308545348611381553", + "ref_pool": "0x5c67a84ec5494ffe24e217b28debb96341014c86", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 6, + "dec1": 18, + "pool": "0x5c67a84ec5494ffe24e217b28debb96341014c86", + "token0": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "token1": "0xf23250dac154d05bb671cb0d0ebef3c635c79ce2", + "emitter": "0x5c67a84ec5494ffe24e217b28debb96341014c86", + "token_in": "0xf23250dac154d05bb671cb0d0ebef3c635c79ce2" + }, + { + "dec0": 18, + "dec1": 6, + "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168" + } + ] + }, + { + "asset": "0xf3081494b87e8d5fb7960f066e931d1d0e6e3d67", + "symbol": "TAO", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "37497530129218277084", + "ref_pool": "0x572d51a3de7c220fdd19451e4e24183b9f2ecadc", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 18, + "dec1": 18, + "pool": "0x572d51a3de7c220fdd19451e4e24183b9f2ecadc", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0xf3081494b87e8d5fb7960f066e931d1d0e6e3d67", + "emitter": "0x572d51a3de7c220fdd19451e4e24183b9f2ecadc", + "token_in": "0xf3081494b87e8d5fb7960f066e931d1d0e6e3d67" + } + ] + }, + { + "asset": "0xf53f66751b1eff985311b693531e3290f600c410", + "symbol": "SHOP", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "70089387976658977032", + "ref_pool": "0x18a1afa849a6940870c2163b7ecbcc85d455ad99", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 6, + "dec1": 18, + "pool": "0x18a1afa849a6940870c2163b7ecbcc85d455ad99", + "token0": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "token1": "0xf53f66751b1eff985311b693531e3290f600c410", + "emitter": "0x18a1afa849a6940870c2163b7ecbcc85d455ad99", + "token_in": "0xf53f66751b1eff985311b693531e3290f600c410" + }, + { + "dec0": 18, + "dec1": 6, + "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168" + } + ] + }, + { + "asset": "0xf6589f11bc40b669e584073f428b05562f568733", + "symbol": "SNAP", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "1798186046511628119831", + "ref_pool": "0x84d251cedecdb949ca392e7bd9dd26481ab91088", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 18, + "dec1": 18, + "pool": "0x84d251cedecdb949ca392e7bd9dd26481ab91088", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0xf6589f11bc40b669e584073f428b05562f568733", + "emitter": "0x84d251cedecdb949ca392e7bd9dd26481ab91088", + "token_in": "0xf6589f11bc40b669e584073f428b05562f568733" + } + ] + }, + { + "asset": "0xfde6b5d9bb419b10c23268c74e369abff39c0460", + "symbol": "RCAT", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "1236195785671282379035", + "ref_pool": null, + "ref_kind": null, + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": null + }, + { + "asset": "0xff080c8ce2e5feadaca0da81314ae59d232d4afd", + "symbol": "MU", + "class": "stock", + "decimals": 18, + "approved": true, + "graduation_threshold_raw": "9178459641793090016", + "ref_pool": "0xd057b1bc54917855bbee58ead58647f47cab35e5", + "ref_kind": "uniswapv3", + "ref_quote": null, + "ref_fee_bps": null, + "trading_hours": null, + "note": null, + "ref_path": [ + { + "dec0": 6, + "dec1": 18, + "pool": "0xd057b1bc54917855bbee58ead58647f47cab35e5", + "token0": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "token1": "0xff080c8ce2e5feadaca0da81314ae59d232d4afd", + "emitter": "0xd057b1bc54917855bbee58ead58647f47cab35e5", + "token_in": "0xff080c8ce2e5feadaca0da81314ae59d232d4afd" + }, + { + "dec0": 18, + "dec1": 6, + "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", + "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", + "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", + "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168" + } + ] + } + ] } diff --git a/seed/seed.sql b/seed/seed.sql index c84839e65d791930ca6617f7de0b7f56fe83e1e3..921695c30174ca38b008c4487823ca982425b227 100644 --- a/seed/seed.sql +++ b/seed/seed.sql @@ -1,6 +1,84 @@ -- generated by tools/make_seed.py: the rows a fresh install needs to trade. Safe to re-run: nothing here overwrites. -INSERT INTO brain_snapshots (id, ts, run_id, beat_id, path, sha256, kind, promoted_at, promoted_by, note) VALUES ('100', '2026-09-24 08:36:20.177537+00:00', NULL, NULL, '/app/data/brain/selectors/selector_20260924T083620Z.joblib', '3c7fa68272b3f27af603c2b617938b2225f392b671238ca09d2786a95f4f359e', 'selector', NULL, NULL, '{"threshold": 0.11570790693163782, "horizon_min": 240, "trained_through": "2026-09-23", "walk_forward": {"n": 441, "win": 0.4943310657596372, "pf": 1.38144462996097, "total": 24.33979777363129, "mean": 0.05519228520097798, "weeks": 9, "weeks_pos": 6, "sign_ok": false, "admissible": false, "base_ok": true, "random_mean": -0.043604739010334015, "days": null}, "selection": {"n": 1109, "win": 0.454463480613165, "pf": 1.7478303945156008, "total": 114.32265460625058, "mean": 0.10308625302637564, "weeks": 9, "weeks_pos": 8, "sign_ok": true, "admissible": false, "base_ok": true}, "random_baseline": {"mean": -0.043604739010334015}, "components": [{"name": "inputs: rug+flow+skill", "passed": false, "reason": "improves the ev strategy on the selection half, fails on the evaluation half", "selection": {"n": 4257, "win": 0.4324641766502232, "pf": 1.3177436910125406, "total": 156.56129668618087, "mean": 0.03677737765707796, "weeks": 9, "weeks_pos": 5, "sign_ok": false, "admissible": false, "base_ok": true}, "evaluation": {"n": 1956, "win": 0.4611451942740286, "pf": 1.2231061387274471, "total": 49.72493856828805, "mean": 0.025421747734298596, "weeks": 9, "weeks_pos": 8, "sign_ok": true, "admissible": false, "base_ok": false}, "params": {"groups": ["rug", "flow", "skill"]}, "trials": 30, "best_seen": {"n": 11175, "win": 0.4223713646532439, "pf": 1.1417425718774594, "total": 160.4559811209051, "mean": 0.014358477057799114, "weeks": 9, "weeks_pos": 5, "sign_ok": false, "admissible": false, "base_ok": false}}, {"name": "inputs: rug", "passed": false, "reason": "does not improve the ev strategy on the selection half", "selection": {}, "evaluation": null, "params": {"groups": ["rug"]}, "trials": 30, "best_seen": {"n": 9960, "win": 0.42168674698795183, "pf": 1.1128948539095427, "total": 120.22805674709161, "mean": 0.012071090034848556, "weeks": 9, "weeks_pos": 7, "sign_ok": false, "admissible": false, "base_ok": false}}, {"name": "inputs: flow", "passed": false, "reason": "does not improve the ev strategy on the selection half", "selection": {}, "evaluation": null, "params": {"groups": ["flow"]}, "trials": 30, "best_seen": {"n": 10646, "win": 0.4281420251737742, "pf": 1.051834368358197, "total": 56.66556845177479, "mean": 0.005322709792577004, "weeks": 9, "weeks_pos": 5, "sign_ok": false, "admissible": false, "base_ok": false}}, {"name": "inputs: skill", "passed": false, "reason": "does not improve the ev strategy on the selection half", "selection": {}, "evaluation": null, "params": {"groups": ["skill"]}, "trials": 30, "best_seen": {"n": 26059, "win": 0.42599485782263324, "pf": 1.0473176405557747, "total": 95.89701989544028, "mean": 0.0036799961585417813, "weeks": 9, "weeks_pos": 6, "sign_ok": false, "admissible": false, "base_ok": false}}, {"name": "strategy: ev", "passed": false, "reason": "fails on the evaluation half", "selection": {"n": 2393, "win": 0.4203928123694108, "pf": 1.4809381595099493, "total": 154.69302676280495, "mean": 0.06464397273832216, "weeks": 9, "weeks_pos": 9, "sign_ok": true, "admissible": false, "base_ok": true}, "evaluation": {"n": 1044, "win": 0.44157088122605365, "pf": 0.9643830832716181, "total": -6.344513000251027, "mean": -0.006077119732041214, "weeks": 9, "weeks_pos": 5, "sign_ok": false, "admissible": false, "base_ok": false}, "params": {"hold_min": 240, "line": 0.07518525362014761, "thr": {}, "high": null}, "trials": 254, "best_seen": {"n": 6576, "win": 0.40617396593673966, "pf": 1.2072819997504896, "total": 176.56922450580532, "mean": 0.026850551171807378, "weeks": 9, "weeks_pos": 8, "sign_ok": true, "admissible": false, "base_ok": false}}, {"name": "strategy: capitulation", "passed": true, "reason": "profit fallback: below 65 % winners, PF \u22651.3 on both halves", "selection": {"n": 1109, "win": 0.454463480613165, "pf": 1.7478303945156008, "total": 114.32265460625058, "mean": 0.10308625302637564, "weeks": 9, "weeks_pos": 8, "sign_ok": true, "admissible": false, "base_ok": true}, "evaluation": {"n": 441, "win": 0.4943310657596372, "pf": 1.38144462996097, "total": 24.33979777363129, "mean": 0.05519228520097798, "weeks": 9, "weeks_pos": 6, "sign_ok": false, "admissible": false, "base_ok": true}, "params": {"hold_min": 240, "line": 0.11570790693163782, "thr": {"neg_ret_1m": 0.0002341159502975643, "top_sell_share_1m": null, "buyers_ret": null}, "high": null}, "trials": 326, "best_seen": {"n": 5955, "win": 0.4213266162888329, "pf": 1.218445230002543, "total": 165.03532723347598, "mean": 0.02771374092921511, "weeks": 9, "weeks_pos": 8, "sign_ok": true, "admissible": false, "base_ok": false}}, {"name": "strategy: breakout", "passed": false, "reason": "fails on the evaluation half", "selection": {"n": 440, "win": 0.4340909090909091, "pf": 2.071096571247338, "total": 55.36811309057521, "mean": 0.1258366206603982, "weeks": 9, "weeks_pos": 8, "sign_ok": true, "admissible": false, "base_ok": true}, "evaluation": {"n": 284, "win": 0.5633802816901409, "pf": 0.5601770324896586, "total": -26.744006780969357, "mean": -0.0941690379611597, "weeks": 9, "weeks_pos": 2, "sign_ok": false, "admissible": false, "base_ok": false}, "params": {"hold_min": 240, "line": 0.05808033749461202, "thr": {"org_imb_5m": 0.7815153002738953, "buyers_slope": null, "log_org_vol_15m": null}, "high": "dd_1h"}, "trials": 334, "best_seen": {"n": 440, "win": 0.4340909090909091, "pf": 2.071096571247338, "total": 55.36811309057521, "mean": 0.1258366206603982, "weeks": 9, "weeks_pos": 8, "sign_ok": true, "admissible": false, "base_ok": true}}, {"name": "combination", "passed": true, "reason": "''kelly'' made more on the selection half", "selection": {"n": 1109, "win": 0.454463480613165, "pf": 1.7478303945156008, "total": 114.32265460625058, "mean": 0.10308625302637564, "weeks": 9, "weeks_pos": 8, "sign_ok": true, "admissible": false, "base_ok": true}, "evaluation": null, "params": {"rule": "kelly"}, "trials": 0, "best_seen": null}, {"name": "meta-label: capitulation", "passed": false, "reason": "rejected (selection, evaluation or random-removal test)", "selection": {"n": 4868, "win": 0.45357436318816763, "pf": 1.3253304914808768, "total": 185.95118421799816, "mean": 0.03819868204971203, "weeks": 8, "weeks_pos": 8, "sign_ok": true, "admissible": false, "base_ok": true}, "evaluation": {"n": 3440, "win": 0.44622093023255816, "pf": 1.0994644592296465, "total": 46.4497674464219, "mean": 0.013502839373959855, "weeks": 9, "weeks_pos": 7, "sign_ok": false, "admissible": false, "base_ok": false, "random_removal": {"filtered_total": 46.44976806640625, "random_total": 24.33979777363129, "removed": 0}}, "params": {"cut": 0.39078361392021177, "line": 0.021555477753281593}, "trials": 210, "best_seen": {"n": 11521, "win": 0.4621994618522698, "pf": 1.220024908037652, "total": 210.5228690501508, "mean": 0.018272968409873344, "weeks": 8, "weeks_pos": 8, "sign_ok": true, "admissible": false, "base_ok": false}}, {"name": "dump veto", "passed": false, "reason": "rejected", "selection": {"n": 1099, "win": 0.45950864422202004, "pf": 1.7780571620545362, "total": 116.1648940028972, "mean": 0.105700540493992, "weeks": 9, "weeks_pos": 9, "sign_ok": true, "admissible": false, "base_ok": true}, "evaluation": {"n": 434, "win": 0.4861751152073733, "pf": 1.3334953482626586, "total": 21.22580524138175, "mean": 0.04890738534880588, "weeks": 9, "weeks_pos": 6, "sign_ok": false, "admissible": false, "base_ok": true, "random_removal": {"filtered_total": 21.225805282592773, "random_total": 23.696868153940887, "removed": 7}}, "params": {"dump": -0.15993250161409378, "cut": 0.6220203638076782}, "trials": 56, "best_seen": {"n": 1099, "win": 0.45950864422202004, "pf": 1.7780571620545362, "total": 116.1648940028972, "mean": 0.105700540493992, "weeks": 9, "weeks_pos": 9, "sign_ok": true, "admissible": false, "base_ok": true}}], "strategies": {"capitulation": {"line": 0.11570790693163782, "hold_min": 240, "thr": {"neg_ret_1m": 0.0002341159502975643, "top_sell_share_1m": null, "buyers_ret": null}, "high": null, "hours": [], "regimes": [], "sizing": [{"lo": 0.0, "n": 277, "mean": 0.10130982453193489, "win": 0.49458483754512633, "kelly": 0.4}, {"lo": 0.006253734230995178, "n": 277, "mean": 0.02603360336582388, "win": 0.45126353790613716, "kelly": 0.1}, {"lo": 0.015874013304710388, "n": 277, "mean": 0.1072833670226736, "win": 0.48014440433212996, "kelly": 0.39}, {"lo": 0.038345277309417725, "n": 278, "mean": 0.17744975688234105, "win": 0.3920863309352518, "kelly": 0.13}], "selection": {"n": 1109, "win": 0.454463480613165, "pf": 1.7478303945156008, "total": 114.32265460625058, "mean": 0.10308625302637564, "weeks": 9, "weeks_pos": 8, "sign_ok": true, "admissible": false, "base_ok": true}, "evaluation": {"n": 441, "win": 0.4943310657596372, "pf": 1.38144462996097, "total": 24.33979777363129, "mean": 0.05519228520097798, "weeks": 9, "weeks_pos": 6, "sign_ok": false, "admissible": false, "base_ok": true}}}, "groups": [], "combine": "kelly", "veto": null, "fallback": true, "holdout": {"days": ["2026-09-03", "2026-09-23"], "rows": 2897439, "n": 43, "win": 0.6046511627906976, "pf": 2.1796729286676384, "total": 6.509838661761023, "mean": 0.15139159678514008, "weeks": 3, "weeks_pos": 2, "sign_ok": false, "admissible": false, "base_ok": false, "random_mean": -0.06539860367774963}, "line": 0.11570790693163782, "costs": "real fees (pool fee by market cap, Jupiter 10 bps, network fee) + impact", "data": {"agg": 4, "features": 3, "costs": "real-fees-sized-1", "selector": "multi-1", "cols": "1ff02dc9", "skill": "h10-L30", "meta": "rug-1"}, "deployable": true, "deploy_reason": "its backtest made +5.52% per trade after costs (random -4.36%); profit fallback (below 65 % winners)", "sizing": [{"lo": 0.0, "n": 277, "mean": 0.10130982453193489, "win": 0.49458483754512633, "kelly": 0.4}, {"lo": 0.006253734230995178, "n": 277, "mean": 0.02603360336582388, "win": 0.45126353790613716, "kelly": 0.1}, {"lo": 0.015874013304710388, "n": 277, "mean": 0.1072833670226736, "win": 0.48014440433212996, "kelly": 0.39}, {"lo": 0.038345277309417725, "n": 278, "mean": 0.17744975688234105, "win": 0.3920863309352518, "kelly": 0.13}], "model": {"kind": "stack", "min_age_h": 6.0, "min_resq_sol": 10.0, "min_vol_15m_sol": 5.0}, "rows": 30340350, "days": 158, "first_day": "2026-04-19", "last_day": "2026-09-23"}') ON CONFLICT (id) DO NOTHING; -INSERT INTO brain_snapshots (id, ts, run_id, beat_id, path, sha256, kind, promoted_at, promoted_by, note) VALUES ('107', '2026-09-24 18:51:23.709522+00:00', NULL, NULL, '/app/data/brain/policies/fly_20260924T185123Z.pt', '5f2313748341ee8e2c8b49c0057d9cb5209e67a249510f3a2cff226cbca2e36d', 'fly_selector', NULL, NULL, '{"S": "2026-09-24", "train_through": "2026-09-15", "calibration_days": ["2026-09-17", "2026-09-23"], "line": 0.015, "sizing": [{"lo": 0.0, "n": 119, "mean": 0.037146079662635514, "win": 0.5126050420168067, "kelly": 0.38}, {"lo": 0.0010151841118931776, "n": 118, "mean": 0.05597637546686771, "win": 0.5254237288135594, "kelly": 0.58}, {"lo": 0.002977347597479821, "n": 118, "mean": 0.0011952295060806228, "win": 0.4661016949152542, "kelly": 0.02}, {"lo": 0.007189072333276272, "n": 119, "mean": 0.017359240755376194, "win": 0.5042016806722689, "kelly": 0.11}], "lines": {"capitulation": 0.015}, "calibration": {"trades": 474, "mean": 0.027916418811289655, "total": 13.232382516551297, "per_strategy": {"capitulation": {"line": 0.015, "trades": 474, "mean": 0.027916418811289655, "total": 13.232382516551297, "hold_min": 240}}}, "strategies": ["capitulation"], "rules": {"capitulation": {"thr": {"neg_ret_1m": 0.0002341159502975643, "top_sell_share_1m": null, "buyers_ret": null}, "high": null, "hold_min": 240}}, "diagnostics": {"mbon_share": 0.3002465268329232, "mbon_saturated": 0.0520833320915699, "kc_active": 0.09762968122959137, "rows": 12579, "slope": 1.0652187606670855, "rank_corr": 0.9376083354709515}, "gates_ok": true, "gate_failures": [], "fit": {"epochs": [{"epoch": 0, "mse": 0.5537799043314797, "secs": 2690.000976085663, "rows": 3286528, "val_mse": 0.35021153676655026, "val_se": 0.0038254940447589748}, {"epoch": 1, "mse": 0.295886658240022, "secs": 5346.020434856415, "rows": 3286528, "val_mse": 0.23602828393799027, "val_se": 0.0024367370898690094}, {"epoch": 2, "mse": 0.2361793302070372, "secs": 7990.942258119583, "rows": 3286528, "val_mse": 0.21071768673751995, "val_se": 0.0020323132598198106}, {"epoch": 3, "mse": 0.2034299299585186, "secs": 10638.009415864944, "rows": 3286528, "val_mse": 0.18947838430085023, "val_se": 0.0017588653591045853}, {"epoch": 4, "mse": 0.1801670785609724, "secs": 13283.329095840454, "rows": 3286528, "val_mse": 0.18435137828233863, "val_se": 0.0015934345099618834}, {"epoch": 5, "mse": 0.16378687861172242, "secs": 15929.834659099579, "rows": 3286528, "val_mse": 0.18715238503235043, "val_se": 0.001756727728581116}], "stopped": false, "rows": 3286535, "training_rows": 9730700, "top_weight": 10.0, "strategies": ["capitulation"], "epochs_run": 6, "epoch_cap": 8, "val_rows": 200000, "val_mse": 0.18435137828233863, "early_stop": "epoch 5 moved the held-out error by -0.0028, inside its standard error 0.0018"}, "teacher": "deployed selector #100: capitulation (line 0.1157, 240 min)", "teacher_line": 0.11570790693163782, "teacher_stack": null, "teacher_snapshot": 100, "data": {"agg": 4, "features": 3, "costs": "real-fees-sized-1", "selector": "multi-1", "cols": "1ff02dc9", "skill": "h10-L30", "meta": "rug-1", "fly": "flynet-3-ch", "plastic": "mb4-ch", "teacher": "deployed-1"}}') ON CONFLICT (id) DO NOTHING; +INSERT INTO brain_snapshots (id, ts, run_id, beat_id, path, sha256, kind, promoted_at, promoted_by, note) VALUES ('154', '2026-10-03 20:14:15.936974+00:00', NULL, NULL, '/app/data/brain/selectors/selector_20261003T201413Z.skops', 'b38f0633ee4a9b693bfb9a01a5e5de044e2ce317e08fb9726e77b48dfbb603fe', 'selector', NULL, NULL, '{"threshold": 0.11529686208814378, "horizon_min": 120, "trained_through": "2026-10-02", "walk_forward": {"n": 2933, "win": 0.480054551653597, "pf": 1.6792328308220534, "total": 260.0641021499687, "mean": 0.0886682925843739, "weeks": 10, "weeks_pos": 8, "sign_ok": false, "admissible": false, "base_ok": true, "random_mean": -0.04572119563817978, "days": null}, "selection": {"n": 2579, "win": 0.423419930205506, "pf": 1.425860298729596, "total": 152.31196360703325, "mean": 0.059058535714243215, "weeks": 9, "weeks_pos": 7, "sign_ok": false, "admissible": false, "base_ok": true}, "random_baseline": {"mean": -0.04572119563817978}, "components": [{"name": "inputs: rug+flow+skill", "passed": true, "reason": "improves the ev strategy on the selection half", "selection": {"n": 4629, "win": 0.42989846619140204, "pf": 1.3337552712670528, "total": 177.56542640196858, "mean": 0.038359348974285716, "weeks": 9, "weeks_pos": 7, "sign_ok": false, "admissible": false, "base_ok": true}, "evaluation": {"n": 5304, "win": 0.46587481146304677, "pf": 1.3353569732988675, "total": 201.20124764325192, "mean": 0.03793387021931597, "weeks": 10, "weeks_pos": 8, "sign_ok": false, "admissible": false, "base_ok": true}, "params": {"groups": ["rug", "flow", "skill"]}, "trials": 30, "best_seen": {"n": 4629, "win": 0.42989846619140204, "pf": 1.3337552712670528, "total": 177.56542640196858, "mean": 0.038359348974285716, "weeks": 9, "weeks_pos": 7, "sign_ok": false, "admissible": false, "base_ok": true}}, {"name": "strategy: ev", "passed": true, "reason": "profit fallback: below 65 % winners, PF \u22651.3 on both halves", "selection": {"n": 4629, "win": 0.42989846619140204, "pf": 1.3337552712670528, "total": 177.56542640196858, "mean": 0.038359348974285716, "weeks": 9, "weeks_pos": 7, "sign_ok": false, "admissible": false, "base_ok": true}, "evaluation": {"n": 5304, "win": 0.46587481146304677, "pf": 1.3353569732988675, "total": 201.20124764325192, "mean": 0.03793387021931597, "weeks": 10, "weeks_pos": 8, "sign_ok": false, "admissible": false, "base_ok": true}, "params": {"hold_min": 120, "line": 0.03766093432903288, "thr": {}, "high": null}, "trials": 254, "best_seen": {"n": 7226, "win": 0.4046498754497647, "pf": 1.2213627086551027, "total": 203.35054916350782, "mean": 0.02814150970986823, "weeks": 9, "weeks_pos": 6, "sign_ok": false, "admissible": false, "base_ok": false}}, {"name": "strategy: capitulation", "passed": true, "reason": "profit fallback: below 65 % winners, PF \u22651.3 on both halves", "selection": {"n": 2550, "win": 0.4164705882352941, "pf": 1.3969661280720116, "total": 144.03126928402344, "mean": 0.05648285069961703, "weeks": 9, "weeks_pos": 7, "sign_ok": false, "admissible": false, "base_ok": true}, "evaluation": {"n": 2917, "win": 0.4796023311621529, "pf": 1.6604187318564194, "total": 252.55368486529187, "mean": 0.08657993996067599, "weeks": 10, "weeks_pos": 7, "sign_ok": false, "admissible": false, "base_ok": true}, "params": {"hold_min": 240, "line": 0.06913333304226396, "thr": {"neg_ret_1m": 0.00023369600821752107, "top_sell_share_1m": null, "buyers_ret": -0.9555114507675171}, "high": null}, "trials": 326, "best_seen": {"n": 6612, "win": 0.41696914700544463, "pf": 1.253637254659857, "total": 216.06627043115805, "mean": 0.032677899339255606, "weeks": 9, "weeks_pos": 6, "sign_ok": false, "admissible": false, "base_ok": false}}, {"name": "strategy: breakout", "passed": false, "reason": "fails on the evaluation half", "selection": {"n": 194, "win": 0.38144329896907214, "pf": 2.0283563531346855, "total": 28.084343573420483, "mean": 0.14476465759495094, "weeks": 9, "weeks_pos": 8, "sign_ok": true, "admissible": false, "base_ok": true}, "evaluation": {"n": 203, "win": 0.7241379310344828, "pf": 0.7808668785674713, "total": -6.714265794442326, "mean": -0.03307520095784397, "weeks": 10, "weeks_pos": 6, "sign_ok": false, "admissible": false, "base_ok": false}, "params": {"hold_min": 120, "line": 0.0904355053529145, "thr": {"org_imb_5m": null, "buyers_slope": null, "log_org_vol_15m": 2.083093547821045}, "high": "dd_1h"}, "trials": 334, "best_seen": {"n": 2081, "win": 0.38875540605478137, "pf": 1.193609019909915, "total": 47.299630237832844, "mean": 0.022729279306983586, "weeks": 9, "weeks_pos": 5, "sign_ok": false, "admissible": false, "base_ok": false}}, {"name": "combination", "passed": true, "reason": "''score'' made more on the selection half", "selection": null, "evaluation": null, "params": {"rule": "score"}, "trials": 0, "best_seen": null}, {"name": "meta-label: ev", "passed": true, "reason": "improves the book on the selection half, passes on the evaluation half and beats random removal", "selection": {"n": 2658, "win": 0.4179834462001505, "pf": 1.3834151621947472, "total": 144.59336371760583, "mean": 0.0543993091488359, "weeks": 9, "weeks_pos": 8, "sign_ok": true, "admissible": false, "base_ok": true}, "evaluation": {"n": 2989, "win": 0.47607895617263296, "pf": 1.6329352687746375, "total": 249.40963365992502, "mean": 0.08344250038806458, "weeks": 10, "weeks_pos": 7, "sign_ok": false, "admissible": false, "base_ok": true, "random_removal": {"filtered_total": 249.40963745117188, "random_total": 136.43414531027412, "removed": 2861}}, "params": {"cut": 0.46649493277072906, "line": 0.11529686208814378}, "trials": 210, "best_seen": {"n": 11698, "win": 0.4624722174730723, "pf": 1.1285983328516533, "total": 113.18220021868092, "mean": 0.00967534623172174, "weeks": 8, "weeks_pos": 7, "sign_ok": true, "admissible": false, "base_ok": false}}, {"name": "meta-label: capitulation", "passed": false, "reason": "rejected (selection, evaluation or random-removal test)", "selection": {"n": 3559, "win": 0.4520932846305142, "pf": 1.2846180967169816, "total": 126.99779967030554, "mean": 0.03568356270590209, "weeks": 8, "weeks_pos": 6, "sign_ok": false, "admissible": false, "base_ok": false}, "evaluation": {"n": 6601, "win": 0.47295864262990456, "pf": 1.457000534598307, "total": 373.215928148049, "mean": 0.05653930134041039, "weeks": 10, "weeks_pos": 9, "sign_ok": true, "admissible": false, "base_ok": true, "random_removal": {"filtered_total": 373.2159423828125, "random_total": 249.40963365992502, "removed": 0}}, "params": {"cut": 0.4471900165081024, "line": 0.030003319680690765}, "trials": 210, "best_seen": {"n": 13029, "win": 0.45076368101926473, "pf": 1.170110820407643, "total": 185.11671483521832, "mean": 0.014208052408873921, "weeks": 8, "weeks_pos": 8, "sign_ok": true, "admissible": false, "base_ok": false}}, {"name": "dump veto", "passed": true, "reason": "improves the book on the selection half, passes on the evaluation half and beats random removal", "selection": {"n": 2579, "win": 0.423419930205506, "pf": 1.425860298729596, "total": 152.31196360703325, "mean": 0.059058535714243215, "weeks": 9, "weeks_pos": 7, "sign_ok": false, "admissible": false, "base_ok": true}, "evaluation": {"n": 2933, "win": 0.480054551653597, "pf": 1.6792328308220534, "total": 260.0641021499687, "mean": 0.0886682925843739, "weeks": 10, "weeks_pos": 8, "sign_ok": false, "admissible": false, "base_ok": true, "random_removal": {"filtered_total": 260.0640869140625, "random_total": 243.56071725736874, "removed": 56}}, "params": {"dump": -0.16902928948402404, "cut": 0.4313309609889984}, "trials": 56, "best_seen": {"n": 2579, "win": 0.423419930205506, "pf": 1.425860298729596, "total": 152.31196360703325, "mean": 0.059058535714243215, "weeks": 9, "weeks_pos": 7, "sign_ok": false, "admissible": false, "base_ok": true}}], "strategies": {"ev": {"line": 0.11529686208814378, "hold_min": 120, "thr": {}, "high": null, "hours": [], "regimes": [], "sizing": [{"lo": 0.0, "n": 1157, "mean": 0.03137388963998088, "win": 0.45980985306828004, "kelly": 0.24}, {"lo": 0.0029479563236236572, "n": 1157, "mean": 0.002502596321285927, "win": 0.4373379429559205, "kelly": 0.01}, {"lo": 0.008467614650726318, "n": 1157, "mean": 0.03428360684746643, "win": 0.4278305963699222, "kelly": 0.12}, {"lo": 0.020865503698587418, "n": 1158, "mean": 0.08523678672043543, "win": 0.3946459412780656, "kelly": 0.05}], "selection": {"n": 4629, "win": 0.42989846619140204, "pf": 1.3337552712670528, "total": 177.56542640196858, "mean": 0.038359348974285716, "weeks": 9, "weeks_pos": 7, "sign_ok": false, "admissible": false, "base_ok": true}, "evaluation": {"n": 5304, "win": 0.46587481146304677, "pf": 1.3353569732988675, "total": 201.20124764325192, "mean": 0.03793387021931597, "weeks": 10, "weeks_pos": 8, "sign_ok": false, "admissible": false, "base_ok": true}}, "capitulation": {"line": 0.06913333304226396, "hold_min": 240, "thr": {"neg_ret_1m": 0.00023369600821752107, "top_sell_share_1m": null, "buyers_ret": -0.9555114507675171}, "high": null, "hours": [], "regimes": [], "sizing": [{"lo": 0.0, "n": 638, "mean": 0.0436464974508308, "win": 0.4373040752351097, "kelly": 0.13}, {"lo": 0.004838019609451294, "n": 637, "mean": 0.00892687059909758, "win": 0.39717425431711145, "kelly": 0.04}, {"lo": 0.012772876769304276, "n": 637, "mean": 0.06240325061294793, "win": 0.41444270015698587, "kelly": 0.22}, {"lo": 0.02930697798728943, "n": 638, "mean": 0.11088952460551786, "win": 0.4169278996865204, "kelly": 0.15}], "selection": {"n": 2550, "win": 0.4164705882352941, "pf": 1.3969661280720116, "total": 144.03126928402344, "mean": 0.05648285069961703, "weeks": 9, "weeks_pos": 7, "sign_ok": false, "admissible": false, "base_ok": true}, "evaluation": {"n": 2917, "win": 0.4796023311621529, "pf": 1.6604187318564194, "total": 252.55368486529187, "mean": 0.08657993996067599, "weeks": 10, "weeks_pos": 7, "sign_ok": false, "admissible": false, "base_ok": true}}}, "groups": ["rug", "flow", "skill"], "combine": "score", "veto": {"level_index": 3, "dump": -0.16902928948402404, "cut": 0.4313309609889984, "levels": [-0.5416608870029449, -0.4038370966911316, -0.266157066822052, -0.16902928948402404], "selection": {"n": 2579, "win": 0.423419930205506, "pf": 1.425860298729596, "total": 152.31196360703325, "mean": 0.059058535714243215, "weeks": 9, "weeks_pos": 7, "sign_ok": false, "admissible": false, "base_ok": true}, "trials": 56, "best_seen": {"n": 2579, "win": 0.423419930205506, "pf": 1.425860298729596, "total": 152.31196360703325, "mean": 0.059058535714243215, "weeks": 9, "weeks_pos": 7, "sign_ok": false, "admissible": false, "base_ok": true}}, "fallback": true, "holdout": {"days": ["2026-09-12", "2026-10-02"], "rows": 4379737, "n": 2156, "win": 0.48283858998144713, "pf": 1.4519683896655893, "total": 111.96499740793661, "mean": 0.05193181697956244, "weeks": 3, "weeks_pos": 3, "sign_ok": false, "admissible": false, "base_ok": true, "random_mean": -0.045112401247024536}, "chains": {"rh": {"days": ["2026-09-12", "2026-10-02"], "rows": 1239101, "n": 2072, "win": 0.48166023166023164, "pf": 1.4511607633956578, "total": 106.26783827815962, "mean": 0.05128756673656352, "weeks": 3, "weeks_pos": 3, "sign_ok": false, "admissible": false, "base_ok": true, "random_mean": -0.04212987422943115, "deployable": true}, "sol": {"days": ["2026-09-12", "2026-10-02"], "rows": 3140636, "n": 84, "win": 0.5119047619047619, "pf": 1.467581146721967, "total": 5.697159129776992, "mean": 0.06782332297353562, "weeks": 3, "weeks_pos": 3, "sign_ok": false, "admissible": false, "base_ok": false, "random_mean": -0.05558569356799126, "deployable": true}}, "line": 0.11529686208814378, "costs": "real fees (pool fee by market cap, Jupiter 10 bps, network fee) + impact", "data": {"agg": 4, "features": 3, "costs": "real-fees-sized-1", "selector": "multi-1", "cols": "86151dda", "skill": "h10-L30", "meta": "rug-1", "chains": "sol+rh-1", "rh_agg": 1, "rh_skill": "rh-h10-L30", "rh_costs": "pons-kyber-1:0.0445583:0.0222791:7.2e-06:0.0099"}, "deployable": true, "deploy_reason": "its backtest made +8.87% per trade after costs (random -4.57%); profit fallback (below 65 % winners)", "sizing": [{"lo": 0.0, "n": 1157, "mean": 0.03137388963998088, "win": 0.45980985306828004, "kelly": 0.24}, {"lo": 0.0029479563236236572, "n": 1157, "mean": 0.002502596321285927, "win": 0.4373379429559205, "kelly": 0.01}, {"lo": 0.008467614650726318, "n": 1157, "mean": 0.03428360684746643, "win": 0.4278305963699222, "kelly": 0.12}, {"lo": 0.020865503698587418, "n": 1158, "mean": 0.08523678672043543, "win": 0.3946459412780656, "kelly": 0.05}], "model": {"kind": "stack", "min_age_h": 6.0, "min_resq_sol": 10.0, "min_vol_15m_sol": 5.0}, "rows": 36259577, "days": 167, "first_day": "2026-04-19", "last_day": "2026-10-02"}') ON CONFLICT (id) DO NOTHING; +INSERT INTO brain_snapshots (id, ts, run_id, beat_id, path, sha256, kind, promoted_at, promoted_by, note) VALUES ('157', '2026-10-06 02:31:53.211081+00:00', NULL, NULL, '/app/data/brain/policies/fly_20261006T023153Z.pt', '73141bf59bd4321c66f28e4c5a7c83f994adf744574de8b17fe5d3ca94fcc1e7', 'fly_selector', NULL, NULL, '{"S": "2026-10-05", "train_through": "2026-09-26", "calibration_days": ["2026-09-28", "2026-10-04"], "line": 0.16517850856645203, "sizing": [], "lines": {"ev": 0.16517850856645203, "capitulation": 0.11595480144023895}, "chain_lines": {"rh": {"ev": 0.1612473428249359, "capitulation": 0.10683617740869522}}, "calibration": {"trades": 527, "mean": 0.045142892621126446, "total": 23.790304411333636, "per_strategy": {"ev": {"line": 0.16517850856645203, "trades": 0, "mean": null, "total": 0.0, "hold_min": 120, "rule": "match", "teacher_trades": 0, "chain": "sol"}, "capitulation": {"line": 0.11595480144023895, "trades": 41, "mean": 0.12749095906208202, "total": 5.2271293215453625, "hold_min": 240, "rule": "match", "teacher_trades": 41, "chain": "sol"}, "ev@rh": {"line": 0.1612473428249359, "trades": 24, "mean": 0.09742691635741114, "total": 2.3382459925778676, "hold_min": 120, "rule": "match", "teacher_trades": 24, "chain": "rh"}, "capitulation@rh": {"line": 0.10683617740869522, "trades": 486, "mean": 0.03819583351808287, "total": 18.563175089788274, "hold_min": 240, "rule": "match", "teacher_trades": 486, "chain": "rh"}}}, "strategies": ["ev", "capitulation"], "rules": {"ev": {"thr": {}, "high": null, "hold_min": 120}, "capitulation": {"thr": {"neg_ret_1m": 0.00023369600821752107, "top_sell_share_1m": null, "buyers_ret": -0.9555114507675171}, "high": null, "hold_min": 240}}, "diagnostics": {"mbon_share": 0.40377666358338027, "mbon_saturated": 0.02083333395421505, "kc_active": 0.10001585632562637, "rows": 50000, "slope": 1.2467359339584205, "rank_corr": 0.908825040513757}, "gates_ok": true, "gate_failures": [], "fit": {"epochs": [{"epoch": 0, "mse": 0.4948816107000309, "secs": 3383.3052699565887, "rows": 4158464, "val_mse": 0.3124759687605339, "val_se": 0.005248413752788144}, {"epoch": 1, "mse": 0.3147444514031629, "secs": 6833.467020750046, "rows": 4158464, "val_mse": 0.2591359709877761, "val_se": 0.0030850056886689205}, {"epoch": 2, "mse": 0.2590653069447602, "secs": 10234.36536693573, "rows": 4158464, "val_mse": 0.21819832194501979, "val_se": 0.0024959954740391343}, {"epoch": 3, "mse": 0.22754745311416802, "secs": 13664.291687965393, "rows": 4158464, "val_mse": 0.20198258384915535, "val_se": 0.0024399089474643077}, {"epoch": 4, "mse": 0.2057440036793584, "secs": 17083.494806051254, "rows": 4158464, "val_mse": 0.19749039311960737, "val_se": 0.002670306554215004}, {"epoch": 5, "mse": 0.19020645904756772, "secs": 20524.88501882553, "rows": 4158464, "val_mse": 0.1723157133250382, "val_se": 0.0023445692390604107}, {"epoch": 6, "mse": 0.1785365387162561, "secs": 23951.43882393837, "rows": 4158464, "val_mse": 0.19201832960320905, "val_se": 0.0017933120401605726}], "stopped": false, "rows": 4158669, "training_rows": 27173380, "top_weight": 10.0, "strategies": ["ev", "capitulation"], "epochs_run": 7, "epoch_cap": 8, "val_rows": 200000, "val_mse": 0.1723157133250382, "early_stop": "epoch 6 moved the held-out error by -0.0197, inside its standard error 0.0018"}, "teacher": "deployed selector #154: ev (line 0.1153, 120 min), capitulation (line 0.0691, 240 min), refit through 2026-09-11 (blind to the calibration week from 2026-09-28)", "teacher_line": 0.11529686208814378, "teacher_stack": null, "teacher_snapshot": 154, "data": {"agg": 4, "features": 3, "costs": "real-fees-sized-1", "selector": "multi-1", "cols": "86151dda", "skill": "h10-L30", "meta": "rug-1", "chains": "sol+rh-1", "rh_agg": 1, "rh_skill": "rh-h10-L30", "rh_costs": "pons-kyber-1:0.0445583:0.0222791:7.2e-06:0.0099", "fly": "flynet-3-ch", "plastic": "mb4-ch", "teacher": "deployed-1", "calib": "match", "lines": "per-chain-1"}}') ON CONFLICT (id) DO NOTHING; SELECT setval('brain_snapshots_id_seq', GREATEST((SELECT max(id) FROM brain_snapshots), 1)); -INSERT INTO ui_settings (key, value) VALUES ('fly_replay', '{"S": "2026-07-27", "data": {"agg": 4, "fly": "flynet-3-ch", "cols": "1ff02dc9", "meta": "rug-1", "costs": "real-fees-sized-1", "skill": "h10-L30", "plastic": "mb4-ch", "teacher": "deployed-1", "features": 3, "selector": "multi-1"}, "secs": 12592.698919296265, "alpha": 0.0003, "chosen": {"mean": 0.023196974769234657, "alpha": 0.0003, "total": 75.43656158447266, "config": 5, "trades": 3252, "governance": {"ic": 0, "any": 0, "drift": 0, "checks": 1365, "shadow": 0}, "half_life_days": 1.0}, "frozen": {"n": 2048, "pf": 1.0961381659648606, "win": 0.44140625, "days": 29, "mean": 0.010135887190699577, "median": -0.019196834415197372, "random": {"n": 39636, "pf": 0.46682219913822093, "win": 0.4265314360682208, "mean": -0.05251633748412132, "median": -0.0052637094631791115}, "days_positive": 18}, "passed": true, "random": {"n": 32155, "pf": 0.46781214203894617, "win": 0.42220494479863163, "mean": -0.052701301872730255, "median": -0.005381436087191105}, "reason": "its backtest made +1.52% per trade after costs (random -5.27%)", "configs": [{"mean": 0.016807934269309044, "alpha": 0.0, "total": 65.02989959716797, "config": 0, "trades": 3869, "governance": {"ic": 0, "any": 0, "drift": 0, "checks": 0, "shadow": 0}, "half_life_days": null}, {"mean": 0.01756637543439865, "alpha": 0.0001, "total": 63.76594543457031, "config": 1, "trades": 3630, "governance": {"ic": 0, "any": 0, "drift": 0, "checks": 1365, "shadow": 0}, "half_life_days": 1.0}, {"mean": 0.020793743431568146, "alpha": 0.0001, "total": 68.8896713256836, "config": 2, "trades": 3313, "governance": {"ic": 0, "any": 0, "drift": 0, "checks": 1365, "shadow": 0}, "half_life_days": 3.0}, {"mean": 0.02111992798745632, "alpha": 0.0001, "total": 61.73354721069336, "config": 3, "trades": 2923, "governance": {"ic": 0, "any": 0, "drift": 0, "checks": 1365, "shadow": 0}, "half_life_days": 7.0}, {"mean": 0.020111948251724243, "alpha": 0.0001, "total": 50.62177276611328, "config": 4, "trades": 2517, "governance": {"ic": 0, "any": 68, "drift": 0, "checks": 1365, "shadow": 68}, "half_life_days": 30.0}, {"mean": 0.023196974769234657, "alpha": 0.0003, "total": 75.43656158447266, "config": 5, "trades": 3252, "governance": {"ic": 0, "any": 0, "drift": 0, "checks": 1365, "shadow": 0}, "half_life_days": 1.0}, {"mean": 0.020104866474866867, "alpha": 0.0003, "total": 52.694854736328125, "config": 6, "trades": 2621, "governance": {"ic": 0, "any": 34, "drift": 0, "checks": 1365, "shadow": 34}, "half_life_days": 3.0}, {"mean": 0.015242534689605236, "alpha": 0.0003, "total": 31.61301612854004, "config": 7, "trades": 2074, "governance": {"ic": 0, "any": 93, "drift": 0, "checks": 1365, "shadow": 93}, "half_life_days": 7.0}, {"mean": 0.014115910045802593, "alpha": 0.0003, "total": 23.277135848999023, "config": 8, "trades": 1649, "governance": {"ic": 0, "any": 111, "drift": 0, "checks": 1365, "shadow": 111}, "half_life_days": 30.0}, {"mean": 0.014811468310654163, "alpha": 0.001, "total": 39.4577522277832, "config": 9, "trades": 2664, "governance": {"ic": 0, "any": 74, "drift": 0, "checks": 1365, "shadow": 74}, "half_life_days": 1.0}, {"mean": 0.011060998775064945, "alpha": 0.001, "total": 18.847942352294922, "config": 10, "trades": 1704, "governance": {"ic": 0, "any": 139, "drift": 0, "checks": 1365, "shadow": 139}, "half_life_days": 3.0}, {"mean": 0.002477346919476986, "alpha": 0.001, "total": 3.146230697631836, "config": 11, "trades": 1270, "governance": {"ic": 0, "any": 184, "drift": 0, "checks": 1365, "shadow": 184}, "half_life_days": 7.0}, {"mean": 0.018671352416276932, "alpha": 0.001, "total": 19.77296257019043, "config": 12, "trades": 1059, "governance": {"ic": 0, "any": 144, "drift": 0, "checks": 1365, "shadow": 144}, "half_life_days": 30.0}, {"mean": 0.01424670871347189, "alpha": 0.003, "total": 28.10875701904297, "config": 13, "trades": 1973, "governance": {"ic": 0, "any": 203, "drift": 0, "checks": 1365, "shadow": 203}, "half_life_days": 1.0}, {"mean": 0.027894049882888794, "alpha": 0.003, "total": 38.716941833496094, "config": 14, "trades": 1388, "governance": {"ic": 0, "any": 71, "drift": 0, "checks": 1365, "shadow": 71}, "half_life_days": 3.0}, {"mean": 0.01914161443710327, "alpha": 0.003, "total": 18.816207885742188, "config": 15, "trades": 983, "governance": {"ic": 0, "any": 112, "drift": 0, "checks": 1365, "shadow": 112}, "half_life_days": 7.0}, {"mean": 0.028120551258325577, "alpha": 0.003, "total": 22.524560928344727, "config": 16, "trades": 801, "governance": {"ic": 0, "any": 196, "drift": 0, "checks": 1365, "shadow": 196}, "half_life_days": 30.0}], "per_day": {"2026-08-24": {"n": 96, "pf": 1.0117903231205847, "win": 0.4479166666666667, "mean": 0.0016304353484883904, "median": -0.04346885532140732}, "2026-08-25": {"n": 92, "pf": 1.5657616101643343, "win": 0.5217391304347826, "mean": 0.06755560636520386, "median": 0.007279674988240004}, "2026-08-26": {"n": 263, "pf": 1.165655794982386, "win": 0.44866920152091255, "mean": 0.014974601566791534, "median": -0.012878135778009892}, "2026-08-27": {"n": 170, "pf": 0.6946020453820136, "win": 0.40588235294117647, "mean": -0.03204870969057083, "median": -0.03053605556488037}, "2026-08-28": {"n": 56, "pf": 1.4862921757026535, "win": 0.5178571428571429, "mean": 0.06174355000257492, "median": 0.010778131894767284}, "2026-08-29": {"n": 213, "pf": 1.011389332575889, "win": 0.43661971830985913, "mean": 0.0011059067910537124, "median": -0.026353871449828148}, "2026-08-30": {"n": 50, "pf": 1.3834646670134814, "win": 0.38, "mean": 0.0530250258743763, "median": -0.08351919054985046}, "2026-08-31": {"n": 41, "pf": 1.6108359675631871, "win": 0.43902439024390244, "mean": 0.07970548421144485, "median": -0.03631661832332611}, "2026-09-01": {"n": 43, "pf": 0.7714416116969681, "win": 0.4883720930232558, "mean": -0.031781017780303955, "median": -0.0033531002700328827}, "2026-09-02": {"n": 31, "pf": 1.8960080329396884, "win": 0.5161290322580645, "mean": 0.09989593923091888, "median": 0.056023046374320984}, "2026-09-03": {"n": 39, "pf": 0.47435046931837, "win": 0.38461538461538464, "mean": -0.0845755785703659, "median": -0.10096535086631775}, "2026-09-04": {"n": 44, "pf": 2.124818757306114, "win": 0.36363636363636365, "mean": 0.17666810750961304, "median": -0.04468941688537598}, "2026-09-05": {"n": 33, "pf": 3.159536891732615, "win": 0.45454545454545453, "mean": 0.11582708358764648, "median": -0.02216796576976776}, "2026-09-06": {"n": 34, "pf": 0.8038220998481828, "win": 0.47058823529411764, "mean": -0.028142571449279785, "median": -0.04691909998655319}, "2026-09-07": {"n": 25, "pf": 0.538075679014026, "win": 0.36, "mean": -0.07897567003965378, "median": -0.057357002049684525}, "2026-09-08": {"n": 24, "pf": 0.5229846306363845, "win": 0.4583333333333333, "mean": -0.0886785164475441, "median": -0.027842504903674126}, "2026-09-09": {"n": 6, "pf": 1.775177140941807, "win": 0.3333333333333333, "mean": 0.064696304500103, "median": -0.0772169902920723}, "2026-09-10": {"n": 10, "pf": 0.0, "win": 0.0, "mean": -0.4276605248451233, "median": -0.40527138113975525}, "2026-09-11": {"n": 9, "pf": 0.4853454520861287, "win": 0.3333333333333333, "mean": -0.07680502533912659, "median": -0.1473425328731537}, "2026-09-12": {"n": 104, "pf": 0.5093969547294807, "win": 0.4326923076923077, "mean": -0.043387047946453094, "median": -0.009646685793995857}, "2026-09-13": {"n": 13, "pf": 0.9683888718696267, "win": 0.46153846153846156, "mean": -0.003039261559024453, "median": -0.011067567393183708}, "2026-09-14": {"n": 19, "pf": 2.3648619908537536, "win": 0.631578947368421, "mean": 0.19480086863040924, "median": 0.07961719483137131}, "2026-09-15": {"n": 19, "pf": 1.9207007670652974, "win": 0.42105263157894735, "mean": 0.12944886088371277, "median": -0.0763954222202301}, "2026-09-16": {"n": 20, "pf": 1.989758339975232, "win": 0.6, "mean": 0.07185642421245575, "median": 0.06811605393886566}, "2026-09-17": {"n": 23, "pf": 2.308382394662165, "win": 0.5652173913043478, "mean": 0.07706345617771149, "median": 0.08088493347167969}, "2026-09-18": {"n": 63, "pf": 1.357841853369037, "win": 0.47619047619047616, "mean": 0.030619792640209198, "median": -0.004897014237940311}, "2026-09-19": {"n": 34, "pf": 1.4133537729578933, "win": 0.4117647058823529, "mean": 0.04197608679533005, "median": -0.05088033154606819}, "2026-09-20": {"n": 40, "pf": 1.2252380260800315, "win": 0.425, "mean": 0.03793734312057495, "median": -0.07361158728599548}, "2026-09-21": {"n": 37, "pf": 1.2097114398883206, "win": 0.5675675675675675, "mean": 0.016092097386717796, "median": 0.01574813388288021}}, "learning": [{"n": 931, "day": "2026-07-27", "step": [0.0, 0.06421024, 0.06420731, 0.06420637, 0.06420581, 0.18729984, 0.18720574, 0.18719876, 0.1871946, 0.58122008, 0.57589885, 0.57586091, 0.57583875, 1.56960882, 1.57073448, 1.57049312, 1.56501226], "drift": [0.0, 0.00123, 0.00152, 0.00163, 0.00169, 0.00348, 0.00437, 0.00467, 0.00485, 0.01091, 0.01395, 0.01494, 0.01554, 0.02869, 0.03589, 0.03833, 0.03947], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 45, 45, 45], "mean_delta": [-0.039099, -0.038733, -0.038703, -0.038693, -0.038688, -0.038012, -0.037925, -0.037898, -0.037881, -0.035483, -0.035206, -0.035118, -0.035067, -0.030323, -0.029782, -0.029599, -0.029538], "mean_abs_delta": [0.189807, 0.189711, 0.189705, 0.189703, 0.189702, 0.189527, 0.18951, 0.189504, 0.189501, 0.188888, 0.188834, 0.188818, 0.188808, 0.187597, 0.187515, 0.187486, 0.187462]}, {"n": 1044, "day": "2026-07-28", "step": [0.0, 0.07277379, 0.07218625, 0.07189107, 0.07176147, 0.20257254, 0.19562552, 0.19169733, 0.19072503, 0.53300112, 0.4800792, 0.46328981, 0.45527648, 1.08817285, 1.01850277, 0.99633343, 0.98662623], "drift": [0.0, 0.00112, 0.0018, 0.0021, 0.00231, 0.00342, 0.00533, 0.00625, 0.00678, 0.00927, 0.01447, 0.0171, 0.01855, 0.02377, 0.03621, 0.04178, 0.04546], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 36, 34, 32], "mean_delta": [-0.027195, -0.026147, -0.025859, -0.025755, -0.025687, -0.02413, -0.023407, -0.023065, -0.0229, -0.01855, -0.016465, -0.015533, -0.015078, -0.006329, -0.001372, 0.000907, 0.001877], "mean_abs_delta": [0.190608, 0.190481, 0.190461, 0.190455, 0.190451, 0.190261, 0.190232, 0.190217, 0.190211, 0.189725, 0.18969, 0.189675, 0.189664, 0.189226, 0.189369, 0.189431, 0.189449]}, {"n": 863, "day": "2026-07-29", "step": [0.0, 0.06034305, 0.05845988, 0.05776155, 0.05721584, 0.16231128, 0.14620346, 0.1383591, 0.13605227, 0.41583466, 0.35392232, 0.33466695, 0.32878579, 0.90389129, 0.76926308, 0.70084347, 0.65879158], "drift": [0.0, 0.00083, 0.00173, 0.00219, 0.00252, 0.00233, 0.0047, 0.00605, 0.00698, 0.00685, 0.01407, 0.01877, 0.02179, 0.02133, 0.03601, 0.04508, 0.05245], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 18, 15, 13], "mean_delta": [-0.012039, -0.011268, -0.010674, -0.010403, -0.010234, -0.009831, -0.008321, -0.007575, -0.007109, -0.006105, -0.002358, -7e-05, 0.001157, 0.001562, 0.00964, 0.014695, 0.017686], "mean_abs_delta": [0.188678, 0.188475, 0.188319, 0.188247, 0.188202, 0.188106, 0.187716, 0.187524, 0.187409, 0.187209, 0.186402, 0.18595, 0.18572, 0.185977, 0.184891, 0.184281, 0.184005]}, {"n": 1236, "day": "2026-07-30", "step": [0.0, 0.10988152, 0.10774172, 0.10559816, 0.10466212, 0.31706643, 0.29190488, 0.28316394, 0.27648895, 0.9586995, 0.81282441, 0.75331462, 0.72414899, 2.05303031, 1.7496128, 1.59959924, 1.50315613], "drift": [0.0, 0.00353, 0.00502, 0.00575, 0.00627, 0.01019, 0.01367, 0.01591, 0.01733, 0.03045, 0.0398, 0.04493, 0.04817, 0.06619, 0.08122, 0.08852, 0.09422], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 87, 80, 79], "mean_delta": [-0.091839, -0.090642, -0.089971, -0.089642, -0.089387, -0.088579, -0.087192, -0.086045, -0.085377, -0.082853, -0.07795, -0.074143, -0.072069, -0.066969, -0.05794, -0.052086, -0.04854], "mean_abs_delta": [0.237217, 0.236632, 0.236356, 0.236219, 0.23611, 0.235612, 0.235062, 0.234589, 0.234319, 0.232781, 0.231053, 0.229731, 0.229042, 0.226722, 0.22439, 0.222879, 0.222044]}, {"n": 371, "day": "2026-07-31", "step": [0.0, 0.04234476, 0.03978243, 0.03882724, 0.03834538, 0.10608156, 0.09335886, 0.09183339, 0.08744498, 0.23009973, 0.18223433, 0.16370089, 0.15161757, 0.3607691, 0.27135012, 0.22326726, 0.20295452], "drift": [0.0, 0.00223, 0.00455, 0.00569, 0.00657, 0.00623, 0.01209, 0.01551, 0.01774, 0.01846, 0.03446, 0.04292, 0.04901, 0.03862, 0.06792, 0.08091, 0.09142], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 2, 1, 1], "mean_delta": [-0.052608, -0.050835, -0.049833, -0.049374, -0.049009, -0.047575, -0.045204, -0.043704, -0.04279, -0.037831, -0.031311, -0.027866, -0.025479, -0.019581, -0.008466, -0.00396, -0.00054], "mean_abs_delta": [0.276031, 0.275416, 0.275123, 0.274987, 0.274879, 0.27429, 0.273585, 0.273156, 0.272904, 0.271009, 0.269555, 0.268781, 0.268237, 0.265625, 0.263527, 0.26297, 0.262476]}, {"n": 54, "day": "2026-08-01", "step": [0.0, 0.00741917, 0.00741597, 0.00676187, 0.00605164, 0.0194067, 0.01646108, 0.01194924, 0.0110565, 0.03528636, 0.01351464, 0.0101124, 0.00974537, 0.03056685, 0.01951861, 0.0138812, 0.01356826], "drift": [0.0, 0.00138, 0.00387, 0.00522, 0.00637, 0.00379, 0.01028, 0.0145, 0.01744, 0.01216, 0.03019, 0.04078, 0.0486, 0.02671, 0.06085, 0.0774, 0.09115], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0], "mean_delta": [0.021166, 0.022723, 0.024943, 0.026034, 0.026946, 0.025496, 0.030874, 0.034262, 0.036473, 0.034028, 0.047387, 0.055297, 0.060891, 0.046447, 0.071219, 0.081994, 0.090024], "mean_abs_delta": [0.26533, 0.265417, 0.265613, 0.265685, 0.265759, 0.265548, 0.265895, 0.26615, 0.266267, 0.265943, 0.266645, 0.267432, 0.267988, 0.266451, 0.269451, 0.271657, 0.273752]}, {"n": 242, "day": "2026-08-02", "step": [0.0, 0.01872033, 0.01859977, 0.0181289, 0.01802677, 0.0558438, 0.05201761, 0.04925188, 0.04794973, 0.17378149, 0.14824457, 0.12593751, 0.1140936, 0.47729315, 0.33874456, 0.24572129, 0.17729059], "drift": [0.0, 0.00088, 0.00314, 0.00483, 0.00643, 0.00248, 0.00828, 0.01332, 0.01741, 0.00756, 0.02334, 0.03559, 0.04639, 0.01838, 0.04457, 0.06531, 0.0875], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 5, 3, 2], "mean_delta": [-0.036087, -0.035588, -0.034009, -0.032926, -0.031938, -0.034705, -0.030626, -0.027369, -0.024855, -0.031729, -0.020271, -0.012292, -0.005555, -0.026377, -0.005737, 0.007242, 0.018867], "mean_abs_delta": [0.214218, 0.214074, 0.213654, 0.213363, 0.213101, 0.213812, 0.212741, 0.211945, 0.211312, 0.212985, 0.210212, 0.20864, 0.207606, 0.211498, 0.207818, 0.206297, 0.20526]}, {"n": 508, "day": "2026-08-03", "step": [0.0, 0.04134416, 0.0371876, 0.03266551, 0.0305573, 0.11778908, 0.08932215, 0.08048781, 0.07565604, 0.31825422, 0.24333699, 0.19889814, 0.18343096, 0.76600191, 0.54425102, 0.4655616, 0.33341111], "drift": [0.0, 0.00086, 0.00272, 0.0045, 0.00642, 0.00251, 0.0068, 0.01191, 0.0169, 0.00584, 0.01774, 0.03055, 0.04362, 0.01263, 0.03241, 0.05504, 0.08091], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 15, 13, 5], "mean_delta": [-0.011377, -0.010753, -0.009203, -0.007918, -0.006615, -0.009584, -0.005852, -0.001992, 0.001368, -0.007048, 0.003835, 0.013027, 0.021712, -0.001486, 0.016475, 0.030461, 0.046891], "mean_abs_delta": [0.206632, 0.206521, 0.206251, 0.206031, 0.205803, 0.20633, 0.205706, 0.205081, 0.204565, 0.205893, 0.204323, 0.203515, 0.203302, 0.204999, 0.203303, 0.203833, 0.206042]}, {"n": 692, "day": "2026-08-04", "step": [0.0, 0.04460403, 0.03859602, 0.03441494, 0.03139943, 0.12362088, 0.09626486, 0.06885854, 0.05953138, 0.38367219, 0.20308522, 0.16834852, 0.131699, 1.01038427, 0.50971773, 0.37668987, 0.27438032], "drift": [0.0, 0.00046, 0.00194, 0.0039, 0.006, 0.00136, 0.00491, 0.0109, 0.01666, 0.00421, 0.01525, 0.02819, 0.0435, 0.01111, 0.02828, 0.05228, 0.08124], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 6, 2, 1], "mean_delta": [0.013699, 0.013807, 0.015059, 0.016398, 0.017741, 0.014045, 0.017069, 0.021082, 0.02475, 0.013845, 0.023644, 0.032565, 0.042299, 0.013797, 0.02944, 0.045704, 0.063913], "mean_abs_delta": [0.183171, 0.183126, 0.183024, 0.182942, 0.18287, 0.183048, 0.182837, 0.182735, 0.182695, 0.182865, 0.182626, 0.182803, 0.1836, 0.182576, 0.182491, 0.184005, 0.187536]}, {"n": 1128, "day": "2026-08-05", "step": [0.0, 0.05903479, 0.05507833, 0.05134258, 0.04764679, 0.17682159, 0.15135335, 0.12787613, 0.11643432, 0.6171573, 0.40581984, 0.34735324, 0.30416585, 1.83372795, 1.00153648, 0.85559196, 0.64182627], "drift": [0.0, 0.00107, 0.00083, 0.00233, 0.00462, 0.00313, 0.00239, 0.00694, 0.01331, 0.01023, 0.0079, 0.01759, 0.03386, 0.01932, 0.01897, 0.03149, 0.05838], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 44, 39, 31], "mean_delta": [0.045258, 0.044848, 0.04591, 0.047412, 0.04904, 0.044154, 0.046767, 0.051566, 0.05613, 0.041694, 0.051311, 0.060843, 0.072213, 0.041475, 0.054393, 0.067997, 0.08784], "mean_abs_delta": [0.173116, 0.173179, 0.173065, 0.172906, 0.172747, 0.173298, 0.173014, 0.172589, 0.172375, 0.173776, 0.17276, 0.172651, 0.173458, 0.173557, 0.173238, 0.173589, 0.176942]}, {"n": 1058, "day": "2026-08-06", "step": [0.0, 0.03945935, 0.03698631, 0.03220626, 0.02746376, 0.12985755, 0.11435575, 0.07488091, 0.05460495, 0.63672671, 0.3500779, 0.1885276, 0.11419593, 1.80572728, 1.57435511, 0.62120528, 0.28846189], "drift": [0.0, 0.00064, 0.00098, 0.00238, 0.00474, 0.00196, 0.00278, 0.00677, 0.01397, 0.00734, 0.00897, 0.01889, 0.03563, 0.01789, 0.0232, 0.03536, 0.06194], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 40, 17, 6], "mean_delta": [0.003393, 0.002579, 0.003265, 0.004759, 0.006634, 0.00097, 0.002713, 0.007733, 0.01337, -0.00438, 0.003119, 0.014892, 0.02849, -0.00547, -0.003837, 0.016483, 0.041025], "mean_abs_delta": [0.168503, 0.168553, 0.168513, 0.168429, 0.168352, 0.168686, 0.168561, 0.168344, 0.16832, 0.169263, 0.168633, 0.168449, 0.169828, 0.169664, 0.169537, 0.168916, 0.172572]}, {"n": 892, "day": "2026-08-07", "step": [0.0, 0.04321418, 0.04232384, 0.03822837, 0.03495312, 0.13560243, 0.12616215, 0.09834065, 0.08321215, 0.49905728, 0.35899078, 0.26573406, 0.22310838, 0.96535605, 1.01828484, 0.73811684, 0.58092871], "drift": [0.0, 0.0006, 0.00106, 0.00252, 0.00501, 0.00186, 0.00299, 0.00707, 0.01449, 0.00655, 0.01045, 0.01962, 0.03622, 0.01756, 0.0272, 0.03745, 0.06359], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 12, 12, 11], "mean_delta": [-0.026824, -0.026974, -0.026615, -0.025306, -0.023529, -0.027278, -0.026385, -0.022427, -0.017015, -0.028057, -0.023258, -0.014869, -0.0028, -0.020316, -0.019829, -0.007844, 0.009888], "mean_abs_delta": [0.165735, 0.165729, 0.165702, 0.16562, 0.165527, 0.165724, 0.165654, 0.165483, 0.165431, 0.165654, 0.165466, 0.165447, 0.166151, 0.165184, 0.165209, 0.165848, 0.168385]}, {"n": 1076, "day": "2026-08-08", "step": [0.0, 0.07095081, 0.07041009, 0.06571411, 0.05991169, 0.21319146, 0.20814377, 0.1705785, 0.13642696, 0.70387371, 0.57735057, 0.42904051, 0.32422108, 1.63103617, 1.54005343, 1.00802515, 0.76446274], "drift": [0.0, 0.00139, 0.00201, 0.00348, 0.00597, 0.00421, 0.00595, 0.00945, 0.01669, 0.01451, 0.01815, 0.02519, 0.04134, 0.03226, 0.04307, 0.04922, 0.07187], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 50, 29, 15], "mean_delta": [-0.031996, -0.031957, -0.031714, -0.030445, -0.028644, -0.031818, -0.031171, -0.027531, -0.022065, -0.03076, -0.027356, -0.019525, -0.007765, -0.026253, -0.024635, -0.010153, 0.006282], "mean_abs_delta": [0.199058, 0.199035, 0.198997, 0.198821, 0.198581, 0.19898, 0.198881, 0.198421, 0.197903, 0.198693, 0.198275, 0.19768, 0.197643, 0.197837, 0.197494, 0.197324, 0.198843]}, {"n": 868, "day": "2026-08-09", "step": [0.0, 0.06866672, 0.0674683, 0.06261659, 0.0572501, 0.19572992, 0.18850666, 0.16446996, 0.13726034, 0.57460789, 0.53087509, 0.43740982, 0.34744486, 1.49970195, 1.28868771, 1.09052244, 0.83394681], "drift": [0.0, 0.00092, 0.00141, 0.00246, 0.00496, 0.00263, 0.00422, 0.00675, 0.01374, 0.00836, 0.01363, 0.01823, 0.03352, 0.0207, 0.03373, 0.04033, 0.06014], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 48, 38, 26], "mean_delta": [0.045081, 0.045225, 0.045497, 0.046645, 0.048417, 0.045572, 0.046402, 0.049269, 0.054513, 0.047274, 0.049228, 0.05597, 0.068074, 0.049162, 0.053368, 0.062153, 0.07952], "mean_abs_delta": [0.223275, 0.223163, 0.223108, 0.222843, 0.222452, 0.222937, 0.222776, 0.222128, 0.221108, 0.22213, 0.221745, 0.220548, 0.219228, 0.220893, 0.220428, 0.219255, 0.218497]}, {"n": 750, "day": "2026-08-10", "step": [0.0, 0.05472073, 0.05364319, 0.04933666, 0.04520726, 0.16364656, 0.1575568, 0.13419577, 0.10970383, 0.5261399, 0.49561711, 0.3807543, 0.30355334, 1.53532154, 1.38939052, 1.08685966, 0.78345868], "drift": [0.0, 0.0007, 0.00148, 0.00293, 0.00562, 0.00219, 0.00443, 0.00785, 0.01565, 0.00741, 0.01389, 0.02234, 0.03848, 0.01995, 0.03779, 0.05147, 0.07327], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 44, 35, 20], "mean_delta": [-0.038261, -0.038247, -0.038023, -0.036867, -0.034865, -0.038081, -0.037483, -0.034662, -0.02846, -0.036775, -0.035351, -0.027956, -0.015405, -0.03419, -0.029574, -0.020264, -0.004209], "mean_abs_delta": [0.197346, 0.19733, 0.197299, 0.197197, 0.197041, 0.197289, 0.197205, 0.197002, 0.19669, 0.197076, 0.196911, 0.196633, 0.196898, 0.196582, 0.195933, 0.196452, 0.197811]}, {"n": 783, "day": "2026-08-11", "step": [0.0, 0.05639539, 0.05519085, 0.05250994, 0.0479547, 0.16113087, 0.153371, 0.13901212, 0.11707776, 0.46874315, 0.42715407, 0.35583433, 0.32026198, 1.04951349, 0.93101532, 0.88466587, 0.79205738], "drift": [0.0, 0.00075, 0.00164, 0.00333, 0.00616, 0.00249, 0.00511, 0.00897, 0.01773, 0.00797, 0.0169, 0.02615, 0.0421, 0.02123, 0.03919, 0.05574, 0.07721], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 18, 13, 10], "mean_delta": [-0.017449, -0.016968, -0.01666, -0.015667, -0.013979, -0.015814, -0.01491, -0.012715, -0.007314, -0.011976, -0.008872, -0.003129, 0.00642, -0.002249, 0.004524, 0.012359, 0.023258], "mean_abs_delta": [0.223001, 0.222889, 0.222832, 0.222668, 0.2224, 0.222629, 0.222472, 0.22216, 0.221525, 0.221802, 0.221368, 0.220954, 0.220967, 0.220241, 0.21992, 0.220531, 0.221761]}, {"n": 737, "day": "2026-08-12", "step": [0.0, 0.05897562, 0.05788302, 0.05464898, 0.0493692, 0.16978297, 0.15688009, 0.13920504, 0.112585, 0.47220465, 0.39037227, 0.34833647, 0.29217996, 1.05956363, 0.93436273, 0.77778303, 0.59210142], "drift": [0.0, 0.00193, 0.00289, 0.00468, 0.00769, 0.00563, 0.00849, 0.01268, 0.02191, 0.01657, 0.02624, 0.0356, 0.05084, 0.0402, 0.05596, 0.07187, 0.09195], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 21, 14, 13], "mean_delta": [-0.070931, -0.070332, -0.069889, -0.068616, -0.066434, -0.069028, -0.067488, -0.064536, -0.057181, -0.064527, -0.058243, -0.05118, -0.039822, -0.0522, -0.04429, -0.032432, -0.017798], "mean_abs_delta": [0.21199, 0.211742, 0.211594, 0.211213, 0.210562, 0.211234, 0.210759, 0.209887, 0.207963, 0.209641, 0.207901, 0.206368, 0.204237, 0.206052, 0.204262, 0.202601, 0.201103]}, {"n": 911, "day": "2026-08-13", "step": [0.0, 0.07731382, 0.07438362, 0.06862943, 0.06052568, 0.20060044, 0.17363393, 0.15290134, 0.12778348, 0.47968175, 0.39746136, 0.34151492, 0.28810784, 0.95362768, 0.76048055, 0.63285901, 0.47991281], "drift": [0.0, 0.00205, 0.00341, 0.00526, 0.00838, 0.00603, 0.00985, 0.01463, 0.02425, 0.01872, 0.02971, 0.04066, 0.05536, 0.04117, 0.06156, 0.07875, 0.09625], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 24, 22, 19], "mean_delta": [-0.009271, -0.008132, -0.007288, -0.005889, -0.00358, -0.005773, -0.003224, 0.000174, 0.007648, 0.001879, 0.009824, 0.017645, 0.028744, 0.017, 0.03027, 0.041444, 0.053666], "mean_abs_delta": [0.213292, 0.21315, 0.213064, 0.212925, 0.21271, 0.212861, 0.212615, 0.212311, 0.211911, 0.211981, 0.211575, 0.211535, 0.212126, 0.21108, 0.211846, 0.213461, 0.21583]}, {"n": 1017, "day": "2026-08-14", "step": [0.0, 0.07309305, 0.0708886, 0.06746373, 0.06293516, 0.20178701, 0.18496513, 0.16956349, 0.14622314, 0.54395029, 0.47835179, 0.42497278, 0.3933873, 1.29079118, 1.11832579, 1.00047263, 0.93534584], "drift": [0.0, 0.00171, 0.00352, 0.00573, 0.00941, 0.00537, 0.01105, 0.01672, 0.0267, 0.01796, 0.03158, 0.04416, 0.06072, 0.03368, 0.06324, 0.08527, 0.10681], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 28, 27, 24], "mean_delta": [-0.033997, -0.033083, -0.032061, -0.030691, -0.028448, -0.030913, -0.027661, -0.024266, -0.017608, -0.022265, -0.014321, -0.006806, 0.002499, -0.008865, 0.006105, 0.01656, 0.025832], "mean_abs_delta": [0.237865, 0.237737, 0.237584, 0.237392, 0.237101, 0.23747, 0.237036, 0.236725, 0.236332, 0.23671, 0.236348, 0.236374, 0.237103, 0.236494, 0.23732, 0.238586, 0.239962]}, {"n": 782, "day": "2026-08-15", "step": [0.0, 0.05105143, 0.04709918, 0.04199555, 0.03510073, 0.13004292, 0.1017861, 0.08234778, 0.0686654, 0.28074686, 0.22478825, 0.18507749, 0.14782099, 0.65555293, 0.46043469, 0.34151427, 0.23708264], "drift": [0.0, 0.00118, 0.00309, 0.0054, 0.00937, 0.00334, 0.0091, 0.01512, 0.02533, 0.00953, 0.02327, 0.03684, 0.05429, 0.01554, 0.04102, 0.06802, 0.09635], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 7, 4, 2], "mean_delta": [-0.021645, -0.02061, -0.019244, -0.017492, -0.014532, -0.01839, -0.014151, -0.009748, -0.001841, -0.011884, -0.001369, 0.008715, 0.021385, -0.005379, 0.015963, 0.033387, 0.049669], "mean_abs_delta": [0.181345, 0.181145, 0.18091, 0.180626, 0.18017, 0.180769, 0.180115, 0.179624, 0.179017, 0.179797, 0.178949, 0.178738, 0.17941, 0.179029, 0.178991, 0.180583, 0.18319]}, {"n": 756, "day": "2026-08-16", "step": [0.0, 0.04780883, 0.04495553, 0.03995133, 0.03440757, 0.13295923, 0.10577023, 0.08685807, 0.0707289, 0.34714333, 0.25140565, 0.19970523, 0.15146923, 0.91328464, 0.58876806, 0.40649874, 0.28196843], "drift": [0.0, 0.00237, 0.00446, 0.00687, 0.01082, 0.0068, 0.01227, 0.01787, 0.02777, 0.01864, 0.02973, 0.04149, 0.05804, 0.04125, 0.05632, 0.07413, 0.10157], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 3, 3, 0], "mean_delta": [-0.078631, -0.077277, -0.075808, -0.073971, -0.070846, -0.074792, -0.070724, -0.066101, -0.057693, -0.068506, -0.059447, -0.048732, -0.035315, -0.060991, -0.046701, -0.028103, -0.007035], "mean_abs_delta": [0.185639, 0.185132, 0.184619, 0.183987, 0.182947, 0.184228, 0.182886, 0.181397, 0.179011, 0.182152, 0.17957, 0.17682, 0.17409, 0.180342, 0.176742, 0.173368, 0.171709]}, {"n": 848, "day": "2026-08-17", "step": [0.0, 0.0558308, 0.05080038, 0.04544581, 0.03801651, 0.14489821, 0.11096367, 0.09344246, 0.07635285, 0.33652354, 0.2530741, 0.21792884, 0.1736102, 0.68794068, 0.54871009, 0.45554741, 0.37407675], "drift": [0.0, 0.00201, 0.00461, 0.00733, 0.01173, 0.00597, 0.01257, 0.01913, 0.02969, 0.01726, 0.0309, 0.04521, 0.06438, 0.03726, 0.06387, 0.08498, 0.11341], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 6, 3, 1], "mean_delta": [-0.03783, -0.03619, -0.034363, -0.032334, -0.028952, -0.033246, -0.028327, -0.023285, -0.014807, -0.024236, -0.013981, -0.003355, 0.010547, -0.006998, 0.010123, 0.023707, 0.04196], "mean_abs_delta": [0.190818, 0.190362, 0.189849, 0.189308, 0.188485, 0.189594, 0.188379, 0.187249, 0.185712, 0.187543, 0.185483, 0.184258, 0.18398, 0.184215, 0.183429, 0.184094, 0.186066]}, {"n": 454, "day": "2026-08-18", "step": [0.0, 0.04264899, 0.04017388, 0.03644714, 0.03388629, 0.11645878, 0.10072397, 0.09010919, 0.07385865, 0.31588276, 0.24178546, 0.1966936, 0.15891182, 0.66131289, 0.47835376, 0.35054658, 0.22946556], "drift": [0.0, 0.00171, 0.00439, 0.00743, 0.01223, 0.00493, 0.01222, 0.01965, 0.03117, 0.01535, 0.03179, 0.0463, 0.06717, 0.03504, 0.06446, 0.08716, 0.11527], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1, 1, 0], "mean_delta": [-0.065056, -0.063888, -0.062266, -0.060534, -0.057736, -0.061668, -0.057426, -0.053271, -0.046597, -0.055009, -0.046043, -0.037613, -0.026444, -0.04393, -0.027723, -0.015299, -0.000573], "mean_abs_delta": [0.263289, 0.262836, 0.262254, 0.261622, 0.260693, 0.262006, 0.260626, 0.259372, 0.25756, 0.25982, 0.257263, 0.25544, 0.253375, 0.256353, 0.253108, 0.251597, 0.250156]}, {"n": 517, "day": "2026-08-19", "step": [0.0, 0.0474315, 0.04435627, 0.04203624, 0.03866324, 0.13560452, 0.1174881, 0.11074909, 0.09387226, 0.3948683, 0.32294493, 0.26289569, 0.19850024, 0.97131711, 0.63472109, 0.49923458, 0.3656384], "drift": [0.0, 0.00078, 0.00324, 0.00651, 0.01163, 0.00211, 0.00898, 0.0168, 0.02923, 0.00666, 0.02246, 0.03943, 0.06472, 0.01641, 0.0475, 0.07743, 0.10934], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 9, 4, 3], "mean_delta": [0.008636, 0.009295, 0.010871, 0.012725, 0.015575, 0.010438, 0.014863, 0.019304, 0.026056, 0.014393, 0.024352, 0.033481, 0.045581, 0.021344, 0.040674, 0.054498, 0.068201], "mean_abs_delta": [0.254107, 0.253952, 0.253632, 0.253282, 0.252768, 0.253692, 0.252861, 0.252089, 0.251132, 0.252837, 0.251192, 0.250173, 0.249123, 0.251433, 0.24925, 0.248184, 0.247839]}, {"n": 501, "day": "2026-08-20", "step": [0.0, 0.0512835, 0.04814666, 0.04310128, 0.03719705, 0.14805485, 0.12118479, 0.09997414, 0.0662997, 0.4322925, 0.30496283, 0.18570551, 0.11847896, 1.07113278, 0.52280753, 0.32041808, 0.22553451], "drift": [0.0, 0.00121, 0.00347, 0.00674, 0.01217, 0.00332, 0.00938, 0.01713, 0.02928, 0.00984, 0.0231, 0.03693, 0.06333, 0.02102, 0.04091, 0.0694, 0.10466], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 9, 3, 3], "mean_delta": [-0.074604, -0.073799, -0.072062, -0.06974, -0.066028, -0.072373, -0.067635, -0.062199, -0.053684, -0.067778, -0.057609, -0.047488, -0.030146, -0.060298, -0.044217, -0.025152, -0.005762], "mean_abs_delta": [0.248258, 0.248032, 0.247679, 0.247237, 0.246558, 0.247613, 0.246757, 0.245898, 0.244698, 0.246351, 0.24496, 0.243951, 0.242646, 0.244344, 0.243456, 0.242858, 0.242606]}, {"n": 516, "day": "2026-08-21", "step": [0.0, 0.06285724, 0.06112776, 0.05891028, 0.0567097, 0.18427345, 0.17233692, 0.16541138, 0.14897121, 0.57162594, 0.50935302, 0.43821617, 0.34439457, 1.37686233, 1.17150963, 0.89756371, 0.65063914], "drift": [0.0, 0.00261, 0.00523, 0.00853, 0.0142, 0.00755, 0.01451, 0.02251, 0.03492, 0.02328, 0.04082, 0.05336, 0.07391, 0.05428, 0.07838, 0.0956, 0.12411], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 37, 19, 9], "mean_delta": [-0.147489, -0.146347, -0.145006, -0.143132, -0.140017, -0.144252, -0.14068, -0.136245, -0.129341, -0.137658, -0.1296, -0.122658, -0.109237, -0.125454, -0.115666, -0.103236, -0.087733], "mean_abs_delta": [0.287041, 0.286704, 0.286389, 0.285947, 0.285183, 0.286079, 0.285216, 0.284184, 0.282691, 0.284046, 0.282292, 0.281063, 0.278676, 0.280197, 0.278503, 0.276538, 0.275205]}, {"n": 984, "day": "2026-08-22", "step": [0.0, 0.07079557, 0.06621022, 0.06018513, 0.05257794, 0.19193424, 0.16152149, 0.13697887, 0.11283919, 0.48820673, 0.35875452, 0.31211081, 0.24346503, 1.00120737, 0.7723155, 0.54850571, 0.24029123], "drift": [0.0, 0.00287, 0.00597, 0.00952, 0.01553, 0.00788, 0.01613, 0.02413, 0.03714, 0.02219, 0.04046, 0.05341, 0.07458, 0.04108, 0.06905, 0.08551, 0.1208], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 24, 18, 5], "mean_delta": [-0.062656, -0.060646, -0.058451, -0.055591, -0.050791, -0.057183, -0.051396, -0.044593, -0.033894, -0.046839, -0.033256, -0.023211, -0.005956, -0.029865, -0.016195, -0.004611, 0.020509], "mean_abs_delta": [0.184449, 0.183733, 0.183, 0.182128, 0.180773, 0.182562, 0.180852, 0.179072, 0.17683, 0.179495, 0.17625, 0.174865, 0.174033, 0.17545, 0.173488, 0.172961, 0.174931]}, {"n": 738, "day": "2026-08-23", "step": [0.0, 0.04704602, 0.04141123, 0.03620002, 0.03101032, 0.12249323, 0.09463202, 0.07637467, 0.06118426, 0.28513528, 0.19911735, 0.15631127, 0.11387271, 0.59940241, 0.38841791, 0.33015157, 0.22280503], "drift": [0.0, 0.00231, 0.00581, 0.00983, 0.01631, 0.00643, 0.01581, 0.02458, 0.03834, 0.01784, 0.03734, 0.05325, 0.07589, 0.03373, 0.06374, 0.08371, 0.12176], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 7, 3, 1], "mean_delta": [-0.04995, -0.047962, -0.04565, -0.042778, -0.037946, -0.044612, -0.038553, -0.032108, -0.021452, -0.035592, -0.023722, -0.013256, 0.004438, -0.024243, -0.010924, 0.000147, 0.030133], "mean_abs_delta": [0.185658, 0.185118, 0.184506, 0.183816, 0.182837, 0.184212, 0.182902, 0.181814, 0.180355, 0.182238, 0.180339, 0.179151, 0.179191, 0.180085, 0.178329, 0.177767, 0.181409]}, {"n": 556, "day": "2026-08-24", "step": [0.0, 0.05199018, 0.04871911, 0.04527927, 0.040009, 0.14488231, 0.12315972, 0.10906305, 0.09013911, 0.40353722, 0.30657548, 0.24763644, 0.19324898, 0.99032098, 0.67078023, 0.56913058, 0.32789244], "drift": [0.0, 0.00217, 0.00579, 0.01006, 0.01708, 0.00624, 0.0155, 0.02514, 0.03965, 0.01756, 0.03639, 0.05457, 0.07878, 0.03532, 0.06671, 0.08866, 0.12579], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 11, 10, 8], "mean_delta": [-0.072144, -0.070691, -0.068244, -0.065315, -0.06047, -0.068015, -0.061636, -0.055142, -0.045122, -0.06035, -0.048188, -0.036675, -0.020267, -0.049637, -0.031801, -0.020553, 0.004401], "mean_abs_delta": [0.236508, 0.236001, 0.235204, 0.234242, 0.232657, 0.235065, 0.233036, 0.231087, 0.228547, 0.232386, 0.229153, 0.226289, 0.223166, 0.228804, 0.224558, 0.222339, 0.220227]}, {"n": 644, "day": "2026-08-25", "step": [0.0, 0.04470174, 0.0406165, 0.03516736, 0.02863363, 0.12024254, 0.0903945, 0.07167589, 0.05667827, 0.29734376, 0.2064007, 0.15222247, 0.12518414, 0.65988753, 0.42234057, 0.33745672, 0.25093043], "drift": [0.0, 0.00154, 0.00507, 0.00953, 0.01706, 0.00428, 0.01341, 0.02367, 0.03932, 0.01205, 0.03114, 0.05043, 0.07709, 0.02609, 0.05574, 0.08201, 0.12521], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 9, 7, 3], "mean_delta": [-0.040169, -0.038763, -0.036122, -0.032954, -0.027635, -0.036294, -0.029576, -0.022216, -0.011144, -0.029791, -0.015745, -0.002789, 0.014981, -0.019097, 0.001565, 0.014984, 0.040777], "mean_abs_delta": [0.196488, 0.196209, 0.195725, 0.195222, 0.1946, 0.19574, 0.194742, 0.194119, 0.194085, 0.194747, 0.19374, 0.193818, 0.1968, 0.19402, 0.194103, 0.195771, 0.203331]}, {"n": 1525, "day": "2026-08-26", "step": [0.0, 0.09167986, 0.08027399, 0.07324188, 0.06339315, 0.2489593, 0.20470223, 0.17083082, 0.13164434, 0.70956551, 0.52298793, 0.3795703, 0.24847017, 1.64649651, 1.0648022, 0.72603168, 0.38225164], "drift": [0.0, 0.0021, 0.00533, 0.00984, 0.01784, 0.00635, 0.01422, 0.02464, 0.04067, 0.01809, 0.0348, 0.05169, 0.07962, 0.04288, 0.06292, 0.08695, 0.126], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 27, 17, 8], "mean_delta": [-0.0269, -0.026158, -0.023768, -0.02057, -0.014999, -0.024668, -0.018612, -0.011086, 0.000449, -0.020785, -0.00779, 0.005347, 0.025772, -0.011685, 0.00583, 0.024162, 0.051822], "mean_abs_delta": [0.191201, 0.190981, 0.190542, 0.190028, 0.189271, 0.190537, 0.189615, 0.188747, 0.187952, 0.189384, 0.188105, 0.187555, 0.188566, 0.187303, 0.187031, 0.187802, 0.193864]}, {"n": 1375, "day": "2026-08-27", "step": [0.0, 0.08118152, 0.06985468, 0.05589185, 0.03914406, 0.1985926, 0.13376636, 0.09940184, 0.07505503, 0.42244928, 0.28212476, 0.21233934, 0.16099011, 0.82615748, 0.60867617, 0.46572436, 0.33367715], "drift": [0.0, 0.00291, 0.00632, 0.01077, 0.01872, 0.00782, 0.01525, 0.02565, 0.04233, 0.01814, 0.03675, 0.05328, 0.08136, 0.04126, 0.06447, 0.08797, 0.12769], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3, 3, 1], "mean_delta": [-0.048981, -0.047106, -0.044455, -0.040856, -0.034472, -0.043466, -0.03739, -0.028902, -0.015966, -0.034957, -0.020119, -0.008064, 0.012922, -0.015645, 0.000754, 0.015879, 0.041156], "mean_abs_delta": [0.173538, 0.173107, 0.172563, 0.171906, 0.170879, 0.172345, 0.171306, 0.170143, 0.169173, 0.170896, 0.169355, 0.169006, 0.17, 0.169029, 0.169379, 0.170392, 0.174663]}, {"n": 467, "day": "2026-08-28", "step": [0.0, 0.04375223, 0.03980578, 0.03540602, 0.03030321, 0.11834353, 0.09898567, 0.08402507, 0.06656402, 0.33475573, 0.26345799, 0.20212572, 0.14414459, 0.72334079, 0.5348303, 0.38463634, 0.21109113], "drift": [0.0, 0.00135, 0.0048, 0.0096, 0.01825, 0.00371, 0.01181, 0.02316, 0.04156, 0.00935, 0.02977, 0.04957, 0.07977, 0.02628, 0.05585, 0.0805, 0.12531], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 20, 14, 9], "mean_delta": [0.001875, 0.003347, 0.005717, 0.008744, 0.014131, 0.005893, 0.010976, 0.018103, 0.029636, 0.010507, 0.023759, 0.035278, 0.053415, 0.022608, 0.040238, 0.053798, 0.07677], "mean_abs_delta": [0.231976, 0.23186, 0.231583, 0.231264, 0.230809, 0.231702, 0.231147, 0.230689, 0.23049, 0.231478, 0.230825, 0.230688, 0.231431, 0.231387, 0.231255, 0.231692, 0.235412]}, {"n": 1316, "day": "2026-08-29", "step": [0.0, 0.08629961, 0.074102, 0.0617587, 0.04749534, 0.2378084, 0.17612329, 0.13548512, 0.10750243, 0.68545477, 0.43172701, 0.33850964, 0.21798338, 1.37938537, 0.94471312, 0.66335817, 0.37055486], "drift": [0.0, 0.00133, 0.00434, 0.00902, 0.01829, 0.00381, 0.01053, 0.02242, 0.04222, 0.0109, 0.02924, 0.04937, 0.08057, 0.02976, 0.05662, 0.08318, 0.12906], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 23, 16, 5], "mean_delta": [-0.014835, -0.014124, -0.011384, -0.007726, -0.000931, -0.012717, -0.006712, 0.00231, 0.016959, -0.010527, 0.005986, 0.021256, 0.043403, -0.000581, 0.02187, 0.040534, 0.07154], "mean_abs_delta": [0.194105, 0.193981, 0.193731, 0.19348, 0.193273, 0.19378, 0.193352, 0.193213, 0.194318, 0.193394, 0.193076, 0.1947, 0.199462, 0.192366, 0.194274, 0.198236, 0.208934]}, {"n": 345, "day": "2026-08-30", "step": [0.0, 0.02939512, 0.02634118, 0.02106314, 0.01576835, 0.08304256, 0.06171494, 0.0449447, 0.031791, 0.23938493, 0.14502543, 0.09164533, 0.07063859, 0.49512937, 0.26300237, 0.20475419, 0.13706382], "drift": [0.0, 0.00112, 0.00389, 0.00859, 0.01832, 0.00312, 0.00955, 0.02165, 0.0425, 0.009, 0.02738, 0.04814, 0.08162, 0.02397, 0.05393, 0.08333, 0.12813], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3, 3, 2], "mean_delta": [-0.05648, -0.055857, -0.053708, -0.050188, -0.043223, -0.054794, -0.049888, -0.040677, -0.026103, -0.052482, -0.037555, -0.022475, -0.000264, -0.042398, -0.021649, -0.001424, 0.02677], "mean_abs_delta": [0.210476, 0.210248, 0.209698, 0.208814, 0.207283, 0.209851, 0.208584, 0.20673, 0.204709, 0.208838, 0.206018, 0.204099, 0.202998, 0.206186, 0.203376, 0.202384, 0.20401]}, {"n": 286, "day": "2026-08-31", "step": [0.0, 0.03179052, 0.02905972, 0.02426906, 0.01884788, 0.09156561, 0.07276787, 0.05196809, 0.03524788, 0.25354667, 0.15317311, 0.10622756, 0.06961744, 0.50010213, 0.29178164, 0.18970141, 0.07194889], "drift": [0.0, 0.00087, 0.00337, 0.00793, 0.01804, 0.00248, 0.00809, 0.02018, 0.04236, 0.00704, 0.02412, 0.04619, 0.08153, 0.01897, 0.05054, 0.08183, 0.12697], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 11, 4, 1], "mean_delta": [-0.019745, -0.019099, -0.017146, -0.013692, -0.006285, -0.018009, -0.013746, -0.004426, 0.011453, -0.015654, -0.001914, 0.014086, 0.03815, -0.007841, 0.015306, 0.037138, 0.063849], "mean_abs_delta": [0.230497, 0.230357, 0.230005, 0.229447, 0.228391, 0.230114, 0.229333, 0.228066, 0.22686, 0.229423, 0.227543, 0.226424, 0.2261, 0.227506, 0.22569, 0.224916, 0.228186]}, {"n": 262, "day": "2026-09-01", "step": [0.0, 0.02220945, 0.01957663, 0.01622015, 0.01220125, 0.06103343, 0.04962636, 0.03571233, 0.02316173, 0.17014137, 0.11125148, 0.06986754, 0.02751291, 0.38155811, 0.19625071, 0.08645905, 0.04951678], "drift": [0.0, 0.0009, 0.00329, 0.00772, 0.01794, 0.00279, 0.00799, 0.01925, 0.04183, 0.00778, 0.02225, 0.04328, 0.0806, 0.01858, 0.04371, 0.07682, 0.12559], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 0], "mean_delta": [-0.050607, -0.049948, -0.048432, -0.045629, -0.039372, -0.048666, -0.045356, -0.038031, -0.024472, -0.045246, -0.035568, -0.022469, -0.001467, -0.037581, -0.021128, -0.002308, 0.020669], "mean_abs_delta": [0.209404, 0.209058, 0.208367, 0.207159, 0.204585, 0.208395, 0.206905, 0.203939, 0.1999, 0.206587, 0.202618, 0.199088, 0.194615, 0.20279, 0.197804, 0.194306, 0.193425]}, {"n": 215, "day": "2026-09-02", "step": [0.0, 0.01950953, 0.01855094, 0.01688687, 0.01535418, 0.05658671, 0.05216318, 0.04597486, 0.03700255, 0.17603072, 0.1492127, 0.11986204, 0.08855332, 0.43408566, 0.33790854, 0.25116014, 0.12548558], "drift": [0.0, 0.00085, 0.00301, 0.00738, 0.01788, 0.00263, 0.00749, 0.01871, 0.04187, 0.00813, 0.02176, 0.04291, 0.08015, 0.02065, 0.04383, 0.07437, 0.12557], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 11, 6, 0], "mean_delta": [-0.01788, -0.017634, -0.016312, -0.013712, -0.007847, -0.017064, -0.014105, -0.007431, 0.005483, -0.015539, -0.007046, 0.005721, 0.026103, -0.012387, 0.002943, 0.022069, 0.04647], "mean_abs_delta": [0.230125, 0.230057, 0.229883, 0.229648, 0.22928, 0.229913, 0.229551, 0.229102, 0.228939, 0.229479, 0.228688, 0.228447, 0.229273, 0.228839, 0.227682, 0.228334, 0.231888]}, {"n": 195, "day": "2026-09-03", "step": [0.0, 0.01902172, 0.01621971, 0.01246771, 0.00844851, 0.05370394, 0.03817254, 0.02533463, 0.01299596, 0.14095247, 0.08077155, 0.04582197, 0.02751261, 0.26999788, 0.14341474, 0.08500843, 0.0433968], "drift": [0.0, 0.00066, 0.00252, 0.00666, 0.01735, 0.00195, 0.00609, 0.01648, 0.04075, 0.00533, 0.01631, 0.03798, 0.07851, 0.01073, 0.03308, 0.06679, 0.12382], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 2, 2, 2], "mean_delta": [-0.074095, -0.073693, -0.07247, -0.069856, -0.063262, -0.072899, -0.070276, -0.063475, -0.048718, -0.070584, -0.063795, -0.050275, -0.026565, -0.068727, -0.055076, -0.034139, -0.003212], "mean_abs_delta": [0.235281, 0.235175, 0.234946, 0.234486, 0.233372, 0.234967, 0.234529, 0.233359, 0.230828, 0.234402, 0.23328, 0.231155, 0.227299, 0.233989, 0.23181, 0.228659, 0.225279]}, {"n": 301, "day": "2026-09-04", "step": [0.0, 0.02933012, 0.02813112, 0.02632646, 0.02038493, 0.0870515, 0.07975674, 0.06365537, 0.03483518, 0.27816772, 0.23210254, 0.1298645, 0.05609047, 0.80538895, 0.49541149, 0.23193107, 0.08924744], "drift": [0.0, 0.0007, 0.00222, 0.00616, 0.01692, 0.00198, 0.00539, 0.01488, 0.03981, 0.006, 0.01373, 0.03476, 0.07505, 0.01537, 0.0292, 0.05698, 0.11641], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 9, 2, 3], "mean_delta": [-0.008926, -0.008796, -0.007617, -0.004779, 0.002813, -0.008594, -0.005971, 0.001216, 0.01878, -0.008111, -0.001724, 0.014758, 0.041606, -0.008391, 0.00581, 0.029073, 0.065003], "mean_abs_delta": [0.231664, 0.231597, 0.231179, 0.23021, 0.227716, 0.23149, 0.230585, 0.228231, 0.223005, 0.231199, 0.229116, 0.22411, 0.219352, 0.230975, 0.226723, 0.221188, 0.218476]}, {"n": 133, "day": "2026-09-05", "step": [0.0, 0.00975421, 0.00895728, 0.00811434, 0.0072122, 0.02755716, 0.02465262, 0.0218246, 0.01928112, 0.08333245, 0.07514489, 0.06559157, 0.05174995, 0.2404971, 0.21020005, 0.16684518, 0.12141792], "drift": [0.0, 0.00029, 0.00168, 0.00549, 0.01652, 0.00087, 0.00411, 0.01339, 0.0387, 0.00284, 0.01077, 0.03097, 0.07288, 0.00765, 0.02306, 0.05126, 0.11268], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0], "mean_delta": [-0.017264, -0.017125, -0.016208, -0.013737, -0.006923, -0.016843, -0.014788, -0.008722, 0.006836, -0.01581, -0.011096, 0.002354, 0.026787, -0.013528, -0.004682, 0.013665, 0.046423], "mean_abs_delta": [0.204211, 0.204155, 0.203775, 0.202752, 0.200191, 0.204046, 0.203193, 0.200857, 0.194883, 0.203664, 0.201818, 0.1965, 0.190329, 0.202774, 0.199263, 0.19261, 0.188486]}, {"n": 246, "day": "2026-09-06", "step": [0.0, 0.02326191, 0.02235886, 0.01986652, 0.01598883, 0.0693392, 0.06285499, 0.05081196, 0.03662, 0.22246512, 0.19159761, 0.14067066, 0.09014761, 0.66139445, 0.5137931, 0.35138259, 0.13267747], "drift": [0.0, 0.00035, 0.00113, 0.00457, 0.01564, 0.00104, 0.00284, 0.01097, 0.03593, 0.00366, 0.0077, 0.02399, 0.06572, 0.00903, 0.01867, 0.03506, 0.10298], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 17, 11, 6], "mean_delta": [0.040899, 0.040892, 0.041702, 0.044247, 0.051908, 0.040874, 0.042741, 0.04899, 0.066231, 0.040774, 0.045132, 0.058354, 0.085991, 0.040447, 0.048248, 0.064342, 0.106088], "mean_abs_delta": [0.235625, 0.235633, 0.235618, 0.23554, 0.235368, 0.23566, 0.235609, 0.235449, 0.236186, 0.23576, 0.235638, 0.235688, 0.239191, 0.235951, 0.235605, 0.236632, 0.243836]}, {"n": 137, "day": "2026-09-07", "step": [0.0, 0.01009121, 0.00953655, 0.00757871, 0.0049945, 0.03115872, 0.0281916, 0.01804591, 0.00879754, 0.11144284, 0.08926339, 0.04007388, 0.01630277, 0.33589161, 0.25725566, 0.12486059, 0.01063137], "drift": [0.0, 0.00026, 0.00095, 0.00418, 0.0153, 0.0008, 0.00245, 0.01007, 0.03514, 0.00269, 0.00692, 0.02212, 0.06434, 0.00599, 0.01722, 0.03329, 0.1007], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0], "mean_delta": [-0.016121, -0.016262, -0.015603, -0.012912, -0.003762, -0.016562, -0.015234, -0.008608, 0.011637, -0.017701, -0.01511, -0.001719, 0.031751, -0.016158, -0.014891, -0.003135, 0.05433], "mean_abs_delta": [0.193758, 0.193774, 0.193658, 0.193206, 0.191866, 0.193804, 0.193558, 0.192493, 0.190242, 0.193997, 0.193409, 0.191422, 0.189044, 0.193852, 0.193063, 0.191096, 0.190321]}, {"n": 167, "day": "2026-09-08", "step": [0.0, 0.01312416, 0.01311906, 0.01243212, 0.01101319, 0.04037402, 0.03838226, 0.03498283, 0.0288442, 0.13334417, 0.12410463, 0.10502601, 0.0544116, 0.36169737, 0.34682685, 0.29105325, 0.09765011], "drift": [0.0, 0.00039, 0.00108, 0.00412, 0.01519, 0.00114, 0.00285, 0.00995, 0.0349, 0.00361, 0.00798, 0.02222, 0.06319, 0.0101, 0.02004, 0.0353, 0.09691], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 9, 8, 0], "mean_delta": [-0.068602, -0.068583, -0.068238, -0.066254, -0.059141, -0.068598, -0.067828, -0.063003, -0.046731, -0.068612, -0.067087, -0.056897, -0.029587, -0.066652, -0.064929, -0.054877, -0.011713], "mean_abs_delta": [0.195573, 0.195531, 0.195511, 0.195489, 0.195772, 0.195447, 0.195369, 0.195454, 0.196446, 0.195167, 0.194961, 0.195327, 0.197649, 0.194516, 0.19387, 0.194313, 0.20038]}, {"n": 65, "day": "2026-09-09", "step": [0.0, 0.00665872, 0.00665723, 0.00619565, 0.00542629, 0.01997365, 0.01974988, 0.01710868, 0.01312009, 0.06583992, 0.05978317, 0.04799426, 0.02451812, 0.18086942, 0.14875548, 0.13515674, 0.04138675], "drift": [0.0, 0.00026, 0.00086, 0.00384, 0.0149, 0.00074, 0.00221, 0.00914, 0.03426, 0.00235, 0.00598, 0.02019, 0.06202, 0.00729, 0.01507, 0.03144, 0.09584], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 2, 2], "mean_delta": [-0.016557, -0.016481, -0.01612, -0.014263, -0.007485, -0.016365, -0.015528, -0.01113, 0.004215, -0.016017, -0.014099, -0.004853, 0.019949, -0.014279, -0.010224, -0.001068, 0.034953], "mean_abs_delta": [0.244374, 0.244347, 0.244295, 0.243738, 0.241291, 0.24431, 0.244242, 0.242924, 0.237317, 0.24418, 0.244277, 0.241599, 0.231702, 0.243548, 0.244356, 0.241885, 0.224518]}, {"n": 77, "day": "2026-09-10", "step": [0.0, 0.01405659, 0.01380972, 0.01371657, 0.01313055, 0.04142935, 0.04116493, 0.04044309, 0.03408544, 0.13658654, 0.13487526, 0.12856365, 0.06083363, 0.37658455, 0.37280872, 0.36325691, 0.05562118], "drift": [0.0, 0.00097, 0.00158, 0.00436, 0.01542, 0.00292, 0.00442, 0.01091, 0.03572, 0.0098, 0.01342, 0.02694, 0.06419, 0.0283, 0.03663, 0.05147, 0.09453], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 18, 18, 4], "mean_delta": [-0.325405, -0.325309, -0.325191, -0.324547, -0.321863, -0.325126, -0.324856, -0.323358, -0.3172, -0.324491, -0.323904, -0.320783, -0.311176, -0.322451, -0.321274, -0.318371, -0.305], "mean_abs_delta": [0.412495, 0.412479, 0.412434, 0.412177, 0.411142, 0.41245, 0.412349, 0.411746, 0.409522, 0.412352, 0.412126, 0.410878, 0.407852, 0.411813, 0.411371, 0.410266, 0.405797]}, {"n": 79, "day": "2026-09-11", "step": [0.0, 0.00534917, 0.00528184, 0.0050939, 0.00442459, 0.01583146, 0.0152748, 0.01371148, 0.01162795, 0.0457517, 0.04556668, 0.04149637, 0.02252047, 0.12201497, 0.11605729, 0.09059171, 0.04589043], "drift": [0.0, 0.00087, 0.00159, 0.00429, 0.01537, 0.00259, 0.00446, 0.01081, 0.03569, 0.00847, 0.01355, 0.0268, 0.06415, 0.02385, 0.0366, 0.05132, 0.09513], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "mean_delta": [-0.12375, -0.123222, -0.12281, -0.121171, -0.113884, -0.122179, -0.121137, -0.117325, -0.101147, -0.118601, -0.115969, -0.108226, -0.084206, -0.109043, -0.102222, -0.094814, -0.067413], "mean_abs_delta": [0.192215, 0.192039, 0.191933, 0.191517, 0.189605, 0.191704, 0.191437, 0.190475, 0.186239, 0.190558, 0.189884, 0.187978, 0.181836, 0.187596, 0.18583, 0.18412, 0.176645]}, {"n": 493, "day": "2026-09-12", "step": [0.0, 0.0370898, 0.03507038, 0.03225774, 0.01932832, 0.10301545, 0.09570568, 0.0697418, 0.03207083, 0.29446763, 0.21732125, 0.14732182, 0.06674761, 0.54893624, 0.33661904, 0.25043254, 0.12497538], "drift": [0.0, 0.00097, 0.00184, 0.00437, 0.01509, 0.00268, 0.00496, 0.01069, 0.03383, 0.00712, 0.01286, 0.02324, 0.0603, 0.01572, 0.02459, 0.04117, 0.08817], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 17, 12, 4], "mean_delta": [-0.052176, -0.05147, -0.050841, -0.048837, -0.040248, -0.050155, -0.048558, -0.043957, -0.025105, -0.046425, -0.042587, -0.033848, -0.006406, -0.040447, -0.03061, -0.019606, 0.010027], "mean_abs_delta": [0.172821, 0.172692, 0.17259, 0.172277, 0.171538, 0.172457, 0.172205, 0.171744, 0.171229, 0.171891, 0.171546, 0.171197, 0.172319, 0.171372, 0.171058, 0.171113, 0.175126]}, {"n": 84, "day": "2026-09-13", "step": [0.0, 0.00483925, 0.00460524, 0.00435357, 0.0029117, 0.01376317, 0.01295667, 0.00991481, 0.00450319, 0.04041816, 0.03158356, 0.02217941, 0.01329835, 0.09713418, 0.06785647, 0.04497612, 0.03795227], "drift": [0.0, 0.00056, 0.00154, 0.00403, 0.01479, 0.00151, 0.00413, 0.00978, 0.03299, 0.00389, 0.0104, 0.02135, 0.05848, 0.00808, 0.02009, 0.03642, 0.0844], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0], "mean_delta": [-0.003795, -0.003168, -0.002474, -0.000561, 0.007693, -0.00209, -0.000259, 0.004098, 0.021821, 0.000539, 0.005203, 0.012976, 0.039871, 0.005103, 0.012472, 0.023773, 0.053932], "mean_abs_delta": [0.129046, 0.129024, 0.128986, 0.128928, 0.129218, 0.128974, 0.128919, 0.128951, 0.131391, 0.128947, 0.129011, 0.129546, 0.136366, 0.129146, 0.129562, 0.131503, 0.141918]}, {"n": 143, "day": "2026-09-14", "step": [0.0, 0.01395576, 0.01309729, 0.01124796, 0.00897606, 0.04010188, 0.03377439, 0.02885052, 0.01906088, 0.12376363, 0.09607635, 0.07550813, 0.05758084, 0.31571325, 0.24308048, 0.18792083, 0.14813834], "drift": [0.0, 0.00038, 0.00114, 0.00339, 0.01425, 0.00111, 0.00276, 0.00818, 0.03209, 0.00348, 0.0069, 0.01847, 0.05609, 0.01049, 0.01591, 0.03238, 0.08018], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0], "mean_delta": [0.042167, 0.042324, 0.042822, 0.04409, 0.04979, 0.042599, 0.043851, 0.046939, 0.059201, 0.043063, 0.046646, 0.052594, 0.071165, 0.043394, 0.050826, 0.059147, 0.080106], "mean_abs_delta": [0.28548, 0.285509, 0.285652, 0.286063, 0.28775, 0.285551, 0.285925, 0.286922, 0.290437, 0.285563, 0.28668, 0.288459, 0.293552, 0.285445, 0.28746, 0.289817, 0.295308]}, {"n": 87, "day": "2026-09-15", "step": [0.0, 0.00589553, 0.00534711, 0.00437177, 0.0023167, 0.0176887, 0.01574199, 0.0092213, 0.00417802, 0.06091992, 0.03922848, 0.02030019, 0.00725556, 0.24855052, 0.07829639, 0.04196794, 0.01774509], "drift": [0.0, 0.00018, 0.00085, 0.00303, 0.01393, 0.00052, 0.00207, 0.00752, 0.03138, 0.00165, 0.00588, 0.01693, 0.05464, 0.00508, 0.0136, 0.03038, 0.07811], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3, 0, 0], "mean_delta": [-0.029959, -0.029943, -0.029289, -0.027475, -0.018695, -0.029969, -0.028517, -0.023939, -0.004823, -0.030586, -0.026434, -0.016761, 0.011357, -0.03272, -0.023351, -0.00884, 0.022281], "mean_abs_delta": [0.204738, 0.204741, 0.204547, 0.204045, 0.202169, 0.204765, 0.204331, 0.203264, 0.199482, 0.204999, 0.203755, 0.201677, 0.197665, 0.205918, 0.202919, 0.199844, 0.196499]}, {"n": 124, "day": "2026-09-16", "step": [0.0, 0.01832044, 0.01804405, 0.0175943, 0.01563341, 0.05491715, 0.05347187, 0.04937034, 0.04400562, 0.18374212, 0.17280494, 0.15474949, 0.11915406, 0.47576329, 0.42201889, 0.38254163, 0.2429094], "drift": [0.0, 0.00023, 0.00054, 0.00263, 0.01353, 0.00072, 0.00129, 0.00655, 0.03016, 0.0027, 0.00378, 0.01431, 0.05184, 0.0057, 0.00948, 0.02491, 0.075], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 29, 28, 18], "mean_delta": [0.022279, 0.022561, 0.023203, 0.025178, 0.035284, 0.023082, 0.024587, 0.029544, 0.051251, 0.024512, 0.029217, 0.038664, 0.069699, 0.028913, 0.036882, 0.049392, 0.085509], "mean_abs_delta": [0.230778, 0.230706, 0.230617, 0.230304, 0.229062, 0.230567, 0.23037, 0.229659, 0.227468, 0.230131, 0.229473, 0.228431, 0.225807, 0.229013, 0.228208, 0.226842, 0.225972]}, {"n": 191, "day": "2026-09-17", "step": [0.0, 0.01182056, 0.01127616, 0.01076521, 0.00703857, 0.0360324, 0.0341499, 0.02775102, 0.01566044, 0.15178881, 0.11024774, 0.07863011, 0.04397652, 0.56895505, 0.39013295, 0.18317343, 0.09496041], "drift": [0.0, 0.00035, 0.00027, 0.002, 0.01281, 0.00103, 0.0008, 0.00492, 0.02848, 0.003, 0.00262, 0.00954, 0.048, 0.01015, 0.00849, 0.01595, 0.06834], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 8, 1, 0], "mean_delta": [0.06222, 0.061848, 0.062237, 0.063944, 0.073174, 0.061108, 0.061989, 0.066574, 0.08627, 0.058697, 0.061237, 0.070202, 0.100012, 0.054439, 0.060366, 0.072398, 0.110221], "mean_abs_delta": [0.211133, 0.211242, 0.211198, 0.210932, 0.209298, 0.211448, 0.211366, 0.210619, 0.208113, 0.211837, 0.211904, 0.21059, 0.208378, 0.211711, 0.211887, 0.211899, 0.208514]}, {"n": 319, "day": "2026-09-18", "step": [0.0, 0.02242248, 0.02217253, 0.02045203, 0.01633869, 0.07067607, 0.06859963, 0.05671508, 0.04174889, 0.25822656, 0.24802058, 0.18150416, 0.11894941, 0.78144932, 0.68174072, 0.54618854, 0.29896796], "drift": [0.0, 0.00047, 0.00052, 0.00155, 0.01229, 0.00136, 0.00153, 0.00396, 0.02727, 0.00401, 0.00485, 0.00799, 0.0462, 0.01092, 0.01303, 0.01686, 0.06539], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 24, 20, 8], "mean_delta": [0.009872, 0.009533, 0.009665, 0.010969, 0.018562, 0.008852, 0.009137, 0.012604, 0.028778, 0.006949, 0.007383, 0.014042, 0.039294, 0.004102, 0.006137, 0.013019, 0.045467], "mean_abs_delta": [0.207011, 0.207039, 0.207031, 0.206975, 0.206794, 0.207103, 0.207089, 0.20695, 0.207296, 0.207315, 0.207282, 0.206951, 0.208302, 0.207783, 0.207602, 0.207094, 0.208569]}, {"n": 192, "day": "2026-09-19", "step": [0.0, 0.02022867, 0.02022888, 0.01973887, 0.01385753, 0.06113315, 0.06201499, 0.05496726, 0.02287643, 0.20289687, 0.20613353, 0.17609878, 0.05229677, 0.56105344, 0.54359421, 0.49684691, 0.1352664], "drift": [0.0, 0.00044, 0.00063, 0.00197, 0.01252, 0.00137, 0.00194, 0.00538, 0.02754, 0.00466, 0.0069, 0.01263, 0.04721, 0.01402, 0.02082, 0.02768, 0.06799], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 20, 18, 1], "mean_delta": [-0.105825, -0.105675, -0.105674, -0.10471, -0.098195, -0.105318, -0.105303, -0.102805, -0.089373, -0.103792, -0.103744, -0.099554, -0.079574, -0.098469, -0.097592, -0.094821, -0.072352], "mean_abs_delta": [0.235298, 0.235257, 0.235256, 0.234786, 0.231752, 0.235149, 0.235154, 0.233942, 0.227892, 0.234663, 0.234644, 0.232686, 0.2241, 0.232779, 0.232509, 0.230973, 0.221783]}, {"n": 285, "day": "2026-09-20", "step": [0.0, 0.03098814, 0.03096187, 0.02952989, 0.01886986, 0.09253663, 0.09229132, 0.07469206, 0.05010612, 0.29653753, 0.28435932, 0.19785869, 0.15575038, 0.63008355, 0.53885392, 0.50478423, 0.42051997], "drift": [0.0, 0.00061, 0.00083, 0.00211, 0.01247, 0.00185, 0.00245, 0.00576, 0.02776, 0.00626, 0.0084, 0.01378, 0.0487, 0.01765, 0.02334, 0.0337, 0.07197], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 19, 19, 17], "mean_delta": [-0.036569, -0.036506, -0.036457, -0.0355, -0.028211, -0.036347, -0.036182, -0.033628, -0.018538, -0.035528, -0.034718, -0.030588, -0.008546, -0.031474, -0.028088, -0.024574, -0.001976], "mean_abs_delta": [0.254601, 0.254568, 0.25457, 0.254423, 0.253539, 0.254498, 0.254512, 0.25415, 0.252636, 0.254219, 0.254275, 0.253819, 0.251753, 0.253477, 0.253899, 0.253214, 0.251228]}, {"n": 242, "day": "2026-09-21", "step": [0.0, 0.02376249, 0.02343056, 0.02027723, 0.00928573, 0.06476467, 0.06307917, 0.0473858, 0.01710585, 0.16063197, 0.14352258, 0.10422501, 0.03475859, 0.23264679, 0.18126442, 0.14457705, 0.06711467], "drift": [0.0, 0.00075, 0.00104, 0.00224, 0.01247, 0.00227, 0.00311, 0.00651, 0.02785, 0.0074, 0.01064, 0.01614, 0.04934, 0.01628, 0.02541, 0.03641, 0.07323], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 13, 12, 8], "mean_delta": [-0.048613, -0.04834, -0.04822, -0.047225, -0.038763, -0.047612, -0.047224, -0.044301, -0.027413, -0.044167, -0.042611, -0.03894, -0.015979, -0.035038, -0.031349, -0.027927, -0.007488], "mean_abs_delta": [0.189753, 0.18963, 0.189622, 0.18943, 0.188027, 0.189376, 0.189349, 0.188778, 0.187194, 0.188566, 0.188461, 0.188063, 0.187236, 0.187389, 0.187586, 0.18761, 0.188787]}], "bootstrap": {"lines": {"capitulation": 0.0025}, "gates_ok": true, "calibration": {"mean": 0.015530285738707813, "total": 19.59922060224926, "trades": 1262, "per_strategy": {"capitulation": {"line": 0.0025, "mean": 0.015530285738707813, "total": 19.59922060224926, "trades": 1262, "hold_min": 240}}}, "diagnostics": {"rows": 5648, "slope": 0.9636524869032652, "kc_active": 0.09993186593055725, "rank_corr": 0.8975306157777404, "mbon_share": 0.48021747339581045, "mbon_saturated": 0.010416666977107525}, "gate_failures": []}, "evaluation": {"n": 1651, "pf": 1.1344105930713058, "win": 0.4476075105996366, "days": 29, "mean": 0.015233186073601246, "median": -0.02210576832294464, "days_positive": 19}, "strategies": ["capitulation"], "finished_at": "2026-09-22T07:27:02.794988+00:00", "per_strategy": {"capitulation": {"alpha": 0.0003, "chosen": {"mean": 0.023196974769234657, "alpha": 0.0003, "total": 75.43656158447266, "config": 5, "trades": 3252, "governance": {"ic": 0, "any": 0, "drift": 0, "checks": 1365, "shadow": 0}, "half_life_days": 1.0}, "configs": [{"mean": 0.016807934269309044, "alpha": 0.0, "total": 65.02989959716797, "config": 0, "trades": 3869, "governance": {"ic": 0, "any": 0, "drift": 0, "checks": 0, "shadow": 0}, "half_life_days": null}, {"mean": 0.01756637543439865, "alpha": 0.0001, "total": 63.76594543457031, "config": 1, "trades": 3630, "governance": {"ic": 0, "any": 0, "drift": 0, "checks": 1365, "shadow": 0}, "half_life_days": 1.0}, {"mean": 0.020793743431568146, "alpha": 0.0001, "total": 68.8896713256836, "config": 2, "trades": 3313, "governance": {"ic": 0, "any": 0, "drift": 0, "checks": 1365, "shadow": 0}, "half_life_days": 3.0}, {"mean": 0.02111992798745632, "alpha": 0.0001, "total": 61.73354721069336, "config": 3, "trades": 2923, "governance": {"ic": 0, "any": 0, "drift": 0, "checks": 1365, "shadow": 0}, "half_life_days": 7.0}, {"mean": 0.020111948251724243, "alpha": 0.0001, "total": 50.62177276611328, "config": 4, "trades": 2517, "governance": {"ic": 0, "any": 68, "drift": 0, "checks": 1365, "shadow": 68}, "half_life_days": 30.0}, {"mean": 0.023196974769234657, "alpha": 0.0003, "total": 75.43656158447266, "config": 5, "trades": 3252, "governance": {"ic": 0, "any": 0, "drift": 0, "checks": 1365, "shadow": 0}, "half_life_days": 1.0}, {"mean": 0.020104866474866867, "alpha": 0.0003, "total": 52.694854736328125, "config": 6, "trades": 2621, "governance": {"ic": 0, "any": 34, "drift": 0, "checks": 1365, "shadow": 34}, "half_life_days": 3.0}, {"mean": 0.015242534689605236, "alpha": 0.0003, "total": 31.61301612854004, "config": 7, "trades": 2074, "governance": {"ic": 0, "any": 93, "drift": 0, "checks": 1365, "shadow": 93}, "half_life_days": 7.0}, {"mean": 0.014115910045802593, "alpha": 0.0003, "total": 23.277135848999023, "config": 8, "trades": 1649, "governance": {"ic": 0, "any": 111, "drift": 0, "checks": 1365, "shadow": 111}, "half_life_days": 30.0}, {"mean": 0.014811468310654163, "alpha": 0.001, "total": 39.4577522277832, "config": 9, "trades": 2664, "governance": {"ic": 0, "any": 74, "drift": 0, "checks": 1365, "shadow": 74}, "half_life_days": 1.0}, {"mean": 0.011060998775064945, "alpha": 0.001, "total": 18.847942352294922, "config": 10, "trades": 1704, "governance": {"ic": 0, "any": 139, "drift": 0, "checks": 1365, "shadow": 139}, "half_life_days": 3.0}, {"mean": 0.002477346919476986, "alpha": 0.001, "total": 3.146230697631836, "config": 11, "trades": 1270, "governance": {"ic": 0, "any": 184, "drift": 0, "checks": 1365, "shadow": 184}, "half_life_days": 7.0}, {"mean": 0.018671352416276932, "alpha": 0.001, "total": 19.77296257019043, "config": 12, "trades": 1059, "governance": {"ic": 0, "any": 144, "drift": 0, "checks": 1365, "shadow": 144}, "half_life_days": 30.0}, {"mean": 0.01424670871347189, "alpha": 0.003, "total": 28.10875701904297, "config": 13, "trades": 1973, "governance": {"ic": 0, "any": 203, "drift": 0, "checks": 1365, "shadow": 203}, "half_life_days": 1.0}, {"mean": 0.027894049882888794, "alpha": 0.003, "total": 38.716941833496094, "config": 14, "trades": 1388, "governance": {"ic": 0, "any": 71, "drift": 0, "checks": 1365, "shadow": 71}, "half_life_days": 3.0}, {"mean": 0.01914161443710327, "alpha": 0.003, "total": 18.816207885742188, "config": 15, "trades": 983, "governance": {"ic": 0, "any": 112, "drift": 0, "checks": 1365, "shadow": 112}, "half_life_days": 7.0}, {"mean": 0.028120551258325577, "alpha": 0.003, "total": 22.524560928344727, "config": 16, "trades": 801, "governance": {"ic": 0, "any": 196, "drift": 0, "checks": 1365, "shadow": 196}, "half_life_days": 30.0}], "plastic": true, "hold_min": 240, "half_life_days": 1.0}}, "half_life_days": 1.0, "selection_days": ["2026-07-27", "2026-08-23"], "evaluation_days": ["2026-08-24", "2026-09-21"], "plastic_beats_frozen": true}'::jsonb) ON CONFLICT (key) DO NOTHING; -INSERT INTO ui_settings (key, value) VALUES ('autostart', '{"pumpstream": true, "runner": true, "train": false, "replay": false, "discover": false, "vault": true}'::jsonb) ON CONFLICT (key) DO NOTHING; +INSERT INTO ui_settings (key, value) VALUES ('fly_replay', '{"S": "2026-09-09", "data": {"agg": 4, "fly": "flynet-3-ch", "cols": "86151dda", "meta": "rug-1", "calib": "match", "costs": "real-fees-sized-1", "lines": "per-chain-1", "score": "teacher", "skill": "h10-L30", "chains": "sol+rh-1", "rh_agg": 1, "plastic": "mb4-ch", "teacher": "deployed-1", "features": 3, "rh_costs": "pons-kyber-1:0.0445583:0.0222791:7.2e-06:0.0099", "rh_skill": "rh-h10-L30", "selector": "multi-1"}, "secs": 38337.81323695183, "alpha": 0.0001, "chains": {"rh": {"edge_p": 0.0005, "frozen": {"n": 1477, "pf": 1.2590082935365987, "win": 0.4685172647257955, "days": 16, "mean": 0.026347791776061058, "median": -0.012595674954354763, "days_positive": 11}, "passed": true, "random": {"n": 26586, "pf": 0.5751959068792711, "win": 0.3041450387421951, "mean": -0.046134307980537415, "median": -0.06115628033876419}, "reason": "its backtest made +2.66% per trade after costs over 1477 trades (random -4.61%; 0.1% of day resamples at or below zero)", "evaluation": {"n": 1477, "pf": 1.2612278943579527, "win": 0.4685172647257955, "days": 16, "mean": 0.02657903917133808, "median": -0.012595674954354763, "days_positive": 11}}, "sol": {"edge_p": 0.004, "frozen": {"n": 308, "pf": 1.4669271249617488, "win": 0.512987012987013, "days": 16, "mean": 0.0377873033285141, "median": 0.005911512766033411, "days_positive": 11}, "passed": true, "random": {"n": 6091, "pf": 0.458730469095968, "win": 0.5076342144147102, "mean": -0.04378599673509598, "median": 0.00037788774352520704}, "reason": "its backtest made +3.78% per trade after costs over 308 trades (random -4.38%; 0.4% of day resamples at or below zero)", "evaluation": {"n": 308, "pf": 1.4669271249617488, "win": 0.512987012987013, "days": 16, "mean": 0.0377873033285141, "median": 0.005911512766033411, "days_positive": 11}}}, "chosen": {"mean": 0.030173424631357193, "alpha": 0.0001, "total": 89.58489990234375, "config": 1, "trades": 2969, "governance": {"ic": 35, "any": 35, "drift": 0, "checks": 1482, "shadow": 0}, "half_life_days": 1.0}, "frozen": {"n": 1785, "pf": 1.2885933084122407, "win": 0.47619047619047616, "days": 16, "mean": 0.02832167036831379, "median": -0.009349760599434376, "random": {"n": 34226, "pf": 0.5107005230232796, "win": 0.47361654882253257, "mean": -0.0425419919192791, "median": -0.0014032165054231882}, "days_positive": 12}, "passed": true, "random": {"n": 34226, "pf": 0.5107005230232796, "win": 0.47361654882253257, "mean": -0.0425419919192791, "median": -0.0014032165054231882}, "reason": "sol: passed \u2014 its backtest made +3.78% per trade after costs over 308 trades (random -4.38%; 0.4% of day resamples at or below zero); rh: passed \u2014 its backtest made +2.66% per trade after costs over 1477 trades (random -4.61%; 0.1% of day resamples at or below zero)", "configs": [{"mean": 0.03036709874868393, "alpha": 0.0, "total": 91.28350067138672, "config": 0, "trades": 3006, "governance": {"ic": 0, "any": 0, "drift": 0, "checks": 0, "shadow": 0}, "half_life_days": null}, {"mean": 0.030173424631357193, "alpha": 0.0001, "total": 89.58489990234375, "config": 1, "trades": 2969, "governance": {"ic": 35, "any": 35, "drift": 0, "checks": 1482, "shadow": 0}, "half_life_days": 1.0}, {"mean": 0.032286640256643295, "alpha": 0.0001, "total": 67.99566650390625, "config": 2, "trades": 2106, "governance": {"ic": 28, "any": 29, "drift": 0, "checks": 1482, "shadow": 1}, "half_life_days": 3.0}, {"mean": 0.03781268000602722, "alpha": 0.0001, "total": 55.92495346069336, "config": 3, "trades": 1479, "governance": {"ic": 17, "any": 69, "drift": 0, "checks": 1482, "shadow": 52}, "half_life_days": 7.0}, {"mean": 0.04063548520207405, "alpha": 0.0001, "total": 48.437496185302734, "config": 4, "trades": 1192, "governance": {"ic": 7, "any": 40, "drift": 0, "checks": 1482, "shadow": 33}, "half_life_days": 30.0}, {"mean": 0.03773646801710129, "alpha": 0.0003, "total": 79.32205200195312, "config": 5, "trades": 2102, "governance": {"ic": 30, "any": 31, "drift": 0, "checks": 1482, "shadow": 1}, "half_life_days": 1.0}, {"mean": 0.04369469732046127, "alpha": 0.0003, "total": 42.51494216918945, "config": 6, "trades": 973, "governance": {"ic": 16, "any": 53, "drift": 0, "checks": 1482, "shadow": 37}, "half_life_days": 3.0}, {"mean": 0.05426740273833275, "alpha": 0.0003, "total": 37.17317199707031, "config": 7, "trades": 685, "governance": {"ic": 8, "any": 62, "drift": 0, "checks": 1482, "shadow": 54}, "half_life_days": 7.0}, {"mean": 0.034157898277044296, "alpha": 0.0003, "total": 20.494739532470703, "config": 8, "trades": 600, "governance": {"ic": 0, "any": 58, "drift": 0, "checks": 1482, "shadow": 58}, "half_life_days": 30.0}, {"mean": 0.04557060822844505, "alpha": 0.001, "total": 51.90492248535156, "config": 9, "trades": 1139, "governance": {"ic": 20, "any": 20, "drift": 0, "checks": 1482, "shadow": 0}, "half_life_days": 1.0}, {"mean": 0.03615965694189072, "alpha": 0.001, "total": 22.129711151123047, "config": 10, "trades": 612, "governance": {"ic": 5, "any": 53, "drift": 0, "checks": 1482, "shadow": 48}, "half_life_days": 3.0}, {"mean": 0.04262426868081093, "alpha": 0.001, "total": 21.3121337890625, "config": 11, "trades": 500, "governance": {"ic": 0, "any": 78, "drift": 0, "checks": 1482, "shadow": 78}, "half_life_days": 7.0}, {"mean": 0.023908229544758797, "alpha": 0.001, "total": 10.639162063598633, "config": 12, "trades": 445, "governance": {"ic": 0, "any": 66, "drift": 0, "checks": 1482, "shadow": 66}, "half_life_days": 30.0}, {"mean": 0.04726389795541763, "alpha": 0.003, "total": 46.224090576171875, "config": 13, "trades": 978, "governance": {"ic": 6, "any": 58, "drift": 0, "checks": 1482, "shadow": 52}, "half_life_days": 1.0}, {"mean": 0.03016720339655876, "alpha": 0.003, "total": 20.634366989135742, "config": 14, "trades": 684, "governance": {"ic": 0, "any": 137, "drift": 0, "checks": 1482, "shadow": 137}, "half_life_days": 3.0}, {"mean": 0.034875430166721344, "alpha": 0.003, "total": 21.832019805908203, "config": 15, "trades": 626, "governance": {"ic": 0, "any": 136, "drift": 0, "checks": 1482, "shadow": 136}, "half_life_days": 7.0}, {"mean": 0.03963538631796837, "alpha": 0.003, "total": 23.345243453979492, "config": 16, "trades": 589, "governance": {"ic": 0, "any": 199, "drift": 0, "checks": 1482, "shadow": 199}, "half_life_days": 30.0}], "per_day": {"2026-09-24": {"n": 120, "pf": 1.2096282469078616, "win": 0.4666666666666667, "mean": 0.019853290170431137, "median": -0.008774115703999996}, "2026-09-25": {"n": 130, "pf": 1.2343648703740158, "win": 0.46923076923076923, "mean": 0.022507645189762115, "median": -0.015695948153734207}, "2026-09-26": {"n": 159, "pf": 0.9110357851981143, "win": 0.4528301886792453, "mean": -0.010142914019525051, "median": -0.02541925199329853}, "2026-09-27": {"n": 134, "pf": 1.4929109749231875, "win": 0.44029850746268656, "mean": 0.046415600925683975, "median": -0.021206816658377647}, "2026-09-28": {"n": 146, "pf": 1.4797542117441214, "win": 0.3904109589041096, "mean": 0.06142570450901985, "median": -0.05863966420292854}, "2026-09-29": {"n": 128, "pf": 0.8905144422024887, "win": 0.4453125, "mean": -0.012050645425915718, "median": -0.02929481491446495}, "2026-09-30": {"n": 130, "pf": 2.164358561365056, "win": 0.5615384615384615, "mean": 0.0869135931134224, "median": 0.03113124519586563}, "2026-10-01": {"n": 94, "pf": 1.0760464155183325, "win": 0.48936170212765956, "mean": 0.008363611996173859, "median": -0.0028990297578275204}, "2026-10-02": {"n": 105, "pf": 0.9110992827614596, "win": 0.45714285714285713, "mean": -0.0093840342015028, "median": -0.004047797992825508}, "2026-10-03": {"n": 89, "pf": 1.5717055834211324, "win": 0.5393258426966292, "mean": 0.046940337866544724, "median": 0.02129359170794487}, "2026-10-04": {"n": 111, "pf": 1.5649065404247118, "win": 0.4864864864864865, "mean": 0.049075424671173096, "median": -0.005090842954814434}, "2026-10-05": {"n": 113, "pf": 1.2647730391586305, "win": 0.5309734513274337, "mean": 0.0248811524361372, "median": 0.017877638339996338}, "2026-10-06": {"n": 101, "pf": 1.4999371942039894, "win": 0.48514851485148514, "mean": 0.04013614356517792, "median": -0.002721513854339719}, "2026-10-07": {"n": 84, "pf": 1.1800436231035252, "win": 0.5, "mean": 0.01683460921049118, "median": -0.005321653559803963}, "2026-10-08": {"n": 79, "pf": 1.9083646078747787, "win": 0.4936708860759494, "mean": 0.07138421386480331, "median": -0.004599324893206358}, "2026-10-09": {"n": 62, "pf": 0.8436231572986813, "win": 0.46774193548387094, "mean": -0.017560668289661407, "median": -0.032155461609363556}}, "learning": [{"n": 2360, "day": "2026-09-09", "step": [0.0, 0.19114103, 0.19111604, 0.19110858, 0.19110422, 0.56869623, 0.56852255, 0.56847102, 0.56844206, 1.84675575, 1.8459054, 1.84566744, 1.84553071, 4.07084698, 4.07035153, 4.07023325, 4.07017128], "drift": [0.0, 0.00364, 0.00466, 0.005, 0.00521, 0.01001, 0.01279, 0.01373, 0.01431, 0.0231, 0.02928, 0.03136, 0.03262, 0.03277, 0.04133, 0.04424, 0.046], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 8, 8, 331, 331, 331, 331], "mean_delta": [-0.03834, -0.036569, -0.03649, -0.036466, -0.036452, -0.033291, -0.033073, -0.033008, -0.032969, -0.02497, -0.024521, -0.024387, -0.024309, -0.018631, -0.018057, -0.017887, -0.017788], "mean_abs_delta": [0.243879, 0.243245, 0.243225, 0.243219, 0.243215, 0.242084, 0.242034, 0.24202, 0.242011, 0.239226, 0.239148, 0.239126, 0.239114, 0.236935, 0.236862, 0.236843, 0.236831]}, {"n": 1490, "day": "2026-09-10", "step": [0.0, 0.1282473, 0.12781086, 0.1276504, 0.12754914, 0.37799113, 0.37522809, 0.37424898, 0.37364627, 1.22299966, 1.21300152, 1.2096912, 1.20772753, 3.09826104, 3.08997546, 3.08711205, 3.085262], "drift": [0.0, 0.00529, 0.00792, 0.00897, 0.00966, 0.01383, 0.02016, 0.02263, 0.02421, 0.03039, 0.04162, 0.04599, 0.04882, 0.04772, 0.06524, 0.072, 0.07637], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 2, 2, 152, 149, 148, 147], "mean_delta": [-0.048624, -0.043967, -0.042428, -0.041858, -0.041498, -0.03632, -0.032529, -0.031133, -0.030258, -0.022787, -0.016992, -0.014919, -0.013626, -0.017012, -0.012647, -0.011101, -0.010144], "mean_abs_delta": [0.203057, 0.201621, 0.201182, 0.201021, 0.20092, 0.199419, 0.198512, 0.198193, 0.197993, 0.195899, 0.194927, 0.194612, 0.194431, 0.193714, 0.192914, 0.192643, 0.192477]}, {"n": 2708, "day": "2026-09-11", "step": [0.0, 0.25055709, 0.24931297, 0.24878305, 0.24843247, 0.7385869, 0.73237111, 0.73011377, 0.72871685, 2.37396603, 2.35785869, 2.35226077, 2.34895474, 5.01956554, 4.99887312, 4.99164725, 4.98743835], "drift": [0.0, 0.00886, 0.01362, 0.01567, 0.01707, 0.02142, 0.03018, 0.03374, 0.0362, 0.04396, 0.05978, 0.06625, 0.07057, 0.06365, 0.08646, 0.09587, 0.10222], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 22, 21, 22, 431, 428, 426, 426], "mean_delta": [-0.066137, -0.060051, -0.057097, -0.055824, -0.054972, -0.05114, -0.045154, -0.042783, -0.041284, -0.038975, -0.032338, -0.029703, -0.028001, -0.031565, -0.024916, -0.022423, -0.020842], "mean_abs_delta": [0.251312, 0.249431, 0.248641, 0.248307, 0.24809, 0.246854, 0.245486, 0.244963, 0.244638, 0.243016, 0.241709, 0.241248, 0.240971, 0.24051, 0.239109, 0.238597, 0.238282]}, {"n": 2001, "day": "2026-09-12", "step": [0.0, 0.1826168, 0.18207247, 0.18186614, 0.1817378, 0.54391342, 0.54233645, 0.54200476, 0.54189227, 1.79139739, 1.78749173, 1.7869684, 1.78686687, 4.26189446, 4.246626, 4.24025719, 4.23618915], "drift": [0.0, 0.005, 0.01102, 0.01416, 0.01644, 0.011, 0.02139, 0.02646, 0.03017, 0.0247, 0.04295, 0.05191, 0.05836, 0.04704, 0.07637, 0.08993, 0.09947], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 8, 8, 9, 286, 288, 286, 287], "mean_delta": [-0.030598, -0.022891, -0.01746, -0.014995, -0.013291, -0.013717, -0.005212, -0.001838, 0.000439, -0.007286, 0.002196, 0.006156, 0.008769, -0.003335, 0.004922, 0.008127, 0.010161], "mean_abs_delta": [0.240174, 0.238265, 0.237107, 0.236598, 0.236251, 0.236032, 0.234395, 0.233809, 0.233435, 0.233989, 0.232292, 0.23168, 0.231305, 0.232431, 0.23083, 0.230277, 0.229931]}, {"n": 2012, "day": "2026-09-13", "step": [0.0, 0.21982043, 0.21626672, 0.21434172, 0.21294512, 0.63683065, 0.62368678, 0.61723266, 0.61262904, 1.99879227, 1.96794189, 1.95389722, 1.9444037, 4.28735923, 4.26309931, 4.25257243, 4.24607554], "drift": [0.0, 0.01208, 0.01992, 0.02411, 0.02707, 0.02774, 0.04059, 0.04736, 0.05236, 0.05817, 0.08155, 0.0927, 0.10083, 0.08725, 0.11862, 0.1336, 0.14449], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 13, 11, 11, 333, 324, 321, 321], "mean_delta": [-0.119832, -0.110132, -0.103852, -0.100391, -0.09784, -0.097303, -0.088654, -0.084247, -0.081038, -0.076759, -0.068202, -0.064189, -0.061405, -0.05926, -0.05074, -0.047498, -0.045481], "mean_abs_delta": [0.264323, 0.26012, 0.257681, 0.256385, 0.255446, 0.254951, 0.251953, 0.250481, 0.249426, 0.247341, 0.244868, 0.243866, 0.243215, 0.242131, 0.240389, 0.239686, 0.239265]}, {"n": 2133, "day": "2026-09-14", "step": [0.0, 0.23188563, 0.22810843, 0.22625628, 0.22503369, 0.67246073, 0.65975472, 0.65391736, 0.64989842, 2.1162268, 2.08203617, 2.06818091, 2.05947671, 4.17329574, 4.17041343, 4.16754732, 4.16457737], "drift": [0.0, 0.01416, 0.02469, 0.02995, 0.03382, 0.03014, 0.04773, 0.05683, 0.06383, 0.05762, 0.08725, 0.10166, 0.11207, 0.07328, 0.1115, 0.13109, 0.1461], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 20, 19, 18, 393, 388, 383, 385], "mean_delta": [-0.09698, -0.082646, -0.07423, -0.069849, -0.06679, -0.065887, -0.053981, -0.048011, -0.043628, -0.041042, -0.025054, -0.017945, -0.013094, -0.03023, -0.014329, -0.008031, -0.003997], "mean_abs_delta": [0.262175, 0.258569, 0.256876, 0.25607, 0.255537, 0.255085, 0.253301, 0.252476, 0.25193, 0.250899, 0.249185, 0.248541, 0.248172, 0.249131, 0.247707, 0.247219, 0.24697]}, {"n": 1767, "day": "2026-09-15", "step": [0.0, 0.13082462, 0.12807035, 0.12691447, 0.1262311, 0.38357165, 0.37626038, 0.37457562, 0.37416996, 1.2576157, 1.24848204, 1.25087272, 1.25437331, 3.13282058, 3.10825873, 3.10193769, 3.09932395], "drift": [0.0, 0.00964, 0.02176, 0.02879, 0.03416, 0.01986, 0.03942, 0.05074, 0.05976, 0.03512, 0.06398, 0.08102, 0.09456, 0.05027, 0.08732, 0.11055, 0.13017], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 2, 2, 194, 186, 188, 189], "mean_delta": [-0.049886, -0.036129, -0.023739, -0.017324, -0.012671, -0.022447, -0.00489, 0.004062, 0.010737, -0.00716, 0.011367, 0.019659, 0.02526, -0.002027, 0.010032, 0.0152, 0.018675], "mean_abs_delta": [0.198057, 0.194771, 0.192451, 0.191491, 0.190933, 0.192258, 0.190222, 0.189701, 0.189472, 0.190494, 0.189427, 0.189326, 0.189398, 0.190068, 0.189223, 0.188959, 0.188849]}, {"n": 2081, "day": "2026-09-16", "step": [0.0, 0.20784134, 0.20151466, 0.19820528, 0.19590611, 0.59886797, 0.5767962, 0.5654473, 0.55716899, 1.87655532, 1.81595846, 1.78898973, 1.7715539, 3.97739167, 3.94241426, 3.92871029, 3.91951185], "drift": [0.0, 0.01545, 0.02817, 0.03538, 0.04147, 0.03353, 0.05398, 0.06625, 0.07651, 0.06853, 0.09982, 0.11663, 0.12991, 0.09462, 0.13238, 0.15425, 0.17315], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 7, 6, 5, 346, 332, 328, 323], "mean_delta": [-0.130949, -0.118029, -0.106376, -0.099784, -0.094935, -0.102888, -0.087331, -0.07841, -0.071425, -0.079286, -0.063549, -0.055869, -0.050599, -0.060387, -0.046477, -0.040718, -0.037114], "mean_abs_delta": [0.248057, 0.242605, 0.238024, 0.23569, 0.234087, 0.236914, 0.231772, 0.229171, 0.227336, 0.229563, 0.225384, 0.2236, 0.222481, 0.224475, 0.221428, 0.220284, 0.219627]}, {"n": 987, "day": "2026-09-17", "step": [0.0, 0.10679559, 0.10547293, 0.10494688, 0.10461475, 0.31651284, 0.31306477, 0.31189373, 0.31149969, 1.04054557, 1.032719, 1.03181024, 1.03275386, 2.65368672, 2.61416605, 2.5994033, 2.5902975], "drift": [0.0, 0.00863, 0.02274, 0.0322, 0.04047, 0.01862, 0.04209, 0.05783, 0.07152, 0.03753, 0.07301, 0.09505, 0.11354, 0.06027, 0.10335, 0.1309, 0.156], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 168, 167, 166, 162], "mean_delta": [-0.040432, -0.02836, -0.01347, -0.004831, 0.002238, -0.016139, 0.005467, 0.01782, 0.027606, -0.001613, 0.023576, 0.035519, 0.044047, 0.003194, 0.023628, 0.032835, 0.039144], "mean_abs_delta": [0.243221, 0.240296, 0.23729, 0.235926, 0.234916, 0.237417, 0.234076, 0.232521, 0.231564, 0.233647, 0.230187, 0.229127, 0.228694, 0.2304, 0.227297, 0.226267, 0.225737]}, {"n": 1103, "day": "2026-09-18", "step": [0.0, 0.10171894, 0.1004097, 0.09985218, 0.09957259, 0.30264728, 0.29896345, 0.2982446, 0.29861928, 1.00028158, 0.9938767, 0.9933884, 0.99473531, 2.53034597, 2.5091195, 2.50166478, 2.49738774], "drift": [0.0, 0.00482, 0.0178, 0.02829, 0.0381, 0.01, 0.03078, 0.04755, 0.06328, 0.01965, 0.0503, 0.07451, 0.09663, 0.03529, 0.07388, 0.10473, 0.13561], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 157, 153, 150], "mean_delta": [-0.011827, -0.004222, 0.01002, 0.019707, 0.027927, 0.003769, 0.024129, 0.037144, 0.047967, 0.013337, 0.032898, 0.042796, 0.050177, 0.01812, 0.033057, 0.039623, 0.043918], "mean_abs_delta": [0.225247, 0.223176, 0.219988, 0.218332, 0.217242, 0.221218, 0.217691, 0.216313, 0.215799, 0.2191, 0.21651, 0.215622, 0.21536, 0.21802, 0.216047, 0.215298, 0.214902]}, {"n": 1036, "day": "2026-09-19", "step": [0.0, 0.11678434, 0.11557956, 0.11478874, 0.11426845, 0.34844892, 0.34460238, 0.34276291, 0.34191144, 1.14770467, 1.14334365, 1.13901164, 1.13590429, 2.75977484, 2.761065, 2.75854016, 2.75362164], "drift": [0.0, 0.00438, 0.01608, 0.02714, 0.03826, 0.01012, 0.02845, 0.04542, 0.06262, 0.02201, 0.04827, 0.07261, 0.09706, 0.03463, 0.06915, 0.10194, 0.13677], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 2, 2, 192, 193, 190, 191], "mean_delta": [-0.054567, -0.049741, -0.037411, -0.027108, -0.017606, -0.044421, -0.027776, -0.01496, -0.003504, -0.037136, -0.025033, -0.016661, -0.009471, -0.032599, -0.024503, -0.019776, -0.015409], "mean_abs_delta": [0.245887, 0.245196, 0.243845, 0.242908, 0.242319, 0.244383, 0.242828, 0.242088, 0.241955, 0.242962, 0.242128, 0.241527, 0.241323, 0.241465, 0.241207, 0.240942, 0.240554]}, {"n": 1187, "day": "2026-09-20", "step": [0.0, 0.13266736, 0.13004195, 0.12775291, 0.12577976, 0.39207659, 0.38208249, 0.37475607, 0.36877949, 1.26922513, 1.24006389, 1.22049149, 1.2042866, 3.04776096, 3.01592225, 2.99810015, 2.98511973], "drift": [0.0, 0.00569, 0.01606, 0.0272, 0.03925, 0.01458, 0.03117, 0.04756, 0.06545, 0.03589, 0.06281, 0.08566, 0.10953, 0.06189, 0.09764, 0.12626, 0.15856], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 192, 180, 181, 182], "mean_delta": [-0.056364, -0.052038, -0.040912, -0.030003, -0.019167, -0.046364, -0.031376, -0.018432, -0.006145, -0.036509, -0.02429, -0.016045, -0.008912, -0.028272, -0.018874, -0.014138, -0.010484], "mean_abs_delta": [0.245185, 0.24333, 0.239708, 0.236471, 0.233578, 0.240728, 0.235891, 0.232144, 0.229056, 0.23563, 0.230898, 0.227667, 0.225108, 0.229873, 0.225418, 0.223095, 0.221617]}, {"n": 1206, "day": "2026-09-21", "step": [0.0, 0.11644781, 0.11481612, 0.11338398, 0.11221946, 0.34545412, 0.34004148, 0.33650485, 0.33410854, 1.12699008, 1.11441112, 1.10691122, 1.10130664, 2.91439664, 2.88506963, 2.86526596, 2.84790216], "drift": [0.0, 0.00532, 0.01534, 0.02674, 0.03978, 0.0137, 0.03031, 0.04697, 0.06614, 0.03342, 0.06086, 0.08544, 0.11219, 0.06609, 0.10559, 0.13678, 0.17212], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 5, 5, 136, 133, 128, 128], "mean_delta": [-0.085746, -0.08049, -0.07024, -0.059297, -0.047764, -0.073021, -0.059314, -0.047263, -0.035521, -0.060101, -0.048115, -0.040621, -0.034493, -0.045142, -0.034915, -0.029471, -0.025509], "mean_abs_delta": [0.233576, 0.232077, 0.229566, 0.227273, 0.225168, 0.230144, 0.227265, 0.224994, 0.223114, 0.227143, 0.225112, 0.223683, 0.222413, 0.223257, 0.221995, 0.221008, 0.220042]}, {"n": 1079, "day": "2026-09-22", "step": [0.0, 0.09756422, 0.09530408, 0.093099, 0.09107224, 0.28622661, 0.278017, 0.27232901, 0.26792277, 0.92025189, 0.89941675, 0.8904902, 0.88611577, 2.42137875, 2.400505, 2.39369573, 2.38974493], "drift": [0.0, 0.00578, 0.01565, 0.02711, 0.04076, 0.01461, 0.0311, 0.04756, 0.06751, 0.0314, 0.05874, 0.08347, 0.11224, 0.05136, 0.09116, 0.12454, 0.16416], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 102, 100, 96], "mean_delta": [-0.077488, -0.070788, -0.060436, -0.048752, -0.035794, -0.060456, -0.045885, -0.033158, -0.020082, -0.04091, -0.025862, -0.016616, -0.009334, -0.022349, -0.006128, 0.001317, 0.006364], "mean_abs_delta": [0.216724, 0.214793, 0.211789, 0.208763, 0.205896, 0.21193, 0.208067, 0.20529, 0.203191, 0.207176, 0.203862, 0.20237, 0.201428, 0.20337, 0.200791, 0.199887, 0.199492]}, {"n": 976, "day": "2026-09-23", "step": [0.0, 0.09426381, 0.09374157, 0.09338934, 0.09337377, 0.28257084, 0.28077042, 0.28023092, 0.28056747, 0.94434258, 0.93987403, 0.93681763, 0.93477549, 2.2962897, 2.29777617, 2.29499464, 2.28755804], "drift": [0.0, 0.00244, 0.01117, 0.02258, 0.03706, 0.00582, 0.02066, 0.03749, 0.05902, 0.01409, 0.04003, 0.06714, 0.09998, 0.032, 0.07254, 0.10976, 0.1545], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 2, 2, 158, 158, 147, 149], "mean_delta": [0.014295, 0.01724, 0.026804, 0.038089, 0.051245, 0.020594, 0.034768, 0.046982, 0.05994, 0.020319, 0.034759, 0.043281, 0.050589, 0.014688, 0.026629, 0.032576, 0.037127], "mean_abs_delta": [0.22437, 0.224002, 0.222885, 0.222081, 0.221847, 0.2237, 0.222278, 0.221525, 0.221327, 0.22391, 0.222332, 0.221141, 0.220453, 0.223597, 0.221879, 0.220773, 0.219848]}, {"n": 903, "day": "2026-09-24", "step": [0.0, 0.08876457, 0.08874128, 0.08892377, 0.08963874, 0.2664914, 0.26678571, 0.26791933, 0.27041107, 0.88996891, 0.89197688, 0.89317843, 0.8949892, 2.18884247, 2.18298296, 2.17738741, 2.16912733], "drift": [0.0, 0.00084, 0.00699, 0.01785, 0.033, 0.00277, 0.01211, 0.02822, 0.05099, 0.01055, 0.02981, 0.05568, 0.09072, 0.01841, 0.05459, 0.09394, 0.14281], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 3, 4, 132, 133, 136, 133], "mean_delta": [0.027675, 0.02845, 0.037452, 0.049463, 0.06424, 0.028344, 0.040858, 0.053426, 0.067364, 0.022923, 0.032271, 0.040672, 0.049039, 0.026384, 0.034044, 0.039349, 0.045038], "mean_abs_delta": [0.21318, 0.213152, 0.212808, 0.212744, 0.213551, 0.213189, 0.212854, 0.213044, 0.213903, 0.213385, 0.213147, 0.21294, 0.212884, 0.212399, 0.212564, 0.212668, 0.212694]}, {"n": 954, "day": "2026-09-25", "step": [0.0, 0.09233516, 0.09085702, 0.0886607, 0.08625427, 0.27524403, 0.27045533, 0.26491697, 0.25959592, 0.90289918, 0.90081693, 0.89529136, 0.88955279, 2.20147544, 2.19533407, 2.19098079, 2.18440822], "drift": [0.0, 0.00129, 0.00665, 0.01669, 0.03189, 0.00318, 0.01136, 0.02592, 0.04865, 0.00796, 0.0232, 0.04857, 0.08559, 0.01868, 0.04431, 0.08187, 0.13334], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 2, 2, 133, 135, 132, 134], "mean_delta": [-0.033727, -0.031701, -0.024465, -0.012349, 0.004022, -0.028724, -0.020267, -0.008991, 0.004833, -0.022034, -0.018706, -0.013501, -0.007184, -0.011848, -0.008334, -0.005493, -0.001478], "mean_abs_delta": [0.202285, 0.201667, 0.199375, 0.195928, 0.191963, 0.200805, 0.198149, 0.194988, 0.19166, 0.198704, 0.197768, 0.196351, 0.194949, 0.19472, 0.194025, 0.193564, 0.192967]}, {"n": 1402, "day": "2026-09-26", "step": [0.0, 0.12922601, 0.12729616, 0.12402046, 0.11979357, 0.37791886, 0.37161283, 0.36364022, 0.35448112, 1.18658264, 1.17125415, 1.15828728, 1.14850897, 2.7113238, 2.68743956, 2.66943223, 2.65527476], "drift": [0.0, 0.00618, 0.01214, 0.0215, 0.03642, 0.01599, 0.02603, 0.03831, 0.05891, 0.0361, 0.05437, 0.07398, 0.10518, 0.05695, 0.08522, 0.11411, 0.15866], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 8, 8, 170, 161, 154, 150], "mean_delta": [-0.091814, -0.086608, -0.081194, -0.071274, -0.056776, -0.078095, -0.071978, -0.063704, -0.053213, -0.061036, -0.056415, -0.05259, -0.049381, -0.049961, -0.045256, -0.04266, -0.040918], "mean_abs_delta": [0.215423, 0.213271, 0.21127, 0.207819, 0.203319, 0.209965, 0.2078, 0.204945, 0.201565, 0.203618, 0.201919, 0.200678, 0.199856, 0.199458, 0.197915, 0.197247, 0.196956]}, {"n": 1387, "day": "2026-09-27", "step": [0.0, 0.13277694, 0.13062171, 0.12782822, 0.12422161, 0.38694882, 0.37886373, 0.37264205, 0.36680423, 1.24036559, 1.21573718, 1.20314916, 1.19559413, 2.91378065, 2.88427031, 2.8693509, 2.85729445], "drift": [0.0, 0.00684, 0.01369, 0.02285, 0.03773, 0.01661, 0.02852, 0.04057, 0.06063, 0.03357, 0.05326, 0.07187, 0.10318, 0.05215, 0.08074, 0.10847, 0.1537], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 197, 189, 183, 178], "mean_delta": [-0.072977, -0.064596, -0.057662, -0.047756, -0.03277, -0.052273, -0.042247, -0.033045, -0.022007, -0.034081, -0.022778, -0.015348, -0.008754, -0.020496, -0.009582, -0.004055, 0.000154], "mean_abs_delta": [0.234153, 0.231081, 0.228776, 0.225892, 0.222164, 0.227098, 0.224311, 0.222096, 0.219981, 0.222008, 0.21942, 0.217972, 0.216887, 0.218708, 0.21647, 0.21535, 0.214617]}, {"n": 989, "day": "2026-09-28", "step": [0.0, 0.13334907, 0.13084444, 0.12786871, 0.12374961, 0.38378832, 0.37389561, 0.36598754, 0.35700667, 1.19671118, 1.16303214, 1.14210216, 1.12409864, 2.72350092, 2.68801324, 2.67189312, 2.66135791], "drift": [0.0, 0.01038, 0.01869, 0.02813, 0.04271, 0.02451, 0.03858, 0.05158, 0.07216, 0.04929, 0.07503, 0.09402, 0.123, 0.07038, 0.10352, 0.12949, 0.17071], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 2, 2, 182, 175, 173, 170], "mean_delta": [-0.143762, -0.132013, -0.12422, -0.11445, -0.099898, -0.115189, -0.103814, -0.093735, -0.08035, -0.08774, -0.072903, -0.062548, -0.051449, -0.066522, -0.05072, -0.042016, -0.034382], "mean_abs_delta": [0.251834, 0.247031, 0.244054, 0.240572, 0.236161, 0.240595, 0.236981, 0.23414, 0.230845, 0.232134, 0.228633, 0.226308, 0.224249, 0.226687, 0.223205, 0.221473, 0.220168]}, {"n": 1002, "day": "2026-09-29", "step": [0.0, 0.09413893, 0.09190975, 0.08997826, 0.08781625, 0.27292635, 0.26521267, 0.26103082, 0.25761413, 0.87567712, 0.85592157, 0.84993359, 0.84860367, 2.21190045, 2.18811738, 2.182877, 2.18202282], "drift": [0.0, 0.00814, 0.01823, 0.02853, 0.04391, 0.0192, 0.03701, 0.05178, 0.07392, 0.0379, 0.06854, 0.09012, 0.12078, 0.05713, 0.09532, 0.12383, 0.16785], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 138, 134, 135], "mean_delta": [-0.09698, -0.086293, -0.076804, -0.066992, -0.053293, -0.071843, -0.057373, -0.046638, -0.032949, -0.050082, -0.031224, -0.020706, -0.010655, -0.033963, -0.017253, -0.009769, -0.004676], "mean_abs_delta": [0.22659, 0.2224, 0.218908, 0.215701, 0.211942, 0.217415, 0.212997, 0.21039, 0.208019, 0.211532, 0.207496, 0.206094, 0.205588, 0.208068, 0.205317, 0.204714, 0.204999]}, {"n": 1071, "day": "2026-09-30", "step": [0.0, 0.10194504, 0.10050518, 0.09937599, 0.09825005, 0.30133916, 0.29625166, 0.29364143, 0.29202814, 0.98718286, 0.97197545, 0.96722052, 0.96552225, 2.5882594, 2.56420826, 2.55513859, 2.5457696], "drift": [0.0, 0.00692, 0.01731, 0.02815, 0.04419, 0.01692, 0.03471, 0.0504, 0.07369, 0.0354, 0.06334, 0.08641, 0.11969, 0.06149, 0.09644, 0.12773, 0.17537], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 131, 129, 125, 126], "mean_delta": [-0.067915, -0.060281, -0.050204, -0.040243, -0.02649, -0.050151, -0.034241, -0.022738, -0.008726, -0.036083, -0.017108, -0.006789, 0.002434, -0.026464, -0.011705, -0.005099, -0.000143], "mean_abs_delta": [0.205825, 0.203874, 0.201727, 0.200135, 0.198375, 0.201546, 0.199148, 0.197769, 0.196989, 0.198831, 0.196391, 0.195852, 0.195874, 0.196059, 0.194181, 0.193739, 0.193529]}, {"n": 609, "day": "2026-10-01", "step": [0.0, 0.05926515, 0.05865917, 0.05822444, 0.05794076, 0.17634914, 0.17416903, 0.17318638, 0.17284884, 0.58035974, 0.57446971, 0.57139737, 0.56842638, 1.51804069, 1.51206034, 1.50497408, 1.49569031], "drift": [0.0, 0.0044, 0.01461, 0.02607, 0.04328, 0.01084, 0.02906, 0.04623, 0.07156, 0.02358, 0.05323, 0.0794, 0.11688, 0.04212, 0.0843, 0.12078, 0.17435], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 100, 95, 95], "mean_delta": [-0.028565, -0.023441, -0.013502, -0.003131, 0.011305, -0.016516, -0.000566, 0.011606, 0.026104, -0.007081, 0.010331, 0.020195, 0.027956, -0.003297, 0.009381, 0.015558, 0.019243], "mean_abs_delta": [0.191003, 0.190211, 0.189107, 0.188513, 0.188399, 0.189233, 0.188107, 0.187631, 0.187441, 0.187589, 0.18638, 0.185337, 0.184084, 0.185555, 0.183718, 0.182369, 0.180954]}, {"n": 696, "day": "2026-10-02", "step": [0.0, 0.07735933, 0.07594488, 0.07460486, 0.07319296, 0.22883225, 0.22280266, 0.21882868, 0.21551559, 0.74457603, 0.7255056, 0.71500672, 0.70750136, 1.94697659, 1.92121591, 1.90359487, 1.88871225], "drift": [0.0, 0.00394, 0.01349, 0.02511, 0.04299, 0.00992, 0.02694, 0.04418, 0.07025, 0.02178, 0.04877, 0.07491, 0.11393, 0.03718, 0.07718, 0.11427, 0.17033], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 93, 89, 88], "mean_delta": [-0.055935, -0.050386, -0.039228, -0.026658, -0.009126, -0.042142, -0.024303, -0.009323, 0.008578, -0.027495, -0.009075, 0.003315, 0.014152, -0.017166, -0.00323, 0.004943, 0.011467], "mean_abs_delta": [0.227442, 0.226246, 0.224205, 0.22251, 0.220733, 0.224583, 0.221964, 0.220058, 0.217969, 0.222115, 0.21898, 0.216752, 0.214464, 0.219752, 0.216397, 0.214184, 0.212456]}, {"n": 558, "day": "2026-10-03", "step": [0.0, 0.06263698, 0.0621514, 0.06166871, 0.06117463, 0.18702926, 0.18505066, 0.18377459, 0.18297868, 0.61843144, 0.61397421, 0.61255781, 0.61330184, 1.63537717, 1.63550191, 1.63811187, 1.64384677], "drift": [0.0, 0.00309, 0.01177, 0.02352, 0.04229, 0.00805, 0.02373, 0.04129, 0.06861, 0.01967, 0.04418, 0.07033, 0.11117, 0.03786, 0.07383, 0.10988, 0.16699], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 86, 89, 88], "mean_delta": [-0.039456, -0.036591, -0.028316, -0.017888, -0.002646, -0.032502, -0.019147, -0.006858, 0.008475, -0.026261, -0.013511, -0.004735, 0.003525, -0.022147, -0.014655, -0.01061, -0.006348], "mean_abs_delta": [0.226071, 0.225439, 0.22409, 0.222595, 0.221284, 0.22461, 0.222586, 0.221257, 0.22087, 0.223118, 0.221433, 0.220834, 0.221022, 0.221351, 0.22072, 0.221111, 0.222415]}, {"n": 826, "day": "2026-10-04", "step": [0.0, 0.09874968, 0.09753729, 0.09601532, 0.09398026, 0.29359792, 0.2878481, 0.28260711, 0.27691978, 0.95724335, 0.93780841, 0.92390935, 0.91154243, 2.32348453, 2.28884165, 2.26555067, 2.24703477], "drift": [0.0, 0.00379, 0.01166, 0.02308, 0.04205, 0.01006, 0.02409, 0.04065, 0.06786, 0.0238, 0.04618, 0.07072, 0.1107, 0.04384, 0.07735, 0.11025, 0.16542], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 2, 3, 157, 148, 149, 150], "mean_delta": [-0.049581, -0.04598, -0.037616, -0.026184, -0.008877, -0.040239, -0.026648, -0.012682, 0.005256, -0.029238, -0.014891, -0.003499, 0.007922, -0.019355, -0.005969, 0.002579, 0.010667], "mean_abs_delta": [0.252912, 0.251617, 0.248853, 0.245207, 0.240312, 0.249592, 0.245098, 0.240822, 0.236063, 0.245219, 0.240462, 0.236945, 0.233666, 0.240948, 0.236375, 0.233339, 0.230816]}, {"n": 888, "day": "2026-10-05", "step": [0.0, 0.09409228, 0.09432181, 0.09478614, 0.09578325, 0.28232353, 0.28357013, 0.28556016, 0.28895636, 0.93958214, 0.94359298, 0.94862612, 0.9555521, 2.33173234, 2.33359715, 2.34122151, 2.35177652], "drift": [0.0, 0.00241, 0.00845, 0.01926, 0.03843, 0.00687, 0.01708, 0.0324, 0.06006, 0.02122, 0.03837, 0.06042, 0.10108, 0.0435, 0.06998, 0.09878, 0.1533], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 164, 160, 159, 159], "mean_delta": [0.039845, 0.040729, 0.047845, 0.058335, 0.074857, 0.041355, 0.05247, 0.064246, 0.0796, 0.037206, 0.04687, 0.054502, 0.06201, 0.033405, 0.041253, 0.045839, 0.050753], "mean_abs_delta": [0.227335, 0.227219, 0.227025, 0.226991, 0.227596, 0.227035, 0.226736, 0.226895, 0.227964, 0.2267, 0.226228, 0.226488, 0.227721, 0.226258, 0.226056, 0.226888, 0.22869]}, {"n": 720, "day": "2026-10-06", "step": [0.0, 0.07442262, 0.07390338, 0.07311527, 0.07213194, 0.22239294, 0.2203837, 0.21838889, 0.21678979, 0.73532286, 0.73350849, 0.73325055, 0.73713676, 1.92132994, 1.92265451, 1.92718839, 1.93436573], "drift": [0.0, 0.00177, 0.00742, 0.01781, 0.03736, 0.00472, 0.01473, 0.02953, 0.0576, 0.01205, 0.03007, 0.05286, 0.09557, 0.02289, 0.05089, 0.08233, 0.14169], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 100, 104, 108], "mean_delta": [-0.03252, -0.030238, -0.024112, -0.01339, 0.004655, -0.026526, -0.017717, -0.006543, 0.008825, -0.017932, -0.012552, -0.007655, -0.002437, -0.005788, -0.001286, 0.000941, 0.002738], "mean_abs_delta": [0.221115, 0.220763, 0.219754, 0.21816, 0.215927, 0.220203, 0.218961, 0.217552, 0.216113, 0.219026, 0.218868, 0.21881, 0.219318, 0.216911, 0.217283, 0.217852, 0.218983]}, {"n": 656, "day": "2026-10-07", "step": [0.0, 0.06619672, 0.06549448, 0.06432556, 0.06277903, 0.19694437, 0.19383499, 0.19051167, 0.18687819, 0.64482673, 0.63616278, 0.63036907, 0.62679891, 1.75893035, 1.74544039, 1.73679071, 1.72847138], "drift": [0.0, 0.00281, 0.00805, 0.01799, 0.03765, 0.00777, 0.01711, 0.03098, 0.05863, 0.02012, 0.03597, 0.05669, 0.09815, 0.03921, 0.06338, 0.09101, 0.14744], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 82, 80, 79], "mean_delta": [-0.065151, -0.06304, -0.057941, -0.04813, -0.030583, -0.059579, -0.05238, -0.042746, -0.029114, -0.052174, -0.0476, -0.044104, -0.040995, -0.046854, -0.044913, -0.043829, -0.042953], "mean_abs_delta": [0.219186, 0.218356, 0.216596, 0.213813, 0.210038, 0.21696, 0.214365, 0.21149, 0.207855, 0.214052, 0.211439, 0.209347, 0.206999, 0.211129, 0.208821, 0.20721, 0.205786]}, {"n": 489, "day": "2026-10-08", "step": [0.0, 0.04758729, 0.04745913, 0.04733163, 0.04734279, 0.1423879, 0.14187612, 0.14172263, 0.14244212, 0.47239631, 0.47090257, 0.47129491, 0.47589861, 1.30655058, 1.30274417, 1.30670432, 1.31761537], "drift": [0.0, 0.0016, 0.00654, 0.01618, 0.03608, 0.0042, 0.01346, 0.02722, 0.05559, 0.00989, 0.02704, 0.04911, 0.09313, 0.02027, 0.04688, 0.07734, 0.13801], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 61, 58, 58], "mean_delta": [0.001017, 0.003374, 0.008615, 0.01847, 0.036567, 0.007282, 0.015568, 0.025251, 0.038331, 0.015093, 0.023941, 0.028631, 0.031492, 0.020845, 0.0295, 0.032877, 0.034275], "mean_abs_delta": [0.204622, 0.204357, 0.203914, 0.203279, 0.202777, 0.203964, 0.203343, 0.202986, 0.203333, 0.203087, 0.202589, 0.202821, 0.204273, 0.202197, 0.201757, 0.202304, 0.203865]}, {"n": 502, "day": "2026-10-09", "step": [0.0, 0.04210724, 0.04156587, 0.04061964, 0.03920821, 0.12507674, 0.12264298, 0.12008081, 0.11740424, 0.40925542, 0.40202745, 0.39771535, 0.39591208, 1.13048631, 1.12367482, 1.12057501, 1.11943064], "drift": [0.0, 0.00203, 0.00642, 0.01558, 0.03573, 0.00551, 0.01372, 0.02658, 0.05493, 0.0136, 0.02719, 0.04723, 0.09155, 0.02411, 0.04607, 0.07484, 0.13661], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 33, 35, 33], "mean_delta": [-0.073759, -0.071118, -0.065983, -0.055813, -0.036347, -0.066595, -0.058104, -0.047477, -0.032392, -0.056242, -0.047095, -0.040388, -0.034335, -0.044545, -0.037442, -0.033248, -0.029847], "mean_abs_delta": [0.174975, 0.174242, 0.172748, 0.170111, 0.166815, 0.173077, 0.170829, 0.168616, 0.166488, 0.170928, 0.169057, 0.167969, 0.167479, 0.169159, 0.168292, 0.167735, 0.167392]}], "bootstrap": {"lines": {"capitulation": 0.016041791066527367}, "gates_ok": true, "calibration": {"mean": 0.07089309626579844, "total": 158.8714287316543, "trades": 2241, "per_strategy": {"capitulation": {"line": 0.016041791066527367, "mean": 0.11632242810015601, "rule": "match", "chain": "sol", "total": 18.844233352225274, "trades": 162, "hold_min": 240, "teacher_trades": 162}, "capitulation@rh": {"line": 0.07225678861141205, "mean": 0.0673531483306537, "rule": "match", "chain": "rh", "total": 140.02719537942903, "trades": 2079, "hold_min": 240, "teacher_trades": 2079}}}, "diagnostics": {"rows": 15402, "slope": 1.0249917184344512, "kc_active": 0.10003621876239777, "rank_corr": 0.9010552624530308, "mbon_share": 0.340462761402956, "mbon_saturated": 0.02083333395421505}, "gate_failures": []}, "evaluation": {"n": 1785, "pf": 1.2904917167018137, "win": 0.47619047619047616, "days": 16, "mean": 0.028513016179203987, "median": -0.009349760599434376, "days_positive": 12}, "strategies": ["capitulation"], "finished_at": "2026-10-10T23:37:21.188648+00:00", "trades_file": "/Users/bryce/Documents/code/fly-trader/data/brain/replay/trades_20261010T233728Z.json", "per_strategy": {"capitulation": {"alpha": 0.0001, "chosen": {"mean": 0.030173424631357193, "alpha": 0.0001, "total": 89.58489990234375, "config": 1, "trades": 2969, "governance": {"ic": 35, "any": 35, "drift": 0, "checks": 1482, "shadow": 0}, "half_life_days": 1.0}, "configs": [{"mean": 0.03036709874868393, "alpha": 0.0, "total": 91.28350067138672, "config": 0, "trades": 3006, "governance": {"ic": 0, "any": 0, "drift": 0, "checks": 0, "shadow": 0}, "half_life_days": null}, {"mean": 0.030173424631357193, "alpha": 0.0001, "total": 89.58489990234375, "config": 1, "trades": 2969, "governance": {"ic": 35, "any": 35, "drift": 0, "checks": 1482, "shadow": 0}, "half_life_days": 1.0}, {"mean": 0.032286640256643295, "alpha": 0.0001, "total": 67.99566650390625, "config": 2, "trades": 2106, "governance": {"ic": 28, "any": 29, "drift": 0, "checks": 1482, "shadow": 1}, "half_life_days": 3.0}, {"mean": 0.03781268000602722, "alpha": 0.0001, "total": 55.92495346069336, "config": 3, "trades": 1479, "governance": {"ic": 17, "any": 69, "drift": 0, "checks": 1482, "shadow": 52}, "half_life_days": 7.0}, {"mean": 0.04063548520207405, "alpha": 0.0001, "total": 48.437496185302734, "config": 4, "trades": 1192, "governance": {"ic": 7, "any": 40, "drift": 0, "checks": 1482, "shadow": 33}, "half_life_days": 30.0}, {"mean": 0.03773646801710129, "alpha": 0.0003, "total": 79.32205200195312, "config": 5, "trades": 2102, "governance": {"ic": 30, "any": 31, "drift": 0, "checks": 1482, "shadow": 1}, "half_life_days": 1.0}, {"mean": 0.04369469732046127, "alpha": 0.0003, "total": 42.51494216918945, "config": 6, "trades": 973, "governance": {"ic": 16, "any": 53, "drift": 0, "checks": 1482, "shadow": 37}, "half_life_days": 3.0}, {"mean": 0.05426740273833275, "alpha": 0.0003, "total": 37.17317199707031, "config": 7, "trades": 685, "governance": {"ic": 8, "any": 62, "drift": 0, "checks": 1482, "shadow": 54}, "half_life_days": 7.0}, {"mean": 0.034157898277044296, "alpha": 0.0003, "total": 20.494739532470703, "config": 8, "trades": 600, "governance": {"ic": 0, "any": 58, "drift": 0, "checks": 1482, "shadow": 58}, "half_life_days": 30.0}, {"mean": 0.04557060822844505, "alpha": 0.001, "total": 51.90492248535156, "config": 9, "trades": 1139, "governance": {"ic": 20, "any": 20, "drift": 0, "checks": 1482, "shadow": 0}, "half_life_days": 1.0}, {"mean": 0.03615965694189072, "alpha": 0.001, "total": 22.129711151123047, "config": 10, "trades": 612, "governance": {"ic": 5, "any": 53, "drift": 0, "checks": 1482, "shadow": 48}, "half_life_days": 3.0}, {"mean": 0.04262426868081093, "alpha": 0.001, "total": 21.3121337890625, "config": 11, "trades": 500, "governance": {"ic": 0, "any": 78, "drift": 0, "checks": 1482, "shadow": 78}, "half_life_days": 7.0}, {"mean": 0.023908229544758797, "alpha": 0.001, "total": 10.639162063598633, "config": 12, "trades": 445, "governance": {"ic": 0, "any": 66, "drift": 0, "checks": 1482, "shadow": 66}, "half_life_days": 30.0}, {"mean": 0.04726389795541763, "alpha": 0.003, "total": 46.224090576171875, "config": 13, "trades": 978, "governance": {"ic": 6, "any": 58, "drift": 0, "checks": 1482, "shadow": 52}, "half_life_days": 1.0}, {"mean": 0.03016720339655876, "alpha": 0.003, "total": 20.634366989135742, "config": 14, "trades": 684, "governance": {"ic": 0, "any": 137, "drift": 0, "checks": 1482, "shadow": 137}, "half_life_days": 3.0}, {"mean": 0.034875430166721344, "alpha": 0.003, "total": 21.832019805908203, "config": 15, "trades": 626, "governance": {"ic": 0, "any": 136, "drift": 0, "checks": 1482, "shadow": 136}, "half_life_days": 7.0}, {"mean": 0.03963538631796837, "alpha": 0.003, "total": 23.345243453979492, "config": 16, "trades": 589, "governance": {"ic": 0, "any": 199, "drift": 0, "checks": 1482, "shadow": 199}, "half_life_days": 30.0}], "plastic": true, "hold_min": 240, "half_life_days": 1.0}}, "half_life_days": 1.0, "selection_days": ["2026-09-09", "2026-09-23"], "evaluation_days": ["2026-09-24", "2026-10-09"], "plastic_beats_frozen": true}'::jsonb) ON CONFLICT (key) DO NOTHING; +INSERT INTO ui_settings (key, value) VALUES ('autostart', '{"pumpstream": true, "runner": true, "train": false, "replay": false, "discover": false, "vault": true, "rh_stream": true}'::jsonb) ON CONFLICT (key) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0x0000000000000000000000000000000000000000', 'ETH', 'eth', '18', true, '4200000000000000000', NULL, NULL, NULL, NULL, NULL, NULL, NULL) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0x03dfbbe0ac4e7bcdafd08ed41a400326b77d8c80', 'JNJ', 'stock', '18', true, '36524326877657062089', '0x7f0ace18d1dced47063cf26e649bc8ab14d09e67', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 18, "dec1": 6, "pool": "0x7f0ace18d1dced47063cf26e649bc8ab14d09e67", "token0": "0x03dfbbe0ac4e7bcdafd08ed41a400326b77d8c80", "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "emitter": "0x7f0ace18d1dced47063cf26e649bc8ab14d09e67", "token_in": "0x03dfbbe0ac4e7bcdafd08ed41a400326b77d8c80"}, {"dec0": 18, "dec1": 6, "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0x05a3d1cd21d0c88145e82600e62e7e496e0f222b', 'AMC', 'stock', '18', true, '3840064684014871241292', '0xaa34fea710a1a737840329051d81d3b0b7c564d5', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 18, "dec1": 6, "pool": "0xaa34fea710a1a737840329051d81d3b0b7c564d5", "token0": "0x05a3d1cd21d0c88145e82600e62e7e496e0f222b", "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "emitter": "0xaa34fea710a1a737840329051d81d3b0b7c564d5", "token_in": "0x05a3d1cd21d0c88145e82600e62e7e496e0f222b"}, {"dec0": 18, "dec1": 6, "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0x05b37fb53a299a1b874a619e1c4c404d52c36f4c', 'RDDT', 'stock', '18', true, '42347152428810721502', '0xa541143f20d7b0643123064abf25f423e375b531', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 18, "dec1": 18, "pool": "0xa541143f20d7b0643123064abf25f423e375b531", "token0": "0x05b37fb53a299a1b874a619e1c4c404d52c36f4c", "token1": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "emitter": "0xa541143f20d7b0643123064abf25f423e375b531", "token_in": "0x05b37fb53a299a1b874a619e1c4c404d52c36f4c"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0x07ebb29a38fbcb41563817e5e19f2cec619c90d2', 'BUN', 'stock', '18', true, '191347838750403524965795', '0x4f22b1a543711903604d5e84333a9fdd4a63166d', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 18, "dec1": 18, "pool": "0x4f22b1a543711903604d5e84333a9fdd4a63166d", "token0": "0x07ebb29a38fbcb41563817e5e19f2cec619c90d2", "token1": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "emitter": "0x4f22b1a543711903604d5e84333a9fdd4a63166d", "token_in": "0x07ebb29a38fbcb41563817e5e19f2cec619c90d2"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0x116f00968269b7bfbad4109ce591d6e74c0601d4', 'NET', 'stock', '18', true, '33075489282385834154', '0xa9caebb1b2fb2b954f192572b309aef1caf6a330', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 18, "dec1": 6, "pool": "0xa9caebb1b2fb2b954f192572b309aef1caf6a330", "token0": "0x116f00968269b7bfbad4109ce591d6e74c0601d4", "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "emitter": "0xa9caebb1b2fb2b954f192572b309aef1caf6a330", "token_in": "0x116f00968269b7bfbad4109ce591d6e74c0601d4"}, {"dec0": 18, "dec1": 6, "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0x117cc2133c37b721f49de2a7a74833232b3b4c0c', 'SPY', 'stock', '18', true, '10900000000000000000', '0xa7bb1ac63bbab0c44316e6c8c455213441689167', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 18, "dec1": 6, "pool": "0xa7bb1ac63bbab0c44316e6c8c455213441689167", "token0": "0x117cc2133c37b721f49de2a7a74833232b3b4c0c", "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "emitter": "0xa7bb1ac63bbab0c44316e6c8c455213441689167", "token_in": "0x117cc2133c37b721f49de2a7a74833232b3b4c0c"}, {"dec0": 18, "dec1": 6, "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0x12f190a9f9d7d37a250758b26824b97ce941bf54', 'AMZN', 'stock', '18', true, '29330291092142702509', '0x8ac92da74ab5f3b1d024dc1943ad7e15dc4179ef', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 18, "dec1": 6, "pool": "0x8ac92da74ab5f3b1d024dc1943ad7e15dc4179ef", "token0": "0x12f190a9f9d7d37a250758b26824b97ce941bf54", "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "emitter": "0x8ac92da74ab5f3b1d024dc1943ad7e15dc4179ef", "token_in": "0x12f190a9f9d7d37a250758b26824b97ce941bf54"}, {"dec0": 18, "dec1": 6, "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0x1b0e319c6a659f002271b69db8a7df2f911c153e', 'GME', 'stock', '18', true, '369000000000000000000', '0xc6bcc95043dc48c204bb2d57fb264a10efe0a607', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 18, "dec1": 18, "pool": "0xc6bcc95043dc48c204bb2d57fb264a10efe0a607", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0x1b0e319c6a659f002271b69db8a7df2f911c153e", "emitter": "0xc6bcc95043dc48c204bb2d57fb264a10efe0a607", "token_in": "0x1b0e319c6a659f002271b69db8a7df2f911c153e"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0x1cdad396db64bda184d5182a97dd9b3c62100b7d', 'P', 'stock', '18', true, '66700197238658777370', NULL, NULL, NULL, NULL, NULL, NULL, NULL) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0x1d11f0496982706c5e14a514d4e79f2e6bde4516', 'DJT', 'stock', '18', true, '917162201643832986520', '0x95def4ea143630d64caa8f55f7570d8023f20265', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 18, "dec1": 18, "pool": "0x95def4ea143630d64caa8f55f7570d8023f20265", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0x1d11f0496982706c5e14a514d4e79f2e6bde4516", "emitter": "0x95def4ea143630d64caa8f55f7570d8023f20265", "token_in": "0x1d11f0496982706c5e14a514d4e79f2e6bde4516"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0x25c288e6d899b9bc30160965ad9644c67e73be0c', 'F', 'stock', '18', true, '711416942148760410079', '0x4dbac19e895322ac5b93abad9008691632bffc05', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 18, "dec1": 6, "pool": "0x4dbac19e895322ac5b93abad9008691632bffc05", "token0": "0x25c288e6d899b9bc30160965ad9644c67e73be0c", "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "emitter": "0x4dbac19e895322ac5b93abad9008691632bffc05", "token_in": "0x25c288e6d899b9bc30160965ad9644c67e73be0c"}, {"dec0": 18, "dec1": 6, "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0x27c99fbde9d0d2aa4f4bfb4943f237843ddf6958', 'DDOG', 'stock', '18', true, '37256385303244523662', NULL, NULL, NULL, NULL, NULL, NULL, NULL) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0x2e0847e8910a9732eb3fb1bb4b70a580adad4fe3', 'GOOGL', 'stock', '18', true, '24200000000000000000', '0x34d0dc122cf9a8eb296fc5e0d3a233625d7d19b7', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 18, "dec1": 6, "pool": "0x34d0dc122cf9a8eb296fc5e0d3a233625d7d19b7", "token0": "0x2e0847e8910a9732eb3fb1bb4b70a580adad4fe3", "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "emitter": "0x34d0dc122cf9a8eb296fc5e0d3a233625d7d19b7", "token_in": "0x2e0847e8910a9732eb3fb1bb4b70a580adad4fe3"}, {"dec0": 18, "dec1": 6, "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0x322f0929c4625ed5bad873c95208d54e1c003b2d', 'TSLA', 'stock', '18', true, '26000000000000000000', '0xc4f0172d6ac8dd294dd1137d047d5e1893760236', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 18, "dec1": 6, "pool": "0xc4f0172d6ac8dd294dd1137d047d5e1893760236", "token0": "0x322f0929c4625ed5bad873c95208d54e1c003b2d", "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "emitter": "0xc4f0172d6ac8dd294dd1137d047d5e1893760236", "token_in": "0x322f0929c4625ed5bad873c95208d54e1c003b2d"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0x329fcaceb9ad6f9580dd5f643fed0646900d043c', 'LMT', 'stock', '18', true, '19417850658193372681', '0x54f160bea60ec5207918c539d86e0b906b4da297', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 18, "dec1": 6, "pool": "0x54f160bea60ec5207918c539d86e0b906b4da297", "token0": "0x329fcaceb9ad6f9580dd5f643fed0646900d043c", "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "emitter": "0x54f160bea60ec5207918c539d86e0b906b4da297", "token_in": "0x329fcaceb9ad6f9580dd5f643fed0646900d043c"}, {"dec0": 18, "dec1": 6, "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0x3b14c39e89d60d627b42a1a4ca45b5bb45fc12e2', 'RKLB', 'stock', '18', true, '162685692553916369167', '0xbbc15d66fef8391f53569e19016e86545be9de3f302b82db93ed6165236d22f3', 'uniswap-v4', NULL, NULL, NULL, NULL, '[{"dec0": 18, "dec1": 6, "pool": "0xbbc15d66fef8391f53569e19016e86545be9de3f302b82db93ed6165236d22f3", "token0": "0x3b14c39e89d60d627b42a1a4ca45b5bb45fc12e2", "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "emitter": "0x8366a39cc670b4001a1121b8f6a443a643e40951", "token_in": "0x3b14c39e89d60d627b42a1a4ca45b5bb45fc12e2"}, {"dec0": 18, "dec1": 6, "pool": "0x84bd4e2d8be11aeb0afc1195b38f587b61e90068548f1063fdbe448fb8cad0b6", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "emitter": "0x8366a39cc670b4001a1121b8f6a443a643e40951", "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0x408c14038a04f7bd235329e26d2bf569ee20e250', 'NU', 'stock', '18', true, '667513667205169680597', '0x0e3faed512e7909758eb924e6919e0057bd6b45e', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 18, "dec1": 6, "pool": "0x0e3faed512e7909758eb924e6919e0057bd6b45e", "token0": "0x408c14038a04f7bd235329e26d2bf569ee20e250", "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "emitter": "0x0e3faed512e7909758eb924e6919e0057bd6b45e", "token_in": "0x408c14038a04f7bd235329e26d2bf569ee20e250"}, {"dec0": 18, "dec1": 6, "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0x411efb0e7f985935daec3d4c3ebaea0d0ad7d89f', 'SLV', 'stock', '18', true, '172796487119437943163', '0x48abc7052509caf820c4c22559f08381f9afd823', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 18, "dec1": 18, "pool": "0x48abc7052509caf820c4c22559f08381f9afd823", "token0": "0x411efb0e7f985935daec3d4c3ebaea0d0ad7d89f", "token1": "0xc9a981fee1f9dec688bb123ccdecc63d0debfc4e", "emitter": "0x48abc7052509caf820c4c22559f08381f9afd823", "token_in": "0x411efb0e7f985935daec3d4c3ebaea0d0ad7d89f"}, {"dec0": 18, "dec1": 18, "pool": "0x26250ba84465454bc731f710e46f1f32b167d66b", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0xc9a981fee1f9dec688bb123ccdecc63d0debfc4e", "emitter": "0x26250ba84465454bc731f710e46f1f32b167d66b", "token_in": "0xc9a981fee1f9dec688bb123ccdecc63d0debfc4e"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0x41f4267525a8aff329540ef24fd83d9044758b33', 'FIG', 'stock', '18', true, '390666925767586478597', '0xca5904c0a9d42f0ec1bf760fdf779907877144fd', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 18, "dec1": 6, "pool": "0xca5904c0a9d42f0ec1bf760fdf779907877144fd", "token0": "0x41f4267525a8aff329540ef24fd83d9044758b33", "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "emitter": "0xca5904c0a9d42f0ec1bf760fdf779907877144fd", "token_in": "0x41f4267525a8aff329540ef24fd83d9044758b33"}, {"dec0": 18, "dec1": 6, "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0x43b07d15ce533bec5476d70c22a78a1b2b662155', 'MRNA', 'stock', '18', true, '66652476626218421319', '0xae6d85e96bee054417d8d627b310a945484415be', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 18, "dec1": 18, "pool": "0xae6d85e96bee054417d8d627b310a945484415be", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0x43b07d15ce533bec5476d70c22a78a1b2b662155", "emitter": "0xae6d85e96bee054417d8d627b310a945484415be", "token_in": "0x43b07d15ce533bec5476d70c22a78a1b2b662155"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0x48e39e56acdba37b09020c0b734a613c9a2f100a', 'BB', 'stock', '18', true, '1259561425061425366930', '0x183304567485e97e68835708f572aaa0e0e71d08', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 18, "dec1": 18, "pool": "0x183304567485e97e68835708f572aaa0e0e71d08", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0x48e39e56acdba37b09020c0b734a613c9a2f100a", "emitter": "0x183304567485e97e68835708f572aaa0e0e71d08", "token_in": "0x48e39e56acdba37b09020c0b734a613c9a2f100a"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0x4a0e65a3eccec6dbe60ae065f2e7bb85fae35eea', 'SPCX', 'stock', '18', true, '72200000000000000000', '0xc61284332117c3fb23a2a56cceffd07f7af60029', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 18, "dec1": 6, "pool": "0xc61284332117c3fb23a2a56cceffd07f7af60029", "token0": "0x4a0e65a3eccec6dbe60ae065f2e7bb85fae35eea", "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "emitter": "0xc61284332117c3fb23a2a56cceffd07f7af60029", "token_in": "0x4a0e65a3eccec6dbe60ae065f2e7bb85fae35eea"}, {"dec0": 18, "dec1": 6, "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0x4d21483a44bf67a86b77e3da301411880797d452', 'BA', 'stock', '18', true, '49724925896506964807', NULL, NULL, NULL, NULL, NULL, NULL, NULL) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0x4e62068525ab11fe768e29dfd00ef909b9803016', 'LULU', 'stock', '18', true, '83716665278587490919', '0x0f4227d27082b3bca6818381b9ea6460275e49f4', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 18, "dec1": 6, "pool": "0x0f4227d27082b3bca6818381b9ea6460275e49f4", "token0": "0x4e62068525ab11fe768e29dfd00ef909b9803016", "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "emitter": "0x0f4227d27082b3bca6818381b9ea6460275e49f4", "token_in": "0x4e62068525ab11fe768e29dfd00ef909b9803016"}, {"dec0": 18, "dec1": 6, "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0x4ea005168d7f09a7a0ba9d1def21a479950e44c2', 'COST', 'stock', '18', true, '8470405930331172950', '0x0a2121a50a09ed0796ae81f9c53ff9398355a398', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 18, "dec1": 6, "pool": "0x0a2121a50a09ed0796ae81f9c53ff9398355a398", "token0": "0x4ea005168d7f09a7a0ba9d1def21a479950e44c2", "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "emitter": "0x0a2121a50a09ed0796ae81f9c53ff9398355a398", "token_in": "0x4ea005168d7f09a7a0ba9d1def21a479950e44c2"}, {"dec0": 18, "dec1": 6, "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0x56910d4409f3a0c78c64dd8d0545ff0705389870', 'Index', 'stock', '18', true, '620133096008220802271782', '0xd29893ffac8b29ec4db2cfe0cdb3fe1377c028ff', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 18, "dec1": 18, "pool": "0xd29893ffac8b29ec4db2cfe0cdb3fe1377c028ff", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0x56910d4409f3a0c78c64dd8d0545ff0705389870", "emitter": "0xd29893ffac8b29ec4db2cfe0cdb3fe1377c028ff", "token_in": "0x56910d4409f3a0c78c64dd8d0545ff0705389870"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0x58ffe4a942d3885baa22d7520691f611ef09e7aa', 'TSM', 'stock', '18', true, '24453231884057971122', '0x91280db3392ea92c08d8134b5760fb4798b69547', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 18, "dec1": 18, "pool": "0x91280db3392ea92c08d8134b5760fb4798b69547", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0x58ffe4a942d3885baa22d7520691f611ef09e7aa", "emitter": "0x91280db3392ea92c08d8134b5760fb4798b69547", "token_in": "0x58ffe4a942d3885baa22d7520691f611ef09e7aa"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0x59818904ab4ce163b3ce4ffb64f2d6ca02c434b4', 'QUBT', 'stock', '18', true, '1294653972257250949347', '0x227bbce9a81b3694b01754298a983be3f9e44a93', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 18, "dec1": 6, "pool": "0x227bbce9a81b3694b01754298a983be3f9e44a93", "token0": "0x59818904ab4ce163b3ce4ffb64f2d6ca02c434b4", "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "emitter": "0x227bbce9a81b3694b01754298a983be3f9e44a93", "token_in": "0x59818904ab4ce163b3ce4ffb64f2d6ca02c434b4"}, {"dec0": 18, "dec1": 6, "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0x5e81213613b6b86eab4c6c50d718d34359459786', 'TTWO', 'stock', '18', true, '26639006882017848277', '0xe69fe23b708362ef5817048f16e4171e0831341c', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 18, "dec1": 18, "pool": "0xe69fe23b708362ef5817048f16e4171e0831341c", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0x5e81213613b6b86eab4c6c50d718d34359459786", "emitter": "0xe69fe23b708362ef5817048f16e4171e0831341c", "token_in": "0x5e81213613b6b86eab4c6c50d718d34359459786"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0x5fc5360d0400a0fd4f2af552add042d716f1d168', 'USDG', 'stable', '6', true, '8090000000', '0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 18, "dec1": 6, "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0x62fd0668e10d8b72339be2dcf7643001688ff13b', 'MRVL', 'stock', '18', true, '48682003099573809148', '0x5201ea77c950abb8d3706a20bf3b83d872e4a98b', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 18, "dec1": 18, "pool": "0x5201ea77c950abb8d3706a20bf3b83d872e4a98b", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0x62fd0668e10d8b72339be2dcf7643001688ff13b", "emitter": "0x5201ea77c950abb8d3706a20bf3b83d872e4a98b", "token_in": "0x62fd0668e10d8b72339be2dcf7643001688ff13b"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0x6330d8c3178a418788df01a47479c0ce7ccf450b', 'COIN', 'stock', '18', true, '52470673453406424989', '0x6707aeac7d0e519b083219d27bb427364363183a', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 18, "dec1": 18, "pool": "0x6707aeac7d0e519b083219d27bb427364363183a", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0x6330d8c3178a418788df01a47479c0ce7ccf450b", "emitter": "0x6707aeac7d0e519b083219d27bb427364363183a", "token_in": "0x6330d8c3178a418788df01a47479c0ce7ccf450b"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0x63c12667638f2ae6fc6ae09b43d98ec84a8586ea', 'wsNET', 'stock', '18', true, '10063226872967693437', NULL, NULL, NULL, NULL, NULL, NULL, NULL) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0x7066a64c24e4206cd62e83bf198c1e7eb361f51e', 'PFE', 'stock', '18', true, '346376981392143376566', '0xc7d573fcda6d2107c97fb582ae18411f9db32e7f', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 6, "dec1": 18, "pool": "0xc7d573fcda6d2107c97fb582ae18411f9db32e7f", "token0": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "token1": "0x7066a64c24e4206cd62e83bf198c1e7eb361f51e", "emitter": "0xc7d573fcda6d2107c97fb582ae18411f9db32e7f", "token_in": "0x7066a64c24e4206cd62e83bf198c1e7eb361f51e"}, {"dec0": 18, "dec1": 6, "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0x75742c18bc1f1c5c5f448f4c9d9c6f66dafaaa38', 'SOXX', 'stock', '18', true, '18167833670600454869', NULL, NULL, NULL, NULL, NULL, NULL, NULL) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0x7f0abef0c07280f82c6a08ead09ded6bae2c13fc', 'EWY', 'stock', '18', true, '55805870522367777811', '0x3cba52145fb271e4a76f44df3d8b86f460a04e767841bcb8288a02729047dbbb', 'uniswap-v4', NULL, NULL, NULL, NULL, '[{"dec0": 6, "dec1": 18, "pool": "0x3cba52145fb271e4a76f44df3d8b86f460a04e767841bcb8288a02729047dbbb", "token0": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "token1": "0x7f0abef0c07280f82c6a08ead09ded6bae2c13fc", "emitter": "0x8366a39cc670b4001a1121b8f6a443a643e40951", "token_in": "0x7f0abef0c07280f82c6a08ead09ded6bae2c13fc"}, {"dec0": 18, "dec1": 6, "pool": "0x84bd4e2d8be11aeb0afc1195b38f587b61e90068548f1063fdbe448fb8cad0b6", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "emitter": "0x8366a39cc670b4001a1121b8f6a443a643e40951", "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0x8005d266423c7ea827372c9c864491e5786600ea', 'LLY', 'stock', '18', true, '8727274137931034486', '0xf212d02146a897f5f686e9d629f6a73da534324a', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 6, "dec1": 18, "pool": "0xf212d02146a897f5f686e9d629f6a73da534324a", "token0": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "token1": "0x8005d266423c7ea827372c9c864491e5786600ea", "emitter": "0xf212d02146a897f5f686e9d629f6a73da534324a", "token_in": "0x8005d266423c7ea827372c9c864491e5786600ea"}, {"dec0": 18, "dec1": 6, "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0x822cc93ffd030293e9842c30bbd678f530701867', 'BE', 'stock', '18', true, '42050779564420924105', '0x1bad145c8f06444e0df81c28257cd20231bd1f16', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 6, "dec1": 18, "pool": "0x1bad145c8f06444e0df81c28257cd20231bd1f16", "token0": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "token1": "0x822cc93ffd030293e9842c30bbd678f530701867", "emitter": "0x1bad145c8f06444e0df81c28257cd20231bd1f16", "token_in": "0x822cc93ffd030293e9842c30bbd678f530701867"}, {"dec0": 18, "dec1": 6, "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0x84cab63bc87912e71ad199ff14a0ba45de68fef8', 'SKHY', 'stock', '18', true, '62965779325786790016', '0x5f2a5025feb93a4c44da4e3b37e4fd3fb0ab5171', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 6, "dec1": 18, "pool": "0x5f2a5025feb93a4c44da4e3b37e4fd3fb0ab5171", "token0": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "token1": "0x84cab63bc87912e71ad199ff14a0ba45de68fef8", "emitter": "0x5f2a5025feb93a4c44da4e3b37e4fd3fb0ab5171", "token_in": "0x84cab63bc87912e71ad199ff14a0ba45de68fef8"}, {"dec0": 18, "dec1": 6, "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0x86923f96303d656e4aa86d9d42d1e57ad2023fdc', 'AMD', 'stock', '18', true, '16665504590959022517', '0x48d284a2a4d3dc1b3da08231fe44317e7e7aa51f', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 6, "dec1": 18, "pool": "0x48d284a2a4d3dc1b3da08231fe44317e7e7aa51f", "token0": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "token1": "0x86923f96303d656e4aa86d9d42d1e57ad2023fdc", "emitter": "0x48d284a2a4d3dc1b3da08231fe44317e7e7aa51f", "token_in": "0x86923f96303d656e4aa86d9d42d1e57ad2023fdc"}, {"dec0": 18, "dec1": 6, "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0x894e1ec2d74ffe5aef8dc8a9e84686accb964f2a', 'PLTR', 'stock', '18', true, '47065125263404356876', '0x61be5bfbaf17ae28bf68006103b2e78fe6112638', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 18, "dec1": 18, "pool": "0x61be5bfbaf17ae28bf68006103b2e78fe6112638", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0x894e1ec2d74ffe5aef8dc8a9e84686accb964f2a", "emitter": "0x61be5bfbaf17ae28bf68006103b2e78fe6112638", "token_in": "0x894e1ec2d74ffe5aef8dc8a9e84686accb964f2a"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0x92fd66527192e3e61d4ddd13322aa222de86f9b5', 'SGOV', 'stock', '18', true, '102816558506191026323', '0x6ba50150b17ffd0972915aaf04ffd5e8f4fa49b4', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 6, "dec1": 18, "pool": "0x6ba50150b17ffd0972915aaf04ffd5e8f4fa49b4", "token0": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "token1": "0x92fd66527192e3e61d4ddd13322aa222de86f9b5", "emitter": "0x6ba50150b17ffd0972915aaf04ffd5e8f4fa49b4", "token_in": "0x92fd66527192e3e61d4ddd13322aa222de86f9b5"}, {"dec0": 18, "dec1": 6, "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0x941ae714ec6d8130c7b75d67160ca08f1e7d11dd', 'DELL', 'stock', '18', true, '23820324705882352994', '0xc30c89cb7815a1488b7998d15eec73961707fc5a', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 6, "dec1": 18, "pool": "0xc30c89cb7815a1488b7998d15eec73961707fc5a", "token0": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "token1": "0x941ae714ec6d8130c7b75d67160ca08f1e7d11dd", "emitter": "0xc30c89cb7815a1488b7998d15eec73961707fc5a", "token_in": "0x941ae714ec6d8130c7b75d67160ca08f1e7d11dd"}, {"dec0": 18, "dec1": 6, "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0x980dcf6766fa79f5cf0c4aadb3ab477ff15a9619', 'IBM', 'stock', '18', true, '43891115360101975839', '0x7502f81aef932e82ccf240f023dd989689a3ef65', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 18, "dec1": 18, "pool": "0x7502f81aef932e82ccf240f023dd989689a3ef65", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0x980dcf6766fa79f5cf0c4aadb3ab477ff15a9619", "emitter": "0x7502f81aef932e82ccf240f023dd989689a3ef65", "token_in": "0x980dcf6766fa79f5cf0c4aadb3ab477ff15a9619"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0x9b23573b156b52565012f5ce02cdf60afbaa70be', 'PENG', 'stock', '18', true, '143586441157738307921', NULL, NULL, NULL, NULL, NULL, NULL, NULL) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0x9d9c6684f596f66a64c030b93a886d51fd4d7931', 'NBIS', 'stock', '18', true, '46093139482053612464', NULL, NULL, NULL, NULL, NULL, NULL, NULL) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0x9e7abd3c9139d14e4c86dce0e455aab7a0c2fb3e', 'WYFI', 'stock', '18', true, '578493600000000022375', '0x2a3063e34c60253abb23c2442f4cdfbc5cbd02c2', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 6, "dec1": 18, "pool": "0x2a3063e34c60253abb23c2442f4cdfbc5cbd02c2", "token0": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "token1": "0x9e7abd3c9139d14e4c86dce0e455aab7a0c2fb3e", "emitter": "0x2a3063e34c60253abb23c2442f4cdfbc5cbd02c2", "token_in": "0x9e7abd3c9139d14e4c86dce0e455aab7a0c2fb3e"}, {"dec0": 18, "dec1": 6, "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0xa30fa36db767ad9ed3f7a60fc79526fb4d56d344', 'USO', 'stock', '18', true, '71798851063829787942', '0x02175608f1b5e6b5ed221ccfdc7be197d111d915', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 6, "dec1": 18, "pool": "0x02175608f1b5e6b5ed221ccfdc7be197d111d915", "token0": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "token1": "0xa30fa36db767ad9ed3f7a60fc79526fb4d56d344", "emitter": "0x02175608f1b5e6b5ed221ccfdc7be197d111d915", "token_in": "0xa30fa36db767ad9ed3f7a60fc79526fb4d56d344"}, {"dec0": 18, "dec1": 6, "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0xaa07a0e9209e16ac99708c3ec70159c6ef3128a3', 'ORBIO', 'stock', '18', true, '193502872457166537181000', '0x34f73f488309208b8cb6012eb47ffeb086ca1c2d', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 18, "dec1": 18, "pool": "0x34f73f488309208b8cb6012eb47ffeb086ca1c2d", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0xaa07a0e9209e16ac99708c3ec70159c6ef3128a3", "emitter": "0x34f73f488309208b8cb6012eb47ffeb086ca1c2d", "token_in": "0xaa07a0e9209e16ac99708c3ec70159c6ef3128a3"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0xab093def657f15df31b33922a95e047add645b29', 'SHROOM', 'stock', '18', true, '1393388248231318945176878', '0xc641a0dc848e7aadd7c69d800bae2fea9b258610', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 18, "dec1": 18, "pool": "0xc641a0dc848e7aadd7c69d800bae2fea9b258610", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0xab093def657f15df31b33922a95e047add645b29", "emitter": "0xc641a0dc848e7aadd7c69d800bae2fea9b258610", "token_in": "0xab093def657f15df31b33922a95e047add645b29"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0xacef2e09adb47ad6abebad9ff06689e60615c2b6', 'INDA', 'stock', '18', true, '206740198138697096367', '0xf5b37a305e7304a70067be356ee611ac29f706eb', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 18, "dec1": 18, "pool": "0xf5b37a305e7304a70067be356ee611ac29f706eb", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0xacef2e09adb47ad6abebad9ff06689e60615c2b6", "emitter": "0xf5b37a305e7304a70067be356ee611ac29f706eb", "token_in": "0xacef2e09adb47ad6abebad9ff06689e60615c2b6"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0xad25ac6c84d497db898fa1e8387bf6af3532a1c4', 'BABA', 'stock', '18', true, '91284676564156947526', '0xa57ab582b310dd6f9e934ea1eeea152741545e6a', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 6, "dec1": 18, "pool": "0xa57ab582b310dd6f9e934ea1eeea152741545e6a", "token0": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "token1": "0xad25ac6c84d497db898fa1e8387bf6af3532a1c4", "emitter": "0xa57ab582b310dd6f9e934ea1eeea152741545e6a", "token_in": "0xad25ac6c84d497db898fa1e8387bf6af3532a1c4"}, {"dec0": 18, "dec1": 6, "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0xae517a2903e68bd929dfd15be875f8369d53e94a', 'CEG', 'stock', '18', true, '35795219915276397737', NULL, NULL, NULL, NULL, NULL, NULL, NULL) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0xaf3d76f1834a1d425780943c99ea8a608f8a93f9', 'AAPL', 'stock', '18', true, '24200000000000000000', '0xaae0d815ee56e4092a5e5c2911e676fea50b2d6d', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 6, "dec1": 18, "pool": "0xaae0d815ee56e4092a5e5c2911e676fea50b2d6d", "token0": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "token1": "0xaf3d76f1834a1d425780943c99ea8a608f8a93f9", "emitter": "0xaae0d815ee56e4092a5e5c2911e676fea50b2d6d", "token_in": "0xaf3d76f1834a1d425780943c99ea8a608f8a93f9"}, {"dec0": 18, "dec1": 6, "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0xb0992820e760d836549ba69bc7598b4af75dee03', 'ORCL', 'stock', '18', true, '74826304376687171137', NULL, NULL, NULL, NULL, NULL, NULL, NULL) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0xb1bf26c1d20ff267a4f93550d1e0d06ac40a114b', 'RIVN', 'stock', '18', true, '643937219730941726039', '0xf9e71558eef115b6c8fbfc9ad5fda00033e9d69c', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 6, "dec1": 18, "pool": "0xf9e71558eef115b6c8fbfc9ad5fda00033e9d69c", "token0": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "token1": "0xb1bf26c1d20ff267a4f93550d1e0d06ac40a114b", "emitter": "0xf9e71558eef115b6c8fbfc9ad5fda00033e9d69c", "token_in": "0xb1bf26c1d20ff267a4f93550d1e0d06ac40a114b"}, {"dec0": 18, "dec1": 6, "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0xb90a19ff0af67f7779aff50a882a9cff42446400', 'SNDK', 'stock', '18', true, '6630443315631494151', '0x995c1ad5eb998b1bdd89f515c4bb64760c411b62', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 18, "dec1": 18, "pool": "0x995c1ad5eb998b1bdd89f515c4bb64760c411b62", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0xb90a19ff0af67f7779aff50a882a9cff42446400", "emitter": "0x995c1ad5eb998b1bdd89f515c4bb64760c411b62", "token_in": "0xb90a19ff0af67f7779aff50a882a9cff42446400"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0xba0cab75495255d0cb58e22b648bfed4ecd1f47e', 'SNOW', 'stock', '18', true, '30736040715515305748', '0xe0ca1dfc1161500fb410c8181f7cc33860ffb16b', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 6, "dec1": 18, "pool": "0xe0ca1dfc1161500fb410c8181f7cc33860ffb16b", "token0": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "token1": "0xba0cab75495255d0cb58e22b648bfed4ecd1f47e", "emitter": "0xe0ca1dfc1161500fb410c8181f7cc33860ffb16b", "token_in": "0xba0cab75495255d0cb58e22b648bfed4ecd1f47e"}, {"dec0": 18, "dec1": 6, "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0xc0d6457c16cc70d6790dd43521c899c87ce02f35', 'META', 'stock', '18', true, '13567895109851171552', '0xa4bdb396a69617eb7f70e2cc1ef526f7340b1b0d', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 18, "dec1": 18, "pool": "0xa4bdb396a69617eb7f70e2cc1ef526f7340b1b0d", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0xc0d6457c16cc70d6790dd43521c899c87ce02f35", "emitter": "0xa4bdb396a69617eb7f70e2cc1ef526f7340b1b0d", "token_in": "0xc0d6457c16cc70d6790dd43521c899c87ce02f35"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0xc72b96e0e48ecd4dc75e1e45396e26300bc39681', 'INTC', 'stock', '18', true, '101528935917721519434', '0x2e5a92f5013a64661a49312111be2e8abd33f56a', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 6, "dec1": 18, "pool": "0x2e5a92f5013a64661a49312111be2e8abd33f56a", "token0": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "token1": "0xc72b96e0e48ecd4dc75e1e45396e26300bc39681", "emitter": "0x2e5a92f5013a64661a49312111be2e8abd33f56a", "token_in": "0xc72b96e0e48ecd4dc75e1e45396e26300bc39681"}, {"dec0": 18, "dec1": 6, "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0xc9a981fee1f9dec688bb123ccdecc63d0debfc4e', 'GLD', 'stock', '18', true, '24940596949573086306', '0x26250ba84465454bc731f710e46f1f32b167d66b', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 18, "dec1": 18, "pool": "0x26250ba84465454bc731f710e46f1f32b167d66b", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0xc9a981fee1f9dec688bb123ccdecc63d0debfc4e", "emitter": "0x26250ba84465454bc731f710e46f1f32b167d66b", "token_in": "0xc9a981fee1f9dec688bb123ccdecc63d0debfc4e"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0xccee82fe024c36fa15e1005ede3e9e4787e23d09', 'HIMS', 'stock', '18', true, '354892004153686420827', '0xeb576c467d69e084a0fdc6ddf744467804634650', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 18, "dec1": 18, "pool": "0xeb576c467d69e084a0fdc6ddf744467804634650", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0xccee82fe024c36fa15e1005ede3e9e4787e23d09", "emitter": "0xeb576c467d69e084a0fdc6ddf744467804634650", "token_in": "0xccee82fe024c36fa15e1005ede3e9e4787e23d09"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0xcec185eb182c47d1ba1efc84e6959e18cd620be4', 'cbBTC', 'btc', '8', true, '13184752', '0xd30e44aae604b42a63f6f9a8109fd0408f35b9fb', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 18, "dec1": 8, "pool": "0xd30e44aae604b42a63f6f9a8109fd0408f35b9fb", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0xcec185eb182c47d1ba1efc84e6959e18cd620be4", "emitter": "0xd30e44aae604b42a63f6f9a8109fd0408f35b9fb", "token_in": "0xcec185eb182c47d1ba1efc84e6959e18cd620be4"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0xcef9027c7d6985b85f0ba431125073529a947a68', 'BULL', 'stock', '18', true, '1054596630934150255151', '0x0a393801fea32a50be77dab6c8bbb503db3a4663', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 18, "dec1": 18, "pool": "0x0a393801fea32a50be77dab6c8bbb503db3a4663", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0xcef9027c7d6985b85f0ba431125073529a947a68", "emitter": "0x0a393801fea32a50be77dab6c8bbb503db3a4663", "token_in": "0xcef9027c7d6985b85f0ba431125073529a947a68"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0xd0601ce157db5bdc3162bbac2a2c8af5320d9eec', 'NVDA', 'stock', '18', true, '41600000000000000000', '0x62ab521f71431f78ac374cdbadc6cda3c8916b6c', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 18, "dec1": 18, "pool": "0x62ab521f71431f78ac374cdbadc6cda3c8916b6c", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0xd0601ce157db5bdc3162bbac2a2c8af5320d9eec", "emitter": "0x62ab521f71431f78ac374cdbadc6cda3c8916b6c", "token_in": "0xd0601ce157db5bdc3162bbac2a2c8af5320d9eec"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0xd5f3879160bc7c32ebb4dc785f8a4f505888de68', 'QQQ', 'stock', '18', true, '11146474875652736535', '0xd60a5d14db690b7afad71f76b108071d7175597d', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 6, "dec1": 18, "pool": "0xd60a5d14db690b7afad71f76b108071d7175597d", "token0": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "token1": "0xd5f3879160bc7c32ebb4dc785f8a4f505888de68", "emitter": "0xd60a5d14db690b7afad71f76b108071d7175597d", "token_in": "0xd5f3879160bc7c32ebb4dc785f8a4f505888de68"}, {"dec0": 18, "dec1": 6, "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0xdf0992e440dd0be65bd8439b609d6d4366bf1cb5', 'CRCL', 'stock', '18', true, '119901670146137782822', '0x654e4143e82a5824445ade0824351c2a9acd95a8', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 6, "dec1": 18, "pool": "0x654e4143e82a5824445ade0824351c2a9acd95a8", "token0": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "token1": "0xdf0992e440dd0be65bd8439b609d6d4366bf1cb5", "emitter": "0x654e4143e82a5824445ade0824351c2a9acd95a8", "token_in": "0xdf0992e440dd0be65bd8439b609d6d4366bf1cb5"}, {"dec0": 18, "dec1": 6, "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0xe0444ef8bf4ed74f74fd73686e2ddf4c1c5591e8', 'NFLX', 'stock', '18', true, '130303046357615894393', '0x59895c0302f41aeaa129d2fa2442cec01e7ef45e', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 6, "dec1": 18, "pool": "0x59895c0302f41aeaa129d2fa2442cec01e7ef45e", "token0": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "token1": "0xe0444ef8bf4ed74f74fd73686e2ddf4c1c5591e8", "emitter": "0x59895c0302f41aeaa129d2fa2442cec01e7ef45e", "token_in": "0xe0444ef8bf4ed74f74fd73686e2ddf4c1c5591e8"}, {"dec0": 18, "dec1": 6, "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0xe93237c50d904957cf27e7b1133b510c669c2e74', 'MSFT', 'stock', '18', true, '16078639417693171399', '0xeb60bcd1d920ad6e102690ccfc6fb488899e1510', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 6, "dec1": 18, "pool": "0xeb60bcd1d920ad6e102690ccfc6fb488899e1510", "token0": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "token1": "0xe93237c50d904957cf27e7b1133b510c669c2e74", "emitter": "0xeb60bcd1d920ad6e102690ccfc6fb488899e1510", "token_in": "0xe93237c50d904957cf27e7b1133b510c669c2e74"}, {"dec0": 18, "dec1": 6, "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0xec262a75e413fafd0df80480274532c79d42da09', 'MSTR', 'stock', '18', true, '79980227385071671620', '0x6f8dc7122b56017c46892f91d38d871acf787f9b', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 8, "dec1": 18, "pool": "0x6f8dc7122b56017c46892f91d38d871acf787f9b", "token0": "0xcec185eb182c47d1ba1efc84e6959e18cd620be4", "token1": "0xec262a75e413fafd0df80480274532c79d42da09", "emitter": "0x6f8dc7122b56017c46892f91d38d871acf787f9b", "token_in": "0xec262a75e413fafd0df80480274532c79d42da09"}, {"dec0": 6, "dec1": 8, "pool": "0x9664d869540e9d0a76f12c6623946c6d5d201e09", "token0": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "token1": "0xcec185eb182c47d1ba1efc84e6959e18cd620be4", "emitter": "0x9664d869540e9d0a76f12c6623946c6d5d201e09", "token_in": "0xcec185eb182c47d1ba1efc84e6959e18cd620be4"}, {"dec0": 18, "dec1": 6, "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0xf0c4bf4c582cb3836e98394b1d4e7b7281101be8', 'RBLX', 'stock', '18', true, '248921514629948368496', '0x6d25417718a8d6c529130a8ccc4bfbf0a18219d3', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 18, "dec1": 18, "pool": "0x6d25417718a8d6c529130a8ccc4bfbf0a18219d3", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0xf0c4bf4c582cb3836e98394b1d4e7b7281101be8", "emitter": "0x6d25417718a8d6c529130a8ccc4bfbf0a18219d3", "token_in": "0xf0c4bf4c582cb3836e98394b1d4e7b7281101be8"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0xf23250dac154d05bb671cb0d0ebef3c635c79ce2', 'UPS', 'stock', '18', true, '100308545348611381553', '0x5c67a84ec5494ffe24e217b28debb96341014c86', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 6, "dec1": 18, "pool": "0x5c67a84ec5494ffe24e217b28debb96341014c86", "token0": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "token1": "0xf23250dac154d05bb671cb0d0ebef3c635c79ce2", "emitter": "0x5c67a84ec5494ffe24e217b28debb96341014c86", "token_in": "0xf23250dac154d05bb671cb0d0ebef3c635c79ce2"}, {"dec0": 18, "dec1": 6, "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0xf3081494b87e8d5fb7960f066e931d1d0e6e3d67', 'TAO', 'stock', '18', true, '37497530129218277084', '0x572d51a3de7c220fdd19451e4e24183b9f2ecadc', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 18, "dec1": 18, "pool": "0x572d51a3de7c220fdd19451e4e24183b9f2ecadc", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0xf3081494b87e8d5fb7960f066e931d1d0e6e3d67", "emitter": "0x572d51a3de7c220fdd19451e4e24183b9f2ecadc", "token_in": "0xf3081494b87e8d5fb7960f066e931d1d0e6e3d67"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0xf53f66751b1eff985311b693531e3290f600c410', 'SHOP', 'stock', '18', true, '70089387976658977032', '0x18a1afa849a6940870c2163b7ecbcc85d455ad99', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 6, "dec1": 18, "pool": "0x18a1afa849a6940870c2163b7ecbcc85d455ad99", "token0": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "token1": "0xf53f66751b1eff985311b693531e3290f600c410", "emitter": "0x18a1afa849a6940870c2163b7ecbcc85d455ad99", "token_in": "0xf53f66751b1eff985311b693531e3290f600c410"}, {"dec0": 18, "dec1": 6, "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0xf6589f11bc40b669e584073f428b05562f568733', 'SNAP', 'stock', '18', true, '1798186046511628119831', '0x84d251cedecdb949ca392e7bd9dd26481ab91088', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 18, "dec1": 18, "pool": "0x84d251cedecdb949ca392e7bd9dd26481ab91088", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0xf6589f11bc40b669e584073f428b05562f568733", "emitter": "0x84d251cedecdb949ca392e7bd9dd26481ab91088", "token_in": "0xf6589f11bc40b669e584073f428b05562f568733"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0xfde6b5d9bb419b10c23268c74e369abff39c0460', 'RCAT', 'stock', '18', true, '1236195785671282379035', NULL, NULL, NULL, NULL, NULL, NULL, NULL) ON CONFLICT (asset) DO NOTHING; +INSERT INTO rh_assets (asset, symbol, class, decimals, approved, graduation_threshold_raw, ref_pool, ref_kind, ref_quote, ref_fee_bps, trading_hours, note, ref_path) VALUES ('0xff080c8ce2e5feadaca0da81314ae59d232d4afd', 'MU', 'stock', '18', true, '9178459641793090016', '0xd057b1bc54917855bbee58ead58647f47cab35e5', 'uniswapv3', NULL, NULL, NULL, NULL, '[{"dec0": 6, "dec1": 18, "pool": "0xd057b1bc54917855bbee58ead58647f47cab35e5", "token0": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "token1": "0xff080c8ce2e5feadaca0da81314ae59d232d4afd", "emitter": "0xd057b1bc54917855bbee58ead58647f47cab35e5", "token_in": "0xff080c8ce2e5feadaca0da81314ae59d232d4afd"}, {"dec0": 18, "dec1": 6, "pool": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token0": "0x0bd7d308f8e1639fab988df18a8011f41eacad73", "token1": "0x5fc5360d0400a0fd4f2af552add042d716f1d168", "emitter": "0x52e65b17fb6e5ba00ed806f37afcd2daa50271ca", "token_in": "0x5fc5360d0400a0fd4f2af552add042d716f1d168"}]'::jsonb) ON CONFLICT (asset) DO NOTHING; diff --git a/tests/conftest.py b/tests/conftest.py index 04fab42d6a9623dd293248a7dee53ba69544e148..1bdf059cfef9632a20983dca01f82aa38fb945df 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,6 +3,8 @@ the refuse-production guard in fly_trader.db.connection can never see the real d import os os.environ["DATABASE_URL"] = "postgresql:///fly_trader_test" +for _name in ("RH_RPC_URL_TX", "QUICKNODE_RH_API_URL", "QUICKNODE_RH_API_KEY"): # no test reaches the real transaction endpoint + os.environ[_name] = "" import pytest # noqa: E402 diff --git a/tests/fixtures/rh_logs.json b/tests/fixtures/rh_logs.json new file mode 100644 index 0000000000000000000000000000000000000000..71dd27800e7bd321d4c8ec41d5db959d245aee96 --- /dev/null +++ b/tests/fixtures/rh_logs.json @@ -0,0 +1,177 @@ +{ +"HookFeeCollected": { +"address": "0xe5e702641ea86f4ae6cc3cdaed2b886f976be044", +"topics": [ +"0xc532c43b3423e14ef72748f1c8291238829ca0af8ba9b67975ad1483485a4b4d", +"0x749d7f24b5aa22a6c1d6b905a71e387be172eb1eb6dfcdb3168c375ba44c755b" +], +"data": "0x0000000000000000000000001b0e319c6a659f002271b69db8a7df2f911c153e00000000000000000000000000000000000000000000000000008bf6c5d13cb900000000000000000000000000000000000000000000000000008bf6c5d13cb9", +"blockNumber": "0x471a777", +"transactionHash": "0xd1265f36ff30d89dcc4ee4be691e5e1f83eda0ce604349d0e4de22b7437321c7", +"transactionIndex": "0x3", +"blockHash": "0x77a9046ae7e8c7aec3d21b1c2f7307d12eef0b878ae42ffdb0bf92a8fbcafc9f", +"blockTimestamp": "0x0", +"logIndex": "0x8", +"removed": false +}, +"TokenLaunched": { +"address": "0x7ed598bcef8bd9edd8c97a195c6d13f40801ec7e", +"topics": [ +"0x8d4aad4953d0ca700d468f3753aa14432d1b35b43ec6409f051fb6aa43a89607", +"0x0000000000000000000000004d804cf7b0e21a95fa3bf8dcbd8c54bac4b44b1e", +"0x0000000000000000000000005fefff9246c16a0035c2c161abb12d16dd5e9644", +"0x00000000000000000000000066666db22163b4a7977a0dab0d52177166be4413" +], +"data": "0x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003a4965bf58a40000", +"blockNumber": "0x471a9d5", +"transactionHash": "0xcf27cdc489a6aef9aa7134cf1ba6c77b67dd432fbba9c2eb9299967a71752848", +"transactionIndex": "0x9", +"blockHash": "0x1b14e951a453c4188c142ce030ead2ca08e0f5d8c712d60bbf1acb12195f181b", +"blockTimestamp": "0x0", +"logIndex": "0x15", +"removed": false +}, +"Launched": { +"address": "0xe33e9e479df8802cb0866d5d05258bec4cf62948", +"topics": [ +"0xdcacba5e347ae7abd91cb519eb877af8fa7774e347b85dd3ddcd24a2ba8cdf37", +"0x0000000000000000000000003a041986e43a68f43fb7bc21010a52dbb289e5bc", +"0x000000000000000000000000f761429c7ac5f7279c7dfc8550eb0a16fa1d3b98", +"0x000000000000000000000000742fb1cb7014caff6167fbd4b1c6477f7502ef60" +], +"data": "0x000000000000000000000000742fb1cb7014caff6167fbd4b1c6477f7502ef6000000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000046f5399e", +"blockNumber": "0x471ab90", +"transactionHash": "0x52fa8ebac685a0b81159b43c5695a6942a5c2fba1bddbec7cb86083320210eac", +"transactionIndex": "0x8", +"blockHash": "0x87d0d90ced59fcd4edb6bd67f184570c88f8385c940a16f3f34ed782758bd2c0", +"blockTimestamp": "0x0", +"logIndex": "0x5e", +"removed": false +}, +"LaunchSwept": { +"address": "0x7ed598bcef8bd9edd8c97a195c6d13f40801ec7e", +"topics": [ +"0xcdb72f157fd3666758a6ce201387ffb52038c7562e4fff352828da1096c4b6b4", +"0x000000000000000000000000c4741f151afec238fd3a17c25b1be83b8e3c1646" +], +"data": "0x0000000000000000000000000000000000000000000000003a4965bf58a40111000000000000000000000000000000000000000000ec565a76cddb7f1db6db6d", +"blockNumber": "0x471e75d", +"transactionHash": "0x3e244355cbc7733c7c9499da5926856080ccdc0a541e3fea6e7230af5bce6654", +"transactionIndex": "0x5", +"blockHash": "0xe29ac613a763933026443c3ef6c00c4686f117e906f45f4a562c44ef9f847e61", +"blockTimestamp": "0x0", +"logIndex": "0x22", +"removed": false +}, +"PoolRegistered": { +"address": "0xe5e702641ea86f4ae6cc3cdaed2b886f976be044", +"topics": [ +"0x01bf263a1db1652580721573296e1a1fa70b3d4c87f61d02a69c4e1109d2d573", +"0xb317f6fcf65d2024b9740ace92876d080a190a96e7a97a2f254b9581f8a15faf" +], +"data": "0x000000000000000000000000c4741f151afec238fd3a17c25b1be83b8e3c1646000000000000000000000000000000000000000000000000000000000000000000000000000000000000000051c6a57de2f1a5d622a43a7ae78d599afe8515df", +"blockNumber": "0x471e75d", +"transactionHash": "0x3e244355cbc7733c7c9499da5926856080ccdc0a541e3fea6e7230af5bce6654", +"transactionIndex": "0x5", +"blockHash": "0xe29ac613a763933026443c3ef6c00c4686f117e906f45f4a562c44ef9f847e61", +"blockTimestamp": "0x0", +"logIndex": "0x28", +"removed": false +}, +"PoolGraduated": { +"address": "0x7ed598bcef8bd9edd8c97a195c6d13f40801ec7e", +"topics": [ +"0x0a44ef75df69c534f43cd6c1aa3ef8983065fe5fe79ef9e79f6494e6f258c259", +"0x000000000000000000000000c4741f151afec238fd3a17c25b1be83b8e3c1646" +], +"data": "0x000000000000000000000000000000000000000000000000000000000033749f000000000000000000000000000000000000000000a8cff7796e78371bb3e6750000000000000000000000000000000000000000000000003a4965bf58a40111", +"blockNumber": "0x471e75d", +"transactionHash": "0x3e244355cbc7733c7c9499da5926856080ccdc0a541e3fea6e7230af5bce6654", +"transactionIndex": "0x5", +"blockHash": "0xe29ac613a763933026443c3ef6c00c4686f117e906f45f4a562c44ef9f847e61", +"blockTimestamp": "0x0", +"logIndex": "0x34", +"removed": false +}, +"v4_Initialize": { +"address": "0x8366a39cc670b4001a1121b8f6a443a643e40951", +"topics": [ +"0xdd466e674ea557f56295e2d0218a125ea4b4f0f6f3307b95f85e6110838d6438", +"0xb317f6fcf65d2024b9740ace92876d080a190a96e7a97a2f254b9581f8a15faf", +"0x0000000000000000000000000000000000000000000000000000000000000000", +"0x000000000000000000000000c4741f151afec238fd3a17c25b1be83b8e3c1646" +], +"data": "0x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000c8000000000000000000000000e5e702641ea86f4ae6cc3cdaed2b886f976be0440000000000000000000000000000000000001b3ab6fd93aa99f6d8bbc21a1f0c000000000000000000000000000000000000000000000000000000000002b366", +"blockNumber": "0x471e75d", +"transactionHash": "0x3e244355cbc7733c7c9499da5926856080ccdc0a541e3fea6e7230af5bce6654", +"transactionIndex": "0x5", +"blockHash": "0xe29ac613a763933026443c3ef6c00c4686f117e906f45f4a562c44ef9f847e61", +"blockTimestamp": "0x0", +"logIndex": "0x27", +"removed": false +}, +"v4_ModifyLiquidity": { +"address": "0x8366a39cc670b4001a1121b8f6a443a643e40951", +"topics": [ +"0xf208f4912782fd25c7f114ca3723a2d5dd6f3bcc3ac8db5af63baa85f711d5ec", +"0xb317f6fcf65d2024b9740ace92876d080a190a96e7a97a2f254b9581f8a15faf", +"0x00000000000000000000000058daec3116aae6d93017baaea7749052e8a04fa7" +], +"data": "0xfffffffffffffffffffffffffffffffffffffffffffffffffffffffffff2766000000000000000000000000000000000000000000000000000000000000d89a00000000000000000000000000000000000000000000006331c062631f59cbe58000000000000000000000000000000000000000000000000000000000033749f", +"blockNumber": "0x471e75d", +"transactionHash": "0x3e244355cbc7733c7c9499da5926856080ccdc0a541e3fea6e7230af5bce6654", +"transactionIndex": "0x5", +"blockHash": "0xe29ac613a763933026443c3ef6c00c4686f117e906f45f4a562c44ef9f847e61", +"blockTimestamp": "0x0", +"logIndex": "0x2d", +"removed": false +}, +"v4_Swap": { +"address": "0x8366a39cc670b4001a1121b8f6a443a643e40951", +"topics": [ +"0x40e9cecb9f5f1f1c5b9c97dec2917b7ee92e57ba5563708daca94dd84ad7112f", +"0xb317f6fcf65d2024b9740ace92876d080a190a96e7a97a2f254b9581f8a15faf", +"0x0000000000000000000000008876789976decbfcbbbe364623c63652db8c0904" +], +"data": "0xffffffffffffffffffffffffffffffffffffffffffffffffffff4a19df0b800000000000000000000000000000000000000000000000020ecc52146f9cf833030000000000000000000000000000000000001b3a6204abe494a4637358e374c20000000000000000000000000000000000000000000006331c062631f59cbe58000000000000000000000000000000000000000000000000000000000002b3650000000000000000000000000000000000000000000000000000000000000000", +"blockNumber": "0x471e75e", +"transactionHash": "0x1d0c6ec9a437b15c2d0d71b4e8f8a07bc01b3e47b8c441a67d9522357eacb540", +"transactionIndex": "0x8", +"blockHash": "0x321157aa3366b04fee7c843965c43e60d1faf73c32453a29177913e39c3273bb", +"blockTimestamp": "0x0", +"logIndex": "0xd", +"removed": false +}, +"CurveBuy": { +"address": "0xfb1097e1b234dcf3fcbb5815afd3291d3a5b472f", +"topics": [ +"0xec36bf571f136799e8dc0b0b8bea4b04d8bd3d43de838aab0d5fc21d4cbfc455", +"0x000000000000000000000000e33e9e479df8802cb0866d5d05258bec4cf62948", +"0x000000000000000000000000933be6f3b2d0b61a940b0a408690e893288cf74d" +], +"data": "0x000000000000000000000000000000000000000000000000002386f26fc1000000000000000000000000000000000000000000000004d88cf7701c61f9d0a50f00000000000000000000000000000000000000000000000000005af3107a40000000000000000000000000000000000000000000000000000000000000000000", +"blockNumber": "0x46fba5f", +"transactionHash": "0x7dbbb118712ffe8fc04825cb62818dcac49530dbb93b5364d8e59902bd12ecc0", +"transactionIndex": "0x5", +"blockHash": "0x15f1a8ddb11b403420680f84b9b9c85ee952533445098d424709c3590f21dbb2", +"blockTimestamp": "0x0", +"logIndex": "0x10", +"removed": false +}, +"CurveSell": { +"address": "0xfb1097e1b234dcf3fcbb5815afd3291d3a5b472f", +"topics": [ +"0x8113d738abdcb6b38357e9d53a54a7157861a09031b453651f0fe7fe151f59df", +"0x000000000000000000000000933be6f3b2d0b61a940b0a408690e893288cf74d", +"0x000000000000000000000000933be6f3b2d0b61a940b0a408690e893288cf74d" +], +"data": "0x000000000000000000000000000000000000000000026c467bb80e30fce852870000000000000000000000000000000000000000000000000011761287419bb600000000000000000000000000000000000000000000000000002d26f99bd0710000000000000000000000000000000000000000000000000000000000000000", +"blockNumber": "0x46fbaa6", +"transactionHash": "0xda3d158ae61fef728255aa011be7401bca1055075fb24ab0f5aa9e341d696875", +"transactionIndex": "0x2", +"blockHash": "0xd8e37fd65523ff715d6305721db33880fcec50cfef63b719c7ca121348716132", +"blockTimestamp": "0x0", +"logIndex": "0x1", +"removed": false +} +} \ No newline at end of file diff --git a/tests/fixtures/solana_golden.npz b/tests/fixtures/solana_golden.npz new file mode 100644 index 0000000000000000000000000000000000000000..29807ea06feff0ea55eaeda209ffa4ef2fccde5f --- /dev/null +++ b/tests/fixtures/solana_golden.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce8546ac191f518c89aa6160dd546f282d10ca20248c9c9fe882e6c3136983fc +size 227838 diff --git a/tests/fixtures/ultra_orders.json b/tests/fixtures/ultra_orders.json new file mode 100644 index 0000000000000000000000000000000000000000..a25f1650b01ad3d70471f885070fb108a6a7990b --- /dev/null +++ b/tests/fixtures/ultra_orders.json @@ -0,0 +1,1671 @@ +{ + "captured": "2026-09-27", + "note": "unsigned Jupiter Ultra /order transactions (Metis only) for a public exchange wallet as taker; signer policy tests", + "orders": [ + { + "transaction": "AQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAQAECn6MCIdgv94d3c8ywX8gm4JC7lKq8TH6zYjQ6ixtCwbyGzu+wbNz6kzWjPSMzF1ZSldIuWarRPXRRMKVgbx8jZFF494g8LS7heptPyCkhKXgsvtOVdYyIiDhu2m7yHjKZE7QEGAQcVkPdYJlMaSAfoY6OYM3YIzoNJFBtf4A2pzebQ/ffN89KMzS1moKr3xD+eaLjFLD93Uu7/6fCChoRW90nlv9rSWGFpqVXVb6QypZU2KTrO7mk03xa0zPTd8obwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwZGb+UhFzL/7K26csOb57yM5bvF9xJrLEObOkAAAAAEedVb8jHAbu50xW7OaBUH/bGy3qP0jlECsc2iVrwTjwbd9uHXZaGT2cvhRs7reawctIXtX1s3kTqM9YV+/wCpjDypHSuOBl0/g2dp71IL5qdiXFjyLAf9gxC1HSlNfm8GBwAFAsT5AAAHAAkD/AYAAAAAAAAGAgAEDAIAAACA8PoCAAAAAAkBBAERCBgABAIQEQkJCA4IChMJAAwEDQILAQUDEg8ou2T6zDHErxSA8PoCAAAAALO/GhAnAAAALAEKAAAAAQAAABEBECcAAQkDBAAAAQkCKb+VBypPvwTfcXWT5zi8zCEnKBXCRxqA2fn1VbDUhCUBIwMAAhdFKDPG32aOo8Oci/jBAstrH8oA9CV36eGaAJtHwuxGbAMBCgwDAgcL", + "inputMint": "So11111111111111111111111111111111111111112", + "outputMint": "DezXAZ8z7PnrnRJjz3wXBoRgixCa6xjnB7YaB1pPB263", + "inAmount": "50000000", + "outAmount": "167773913011", + "otherAmountThreshold": "162740695620", + "router": "metis", + "gasless": false, + "lastValidBlockHeight": "429027252", + "taker": "9WzDXwBbmkg8ZTbNMqUxvQRAyrZzDsGYdLVL9zYtAWWM" + }, + { + "transaction": "AQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAQAEFX6MCIdgv94d3c8ywX8gm4JC7lKq8TH6zYjQ6ixtCwbyGSrS4ZpsR06oNxYI1gGfJ2twE6ba5XNTDjbZB5STmJgte0lGKEdXpSl248IHhrWb1nBWLmsljV4TExnVSsDJim0P33zfPSjM0tZqCq98Q/nmi4xSw/d1Lu/+nwgoaEVvewyBzBHFjVc1Z5KzzDFJeblzc/IJ3lvUJlfMgtQLrP6EFWnhL8NuFfR9EbgFt+oVPyIkPnyKI99wPpgNBGzvg4wTbVpWxtK9Mb1LFOJsDYZQ2xqz69/zddWBoAVRqRGKp/bk96C2ZIJI8sXQi6Ab6XKIQH5TKc5oAyR7CtRIdbq8oSIMEYNstPNiGfaG1oTWewn4XkLWLEoZx1N4Phw/6LcRI8NhD7sXUippT7vJJZ2/ErsJdoqxZ0u2kjFeWMpct6OnmCEsEa8E5FIr9h1qJ+n+JhIkIJtJHKm6gzeuJf7RcKCotDRHaabLMsftt8O4fvVu3qX46om4RJgRRKfT/9PqjPWsrKjNBSB1EhdcQ871Sl3Znt4goWtVJTc485fc18cVM0/bpfzWiPr+pvM0++R3DFbY1R3iNpwvTg1dYHTe0Yx8zCphBxiQ/adACxP33W/x2vpFzKJwSrKc0nJh7uEq5SHGt7mBf4LXlb4yHEBrmsCw9wmQhVHxSENqXKvF+oDuc+XIaFbuPpt10LxIira7B8Vrdu/n4wwKucesDtMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMGRm/lIRcy/+ytunLDm+e8jOW7xfcSayxDmzpAAAAABHnVW/IxwG7udMVuzmgVB/2xst6j9I5RArHNola8E48G3fbh12Whk9nL4UbO63msHLSF7V9bN5E6jPWFfv8AqR2ESNGXvNNrTeUiUax8Q7cAZmm3gAyLr4NRlmRP9HbYBhIABQJv+QMAEgAJA+YFAAAAAAAAEQIAAwwCAAAAgPD6AgAAAAAUAQMBERNQLwADGRoMMzEUFDATGSovNCAZDSIhHxQGIw4EEyovKSUNGignJBQJJgEKEywYBRcWGRoxMxUsLxQUKywQCwgTLhQULS8cMzEZGxoeDwcCHTI50ZhTk3z+2OkMgPD6AgAAAADvdV4AAAAAACwBAgAAAAQAAAAaCxUAARoQJwEEJgEJAAQvAQAECQAEFAMDAAABCQUnaKf1QqB03GbZZkhj3JNcW6jBN7X+O4gQz7S6baZdlASyuLavBgUCs7VBGSm/lQcqT78E33F1k+c4vMwhJygVwkcagNn59VWw1IQlAiQ1BQ8AKAIXM2aDRp4f180x1sPbf5n8pGmJvVBxKcCXbOmQTrO9tIwEHyIjGgBcNresWGU05fQQ4O6z8cSOx6uFYMkzz7j1xnVAVzn/MgXi5t3n2wHlshw2HF5m19mhaXfEW5yL5HE5/BSMGU6/40bsIv07UIUFoaKghIUA", + "inputMint": "So11111111111111111111111111111111111111112", + "outputMint": "EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v", + "inAmount": "50000000", + "outAmount": "6190575", + "otherAmountThreshold": "6004857", + "router": "metis", + "gasless": false, + "lastValidBlockHeight": "429027257", + "taker": "9WzDXwBbmkg8ZTbNMqUxvQRAyrZzDsGYdLVL9zYtAWWM" + }, + { + "transaction": "AQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAQAEB36MCIdgv94d3c8ywX8gm4JC7lKq8TH6zYjQ6ixtCwbybQ/ffN89KMzS1moKr3xD+eaLjFLD93Uu7/6fCChoRW/T6oz1rKyozQUgdRIXXEPO9Upd2Z7eIKFrVSU3OPOX3AMGRm/lIRcy/+ytunLDm+e8jOW7xfcSayxDmzpAAAAA/NE9oEFVkBPPSuuhqE7XAjrU/yGoR0f/OwWrMPqCB8MEedVb8jHAbu50xW7OaBUH/bGy3qP0jlECsc2iVrwTjwbd9uHXZaGT2cvhRs7reawctIXtX1s3kTqM9YV+/wCpKD1O6srVhq2lqW7kmBcw5XxRZk/9usf2NGDmeaxcmtIEAwAFAueaAQADAAkD/AYAAAAAAAAFGwACAQwOBgYFCwUHEQAKAQIICQ4MDxASBgYEDSi7ZPrMMcSvFEBLTAAAAAAAQwVoAgAAAAAsAQIAAAABAAAAlwEQJwABBgMBAAABCQIpv5UHKk+/BN9xdZPnOLzMIScoFcJHGoDZ+fVVsNSEJQEmBAAoAhe1Dh+3xL8b+dTiI1DcM69hfXHQaeljCtwigIUwexkpUQPq7/EE8jGFLg==", + "inputMint": "EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v", + "outputMint": "So11111111111111111111111111111111111111112", + "inAmount": "5000000", + "outAmount": "40371523", + "otherAmountThreshold": "39160377", + "router": "metis", + "gasless": false, + "lastValidBlockHeight": "429027262", + "taker": "9WzDXwBbmkg8ZTbNMqUxvQRAyrZzDsGYdLVL9zYtAWWM" + } + ], + "lookup_tables": { + "3oy9ojnsDzqmMNi87Gs7Hn5v3MPVqnWjG9k8BmzKR7yW": [ + "D8cy77BBepLMngZx6ZukaTff5hCt1HrWyKk3Hnd9oitf", + "jitodontfront11111111111JustUseJupiterU1tra", + "jitodontfront1111111111111111JustUseJupiter", + "GGztQqQ6pCPaJQnNpXBgELr5cs3WwDakRbh1iEMzjgSJ", + "2MFoS3MPtvyQ4Wh4M9pdfPjz6UhVoNbFbGJAskCPCj3h", + "BQ72nSv9f3PRyRKCBnHLVrerrv37CYTHm5h3s9VSGQDV", + "6U91aKa8pmMxkJwBCfPTmUEfZi6dHe7DcFq2ALvB2tbB", + "4xDsmeTWPNjgSVSS1VTfzFq3iHZhp77ffPkAmkZkdu71", + "CapuXNQoDviLvU1PxFiizLgPNQCxrsag1uMeyk6zLVps", + "9nnLbotNTcUhvbrsA6Mdkx45Sm82G35zo28AqUvjExn8", + "6LXutJvKUw8Q5ue2gCgKHQdAN4suWW8awzFVC6XCguFx", + "HFqp6ErWHY6Uzhj8rFyjYuDya2mXUpYEk8VW75K9PSiY", + "DSN3j1ykL3obAVNv7ZX49VsFCPe4LqzxHnmtLiPwY6xg", + "69yhtoJR4JYPPABZcSNkzuqbaFbwHsCkja1sP1Q2aVT5", + "HU23r7UoZbqTUuh3vA7emAGztFtqwTeVips789vqxxBw", + "3LoAYHuSd7Gh8d7RTFnhvYtiTiefdZ5ByamU42vkzd76", + "3CgvbiM3op4vjrrjH2zcrQUwsqh5veNVRjFCB9N6sRoD", + "GP8StUXNYSZjPikyRsvkTbvRV1GBxMErb59cpeCJnDf1", + "7iWnBRRhBCiNXXPhqiGzvvBkKrvFSWqqmxRyu9VyYBxE", + "11111111111111111111111111111111", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL", + "So11111111111111111111111111111111111111112", + "g7dD1FHSemkUQrX1Eak37wzvDjscgBW2pFCENwjLdMX", + "H1qQ6Hent1C5wa4Hc3GK2V1sgg4grvDBbmKd5H8dsTmo", + "8ctcHN52LY21FEipCjr1MVWtoZa1irJQTPyAaTj72h7S", + "7x4VcEX8aLd3kFsNWULTp1qFgVtDwyWSxpTGQkoMM6XX", + "6zAcFYmxkaH25qWZW5ek4dk4SyQNpSza3ydSoUxjTudD", + "91bUbswo6Di8235jAPwim1At4cPZLbG2pkpneyqKg4NQ", + "A8kEy5wWgdW4FG593fQJ5QPVbqx1wkfXw9c4L9bPo2CN", + "BuqEDKUwyAotZuK37V4JYEykZVKY8qo1zKbpfU9gkJMo", + "2p29nqD7DN1PczBMmgrFdtYKTfv6rJ7H3yMut4eu7nYT", + "EUvpCGh4qiMtq9wKgp28f9Bjv5Xz2WJqrM83XmYAqkEq", + "qqdJ4z1yu4sTbAitwXZsGNDoGZFgL2HfVKSVwAXWCfq", + "EaUghZfmuhgtZEEwaZwX5EBroz4WyH8VM5NSXi4tam5A", + "6AMWTvaL1pscDo5CAaBhPfv7xnSLjJ3ZXScV9yt5Gr5G", + "39FWQJxqTmUkFqHKjEs3CawQ1b1rqLcKmHfJ1FWMAWdu", + "GyY4VgEpJQhiKZRAJJmoM4hv5Q2xC4pvX68MGrGidxyG", + "2oL6my4QDDCfpgJZX1bZV1NgbmuNptKdgcE8wJm6efgk", + "EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v", + "DVCeozFGbe6ew3eWTnZByjHeYqTq1cvbrB7JJhkLxaRJ", + "JSvtokJbtGsYhneKomFBjnJh4djEQLdHV2kAeS43bBZ", + "7u7cD7NxcZEuzRCBaYo8uVpotRdqZwez47vvuwzCov43", + "59v2cSbCsnyaWymLnsq6TWzE6cEN5KJYNTBNrcP4smRH", + "m3BrPbv2TFmZZTPpyB9NgsCXqGNujpXvzvGqj8ksars", + "Gjmjory7TWKJXD2Jc6hKzAG991wWutFhtbXudzJqgx3p", + "EXrqY7jLTLp83H38L8Zw3GvGkk1KoQbYTckPGBghwD8X", + "G4CD7aqqZZ6QKCNHrc1MPdS9Aw8BWmQ5ZkDd54W6mAEG", + "5SPztfEn1VAaWDBAXjQKwVrGbr6e8g3F6JJnUc9eCuSe", + "FbruxBVHi463Agw2B3Vy27cBkGnEN5g1f4NcHe3REXfe", + "E6LAwCLSHLkDCoMXZPtnDtpcvCYWcs3ZZLHLreiFwjUi", + "DY6pE7aiDafuk35REZF9p9av3vbV2VQrvdZ4YyB1pZ4C", + "819hAmNKJ4MyEF9fEYc31vzqsnwfxN7NtrM2QpovgvtS", + "4pCDJv3V4P4WtvK5x4f64DS4wDctkMZDAjz6CxTdqPGj", + "HoBCz6z9AG92GGozMWEkBPE9UhQWGZ5cXhYcjoGJvwP2", + "EpdaePzdqRkMtdZJquVPUWgyoJ5YEEpYALki6dv9VBrt", + "6zQecXhjYTifDGYxbW7vRTQBrBYsi1Uac6BEJ4WzefWS", + "GgY8theL9n9hQPoz2keQM8y6z8T6G6BH9FPLjBtkF9Hd", + "G4FUwFD1h4tb4R6jkZXuoyst7YNbYTcJH3MvCUguss6E", + "F2Xjd4ZJYz6SfszyUVGzLUzAHRRhfU2iJacCfW5GCJHM", + "3cHRcBKWbJeL2qyjgQ8wdSYxmRYW1ZyC2nVqTakAj57G", + "6Ugimjtgk7rk5SbZNzcYvZiM3P6ki4Uq3QGtTHWNn8co", + "4QKRxAfawktf6szGUP456AqBvaKSnmuGy91QnqdBDSke", + "9kiYqGSb1nbYMc5xxZQHhKvJR57LLAHVyDvSQ3FHjDPK", + "5xgh5nvdQivmBPCsK3oCLGTQFyQHytQ8BWox7zpXAuye", + "5eGuZuVysi3Uksa5jRujq9XWWyNiNsosAme4xX8GM216", + "6NC5sEHoBEyhJTW6mUxzT3CRerMNyjsxK21yjVBMqFY", + "J3Up4p7i5LncgWbwytNsH4vrwco3qWoBTVQZnMJYxe7W", + "75P62FyYr8sHjwUCCcUc2bT1MDrAVcsC6GkwkDyMoVVn", + "GzSCp13VYUj2kpCBVpHnXaPS7SVE6R5Bq7D7tCbZckxP", + "75YeSeaeVKiNvskBNZqmZ5EeuktgkfUq8YBqXoF4cTYu", + "5xgSuPHiMxnvw38P8MxFJJN56WdSYTLBeq8XrUfKWaJE", + "7hZdUKF3mNFm8NxYfBruNBEUHa8zwHr8SGk2jzP8W3zz", + "GnqhCoAp1LbSbB5Vti7eHX3sZmFeQVV4y8aPy6sjsMeQ", + "EDnz8jbGoLNTwxbHyT7cDvDqYLrgZrYzZS19kszebzVt", + "7Svve3shNti3WaY2MdKtFY4H7eNYRiF9nLCXX6KBhGrq", + "A2Kj8EzBSif36NNvs2i35GjdYGcs1RAwuXRrLpzfwnFF", + "Ec6y749opoW5JKbJEip3PuJaZhusQHQtUxtDzWysmTPi", + "Fja7LBfyWbnZQxrnQnw2UHKzknXyUNFWXXdQVibnrzBk", + "GBRPRss7LgMQf4rxXyP8YCjHpdpEWM5BDXUCHHt6unEa", + "EkV7V23YfHztteeNz1tfC4Um3MwWfVwMC9bhotoZtJKv", + "6EF8rrecthR5Dkzon8Nwu78hRvfCKubJ14M5uBEwF6P", + "4wTV1YmiEkRvAtNtsSGPtUrqRYQMe5SKy2uB4Jjaxnjf", + "Hq2wp8uJ9jCPsYgNHex8RtqdvMPfVGoYwjvF1ATiwn2Y", + "8Wf5TiAheLUqBrKXeYg2JtAFFMWtKdG2BSFgqUcPVwTt", + "pfeeUxB6jkeY1Hxd7CsFCAjcbHA9rWtchMGdZ6VojVZ", + "Ce6TQqeHC9p8KetsN6JsjHK7UTZk7nasjjnr7XxXp9F1", + "dbcij3LWUppWqq96dh6gJWwBifmcGfLSB5D4DuSMaqN", + "FhVo3mqL8PW5pH5U2CN4XE33DokiyZnUwuGpH2hmHLuM", + "8Ks12pbrD6PXxfty1hVQiE9sc289zgU1zHkvXhrSdriF", + "LanMV9sAd7wArD4vJFi2qDdfnVhFxYSUg6eADduJ3uj", + "WLHv2UAZm6z4KyaaELi5pjdbJh6RESMva1Rnn8pJVVh", + "2DPAtwB8L12vrMRExbLuyGnC7n2J5LNoZQSejeQGpwkr", + "LanMV9sAd7wArD4vJFi2qDdfnVhFxYSUg6eADduJ3uj", + "WLHv2UAZm6z4KyaaELi5pjdbJh6RESMva1Rnn8pJVVh", + "2DPAtwB8L12vrMRExbLuyGnC7n2J5LNoZQSejeQGpwkr", + "62qc2CNXwrYqQScmEdiZFFAnJR262PxWEuNQtxfafNgV", + "7VtfL8fvgNfhz17qKRMjzQEXgbdpnHHHQRh54R9jP2RJ", + "7hTckgnGnLQR6sdH7YkqFTAA7VwTfYFaZ6EhEsU3saCX", + "9rPYyANsfQZw3DnDmKE3YCQF5E8oD89UXoHn9JFEhJUz", + "AVmoTthdrX6tKt4nDjco2D775W2YK3sDhxPcMmzUAmTY", + "CebN5WGQ4jvEPvsVU4EoHEpgzq1VV7AbicfhtW4xC9iM", + "FWsW1xNtWscwNmKv6wVsU1iTzRN6wmmk3MjxRP5tT7hz", + "G5UZAVbAf46s7cKWoyKu8kYTip9DGTpbLZ2qa9Aq69dP", + "GesfTA3X2arioaHp8bbKdjG9vJtskViWACZoYvxp4twS", + "4budycTjhs9fD6xw62VBducVTNgMgJJ5BgtKq7mAZwn6", + "8SBKzEQU4nLSzcwF4a74F2iaUDQyTfjGndn6qUWBnrpR", + "4UQeTP1T39KZ9Sfxzo3WR5skgsaP6NZa87BAkuazLEKH", + "8sNeir4QsLsJdYpc9RZacohhK1Y5FLU3nC5LXgYB4aa6", + "Fh9HmeLNUMVCvejxCtCL2DbYaRyBFVJ5xrWkLnMH6fdk", + "463MEnMeGyJekNZFQSTUABBEbLnvMTALbT6ZmsxAbAdq", + "6AUH3WEHucYZyC61hqpqYUWVto5qA5hjHuNQ32GNnNxA", + "5YxQFdt3Tr9zJLvkFccqXVUwhdTWJQc1fFg2YPbxvxeD", + "9M4giFFMxmFGXtc3feFzRai56WbBqehoSeRE5GK7gf7", + "GXPFM2caqTtQYC2cJ5yJRi9VDkpsYZXzYdwYpGnLmtDL", + "3BpXnfJaUTiwXnJNe7Ej1rcbzqTTQUvLShZaWazebsVR", + "5cjcW9wExnJJiqgLjq7DEG75Pm6JBgE1hNv4B2vHXUW6", + "EHAAiTxcdDwQ3U4bU6YcMsQGaekdzLS3B5SmYo46kJtL", + "5eHhjP8JaYkz83CWwvGU2uMUXefd3AazWGx4gpcuEEYD", + "A7hAgCzFw14fejgCp387JUJRMNyz4j89JKnhtKU8piqW", + "94qWNrtmfn42h3ZjUZwWvK1MEo9uVmmrBPd2hpNjYDjb", + "7GFUN3bWzJMKMRZ34JLsvcqdssDbXnp589SiE33KVwcC", + "X5QPJcpph4mBAJDzc4hRziFftSbcygV59kRb2Fu6Je1", + "Bvtgim23rfocUzxVX9j9QFxTbBnH8JZxnaGLCEkXvjKS", + "FGptqdxjahafaCzpZ1T6EDtCzYMv7Dyn5MgBLyB3VUFW", + "CGEWR6pxwgQvYKeX4pZDqpZtWYPvyTjiAsw86SNzJtGy", + "7xQYoUjUJF1Kg6WVczoTAkaNhn5syQYcbvjmFrhjWpx", + "BWXT6RUhit9FfJQM3pBmqeFLPYmuxgmyhMGC5sGr8RbA", + "C93K8DX4YsABYJtHX9awzgZW3LWzBqBVezEbbLJH4yet", + "Bckr4rY4rUGvWtwqC9mnWN35LKzfPRcmsoMRYq6DFEjN", + "9tvBjCQ4m3954ESgxW9PnBPvW8afHEjKpkNEAdfN2D8U", + "CoRuMWRuDAsjB5SDMmYEcDuyQyaH3L2vVFvwBy1FF8Kr", + "21L1QB6DJ9HBvePgwN6jFuK3UT9mq2SgTq1oJ7D6cNiV", + "2i3uCDzDhr3pSvMPF5uBpUckLAv7XGdo2a8soiqNJMHk", + "BZeq7y8ajn5Pjpy5KjEhF6eBY2H9ippTpqCtQLQPsmJf", + "ghSBUgyxyvyurm1vJBkU4rUyLJoUipCZhFeiBogKCSy", + "HjQjngTDqoHE6aaGhUqfz9aQ7WZcBRjy5xB8PScLSr8i", + "GAFuhgcd328SkkBYHpfadzmef9hTGAFRCi9QoCnsZQug", + "AktftA98kSWAxn6kVSoqBXBELUArjKu2H9WmKB48ULFY", + "6rVkF4HSgy1jrnC3HogfRgPHrq4CtLg5f11URpsC4i9D", + "GYH1Gae1wJytMSvMvw8JVcv7nuAbxi8i9erNVbERnzXd", + "CA7v8gHfbquYXyDnDx6QxWW8hmL1H7X6Y2RYDrGLnuck", + "CASRL2zkwDnppxEFQ4LgdwgR9pdz5Q8R8nEMKVZ9QoLp", + "qkYdTGRPHbWTWuBMz45bCiU6a23axRqf6sBHm9295WY", + "BqcWAXkSdknwQxvqXYVGKtttZynYNHACPVJmTaoqgfv8", + "3beutiWC6iV5Hz2RC711oXTqWa93rHUwsS58xWBHyTd6", + "FC6zaBZjnJ1tF5nY4b2nrPgu62thjXdRkk2sEtjxU16E", + "C5bwoYa7RD7Prc2u36idJ3hDjTvvoXPdBdx4iYeDVaQj", + "APnwGpYPQJqpndpjZFUFUrzsSU2sd2SG9qKtpXQgRimu", + "CN371Div8bqcEqq2grrGQfBX7geFLgHATEFMNLEuQs1U", + "2yC9PAQvtxFjdV2G79N7cGsFhitbNiEQmZ3Z6dmLWfQg", + "BMqY71czEnfwxTp7zTc3Wdkushpn8VfSJ6NGZX11djM1", + "41xY1DU1zzo893bEg2HzTFxPVVM84UvDCNsQm6aKRq8Z", + "GYahGz4ts13NYJBJfwFfMsN7aWv6mcT5D5Jd9znx18QW", + "Gzc5K38syMVA8V7Q7czVJqQGxBiWhJ3iUqZdGGdSvqEP", + "AupmCV3PGYcm5efdXS7Yd9TvvzfUP4wKabvmFoDe9LAf", + "EQkDDtyzVwY3RCBNJSd68F8c1WQP6njSbcPLpnYFoMxU", + "A9KuWNQUZBT5ZTp3HB5tKxCkSwxjUrW5fFqvk6JcKyo6", + "D3ANwvaijotpP2BZJTxHs7mdhKaNDZ8JgWAZRAGX5wva", + "8i5djNGuUXSAqang3mcQMPfAg7ynPdLLyecDx1D393od", + "6oCkp6gpyjxVTeL6ahMYcekN2x2pzt1KY8g2LqemaTNE", + "DxvbV1rR2hmFJ2gYGXmz7jnMPsvf39M1BWd3Ejshd3Zj", + "H2CUXP4v2ZSWEFvnj9C6RbbD8cNNZPLK3H374nKARN1t", + "9JR4rG7BK32TVENGAcKMseS7tdoz3Y5pXeSq234MEowH", + "4EcDKGwpgYLVnMmjJCDrUN2DVLQKSpSKyMhqU1GbuMsv", + "EZbmj4jpfk9GGgRNfzX3e13Zo4ZaNMHQ5UUmRVcZQyEF", + "BJQ1HTx43bBDF1ba8GfZAfxSMZneTmQNr5m9yUfx6vAu", + "fewxWzSMHpHhDT9c5FysEXnHXtxvWVeHPvFVTyZdPwh" + ], + "5exgoTBDmaCRsZh2nWNbr6UvQVofk9PjK2H3SKo6WoBR": [ + "So11111111111111111111111111111111111111112", + "4dmvFGeQH2eqa3ktNHMgm4wZ8vuTukBiK9M7gxW5oR9F", + "DezXAZ8z7PnrnRJjz3wXBoRgixCa6xjnB7YaB1pPB263", + "BJrH1efsve1wWmhQnUWmu9WJvEPCvAqoSpLjRcLbcvDa", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "JAQYAereskx8w6viL9DRGabgZKYs9r3KpeamkUYojLNy", + "G5zYHLwbi1cMyzL1JJAX9Cji53rvbXf3cY1uBxzX3zU8", + "EXpqcVuTTPEmyeVu2LHM75ha7FwmjJ88MnhSMRRdy5Sh", + "FaCkqVAWobxXRX1weXiFNBWpszbNXgRjq2TMbYhkoEQg", + "MemoSq4gqABAXKb96qnH8TysNcWxMyWCqXgDLGmfcHr", + "5zpyutJu9ee6jFymDGoK7F6S5Kczqtc9FomP3ueKuyA9", + "whirLbMiicVdio4qvUfM5KAg6Ct8VwpYzGff3uctyCc", + "ES7yhSrYeFo4U1PfJHNRkbfCWxCwPLk2DjrEbmN8bg58", + "A7xxBQk2BAEX3ewQrtDA28Mbf722nDmPVtb2UNu9gwCU", + "So11111111111111111111111111111111111111112", + "8dgti1CZafWd2UtFVsnLpzpFGEg8crK4E2bBVbGDbYKy", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "BWaFDZSuPeo2PHpXctnFw98nW1KsVf3EKqJJkqcszmSt", + "Atoo31xyUXhJpS1AUZU75cT8suAgLrSBRxHgMQHfvrzW", + "EQ8VvdSYovj85SCc6nvwKahqY1sCHJZ8uB4DP46nUmF9", + "6HPG2keBDw8bTzb39HLX5nZxetYnNMxWk4r27AP8D2AY", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "So11111111111111111111111111111111111111112", + "8J8RMxfaVQdgqqxZKuoySY1BwkAPnYp5hSSRhmVXhp2P", + "AFsrSzRUtwX31uSR8rPxwJUng2j4j13LPBVomntR7D2L", + "H8daou9oiyWCmt6LYqA8K8q48rB6WC4gYcpe5evSZuBB", + "Cp1tBtADQYeKLJqDEw5qa6eMnbbmBzEbJPytzLuMeQJf", + "So11111111111111111111111111111111111111112", + "4grZ36YFUu5ScJVpV2jTiTJutkKeoKPsDJQU5XsCrzWP", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "2rtyJXmbb4ckEikJc1kfaWjVsAdLY1rxprgTKv49G6NA", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "Hx6BpbY2bVgWV5gCpPQkJLJrCcaUakkUBbZzTF1dWhMm", + "G1hTP8SypQpm3uMKRqZWDvR2UEu7r7E4exSEKrMMZ69C", + "7PUqP8vd1kJqjaz4Cqc8NeWctV2NHNfcLw57mFA1j1wq", + "7QE6vT3MMtJg4CauNn1XFSHXxGqvxrKV2aYwGUf5Mp2y", + "So11111111111111111111111111111111111111112", + "D7FZQzgt5DUbDpWAhtUtqiaDZiQKLN9Af5Hrj1ZiaKXi", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "2w4tBVXnMUAjhqNwr99nJmMbxEHrjoi12qYaDcTa4ha4", + "919LGHhb9cRhifu95yVg7FMALJivj5i3sLqHstwCtCW7", + "8gL3Dad6dPmzQBAT7mGagC9jYpb21Uk9MWCqqyQvCfNv", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "GkA99AbGF388EsvYh7JgZghaJmtRQBVYnTWEeSn2929D", + "A1QRDQM6x62xKGvcNXXZoQ9JNGVnCox7Eo3BNh39LhLa", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "5j1gjrrPbDL6GRvX8Qndp2jWYeWDbNuStSGVNpaMKCKN", + "AfjCkq8zSBACTVLnRd3yjjMZgMwct2e4EWjBQ48MSmCN", + "AL2P4jVKZMuKXhgGcVxr51j9ynQmL7DQ3Knknc6uWHDn", + "35ZvVk8pc7eHpex8igyDa8fZNPxWq7oRCk6ADiTsKkFu", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "So11111111111111111111111111111111111111112", + "GHAqeMB7tbd9BNKbXPrxbKFW56L86Ji6wonHAfP8DVTF", + "72muukjweYVZRSStNY2imCJTAYcND3WoDqJZjNfTSv9L", + "8K8T7rt8BWGeYmYkriqeRVrzCDAKpM2YzxY4XpEDLJC1", + "So11111111111111111111111111111111111111112", + "Fc9eiLMVWU9RtfKDxCWPK3wbeEVAMW4jS7adkhrmbBDK", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "31tB9ps6izMTXmubpRDLEguNE6y7YNw1ioqS6wUsdGty", + "B8NFACoeh8UvhEoPP8jA45gd2KQ7fcfZbtrpBb6cNY6N", + "BYaq2Hszb7Xn6y2sLwUGze8jTsdcS7C3mbDeGSvP65bR", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "ETNejKCgiRq3z5PwVuGfnpzFuvuiyQ9DXsYTAVbi6pYj", + "BTRTzExE8MSpLeGiX7fCujUjKnfe1x5GGc7f45pZ79Ex", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "CnR2QXawC3d5eArZGkYNSNGU4U2PNxyQUtArx6WteLuu", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "EXD4xS8qXzuYrpsCd5aCKkvWAtcoSbNkg2Dafviyjfqd", + "F1t1NDZYjyJDhcjF22GvHQG7t9Poy6wtRT4rZb7s7K64", + "H2Ng29n93dHfBGbXhPW4wtgVUwqLq8jLrZ32Xo3gYkTD", + "JAWaNbyDHrjsmhxfWtysPWF1vDqbj3ZmBA4VpUHd72sh", + "CFsDQUXKrf3cp1faLj87AXdAnbeco8Nb9EEKpzWyGv84", + "So11111111111111111111111111111111111111112", + "9aEc1YzSGW7DAcG6msMRXWSstPn6RAWsB6Q44JwhNJ4B", + "EvufvpqUDRpo7ThSFNxm42dQfS8tcDL46QE1XqFXpRoM", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "2BZcW2MuwhS12aEDx56Hrj8oM35jf3RbDR6k1JdWELZS", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "5x9yvMFesrFbbyaT1TPj2bjyNAbW4nQECjor8ohWzeM2", + "4jJ1W5LJAcg7TcXm5ocSgPsE3qNFbkGtX2GAW3A5vCvg", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "2S1r8cdemh9o2sHS3gF4inoqwVdprQEg923UTZN85YY6", + "3QnimrTMwT4yEHs2ZN9J5Q8tz5dNbc18x71wbXjHDays", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "Q3nPEjQZKFyJXLrhjZao5BSRJjTV4hRuaaE4TxmCJjh", + "DNaXEUhEenjh1mjE1u4g1GsAJUYtaDrnQqmQ7TE5JB6P", + "7e1U2xScjU6e9NykNqHvJQ5twQzLUiB1gioGsHEpEUsF", + "9z2AFAD6W6r9pp22mT9CU7baeTdkEWPG1EMzKMD5DqFm", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "5VUZbYQSi5mUB3o3EkCTrXtUwz2xc7U55tAy9rTprj4Z", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "So11111111111111111111111111111111111111112", + "Fcan5JcLhUWDveSknM6rrx6p5BJ7nRFkDab38NNh4YLJ", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "DubhQRHEiCJghxuCtmk7eAceeuQSaZDVARvFSKt1f9zu", + "So11111111111111111111111111111111111111112", + "DFA3U5ZAd1ebKhoHrXY52JbHkBmFYskbkxnbyegPhn1P", + "HpqrZew67QuQDmfs7YK1xVUWHzRqU3Syswfyz38UBcB3", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "7u3sZujZ3Tj2VTYacqT1j8Wzz7iMSQbjennqt2K7LMDd", + "8fbM8HzrroVRShgibxfgMuJPYvdiLyhyoJcxYQjM7bH1", + "5Q544fKrFoe6tsEbD7S8EmxGTJYAKtTVhAW5Q5pge4j1", + "675kPX9MHTjS2zt1qfr1NYHuzeLXfQM9H24wFSUt1Mp8", + "6w5KYZB3JnEUmmZk7SE1f9afF6WyttojntHbJJxMERaJ", + "So11111111111111111111111111111111111111112", + "8RoJcgV36yTrDcdLmFKaASA8z6Py1Qw9tgndoiRveAgd", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "9KVaXgKi2kNk5m4UdqsfCFfaf7JKHG4Xim2pe1jRhdfR", + "6KnarJ45V9ZmBUsjjnGL1VFprKPtu49Z9y7b4ajr1GzH", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "8yXFGQQF6jvig4i66yDvoPs6jScr33SkUZYBaFEL9L57", + "CzdKzo5cTN8akST59jWDikQ3c1aiUhRx3MYw3tnjT2Pj", + "A1oy9T11uBcXqQCBuqrRMtxti6qZNatDPZQg9FgFSAve", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "GxnVUqcq3FwSeoSsVEnZVeohMM5xTg5gC8gYEziw8qnm", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "HAdfFxqByNd6ygNy5LJWDux4GqvKmNZ59fwctwEpuDvn", + "7SuboMpN3nhLv49qxLyGc4Hf96U8spPrdquQT2HQuhyx", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "CJvpcPBop3MjxfYXpsq4M4UPAuLnKrTkPPaMjkhBfGFg", + "So11111111111111111111111111111111111111112", + "gHyWWu7TaVGUKrbhgyUZTWrGEKHUwhdQye2mVno7Pnq", + "3Y2RYDQ8zzkTuEfHQ6t7cxQiHLpaQx6X9Qkk9QAK1ndn", + "Lkb8BGVMsv18TpuZ9wQnxV1fW6tQpXc8cpqhcZ2z5cW", + "ASX7aD1fpLnQeyZaL5iUFqUeJwvcS8gxjBcihqpPX78i", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "5CDYuiTUA4SDWJsvcTfkTyirVxFPnLt68M4UfUWiMpQe", + "8XZdkZjuTYpvhjo76KAAgmbkCiToaC154XCFWAzpsATF", + "So11111111111111111111111111111111111111112", + "2TGS94ZKX8BRagWeT7yVexnx6hb6GF6bSLTr4Q2vXKKW", + "2gFrb4Q9yJx1RGQhR5Z6ARQybj46mwsG2feqG9MxjXN4", + "j1jEpGVrQqcg68EgmetPiucawXptgpu8hCduTtXABug", + "46DabgmBuMbpLom15AvuNoBuwVqxzaEKDQTQJLL8mV5b", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "F1G14Lzwhm2irk89ZF2LDhwcQRbz7a6hLAs8g3jZVkf", + "5wU7ag4kDzKXSyXLvQ7Bmc3quTQKy9nfA3LE63PHt8Q", + "De4h5vSSvwcyK4P3gT1F6oBvpr9UmSsWWP2tjcfUrErp", + "46HbgwmDw9Tio1U7zoezNTLP5tqepTbhTQ8MMMkDSmW7", + "Dqy8FHLsFWhKSscZDynHXmSBB7ikshZC3XHV7p21MYs9", + "B1gUFX4rHbKCZDkv8m2Ge1AQdaFpRvC87eYsHg21Ekrq", + "So11111111111111111111111111111111111111112", + "4eTFdhAiejcUoVet9q17aHRDfiCkLP37BFZMtyJNwzfC", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "rYh2XaJsu3p4VUCwDVM6k6afLhXye3i5DmzyK9yFBbo", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "DEZfNa2DrJX4BdfTUTynufngwcij4dbcxH7gaWsTDMPk", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "7zjZxadgKoKXPpEBAB7pDTJ3HD5ai63EC1FMiNa6VPaq", + "2cL5iGn3pbhWjqb7pD6EUT4VhjTs6zy1YhuAn54A85gy", + "CwkLMVg5MWN3cvWWV4Rdq7aEnhkCAjgHqxwjvSTUSDiZ", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "AcVMhNuSz9Nu9ZsyXNGKNPJPhdyppnQq1N8459dTVdpc", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "E5NuSTB4PZ1txu9X3ussyEHPqUJdAH1dECUhPiuZs73j", + "3TpCEXWCPWq9St1YPcAzdgjpeZkEWDXT1h2oLZWg7zGW", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "DcaoCsBCrK5BVMdSgf2U45VpExcpHZuwE9dpysazyrgn", + "2sG73jsPxtrNvgAwruMepFovxC7oAq1uThegh9xdoWP6", + "HGPhb4pZS7AyDdpGwtpARE9rSm4EvdSmx9ERCb7pY1u", + "So11111111111111111111111111111111111111112", + "E8CqoZhA9zvxTpxQg84pcrwtHMaAy3ZSjfNekGaumDvG", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "AshuMB9UaXRc1oXZw27c7pdxjyJLbTJffYWkvNrq1qJC", + "Cg4g2imQGdKYx1YusgV71p8A4ya2KPih9hTt7pnhSHoH", + "DKrZwTK38NWuEn1ipzuBoy958rvyWm566ui9AoB16vMX", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "BUeCvPp4E7VoEgfSVBGemx2AGs1jfeEgwpL2WjJgyUnt", + "So11111111111111111111111111111111111111112", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "2aMJwqB1a54on91nvS4MZ1n1FyGjpijv91AQwVFRYGRk", + "AkJDXj4CaM1PVBfLXDsNLmhJLKNcV3dLqrge4qqMEFXL", + "BNERLReWQUxjPkiZSfco4TZqEhDz6Z4ApLWf9VmQ6bLH", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "FwZXa1FAZpdHau8on8qzXHokGMUcFir2PY7r7JLJJcn3", + "AbmZokyUZG7Kza7U6m4QCmf63cK6ytWtGD6xKeN7b6Xx", + "7ieaU6dCJgRCuzBY8XMbwWa5pewJbuBbQYxFZsemvdY2", + "2LWL7yZeQjYPMcixfCC4Zfyh6uAWxnwSJBV9iFMFX1kX", + "AaF8HNumboJmm9uTY6taV5dshvJJZbBjryEuWisq2sQ5", + "So11111111111111111111111111111111111111112", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "65NXkhJJwAPBJiEDcEMjiWYR34HN3HiCKzzKccXUCK9M", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "ET1Fy7EPpFJaYxkAvZkKcv8wzUFBoandWyBH2M5ts1Ti", + "AzxBt1mAszusPfaQsj1VSMLWDK7sj9HoZAxcg6qrCo4p", + "39X2ShQmpg6TZZaYZTQ2R6L84PHXjZGs4C89ZWqKjGPe", + "7mZqYC1kHTRT35XVDDBjsG8fmD5SDefZs8R8ZaUKEVse", + "So11111111111111111111111111111111111111112", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "FVU5KyajaoBQp64mhApJZihUrcUCarKt7QQwoJRwk8AQ", + "KgmuF9GFTLwG3YHkecQWruKDev52DYJAgruU2UZ929j", + "4FrRMf1jo71hZ6HTjKvnSwhktptGDUg4qhMDzEftjfsm", + "wHdhAGfErfQzdxYKxmpTUvVmhAqEPW4FQRfVMDaXuie", + "H77PehbYZCXHTqkRUjKDBNHAVtQ5ycvRgY6G6NjU9XyG", + "DuFbdb3Yop9x1Ak98jHTkhWLfnfqRGe9q5NFbUb4SPxy", + "2w9tVS6eaoTGSXgEaZ2xKjp55J4FkUjNP2EReLepLWAP", + "3AQ8ba4os6EJTVeBy4WXx14gbfEmASaet9wgmyu3zPeG", + "E31FPvAPjNS7xMWXQsaWgNK7K22FhDm1uZs3b2GSTsw7", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "EfcyPmNT4ieBjU69eTDB9i8w4NhBrqJaRwq3WuthY67g", + "FMiowhb5cBPZt8ieEyRU7GjePuS5HiSnEZnW1N41Tzt6", + "4vK25dXBruZZXtn39DBDRMWRkj5a4PafMQ3YSVonsuvS", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "So11111111111111111111111111111111111111112", + "GJqsmm6ADfW94wmxt4Mxqn3P1eTaHSLMbnnCvDbQuY7v", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "5GLqdaxTcWYzxDBMU6187gRUcGgibquWoLywGb4QLgqR", + "5Q544fKrFoe6tsEbD7S8EmxGTJYAKtTVhAW5Q5pge4j1", + "675kPX9MHTjS2zt1qfr1NYHuzeLXfQM9H24wFSUt1Mp8", + "2LuPBqxn4WhMvFMDLMycngWdmNa4rJXi5Wu6mtN5Vgt3", + "5kqrSFsQCZAP4P2Ve9hM2riH7G3n1kD6wyXZyLGo6JTw", + "8c2XuwNrhG6zsono9BxHortTVpgdrtj8vvLZP2DLry54", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "8hKqv3RfD2VXbSU1b9K1c8EtLknFkNGwikrb8FLCLJbV", + "So11111111111111111111111111111111111111112", + "7oVCyXmd8RKtCVDgU4tGE4xKn64i9xhqRCfmpgM7aJap", + "xfHYCFSo9cb3LKMocgHENq32Ucyu9gHko3mN3qQVsYE", + "7mh1SVSKvCh4qr4htr33ysEobNpfzj8aPNpL9vmYG2Rr", + "9VYaRqss8ProQ6YV8yMao3u22mGUvfe3ysVAhDQtNzZW", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "675kPX9MHTjS2zt1qfr1NYHuzeLXfQM9H24wFSUt1Mp8", + "FyA4XQKz7H38huartTPKo4819s788dejYcGZ4a2itTyw", + "36XqS3LahrhrvEBW6sYNhKdqUhTdM7da6Y4qkRcVwy8t", + "5Q544fKrFoe6tsEbD7S8EmxGTJYAKtTVhAW5Q5pge4j1", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "FGFm7amyZMxVbxKTWbp4dibvXVqe4uqF4sMRmHY9cNn6", + "5QeytZoTLWduJc15dKHjFdN4FQM1Da2GrtLBKJXZciiA", + "TVMjyy1rqojcz2YXWQqmza52nrbFH1gBuBdzGtnn2g6", + "4pkeSvocnRUWKepH7jSTzj7zT1XdEYydz1FVZ2HjiusP", + "Fv39wDBCarLryjq2gWmdsKv5FiNFxDW5BZNcqKtpZZQv", + "So11111111111111111111111111111111111111112", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "4MHWqZpRJKokKsS5YZv2XRtWUJY8KG3x8kAfWKTmQo9q", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "8JiAqptQF3Z7LDP8uNyHbYToteHDAkgeTjS3wUvRj3an", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "2XqZnRGW78v6cdKb1fmFCAEBkwpAikQoKWiNANzD9Z4p", + "8av5EHRAu2Jk7rBGHE6EE1owrA9y1sfPH8RwBwsHuGmT", + "GTpxmzh2gLxjsjGfS6dQdhSjAb21mKH9c35LZT9gV7ef" + ], + "3eqTbgbMhbAQXEYLaw5BSgpJmNYJ7gjtyiukpTmeSAaX": [ + "Ec26W6va9wt8kRGPYs9s36S4DjSAMLJvvf7VCyPPcXzW", + "A8bumh3sy4P3ncEbmNXK8LRcjNkLtY6475AquQT3jQGi", + "CAMMCzo5YL8w4VFF8KVHrK22GGUsp5VTaW7grrKgrWqK", + "7eiwmjnQZu22AXJ9D16VSxoCMbYuhVrdypttHCfXEzyy", + "9mCSvHfbY1BqfHeWTMkwdWgHxd5xHgPjqucbsqeLdaf1", + "9iFER3bpjf1PTTCQCfTRu17EJgvsxo9pVyA9QWwEuX4x", + "4iqDL1WHxSVKpj2Dv4No4DqRMXiU6QPcYjHGBN6j2sch", + "BiqfgY1Xt46omtG5zBMj2rMbsJdWp9x5meA16TU4tPjk", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "CJbrJkmXxbwnPmayLsD8MQuZnDVbabp5Ja7pmCTLthaH", + "J3EnkjsBdsmLasQMThyq8WjzLFC2jwCg5FWVvS2ihj5V", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "So11111111111111111111111111111111111111112", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "Fdb2YDbyQshje7ext2QHsagNwJ253PxQP6pWgu8ZPGeP", + "4tLYEBU1W2bZ1H3an1Apd9inTRWk2YWdrSGwcv5aqpvW", + "FpKMwydoSiWDxKbKxwXtW6xW9UwghwFHrDNka8oYsBRj", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "qrA6Ut5C6ocLYkeU5tpAwQs1CVqxvibiA4Z79SJjae9", + "A1Z1tHCk6GpTQaPWquSaKZCMCw7KQGW15yU6KDzKsmiL", + "3waozYwXuucwdjhfNadZoHyAB4r51KfpWqLKKXkrEubo", + "3rmHSu74h1ZcmAisVcWerTCiRDQbUrBKmcwptYGjHfet", + "6Zu6XLExL2t6b5GGATGwjC5EygFG7bYnUJSboDUnaotN", + "2KAuRDpvPip89BNDDkF4J6jFq5mtKHtR4yYaBzQ43BNU", + "57m72PTCutbS1W1AdDQrTgcHDQLQqJbgJt5oeKNnFRY2", + "whirLbMiicVdio4qvUfM5KAg6Ct8VwpYzGff3uctyCc", + "9wRMmfBd8fCZoZp4AFZk77xiBT7kLwDdjV5vuukfieWL", + "7vYABoQsXds4F4q4FaQ3EAdn9fhxMZn1qv1CS7MTJWX2", + "2G48WUjRX62PdVdtY51Jc3vQ25UaNybbTJftqsAh65s8", + "BqFXrxkC5vM26hkdbTdU3xdNyuNmSrkJnLwjERzfUJSb", + "CpdPePCYknw1Dy73NGxaP7Rjri5RdqpLJvqbZ53qncfw", + "J9NnJwJxPP4iWV6u4vyQgUcowNmLorvzdjHkiHMPCxpK", + "3F5fRYgTHr8SGKPczqjgy4N3PiNEyFUmKsex77329inW", + "8N3GDaZ2iwN65oxVatKTLPNooAVUJTbfiVJ1ahyqwjSk", + "EZVxM75XRAaDx1pZ1pADxDT3wbxis56i9pNxByycsNja", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "62qc2CNXwrYqQScmEdiZFFAnJR262PxWEuNQtxfafNgV", + "A6S4y1ZtzLgPbUouoGHLb69WDhEqgaqx3ft8LzbWw8kg", + "So11111111111111111111111111111111111111112", + "ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL", + "yEbpgkCrNYq37xdsG5jXfFspz5wXEF37GczLGwnJE2U", + "JCRGumoE9Qi5BBgULTgdgTLjSgkCMSbF62ZZfGs84JeU", + "YQg5mh74GSRpZUYjhVK8QeyecWVukpUeMMdi7NnpJSL", + "pAMMBay6oceH9fJKBRHGP5D4bD4sWpmSwMn52FMfXEA", + "BYgyqqPaxmYXCTNYL1sTcENdMVUEfiKkK7vr4eLvVXCD", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "AdcEUouofsKN8feyWKxeN348Z1Ek1e7Vo7nk4cn57Ren", + "rJKBvChp1NPL7mJ2XJmYEzLEQyzyip5CmpiW8LRJdJw", + "8zBCuExbj1syE1UZxrrYsBW8dJQvDSGZXpxEVrrtft9C", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "586RthpB4QXAd2ZdnCNLNNjN5FNxvWKJGcYaaAG8nFHs", + "BruJetaMiRu9zswenb7sssDRszHQHZ9MZnFPWfPqCioU", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "F7kmy4N3G8csHUePbr9Ut92hX13jNQjW7RT2gyjre4ii", + "EuRRiY4To3R21nf2yNpmRmdjSspG1ScQiSKuvyi8cn8N", + "So11111111111111111111111111111111111111112", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "AdPAnUWykk27XFJmti6CyhaJPZahwP7gCYpUpe8NszBd", + "sTBXZtS7ktBSkov8ad3Airrzzzapa4DXcJrwxHrNJAz", + "BixHeoBJHAsUmJUGdJJ1RXd9nFunXyVMHrUsrKRAW2N9", + "J26cEVxCnZBVwkZqD37VwQgRfmCZvYt72sRfacxZBhwe", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "GiS4hfot8Tsuc8iMNX8ugEboXMQL9S5zVZLUEtUpETbE", + "3NZ9JMVBmGAqocybic2c7LQCJScmgsAZ6vQqTDzcqmJh", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "MemoSq4gqABAXKb96qnH8TysNcWxMyWCqXgDLGmfcHr", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "HLnpSz9h2S4hiLQ43rnSD9XkcUThA7B8hQMKmDaiTLcC", + "So11111111111111111111111111111111111111112", + "9KTtaTjLJLUnVs7KiMEgtYPkjh34cNPd6XQdb78xErmw", + "DFSHkUtBjQavw44ZSVMkqbNZjtJNPLAs8Vr6ikD7n85s", + "ATygpFdY1KUVn5YLB1CU6L9Xg5Z1FAY9qimtEae3wBLV", + "cpamdpZCGKUy5JxQXB4dcpGPiikHawvSWAd6mEn1sGG", + "CNuDA95yVxps5nSndV66kYLDziQmykeDraa4gFcHMqn4", + "So11111111111111111111111111111111111111112", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "3LrZxTubnZg5RQ7Bjy7gmfwdt3nmBacTgBP82CV553SQ", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "BS1HNiCpWF8Q7mc7qHefJWqiVppHUH1JGsBbWwEfp11W", + "5y6Av4Ub92PVzUSTJiht2sT4onMsGEjk8UfDsqs7gZZP", + "BAjh9diJwFjashVrzftfHtf2jYEZpqKR6g6gXzHHAdqC", + "DeSnT1JzoD81isq637kku1yeSrmz4UtegrQV58NcWM7b", + "EoBiVnkuFhAt8ngL4P3jq2khxGVJ1atNMT315aeZAbTC", + "DHcAJuitD3Q6TFcfATxUgwVHMnxYx7bHeVJYj9qQFTaS", + "14oQk4PESvYKrUdZJadbC5yvdKiGUwGeCwFwD9hrHcw8", + "So11111111111111111111111111111111111111112", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "6qbAWgf9LCXcWmzBEqzg9c51kYPrb2zVuWAZiNZNsJ2", + "GS4CU59F31iL7aR2Q8zVS8DRrcRnXX1yjQ66TqNVQnaR", + "DmAA38D7piqnutK925ETc3qwnVTBas6gPNQXbt5nBfpS", + "9txxJ836MPsrhydDzYWxDc2FuXPoZMT1KiSibVMRBZAW", + "ADyA8hdefvWN2dbGGWFotbzWxrAvLW83WG6QCVXvJKqw", + "AVmoTthdrX6tKt4nDjco2D775W2YK3sDhxPcMmzUAmTY", + "Bvtgim23rfocUzxVX9j9QFxTbBnH8JZxnaGLCEkXvjKS", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "8NZmyZLfN6jbzxdxn51WD9BvWGVWuouHik77teCvZXhE", + "9rPYyANsfQZw3DnDmKE3YCQF5E8oD89UXoHn9JFEhJUz", + "2gqGdTCkWa1ewXubSG1Yhtgzf8SPcbYhqADGsfYkVYZQ", + "ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL", + "FsMKJcXP6iTMvYtw4Mnkz172JNqjJ9Qu778tpvgqDBrY", + "GS4CU59F31iL7aR2Q8zVS8DRrcRnXX1yjQ66TqNVQnaR", + "So11111111111111111111111111111111111111112", + "FGptqdxjahafaCzpZ1T6EDtCzYMv7Dyn5MgBLyB3VUFW", + "82pwMUKGkN9VevtvsrpZxhJtW9LaUV1qpgSNB4qgMPi7", + "pAMMBay6oceH9fJKBRHGP5D4bD4sWpmSwMn52FMfXEA", + "CgPqX6ZpwPTFeWxNCMAq71dF6bPTVxjrwN4EyXFphbTo", + "HdFoqD6mup4MRpkNxgHktaA7tgFy53iHvDt2zD3Upump", + "ADyA8hdefvWN2dbGGWFotbzWxrAvLW83WG6QCVXvJKqw", + "ABod6XmUeMFbBHK43kBMsSkLYuMCUTo54ppoUQZRMUwJ", + "AVmoTthdrX6tKt4nDjco2D775W2YK3sDhxPcMmzUAmTY", + "ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL", + "3ob474g22GqugRgcjE5pyWEjxTxiZ1vMo1RWeEHrnYkN", + "GXXFT9Nt733yBFiW4Frw1C2kMPnJs2F6Yu8aHyNv6GnA", + "W1qMgw5UjA3zRsspbVfYqHE1Pxsh4duZh4R7vJZ9UNo", + "29P5ejJFSGHzrfk6N3ZPPYKfyoMDsfDPRrhuwL25F1uv", + "E5HLGgxAfbhX1M2S9sqmU5K98SXP2G9ecSwYg9r17hcx", + "So11111111111111111111111111111111111111112", + "GZnKxgCQj2GkpjBU8JSUEdZwDoyVg5g5udtEsA2osZ2", + "8bHWkv4R3UePbMvjZ5ydZ77SSJAzZRU8XJ6U5eBL4LX9", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "Gpfyp3QEWWAPqfKmohCyZALbYEhwMWT8TCLQRgSu2Y1K", + "CPMMoo8L3F4NbTegBCKVNunggL7H1ZpdTHKxQB5qKP1C", + "8MsFHfwamHjRZoT47E9LjZjPtMjMBn4YS931Hjzsfq3v", + "GpMZbSM2GgvTKHJirzeGfMFoaZ8UR2X7F4v8vHTvxFbL", + "CXiS4QjTEcRdTtQTNTwthjGWGSZXR9SD5TXpT8ESDP2Q", + "So11111111111111111111111111111111111111112", + "C7Cx2pMLtjybS3mDKSfsBj4zQ3PRZGkKt7RCYTTbCSx2", + "5riRDsGpuA9Yn5jwstTsB3FQYpJqtzACRxE4Hw6zfgYk", + "So11111111111111111111111111111111111111112", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "EeuJy57CadMdTZrjASAWeo6HLCwzXrYj6gDbyc6ACTme", + "Ba2MdmtwiPcvtvEKabaddB4iXrHn2YaB1MdpB42JtLFJ", + "G3UKtgUcA5UYX374mdzoAcGGT4gsdnofZ4et9WVzPHRo", + "FWsW1xNtWscwNmKv6wVsU1iTzRN6wmmk3MjxRP5tT7hz", + "ADyA8hdefvWN2dbGGWFotbzWxrAvLW83WG6QCVXvJKqw", + "2sB3AjdXNnVZaQRKp3sy8ocqE2QMhPXDPdAdgCP2bZPP", + "7MqQx6CzEMTYj1r39kvvcqodsbohH3G81V3F48jYbc9q", + "pAMMBay6oceH9fJKBRHGP5D4bD4sWpmSwMn52FMfXEA", + "CT2T31frvjVTriVarsxZpts75umap11LviHNoqwEPepo", + "8N3GDaZ2iwN65oxVatKTLPNooAVUJTbfiVJ1ahyqwjSk", + "GS4CU59F31iL7aR2Q8zVS8DRrcRnXX1yjQ66TqNVQnaR", + "HdchsRU1esHmpdjbp4h5NYsJZCCFhtR2rQKED54xNfHj", + "5Dq9XnWH5UMMhaEWkzxf5KEz5PdAoQHfEeadVinZefDs", + "9eEDEXNetzSCdZg3gqkPC93gzih7YmughNqaS2KpLn5C", + "CwetjhdZUXy18uxaV3Wjg51SQViZP8JKcMLQw7Zs6RDj", + "CAMMCzo5YL8w4VFF8KVHrK22GGUsp5VTaW7grrKgrWqK", + "6g2NyReKT8NHdK3osUAGDyvrN8B3QmESrjNKBQZtAbMB", + "9iFER3bpjf1PTTCQCfTRu17EJgvsxo9pVyA9QWwEuX4x", + "8stt8bt54QFtBvttupXp4bnhaVyWiY15VBVaYvFYFiAN", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "EbVHARuUgfYZSnueLAs3MxVjHBxwWC7sDBLhRrb68s9Z", + "gE7ED5oWauF2M4vPXt7X2FfvXpMqq7hyqB4ksF1Su7A", + "4y6qABfRyb86VPjHYD1yb5J7kdZ36F2WRD1FhTWgAh1B", + "CWMivp3EUm7PM8htujcj5M6rEvHnN745gxhkydgArfw1", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "4CieeBTumFHzuEd7cdKaXN7HGDiYBtWEzr6cTPfLmMdx", + "753KsdprPzZMt7QsrxQZYVvaLEJ9G1e7LSS4Cxtv9QP", + "EnpZHmNze4n5dxeRqGy9PaZNYCvveavNiS6XRE4vn7N4", + "So11111111111111111111111111111111111111112", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "9uTvFnmhLmc8gkioJBQd8fy2HDmyecX1hnPcyzHL95vJ", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "4bFbYJKHBLWEtUF7knAZ9TnkhP2G5b8kRwb9o6t5b75R", + "AwrPqQZxhfGiWu6DZAUz5CoyRtacgBViqGavGXkvrSLN", + "8qKQuZJe2TZPa5v7L9FefnoRbK5pwBrwvNbAiDuQ9Nuk", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "DXdNfnVz9sBBHtzvmRWbXHZvwtS4gnLsd73GsHmk3iPB", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "HYojQMcytx5VVtjkRBF2PPeqjU9rGA3vqGgckkzeQ7Yp", + "5vUw9VqWouVZoWRceXhgGpkjtBD7SazoDhVX6W2R4wMU", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "9XQEV3CWWdU3xKrEWfochrfK4kcTX8ejVs7zC3ZH4t6H", + "F5Lb1Hamu7SBA3fPDwmajtACqaYNLrk4AQ8h8aL5936g", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v", + "6ScXCzzvzQnpJNQeM55cABJXjSxTd4F38RGRKXbsvR9c", + "D1ZN9Wj1fRSUQfCjhvnu1hqDMT7hzjzBBpi12nVniYD6", + "EpETTzp2YFneY5zbMZJL4texnm5L8Wje6jgFhboHc7A", + "LBUZKhRxPF3XUpBCjp4YzTKgLccjZhTSDM9YuVaPwxo", + "ChE8pMhGyNwXXk6doue8smuDjbUR2ji8Nhb7i97chDSV", + "5BK1umY5CkaBaJyM4YMN3UAKUXi9SB5ZX26Z5F4YyWi8", + "CAcNzCBJN9uLPRyKsdAWqYExyZUd1bxovpXCtnmYHjLL", + "So11111111111111111111111111111111111111112", + "4ibRL9tWdxDVT3xXp13tr9KxMGvnU8nC9onNh6Qwc989", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "2LKX1RKFqU6eCigwf3tNicpVUhGci6No5mxucCQZDndt", + "5htHvwqB26D8eu68JXSWAUSX6zvfFWXNcCLf92jvgw3i", + "EiwED4mgv7FPYm3rDav5Uavcm9qEo2ygceP7vdiqjYyg", + "8rkEyNvR9B52noTbV3LKq2RU13k6RvB1A7kLVvDtDcAk", + "6j8o8uByeF5bMam5EJmEpUY877derTe6997cpijwmS96", + "So11111111111111111111111111111111111111112", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "AAmHwkhG92HMhJp4AcZ3DutWAL8gC2vekDigSm5YFb4p", + "Fn5ytxfEyajR3WiWU7Xo4JpEsgqNJvYaW522fSYb31ET", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "A9QsyjnSZN1qKjZrGPNQJ1sfpUVdWs8UBG7MSiP33e55", + "E4LkSYRGLZJg7ibod8iAzQzrmD6trHEJqWtigTuTGaqL", + "EtE4ir6xkhdrVs6LKmtTJnQr6vvhucQN8be83usMEYFt", + "CPMMoo8L3F4NbTegBCKVNunggL7H1ZpdTHKxQB5qKP1C", + "GpMZbSM2GgvTKHJirzeGfMFoaZ8UR2X7F4v8vHTvxFbL", + "So11111111111111111111111111111111111111112", + "9EytaZZnrwDKW3DVxApTYhBc9gycvpSHydTr19dX31hG", + "D4FPEruKEHrG5TenZ2mpDGEfu1iUvTiqBxvpU8HLBvC2", + "GKs8ry6M6C5nmzLz4QpmZEKFmDNx6SmwiBzuqnjYVwDd", + "AhYuYdZKisHBVMVMWfRxK7Ui2Wjt1uvQeq3Ac367haQ", + "So11111111111111111111111111111111111111112", + "ByCim2wgzHo9ZoMVAPXdmr9arZbo6yhWfJPjaqVZELn1", + "pAMMBay6oceH9fJKBRHGP5D4bD4sWpmSwMn52FMfXEA", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "So11111111111111111111111111111111111111112", + "7VtfL8fvgNfhz17qKRMjzQEXgbdpnHHHQRh54R9jP2RJ", + "J2ZRCjfPA5LYNUf9bomBv5xwF7v6V27KG5e5HWQi2279", + "Ao512PfkUuWKcr9E5sVYoFBt3SKGdNDmibAcz1cUCUVP", + "GS4CU59F31iL7aR2Q8zVS8DRrcRnXX1yjQ66TqNVQnaR", + "7GFUN3bWzJMKMRZ34JLsvcqdssDbXnp589SiE33KVwcC", + "ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL", + "97HYhVyT5KPF7HMnvThWN73UzyrRYyxFDEJJzo4Ko6Bn", + "H7zh7kBJY8cGHcbHgKpJgRC9vmQGf3Bk4m8fdbNdy3hL", + "WC7Awj7NppEf2YjonhexE71UNrzK5aBZnDR1MVmpump", + "ADyA8hdefvWN2dbGGWFotbzWxrAvLW83WG6QCVXvJKqw", + "9iFER3bpjf1PTTCQCfTRu17EJgvsxo9pVyA9QWwEuX4x", + "CAMMCzo5YL8w4VFF8KVHrK22GGUsp5VTaW7grrKgrWqK", + "BtR2EFMhSnmU1cTD977YKtASgFQLL6kic7NhgeXGYWGj", + "D2iM8Kuy5cZJscJcLJdQy1yMXvWXZREvD6m7hZWC2WWF", + "8tSjriEr6fbekKe6H8xW4WzHw4aSAT2ZJikzHZ2ZSufQ", + "D339bAiS95BSeVNxXhpeuhYLZ72Z7PCRAQvvbu6xxTCE", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "Ab2qsKsCrszLG8oHEjQMcTFoBonNWj7Mzzq6Yqcz47Vu", + "f99fpvJcYKM2RWwxYXuGgDkBd9o3BwWNXGBQyp2nCpY", + "3XhbDThJKdjADayMvxYmFFWBkGKaP64RycWkhewXnAn8", + "6XawyWTxGcFDjC8Koj2StLGFQxUyHJFESNqPsAY6dJLp", + "ApRBRLupmobhG9TgDYUv8NrHQ7mBq7znKv7VWRrLK8dw", + "HqvuuKtZSoSfGzoNM6ncTMnGjudAyDnMhG1KkMGMZYaZ", + "AvsSadXBhG3b78KgoV8waadG2kmUnxXzK7qEaectN9Ai", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "So11111111111111111111111111111111111111112", + "BhH6HphjBKXu2PkUc2aw3xEMdUvK14NXxE5LbNWZNZAA", + "2AxuK7jdtjeJ2heHZ8YZjWtW6uK3nGAPyapD3fVsNhtV", + "8w2DV5wqiKSZMqznRNnroT2KnKGGRsKiFwhCeFjugKdC", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "GLD8zcmQqwhLCg3wtF48EpJbWWFEbiRWKkKQq57PLLi3", + "GpMZbSM2GgvTKHJirzeGfMFoaZ8UR2X7F4v8vHTvxFbL", + "8WorQUdGxFwkgM4XfeUzXFV9tJwYpHQkdVK8oqgVADRS", + "CPMMoo8L3F4NbTegBCKVNunggL7H1ZpdTHKxQB5qKP1C", + "FWQLkHNVdFt9rM3BYgCYu3QeSwburz9aDm5jPi7v7REV", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "So11111111111111111111111111111111111111112", + "DunHozbFWYrRLefrX8Ae9KuVji4Vtb8fES462p7tDgkA", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X" + ], + "4TeTR3QLmwPdnw7H3oUgPSA7Etx2W3WEooX4wC62JLB5": [ + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "CK1Gpzf8cuF5ZRHPLAUpLGhn8B2FqWgeDzUwLbyXgbtN", + "4ZBiY4gwWEGbfhYGJkfm12dJ9BW8u4FNSwLJMzmEL18K", + "So11111111111111111111111111111111111111112", + "HA1yUzwBApbTS8qfFRDywg5jmFro1VeuwBvJYDZkKno", + "HoqD46vacuNzXgfy8ux8udPkVQ1PEzFDRSCEeB4Lqus7", + "GMEQnLBgWx1b7TwhayC3jErK1QRdoveGL4cQMxa5C6SD", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "3u87TQJjwucQmj6RhZjD7cHjn489EkPQrpR2cV2Aak8F", + "BXhHbEwHuhfQjp9VjL9DJ8NJjUYcvf9j8he4wLXZ5Z42", + "FKeD9LjjRtGKsxY8F74odBsFY2msgYiDk5RfcAem6C8F", + "7PPYq1xSNLVUJzkcaZkTcQN8bvRsxJ6W8gphmakwB8xy", + "So11111111111111111111111111111111111111112", + "CnCLjt62Jb3rxtBSM73LyHo4FYMPM5Ke6t8Po514yPSb", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "AKTDi57gtZayUzD4akYMKxGhyUse3Dahem15MLHZyMMY", + "hzsEb1JqQd2e1P6An3mg2wisFoj2hNb6rW7ccPxSanY", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "VEufPpFRBRkJMTDt7eURCpmYka73LLqs3FQ7r3ZsFHX", + "6juQtBQZNqz9EqZyZ9JdzK6NKrhAt3jDHy7raQTJJHpj", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "2VZTnDCMuaT5kfeC1CNiN892arLyc9vipDyZ1EnvdUnF", + "3KvFKFouye1cHTXSBC3gEJvLNkNJTRrTKqBbyC6QrXrU", + "whirLbMiicVdio4qvUfM5KAg6Ct8VwpYzGff3uctyCc", + "EJSUjwhvMmbYJEj31yGuC1P4tsdwtmyw4Pbi4qTsdvP5", + "iMH9YW3kAgk4kkB74ZHPHESHAQQWPeGWn6Rmty3WJnq", + "2KVJNFs2TB1yFPbz4NqZv3euqSyzJSDXt1LyJReTAANi", + "MemoSq4gqABAXKb96qnH8TysNcWxMyWCqXgDLGmfcHr", + "EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v", + "So11111111111111111111111111111111111111112", + "5fEq5SYpSpnDUDn8dmXUmZeHVaXpL14UirnGajF9QMor", + "4ih4F7MGLKnx7j1dJyQ6a7id9NkkZZRRDkxHgkpbDTFe", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "Esvfxt3jMDdtTZqLF1fqRhDjzM8Bpr7fZxJMrK69PB7e", + "GLQdMoJ6RJzWSBLiguhKVqt5K7B6EBSk6xiRBxiLRpbz", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "E1DyqaSk7FhBKoNoSNPHpcCXuRXZsvbUhaQ5A7TPgysB", + "7kVvUjExBjz2gBPXjE89iKEzfyztEqLR27nQewoPNB8", + "EcWYMMBieyNoK7aWv7hKJCN5HpESxBLP8Tc8B7tR6qFV", + "7Mbmf1gL3ZF3RhHsKvmLy8UqDPxrWk1UoeV19FudgAyP", + "6WZmV4kJVpMafv2DhMqSmcGTNjCuYoAPVpm3TtAPw21E", + "6uBzoneF2MnHDwJPj5RzQAbdWW1uPtZyTxFcniu5QXBP", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "3mu2YxGEqkGEPq1kH65iqmM4rwDeaLJMuDDQ9gnWjBg1", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "So11111111111111111111111111111111111111112", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "CnfY7kvDDcnbhqLKj4VWuaTr228guoQCKksTqtS5KwQm", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "HcYwCBZeNHP961u6XjZkBfEqRyCK7YrtemVa6KQrEN21", + "G8E65L4k9iSgroLwF7QKzpiFE8vNK2TZa3XNjXMSsqPX", + "So11111111111111111111111111111111111111112", + "EjmHq7tZQfVABPsBFTUjFTjRnU45CTDrDU5EAkjrPA4L", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "Azn756NdoAv1R3Xk8UFcVgfrRJR8q95jCQJXVxxj4tZ8", + "8sLqCRN8XGYxn7zZzscnRWFrcgGqFb2myFRcttimviur", + "83Pg68Gj5uW5utZJWF5YdY82k6HxwNSLpruuRWfHdemP", + "9bFxtbqTDJxHnKwyDJkguxCwAqw1YPN5obd72BQKckCg", + "So11111111111111111111111111111111111111112", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "CtmmfYF7xYHok24SFpbuDaLenK3WTy8RD6n1YB1ez3cL", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "HcJFFLxxmaZSzRmyZecRhr4wBm82tUL5jwDqyMyVCcAt", + "DxAGtkfWr7eQFtkNTbznEBzBpD4661AcZtZkfFyWxnqB", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "e2abUh1jyPJUd5X9mp7B1X5gWiwm7yDqHsTkShYm26U", + "A9tZkrauspSGjyGu9re6DvQRxRVRHxxgQALPiHiDCQRx", + "AyNgyi3cjntFtT9dLc6Jz4J4PPTYPH9umizEcHv5D6HZ", + "CAMMCzo5YL8w4VFF8KVHrK22GGUsp5VTaW7grrKgrWqK", + "HvWnRCey5GyHZjy7smWSUhqt97Mqap9aGiNfrvSHPAq7", + "847wKHMKBoY59JNWh5J5oPppHgDBd3cFwtPtJBjc8XH3", + "B1DNxGbbZyshCbYcUGkzcBVsbF3vr9RwwoT8XoeYyq8Y", + "9iFER3bpjf1PTTCQCfTRu17EJgvsxo9pVyA9QWwEuX4x", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "7D5ZoZarSZX7zNwrFNKr1EME1Hqfrvz55FH7vvVnwdAQ", + "Cfgza6xM5xoeuaCREohKRAyWZmLnzWLpyQLP4QKtjyJT", + "BRt3BmpiGivHviUvP85PE548u1LABFJDtKbeGEA79zSc", + "8Q8TkiGHhGzYKn23yMqiGYAMBqFEUHBMUetY3mqgn5c5", + "9iFER3bpjf1PTTCQCfTRu17EJgvsxo9pVyA9QWwEuX4x", + "DWV2u9b65i9K3pMDPdfpEcF6MqArL2xkb4kzc4yrjofg", + "3srpQQxw5SsimfkgajPJNEgbod5UR44iR9N2X7ZvXmUb", + "3XaUVKdMUdhJDi9SwaBz7A2GkjrMp1cxtpr11bzXG3ra", + "CAMMCzo5YL8w4VFF8KVHrK22GGUsp5VTaW7grrKgrWqK", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "GUj7Pijv2hT1XyhNaEcXgnCTVXdjHBFhDmGS6CX15eSm", + "2VMLKDebvzcGAjUZzmSSnJAqL27PZQzKNmU47o4cdvs7", + "5vfCmpFC796j1V3VAvc5wv74f4agmerth84eaCt78YDk", + "84jt4QgLVmDmXR9xRUDLL6brqcE52BLH1zUeXiV1Enfq", + "8dA5zTtboFwZ8q51J7sBKcJdbmt11RjzApy6Vt7mQbEE", + "8F52EQJtHcbX8rBPuFcWSVrmYrPjEEFMkA9Rk52dW7e6", + "GewxbY7zkt9PjJfAAptdkKxxjtPzzDS4ZoHkjuDyJiwB", + "Ft8iUFtCNpKXvDiiBXGJHD8aaxhA5HekEYmxpf89pzQp", + "So11111111111111111111111111111111111111112", + "4SkaGqLh5kRWmFZj8aMtS1P2cNqazeShUwHY8mxKmBhp", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "E2fgRSdps2YnSsYFffuaVJ1XJ4ASQmStHp2zzBxq5he7", + "7tKhQNNiriB4gXq1s8gPQ4xwvCWL4Ut2ZozTD7S3i7mh", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "EdyUCovPMhZBT7bgoaQuxVizXxbQL6ahVGe5GFjgF21u", + "BPRMCz7ym4PnvsKzNMHzZPdPLDV11B5vdtXzCiE7Yz3J", + "A2PzHH8WyURDPanSbUesestCun74FAdYjTbRAjPN42p2", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "HwCrhdANj62q6yKUQhycnLGFK4PGsHP9mYZv5S7Y5Zt5", + "5y2xCF4Y4BxLjcDanAn1bQVPpKbNQH59UocjcMT53Da", + "8T6FuZe1m9MQJTmFkNYXb8dP4TVmaRicyAHj3uznwe2K", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "So11111111111111111111111111111111111111112", + "2a9UdFFe6zqpyYBCW1FawipfHY9ax2JmCnE8VQsdBNM4", + "FUE6Lw22boS11E5rQ2FkzCTuVQNnQ6F57NvwNPFdmntW", + "HdyJjm46C2HXMsCFPfBgZ7hqHAN4NKaXrfgQJD1CEiy2", + "22SdB1ZPJTbc9aAKswWBP1dVfjV4Q3ninDSe6PrjHzpG", + "HWyDTAnBrET6y7ykNHaZwh3gu3qzvcAyqsgZSydvQoN1", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "3rWbg8G3NaqGssatD1CFpMDbZ9Gt6CRTkFvKCyy49xXt", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "So11111111111111111111111111111111111111112", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "4kvaCNzrwnce82uDJkK6WQmuC6xmLfKeQzGQ6kxK8Mwh", + "5Q544fKrFoe6tsEbD7S8EmxGTJYAKtTVhAW5Q5pge4j1", + "675kPX9MHTjS2zt1qfr1NYHuzeLXfQM9H24wFSUt1Mp8", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "9aLJMXQ1vVKnkPGzzqDTJGrAQXKDbjWKw7418AroaGSF", + "9xXu1jrkhTVcXNCCN84AeNZaHpevrwwHVwgJyNNUPZxg", + "RfViMNXrxZTN7oRset16UpX7EZFEnTC6D3hhzGqxu75", + "CBqr8SRcf8chXZS19ivcs1ux7urbeybeQfcoj3vf86iJ", + "6Ht1fQAAPUhFB1THLhuSZzk8QrfjcgcRwSyVShrnSbtp", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "3HjL2RUkN1QLC1zzA9sHd91q5LBjKTRKyHWugCdhucpQ", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "6TXdNkGRENz382BiS2vRwG3hPqTP9aso5TnNEBS9TzuV", + "CGvAmrJ5HdKpYfxRonBdWdiYPkfw1brGwkVEabPQEF8G", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "8RgC9SGDAMgNZgkPsSVE733fTVV2j4jRRDcAEDiJP8K1", + "GoSBQMhvbUcqbGPMVnoPAyaajVRmk3JFGK5xx9sLGAh5", + "Bx5Pkp3CjHmjz5vmmoajdncozZFgCc8PoFqhAVbSFKHj", + "CfdFC6g8v11qR9REjyjtdQYL6B5GqFDxgcCfnKq6fNHH", + "DthKeSnTsHujwgi2LVxU1ZyTW8u7eQ1m3Mwp5ybLLfhs", + "Emz4V8hnVJEnv269J3D87tNzhfVuHzmgY6ybW8CKDjeu", + "CAMMCzo5YL8w4VFF8KVHrK22GGUsp5VTaW7grrKgrWqK", + "9iFER3bpjf1PTTCQCfTRu17EJgvsxo9pVyA9QWwEuX4x", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "GJhMGy7nMWmZ46NUKk94dgU2QVJYFedXorALW1sNVcTp", + "14g4QyjtHjpMao6YiZPzGGb8qc6wAw7QpGzsfXLg6gQY", + "4tbvH14bm9guy2ET1RJns891cVRW6h8ktoB3a1auZVzA", + "6wvaxUzYiZiyB9bBNydd64xggF5rfpoiAa7EapPMBMFH", + "9iFER3bpjf1PTTCQCfTRu17EJgvsxo9pVyA9QWwEuX4x", + "GUTpdjwjQsBnY6yf5zXaNayyVN3tkckuhvdAPmSWsoCv", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "GnJbapss6wbG1LB5Df4UcxqtGUiTKCoFuJNDbJsqrn68", + "CAMMCzo5YL8w4VFF8KVHrK22GGUsp5VTaW7grrKgrWqK", + "44KDmgijGsPBe9ztRPFF6N8A3XaNGiLmmobcPGVJM3uj", + "8NrghUL28zfkmXkP6NtrJdpoSWdZ4yN555QrJSZxXAdo", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "6S47qmK9eEAHD7NFiLsJajthTwA2BfArmCK9UBcwMkFG", + "So11111111111111111111111111111111111111112", + "FeLgsiaqVW1GqyrBjiB69g1CtC8oF82XCfpFbpELBaMC", + "4GvrvJGYHZjo1SYxiQEAPJ1V8mhyE5J7SagZAwNQPUNS", + "FuLEHDXZy2N5dUB6CT8qamjqA1KtntAsT7kgg5jcVujZ", + "7nfef168ujrqNeV7yfnQBc4wGZpm8wggrWetBamkGozs", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "7YmYagMoQ8qD6Vt4u9DpKxmPU7JC6Sa1nDkvis1eDxJx", + "B2bpwHgWV2JVShAdzBiqXWkdnjYrnALx3dzZoF4DUd9f", + "EBDekWSv8z2SzeofpRdPzBsPE9c9ouzqYNvG7sj92YEv", + "3sX4N2y3gQinZL2f8DJ58iPbcXDEjGM5NpihZFNvyMih", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "9iFER3bpjf1PTTCQCfTRu17EJgvsxo9pVyA9QWwEuX4x", + "8FenGXh4cHXYxXa9rQKgGv8RiiQc4tdYVcUWDAVQsyk6", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "Bn2GssqngrSsYQq1HPXCminMdcpVBt9R2JDciPN7C6pP", + "FT9vxeqic1azeh2TK9ogRZmBZdp78sXkhEiomxLEJXJi", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "9dcBqGS2k5rspdrqaiscXLmjcEysJYEFzB5En9gNf2Qy", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "So11111111111111111111111111111111111111112", + "eCDXHACCyyZndXAxgi5ghQpGk4MSF2yGBXifHS9tWjs", + "73kktzTKkHMeStJCwePWvEBoaBimacfHGxoeY4GtgA5y", + "39iH2JLwNVFWNHYY4aAvUccpC39zVQqbhKrvPH5A6vUS", + "3ggFGqSkshuYBBZo3qQcYWAcNfJrmUgsGRyKRtoynsDK", + "G1cU8gq8okTig7CEkonWQPcNz6S79rL8GgeByMMMefhC", + "7i6MuCsDvzXrWSHLFBd8QasmKFsnTve4Lg8a4ffMpoSy", + "So11111111111111111111111111111111111111112", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "C6ezQBjN5sJEt3U7ZDYngQyVQJPKsw4JeGcrh8Znr7f1", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "7emfiTxpRdphqzwG5xbHzLZz3kK4mQNZBf9j7Z8aBF8R", + "So11111111111111111111111111111111111111112", + "9ohrXQwZQrxZhyJE9N6BLUXKajhBckWKdKD9NVwiMPSR", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "4KEvJyb9o51eLRjN9n1WLx19WvsiUgnADQN5jnU7cLKN", + "6BvbupM6hTu24mfhM84nMdb7FjZLL3wcE67UFwGtiHT5", + "HbXXXqQYH5MR5JDukPYEYrL6UvzmMz6Vep3tJAUEasEn", + "FJgupMVa7HDTHde85BrUYw8Ksce3MaPTv2nyYxTcJXp9", + "So11111111111111111111111111111111111111112", + "9nudrEceEZAvs7EKhYxzY8jeJ8cKAsgbY4srtV9yhe6J", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "H8zNYPGPnLW2KK597TrDqv4rWudX5aSRpLUX76ww1arA", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "F1b2vtcoAHDcFHLobYUMsFNYCx2kbhFc6ZBhqC5NRHDF", + "So11111111111111111111111111111111111111112", + "DNBDHLkZkv8XRxU6YYZG8BZpac8dyzGVtLRtNu6GfvZ8", + "4sWF6QRQR5p4MB4QyVThaCbmmvvhhHoAuTkUVQi7R7G8", + "7z99Tk6utgsukyAf2vHQUFd97SiZL3btKbf2J2x1s4TB", + "3F2PEjNyVSbV3BZA4CLL1oWJ3MXtAnxhvNedPE6B8mwh", + "68VxRCoZ5ypUHQD8B4CoTsJ1aHZiBi3W8pD9Qy8nuLmG", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "4ecdt4WSchBXezF4qSRGZLVbuhjrmg5HZz299TsVXfJt", + "9mZq2cqquJbXAgWRFSkqDwKtB45NEVrSmLgBk32CUKJH", + "41zC6r7iXWHpLesL7o1KR996bv6Cc8AwePPGNB6WRF7M", + "CsPVHtvBQKHTFWKEwZ2gccLo1byfN4UskobPjASTrz2J", + "So11111111111111111111111111111111111111112", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "D8R8gypDw5XA2GWLLc12e7oR3BaaMirUgKbQe3QUMSCv", + "DDqVeFtYBT1xNYpQoUP8MBCXbgJGd4XrN3HN864k2GHm", + "BkfETjRt6Yri6iSaJbEHN1XXbAi4eYqa9GDCaWzi4zhA", + "74Hj8Cj9GgHPWXvmgf2Wcy3DmAhGTKrauesZ1voBCF78", + "HFqPx3ueSYqLoo4WukE4pnnmC5ba39UvCUJZrMpL7uzD", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "H4CgoaPyF4ePVsZzjyU6G2z9nHYFCqWeG1Dkca7P2xAs", + "EtdqE87KQD35gSZwDN5czXescm32sciVawiDKAdftq8m", + "4rFgEwcnvKcMTYRvso7UTZZ5D6xXGnA7sKTS2j4YTDCi", + "AgEe3SiFDnzb4u17fMt2av3AxP9HLtNfDudeSCdVD2UJ", + "Ae2PsS7Vj6375n6jrBhcdvgRY9BJtLfzB6YHY4xymyAT", + "3zsEG1k4p4PaqLZaDEkyPqiEUkAxGuxABi42JUpyxze5", + "CAMMCzo5YL8w4VFF8KVHrK22GGUsp5VTaW7grrKgrWqK", + "4qde2K6mNsjiTBQmSnp9CL8EHPJSukn6pNAaiWoCLj6j", + "9iFER3bpjf1PTTCQCfTRu17EJgvsxo9pVyA9QWwEuX4x", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "91Rw5a5P7gLYdTv5izPqJ5qVN1qQXRyZKWrLTUvEZFr3", + "587aqQX31XcaW8pi9z1CkBpFFq8iw4vFE7KGv4xYkBDi", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "J8tUjhojgWRvpeYgi5Fzh7tYqzWEYBPf3XbTBREMLzuy", + "9r3RtH2yhz7TbtsNvvQKHuHn2FUbZ8HtntUCTgPXmmkp", + "7uvN4P1qEKjCE9NayhYVGaQ9i1gZBZiJ7eGZWKcdTER2", + "HLtcMLoCza2kzG3BEq25CkutM42XVc1ZC41AXt5YTeVm", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "7a2qiW3xKCPXVwNuM1bzTGgZDU1DPLNr1a5qLG5WNSBy", + "fTe8FW8ni8geJ1UkdjNPTS7VxmmkNwBFovG3MJDosgN", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "So11111111111111111111111111111111111111112", + "CotFh6YkYHQg1Fu4tN4tM6Ci2GrXjJXyWEWpUeLSeEEH", + "7eDaHy3xgkwwTceMyejzjh1QBP8FGHuLaU3HHSVSqHwW", + "Dkr7BUktJXWz68GZx2SDWASNekF7wCp7aTjX57jzQRpv", + "CDnfaqqQXiaeiDE1kJ94tFMJJ5rJ7pmGwnmKJTsAmmt8", + "E9jZoVLrJsG3aHpzBdCbEiZci7xrQM7FnPD2eEQpmYji", + "8AqVGsSaCDGwhRtkswDeEk4WKHi2M5YKtWooefAzriu1", + "J8BhbMHXTpSzij7Fa96dXfmyM5BV8cEYHntW9eBfyi41", + "CAMMCzo5YL8w4VFF8KVHrK22GGUsp5VTaW7grrKgrWqK" + ], + "7CxuEdqys1J36z3BGKKRoLjdqwEf3GxBY9zt9ZdBxyCV": [ + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "DYQ3e1BT15GV79D5DQL2t5tH29jdpoiNihfV2EaMEwVg", + "8d48X4AqXc21qVtDbyMXikxbA3yCP69MYTbxPNaD4qpK", + "675kPX9MHTjS2zt1qfr1NYHuzeLXfQM9H24wFSUt1Mp8", + "3wkkhMqTgL68eKq2YPwk44DGTKpCX5ovBfYv8XuLoP42", + "5Q544fKrFoe6tsEbD7S8EmxGTJYAKtTVhAW5Q5pge4j1", + "5Q544fKrFoe6tsEbD7S8EmxGTJYAKtTVhAW5Q5pge4j1", + "24chtHr3EWJHL1D9vVQ78PAava7Yra7qj9AY7SwgaS6s", + "7vZ1x7fu9Uhm4ANtfigmxPyterHhYsPPWkDtzDL9dPd2", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "675kPX9MHTjS2zt1qfr1NYHuzeLXfQM9H24wFSUt1Mp8", + "96uGLUCWw36b7d8tfjHEPJMCtgNL5bT7qyLBCkLRBkxv", + "So11111111111111111111111111111111111111112", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "D4FPEruKEHrG5TenZ2mpDGEfu1iUvTiqBxvpU8HLBvC2", + "CPMMoo8L3F4NbTegBCKVNunggL7H1ZpdTHKxQB5qKP1C", + "2HQPfjkEzSueZKZmwqPRJgL9Z3jgf875wsoBqv3K41Fr", + "A5JEPQQhhkRWE5S7o6Q1hx9VdjtB5ZAGhdMyYdpJR1r", + "2DDgzcPCmhhmPQwo69GfajtMF7UwwHv7dR3Q4e3sQ39n", + "GpMZbSM2GgvTKHJirzeGfMFoaZ8UR2X7F4v8vHTvxFbL", + "9akdJUw2o1xL3CcGgvsnEdFzbYuChUs9j3k8TrHESa5w", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "4QcjGDBBE2B6Qv3Pkt8PHT2LdZ8Pqy3HZm66LEdAXPr4", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "2zyKsJf1Q8whpKVYxafemvpdxYte1rLeesM66owf2r6w", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "Hfya6PPdBGnHK7rwg7RnbZ8s19iaFKrShDwkjNkhacJF", + "28CWdnBhm1j54WH6QuJoZWziY99kzdbf5g3CswMn8ueJ", + "5QSEQeHaasLMyy8QRwwTtB6rZvk9WtZNpwwWwR4C3nio", + "GnvNAgu3AQeWk8BvGeerHkYuW338HAabHR1yAMvYgycC", + "3hADWSgzWX5fw5TXvMufr97Gu4Hj1yVmN2nAGGFzkBub", + "So11111111111111111111111111111111111111112", + "G4MUKoqnxe7qnYMbbY9XRnunHj2YApcr7pu68ahb6hvc", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "eViTQPsnCZvwMrvrKLDpVxvuuRZVMou59nY1qpnWnYT", + "5Q544fKrFoe6tsEbD7S8EmxGTJYAKtTVhAW5Q5pge4j1", + "675kPX9MHTjS2zt1qfr1NYHuzeLXfQM9H24wFSUt1Mp8", + "8M2vi7fUG3drpp3ceqWS1FYJABbfuN6rTovqb1LebBng", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "6ge8XwpH6JRJu1cwsuueXXTKLkVgfJF97P7K271ziYZ8", + "8pyTHXE5aj1FiagQuZQdLSAQ6NnzRpfx5DQmdVF3R3ou", + "2zFq1Gx86AdfxEUbdkwy5p9yNufsCTngmb6ynNgBSFvX", + "AGR3AuTi6Hg4CExRRXEypkuC2JFmb9i4voQATuhg5pzP", + "5dEj5Zj7UDTYAT37eEFjUqMPN89SBCQ4FrtQRnP7Nna7", + "So11111111111111111111111111111111111111112", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "9R7CF43vYjVBETyLc3cKXoD9gYDZpNNNRqA7zGqkbXaw", + "9uPtC3Pp2mFVGqeUFUhDU5hQQ4kHaZTCgYcw4xkBgW3k", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "8vK6tXTjS9HxEyiqR17EXpCynN7UEpjSZ1j62WYDSP3c", + "DyTrCG6egN7vBkJMvjBqevq88oF1Gez6fVER6oers5RY", + "BnFXV9EQze9hK2CxhyS5dt6XUxMHG59RHqr2hSd7bPrE", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "4AcTMy9kqmeMsRU1dKPnTxEDtV4b5pLbzGThMvuP6kYW", + "5Q544fKrFoe6tsEbD7S8EmxGTJYAKtTVhAW5Q5pge4j1", + "675kPX9MHTjS2zt1qfr1NYHuzeLXfQM9H24wFSUt1Mp8", + "5Q544fKrFoe6tsEbD7S8EmxGTJYAKtTVhAW5Q5pge4j1", + "8nZGAZcLJkNmfpEKmF528g2xtWfq2QKu5BER4wQiuSGJ", + "wFGgVn1kA4EfXjqTRMZQKB284aB38uQu9M8es9EqYir", + "8pDjYSSnCMkgyBw3MzTgjMzrKuMCy78AA9J8uE8DXNwr", + "675kPX9MHTjS2zt1qfr1NYHuzeLXfQM9H24wFSUt1Mp8", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "FyB2Hna9N4fRkFSeJfpMQNnHvwa6pm3NASFwedejgsQz", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "5dM31fDicQFx6EcxQkkZULbHvYDpa8biW4ebRCqxhhEP", + "AP4RrAALtGN6ZfmAcuozvjTGteXuDHWnG8HoFJJJikrY", + "6ME8x7DdRpeMH6Uo4hDFMGYqdWL2FZz1FENTC6g2KrXX", + "A2KTgfny9DNoEzLKG3XuYzrUj4nn3VHPfhUsL1zp5UsB", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "So11111111111111111111111111111111111111112", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "AD9XuRMvVjM6htP7Joemmb4iR1DjVpsa9R7ph2LkMiQm", + "2sRRbjYczEkP4NYgCpKgyecnbNkRL6qzGkAiLzuEh9CJ", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "AAfbQKvhupmNxLm8b9c3L3LuV5vay9gsptJstRq4u9K7", + "72UrGZ2mWPRQdEz7kxc47d3C2UaxTcx8jKGQtYHR5TFY", + "So11111111111111111111111111111111111111112", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "CEiFX3FW79p1bK2W3AUvqsxVw5j6HbmYD3MnA9vZZqpV", + "6gMFPP8LUgxBmNDVjrt2Zh9ZpBN4eRRafhJkBHPAJf9C", + "5fNLCnbAwdez4J192CS99VkS95cknSmcagqzPWiJViTh", + "22D7a1TzdBbzYiCqcsoGt9TLB1aGkxVPSEaxfDCw6Kgp", + "8yQ73Jhkujg4U1euSsiMm4CQMS4EgRviUCijkRZUdQth", + "HCH5RbY3Re6sAKQWibyniGEjvkWwHdmYLepkurZTmU3M", + "BT4AHn3S9PFcHttrxdvheAsRj3xSg9LbGwSbpgt9oXqu", + "G5h7Q7h1yQPQXUELZKU8vaozXe8z7hUUb1dUKVznz78p", + "DidXPARUqi7L8rFjmjtXAdz2kMdFnW1ULFNWFYUvwyDG", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "5FhbPaEY4Xiu7wYs6gdXkEZejt27eyzzzzAdNHFKUrpz", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "HqHoNmB6PEQhh5Wxqh8jt1XBHursxjVemabrs85yTbiK", + "E6aR9mLHPRNKQ6yQrrd8LpisNPcXPmX4BAg7k3PF96Fj", + "Dfr5rLAZNGttUGe1GWtAEnVQTYtDbgKqFAvYzLAous3b", + "84EXYcJ2buSchArKuRgNrpGLbY6L7sjnzUadtMFKy4W5", + "J2YLNZ8mcTVz2ruAVmAp4W5YoKe4LJ9YfMdmwP7TC2gf", + "So11111111111111111111111111111111111111112", + "HSDw4TCiHXyz4E1GMhPP3uYEvDkot7BnMKCpzWen26J5", + "Hx9Tcx43asJUtoXMsnPwHa54b2nrULHupbsM6GYe75bA", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "DRX8DnstFNhdNLfX5QQkEKauvccZvDVmLPwAVWiYkUym", + "So11111111111111111111111111111111111111112", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "EtGnBapHRZ7nhHrLNMifWcrM4LxVDfDrrgGh4eMboCYb", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "7rLAb28sQBmjLSGJFFZEs7iArWv8y9YREA6e5DpbY92b", + "A2xbmUKWLEAf1mts3zEsVyS5Eo9dGGgABHVcDRxaJ4pR", + "5WosWmtG1kaTgtNte3KGkUUm17c9CYwRkRPh7eDA5aeJ", + "BgDXhbzKG8hSdmvpHqhGWmLHB55rV4ALHrrQ1AipVYU4", + "BZZGRhQKmmwXV8LcCHyesfUATcXr12PZbXuP8LaLaj94", + "3Gugj1v2LcnJdhAwqhFJLYn7wDMNeEuzjQzRWNZY6R12", + "675kPX9MHTjS2zt1qfr1NYHuzeLXfQM9H24wFSUt1Mp8", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "25gS8TjPMGEn11jiRuoAoVnrmkjNyfEdB6UudWcdxvmo", + "5Q544fKrFoe6tsEbD7S8EmxGTJYAKtTVhAW5Q5pge4j1", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "So11111111111111111111111111111111111111112", + "HrXMJUoZE1sxpyWKs1EKSKo4jLvrpbKqRjDG5q7NYU6E", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "5S7AFtW51wyYCviqSSKmAGB3utrw6KfKjYpegfP4b7kn", + "75v35atxNsS2r3f2C6cx6Tg9n9KBaG6gzELANkjnscsq", + "2w6dutfnvDVCWPzgXs3buPGmP9CZ945Yvqp8QN23FMkW", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "C4ASgExdDGB39YWBKRSLt2mZtawbqmAw4LZNnPrBHCJo", + "4RBRYfjB2TbGySbQYCotD9KwSpuK3nf3fzpDeJVdj5hX", + "EZnFe85fzRu1PGgbjhgr8pJwhfUkk1GjbFYYWwvSqPAY", + "7kuWyHexZMpELvGEKQ7rdwoRs4X3gWRtKx6Y2rrbZBDZ", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "JAC1q1jFQWEAXhUymdn92J462S3Xxa3QiYGvkRUZFSY", + "So11111111111111111111111111111111111111112", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "5Ht7uh8zbS2xRwaUAQPUrBZSPhgijtfPXti5dGcbD9kJ", + "9cxpbz1tNgXghtUeotsks2wb7eM3uJxPiFoWBddzqXJX", + "AthfdSs1qS629Q6jg8uR8kqq9oa1wERcsU45xds9Upe4", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "6xtmqkeEtUfatyoJ6Ca2eddwJPSYhkiNDWuoNXab4Lrs", + "BJ3JHWf9FsWenPQjtanjDVtkVW5nGwyUsDULauS1UVXd", + "So11111111111111111111111111111111111111112", + "69X4KDTKc7bmF5egpyBQiqpkXVa3tFT4m1Z6gxgZH2gV", + "HLGJNF7hEKLNVbQnPiBfHjVbEFYsHgVpqryDT8pZLNBm", + "BDRWynfVxj7RCQTLT6FnYv1YPbJhsFc3GnDCnbFiaYAz", + "So11111111111111111111111111111111111111112", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "qQvYLq559KSwANCY16NbW9EVrVNoEtmHcdhZaUsFV1o", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "3C3WRamsi6et8RgbbRtBmU72vKpbR9XeZ312iZ6pid9v", + "4DHZxWy5kAibVBa3qGpHGxSX1TJZqYceG92czfgRxGcn", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "3ccnaAvBpRrWaKJPCmBGhuFvLp6bBMrV9jh7MeRQvqsf", + "HHY2S72fGahdChph4p49utPCFduenmzezmRyQamsvze8", + "H96ciFcq4bPQVXefpbLuBj5JBZYCKaEvW7oHhM9DNqGC", + "So11111111111111111111111111111111111111112", + "3vPyCb43FTxyteivi9KshQH7uLGfwDprmpVDc8gjd5VX", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "FxstiPnZtTiAik4yFgY9SELaUu9npzxpJhVvRgM99PYx", + "HxppmLhwsNDYNKsDzouEoHy9EDT1NXfmix6KS22EvBBJ", + "WuadYTGARof5YYxd876aBn6be3dCN8k5mWTJyW3Q135", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "BW1biC4GAtCmBDzQ9UqZ1Q3MpoRnz6pLmyxJfx8eDtyZ", + "91adT3fDiVUgnanLM4rwpiM3FUb8nR3ojFe6bYdsfXSA", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "HNLypx8BWKFf1DzsjaKNmRHTqb7MXEfEeLTBsmxi93VU", + "7oT7tHw2CJ8atUtbTYTuAaFgae3p8JA48TWna9g62j1D", + "AKRe2S5h8VuAUPtTZPr63zp3HU6rFuKCgA3ATDKkFbqL", + "5ZbCvmLmpDsVEmUBq65ybUS7FcuxqxyCj4JaXFJxiisw", + "ELjqv8ypVHxfuV1GghA4LwSjtQVCtrg76EMr47F4UGyU", + "4oMjz3prvTM3msvmkuLFTEpwiqEMnr9PTvpsS8Peufgs", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "DyFeoPZAUsRA21HBwAKhUbGaBkeXeDRqeDxvwE2pLUao", + "C3ND55ndF6EwLBBVQN5gNrZoor5mkwdZaKB8V5ttp4rr", + "6XxEH5vXYfLdmah5nGDDDzDTN4Z1otzUFtd659FBJ1K7", + "FSHZYEUMgP1RrbcuQJFu54kPWzaxpodvqdAadNjubwwd", + "So11111111111111111111111111111111111111112", + "DgyH9xpdXzM9vnKPJKFv7MZpPGxi9VvY8qWNnf7gGbUc", + "GLgfkAXeKJaw9exZcfLBReN23oCSMnqreCgBZivJghJ3", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "Db8GCqMeT3iTjvAjGxTpUZq8hntViNGPrxsHTsmEtiVP", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "4QEA3xeLVf35dFQRrrWziHaYC4rh8SL57SpN15yirmAt", + "675kPX9MHTjS2zt1qfr1NYHuzeLXfQM9H24wFSUt1Mp8", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "5Q544fKrFoe6tsEbD7S8EmxGTJYAKtTVhAW5Q5pge4j1", + "HMu3zRg4E35Bp8uwHCyfxCSRYzbFX46tWDRF8JV7KJVi", + "AwHVPct2LYNXvxHRCKKEEsBYEZ3B9snknTGiucGMe2Rm", + "Aa7BSLq6q4SCnU5aRHTsHpEZQT6Gq6a6bAd5t2vJq1Zg", + "5YnVJxj1ks5UsKeTsA442g8M4CK7Bk7ufJ6YfPXb7VMG", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "6cxhM917BysbRbkNmXw8LzpBmtjXgGc1v6rZSfWDBq3X", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "BHMk6S7kkpkZmQyMz5MGVsBczBBvJcGAyWoiwEnEcvF", + "APcNDLsSxpCdYNWgSBSAeLxjqhuLH5SE7vL2PtfywMU5", + "B3v9Y6MJhU3pBN7QvxN1CrnGx6LkqwrcYkpQy4onYuoG", + "8cAUKJmamdFC26jJfUyGJ8An4w35ZkzRQdm5BbmCsAS4", + "8vAtVAC2x3xPNKtkzgS9Cq53HWFfsrdBpTUiVcdHL1CH", + "3LGkwrCmvmKLqJMwW8qbKqesjjXvnVNb2wx1gkscfjex", + "So11111111111111111111111111111111111111112", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "3e8Wy2HDbp5CCXZMqomiYn8a2Z9SkoZjxrPyVtTPWNq5", + "2B7u71AgybWwe7kG3DMpUpK9LJAk6PASjvS6CmkAyJvJ", + "AMEniSd9c4e7DodeaKsfVsk6j1AFRYFLpcP5QEjTTtga", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "8EREkeiTc6TvYgqBb7kzajUSeKsV4zH4GQ7Mw5bvvLZ6", + "DAT6ZuLMadPU19ZJJSNcLKSvcHXzA75vG51Q8nYDTgNq", + "HnN3a1YqbubdGAwomBttivJdzJZZ6JJocRLBVnj6zks7", + "8beJ6UbxjLNQMRJtoi8WjN9R6syans5iK9Er5U2jS7JB", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "FKBoWdGHEvU1BDv3EoTVNQhBpSqczpWihmDzWG1o2fZr", + "FDbmsRfPijgLbKVTfTCAQgH99LWjyGc3JpryX3zrdspu", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "So11111111111111111111111111111111111111112", + "EafESzBT1CpqGvr5dpJT6L3vGxr9THe58uVeGVDQoi1z", + "9Z7zVofxG7Vy1BftLkJuGhW2zFqC1VrAT892v6gKE5oq", + "GrVnBcsVVZSTjK8KrWkF4wRvfMWTrEmsRvzn8nFA2Tt3", + "J9i3NSzjjtn4SugHrSAg79sMxDqsukdDgF2bELQvDooF", + "cHxJ2uC6vgcCfoFSfupkfCWbKHAkekrGfG39DXRamXT", + "BpzrgCPt1p8mF7CGqztKoSasmina8U5PrwL9NHpEWY26", + "CAMMCzo5YL8w4VFF8KVHrK22GGUsp5VTaW7grrKgrWqK", + "76zcEMk8Am9jsAyxAPGp7Xat3MeWiUUtYuGVXdUxfEpR", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "63KLVrUJSertEZe5n7mMZGZAe8DRVdx1FuNtmmbPb25W", + "3R9kpTXH5uwFjnAPgkyaQxJywK2CbUam3eRPSP9g5yDk", + "4wMF7JFeBoh2yeSedsH3Gh1SM79HuEFs7RpN3tpq8Bdq", + "3XCQJQryqpDvvZBfGxR7CLAw5dpGJ9aa7kt1jRLdyxuZ", + "AwaVyGAF3N6K4YRUJk55Ui1GquKhKdD6vTCmVvCzXpKf", + "Fa9ZBfhcGaF5goMa69cjjyn5yoM7Cpnzs2xBpCgwQUN5", + "2QCh9jDuZM2W6CwAGXpVNpMcUUUp61etEu5ut7MYcWWX", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "Guw7tnpdxH6JaQWNQL1AP9TJMfASs34qWuE9NweSnUR4", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "2VbuXWfvWGg99jWVmGtvUqs6GKV6GJLA9Rfhzpsoqjwj", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "So11111111111111111111111111111111111111112", + "B6N3SVqhX4VD91irciEii4ne6j1NggiUMn9fVBZpqSwa", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "CbpnmkvmcDSxPZ3h3YCW3B12E7BFFwLGxT5jAvZmBJhi", + "So11111111111111111111111111111111111111112", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "EUcGWAwzywNZRTdBiCxbsGZCs3jFKRC9JirnUhhc7yqM", + "HdzH1U19c5TRdpsTCfnSMrMMgHMvc2xGABej73mKNGtd", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "HSRiA7zH943cJc6fK1C6kn5xhNJTb8aEccrZYzDa7ruy", + "DVRxk9HjK6hDzQnuuzZnSMbFkVt813eGGDWz4Cy8V65j", + "UE5Nuf2rnhRgEcYgF74rdVezDyftF8ig7VbN82QyMUX", + "51iQfWt31AwvEZHhgFoFugwaYbMRRA2cvDrKKfCvx2q8", + "So11111111111111111111111111111111111111112", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "5oZ7ewKiF9LJ6j9DcVUVUz9XR2RjWg8BxAU1txfnnQMd" + ], + "CzGXBtzCo3SfE1y9erkwa7t7ffT7q3GskAeAp9o3JV3S": [ + "EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "GL2Mk1vT9bMR6jXyguZzbpQuXoiYGUYUf7kivqBAfQNC", + "BjXnXhAhXAtuqJnU9ux3RGHVXTnBL9sdxqeYoaJKu56J", + "8gPqbUwxpowytL8MygKmJEiX9NacK7GfUJC2zQMVwE9n", + "vmaHUDMpAokmx9oATcTs98u2kjYMq7H2Xt9ixPDVwpc", + "D1ZN9Wj1fRSUQfCjhvnu1hqDMT7hzjzBBpi12nVniYD6", + "LBUZKhRxPF3XUpBCjp4YzTKgLccjZhTSDM9YuVaPwxo", + "35ubfx6k2L4bt2pKC1UmRfrgoE9A9wjuiL4E6KajCN5A", + "9uTwi12PcJ2Zww8ph28FKHYm3bpBt6CQGBjjHNM1iQp3", + "BmHjxevUTigfnvubFCT7jkacwovpxYpiVB8Bskt8PeRx", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "Czsxx6xYXkdJsSGkz5A743hTCrUDtw6w8VCKuV832TXz", + "8FmpaWRBiecmZmr5DmhGQfuKCK7W643ecgx7mrheR5XY", + "5oFVHVQkYziZ8QypUToVKA8xwbBuGs5KBwE9fJ4ptwkm", + "DAqKBpXcFEjXwWymaSwqFNpVQcts6fb34bUzRWHfndGV", + "9JAvfhbAt8a5Tfc54MZnaFrrGQuHkek5Zs3D246jcYQt", + "9cczbSXSCVXnUnJ48MXwcvyN5TwgUD8nL58C5WJEnkGT", + "whirLbMiicVdio4qvUfM5KAg6Ct8VwpYzGff3uctyCc", + "2MDeHweDJqwbMj85jTDoGceWd3ETLvbfdfEP1QSWwi37", + "HUL48DhWrRwp1vS64MyWgpR4m8VHBCTTcQq2v67kJ8ex", + "5Lq5CGAr6KDrZrTnPA6GJtgPkpvB5PjBC9FieMfpvFrR", + "AQGHa4DGJiUNx5ZxMrvDK3ggCu2rdc6L2TBtV9ivSFKe", + "7WwrbDnbFgDuavKXTQTeQpoEPCntzsKHVTzLRsfSWmfR", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "DxtF5yPWMWunDKRTdN3BMLsRAdieRPa2c98yQiKXk3jy", + "675kPX9MHTjS2zt1qfr1NYHuzeLXfQM9H24wFSUt1Mp8", + "5Q544fKrFoe6tsEbD7S8EmxGTJYAKtTVhAW5Q5pge4j1", + "EUdC3fTipZTAg9yPfY2MK9b1HqnwGb9J8Hi9mEPSQ6Nq", + "DN5Vy8R9EtFnqN8RoB9pzSz88h8a8EnSzrrVTx93BFQ5", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "xQyoLcArag87HQLwrwEaszUugUPo59YkzRJMCrs41Py", + "hpLC9YSTwqyLQ98sTG5f9JxsAKC5u5fbPxFsFCBgPn4", + "FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X", + "4EaAtVJoxJQprek3Bdj6wuCDuGEbZE4MLPMZwjRnrLYK", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "ELWFd489g6Wu4i2DHWLULzrH2XqRhrbhktVjE97UaHy9", + "6ydfYUQNrnQ7XEXa5WTAzWHeSHE18iyhZhY75EVk417h", + "7cmnHrnfruq6TdZW12V7DUJswh9LqMHv3pAkcdba1dwL", + "imf4uXJcxv1kuKB2uNtHd36FEVfsXUNB5WKnKtYrFLX", + "DMLjiTJUwdNZBCdeffZkWW6EcmXhsBuk6GYvzUu7qUAN", + "3jnKcNuCVUC68xLrKmUuUCnHLwWNbsPeFNJtGidDftG7", + "So11111111111111111111111111111111111111112", + "675kPX9MHTjS2zt1qfr1NYHuzeLXfQM9H24wFSUt1Mp8", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "5Q544fKrFoe6tsEbD7S8EmxGTJYAKtTVhAW5Q5pge4j1", + "4JWkj4Q2zibXbKFTUASSthq4t5mFrUZ55TKGuBSTRj9X", + "BkZto7TbiHi5U1kVyzpr9JhHUYSDZu7hQfZ1p6tg1fTJ", + "GvgjTVbzjv8UGZ66SQwdbNFBc6jLjt8esW6SiQTCgNGR", + "4pKEhRDDednykwHSqF1TPe2e3HWEqM482HTtZdNC2eGB", + "5Q544fKrFoe6tsEbD7S8EmxGTJYAKtTVhAW5Q5pge4j1", + "AxKfTu2gLEVFyz5SUJtPPakCjSSgUowDRSiDkQaLsDvx", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "Ftt8TtCv6qobzHrEx7XBD4qn5XPNtEAXkgcyjt9HMbU4", + "675kPX9MHTjS2zt1qfr1NYHuzeLXfQM9H24wFSUt1Mp8", + "4H7vGEf7U8GCsRhYwBWAuJtnEHE9g7BEnDZ8ph8K3Eby", + "5Q544fKrFoe6tsEbD7S8EmxGTJYAKtTVhAW5Q5pge4j1", + "8hoAaPXbwRfUWQB3eEieacGQjsQ26aGrJKNX1UnNJ8Mu", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "675kPX9MHTjS2zt1qfr1NYHuzeLXfQM9H24wFSUt1Mp8", + "62AbWqKiCaccLjh8Gnf2fmva2Y2YUshnqEsPXeEp9Ved", + "675kPX9MHTjS2zt1qfr1NYHuzeLXfQM9H24wFSUt1Mp8", + "5Q544fKrFoe6tsEbD7S8EmxGTJYAKtTVhAW5Q5pge4j1", + "GnSsyGgxPkXaGJVFjk3s6EVDDChe5KkAwvLpamrJBqZ", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "CnJ1HwC33NDy5CR3MPFvWp1HWTWf6n3V4sNMSoEgwGMz", + "GjqDEppSu9NSVgL9cWZm3pcgNcjonbuRxvvEDafSmB5a", + "675kPX9MHTjS2zt1qfr1NYHuzeLXfQM9H24wFSUt1Mp8", + "5Q544fKrFoe6tsEbD7S8EmxGTJYAKtTVhAW5Q5pge4j1", + "DpPLLQ5h8Zk5tm5iJqUm6wUk4YVWfwy6X3nZ74kZnJsu", + "66LVU82NYzgVLTUu4DBB4HkxL3N7V6L9hE1bNZERQjve", + "675kPX9MHTjS2zt1qfr1NYHuzeLXfQM9H24wFSUt1Mp8", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "5Q544fKrFoe6tsEbD7S8EmxGTJYAKtTVhAW5Q5pge4j1", + "FYdzffNt2pq1pNRhBW1xU3TEeuaedWJi62JRZvtvvmkr", + "5Q544fKrFoe6tsEbD7S8EmxGTJYAKtTVhAW5Q5pge4j1", + "675kPX9MHTjS2zt1qfr1NYHuzeLXfQM9H24wFSUt1Mp8", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "7NeiiocJxjHCnLGi6Btbb8Tttn69aCmxSrMv2eZyWoz8", + "E2G9wv6rsWtfQAtbJRR6nksEWoxcvcdHYSSNdEWKBbNM", + "5Q544fKrFoe6tsEbD7S8EmxGTJYAKtTVhAW5Q5pge4j1", + "FJHjNMqjEZ3FgKnKmXmaymFTy9eQVJZYghB1u8ivVRBE", + "6WarRfSVZAtncGv75VKYF8wsdbzr43gyJFbBhx7BMwXc", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "4KUGxXGrJX6vMAkeK8pLkaGztFT4gLpmDe5y2ZQfZGyh", + "675kPX9MHTjS2zt1qfr1NYHuzeLXfQM9H24wFSUt1Mp8", + "6X5fJLeCptf5RGAe1HXbxsW1zqzaHiAsDJ23hgUkBhmf", + "8gBZynFg3jRxB2LSCHxJoLgPZoCV2oadvq2T4H6nDFtK", + "2EcfU7EtvzyuGKvA45xrUcKGs2SifyhGm4fvMn4R9hqd", + "FMnXaGzpb96Wsj4k7Lniz1Vpb6TRj678YUa7fAM9k6GC", + "4q2WtRW2Z6gmNjNpHx4StvjNtfeuHVkSGWZzWeBh3GUj", + "LBUZKhRxPF3XUpBCjp4YzTKgLccjZhTSDM9YuVaPwxo", + "D1ZN9Wj1fRSUQfCjhvnu1hqDMT7hzjzBBpi12nVniYD6", + "5Q544fKrFoe6tsEbD7S8EmxGTJYAKtTVhAW5Q5pge4j1", + "8MHfb66n8XNniCW3tPP7Rf2BS4dghra7MtSrscaaACmN", + "ASTMdxyZFiHCTKdZZRyXBZ3FKugLMrn3kDTSgL7iapqT", + "675kPX9MHTjS2zt1qfr1NYHuzeLXfQM9H24wFSUt1Mp8", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "339bpecxp5zfZ8aFojwtXQj2yg2TbP5UygBe3QnGhrUz", + "65Ya5QevHfXW6PqpWK6pLbjGHy1n5jG7FVGpnuHfX1i3", + "675kPX9MHTjS2zt1qfr1NYHuzeLXfQM9H24wFSUt1Mp8", + "5Q544fKrFoe6tsEbD7S8EmxGTJYAKtTVhAW5Q5pge4j1", + "AKJfzCcvBbbDKJaNRtuNhhw5N88Te1HsAEq8Zwqj4jgV", + "675kPX9MHTjS2zt1qfr1NYHuzeLXfQM9H24wFSUt1Mp8", + "84anaN4A25vunCMRp8AqVbzgesxhLvg9XYuPMknD1hHg", + "5Q544fKrFoe6tsEbD7S8EmxGTJYAKtTVhAW5Q5pge4j1", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "7idkUsRM7WFEChuSFER93Kw3eummV7krFwJmNzWnrLxJ", + "ADp6ZMCawqKwGVpHdGBdDch42si8mbcCTC3mqGd5aUC3", + "675kPX9MHTjS2zt1qfr1NYHuzeLXfQM9H24wFSUt1Mp8", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "F3ePq9ixrLgFK1bWGZPZ4bWNkxCPwqnhofjvop8M4Jyi", + "HQYqe35aHJjXXYtwUYPzAJvBzKphwLp5YjEeZbwsHZ52", + "5Q544fKrFoe6tsEbD7S8EmxGTJYAKtTVhAW5Q5pge4j1", + "5bLytqFhrKwauLojZMUYKVe3ZCFuKA8WgVsLxQDhGCvR", + "3mj75ZnkaYEjYGgsxbYEESRDzN3S2Lkg3HKhdhVgug1x", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "Es9vMFrzaCERmJfrF4H2FYD4KCoNkY11McCe8BenwNYB", + "JDMB7phiBywTJG1FnkwNVV1ceUipz7uVMvQCBBn38erg", + "DWYRBXTXpMuxHqvLKkTCKXCzcMpqiB65Xc79ypb6pcfC", + "So11111111111111111111111111111111111111112", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "EiPJofMWLJ5qCkbU5Fx745Rf8D3XHEXtijw7Cuiqtmk1", + "LBUZKhRxPF3XUpBCjp4YzTKgLccjZhTSDM9YuVaPwxo", + "D1ZN9Wj1fRSUQfCjhvnu1hqDMT7hzjzBBpi12nVniYD6", + "EaUNXo9fsk1VUmZGdGVdnHhbQ52SAhisAiTKfh8p67VW", + "5R7BTFfATpuPDWrzXVHWRQugMtRCZyW7TD3qZgQmTfp9", + "EnLbS3xSzoVNXgFcnxKtfxgo3AMaHfrtCeACPWxLgQh", + "9wToKSvHSCjNuN79aVsAT63mA2gHUDSHxkYBrMdmPZKT", + "F94iVB7PgiWkuzaVNGsimExUyZdTHXXYzAdRHSyKrWcp", + "5LgNLDTvjV6nKQHBEZ783VDoYN2PqGNAKDWPg8wCpump", + "ECw2X1TYbsrqFgdiYApNpn2ggznbj8pL5tREpY9Fb8jY", + "GFwsANMCPK8W3WhqTnwVP8JiwHaAr3cNDe5TJqgHHSPe", + "FJknsCSwypT6HRmnqZqbm8YYhHcyduJEM9yqQvasN9N", + "CAMMCzo5YL8w4VFF8KVHrK22GGUsp5VTaW7grrKgrWqK", + "9iFER3bpjf1PTTCQCfTRu17EJgvsxo9pVyA9QWwEuX4x", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "75NAey24NfSn7EM7RcqJANh4yuPSCTQpTzNDTbsXABwG", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "DiUQsNo4isWFEHWh39VxcQ16JEmpBYCoD7wsTcmpraqw", + "AgQKDVfdJj1cSJGiVBa6RzMTjQYUPTUR4TrZQrZNvsj3", + "FhRMZ1iYU1AD8huPJHu8GdjVc19tKZKRtrqQ5iMgWzjf", + "675kPX9MHTjS2zt1qfr1NYHuzeLXfQM9H24wFSUt1Mp8", + "5Q544fKrFoe6tsEbD7S8EmxGTJYAKtTVhAW5Q5pge4j1", + "FYZT74kDKrSjymWzAhkEbHYmss2Xb5ofnu2v2cAfpAX3", + "5ACsCWMbzJvaePTeXiGhLvTrrjThf3SDNtr5FoxBDtkX", + "5Q544fKrFoe6tsEbD7S8EmxGTJYAKtTVhAW5Q5pge4j1", + "J1iAfYcofFqMw8sUfSeKLqqgRPUT3EBEeoSpjudDxkEt", + "675kPX9MHTjS2zt1qfr1NYHuzeLXfQM9H24wFSUt1Mp8", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "DDcW8qVVNSEPx8VcCSg7TvUxWZXuqLdRmUP3CHKQXjHy", + "AxF2noe56hoCyuFTt97DYb6Dyv9vJWQMWNyPY8y52FGG", + "HskbQyJvUMLhFAU9NHHz48xwvWHSAaaNBcvdQhqpusHx", + "675kPX9MHTjS2zt1qfr1NYHuzeLXfQM9H24wFSUt1Mp8", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "5Q544fKrFoe6tsEbD7S8EmxGTJYAKtTVhAW5Q5pge4j1", + "DbiEnmwpBUDuB7mXDQ6eiCUeNkMKoH8yVuLB5TFQT948", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "4HSR9WBSHgw7n5V8WGYYhLW8g92RPSrgnf2CHzbeQrrr", + "2UQncszfVU7igwDiGN3jq2sUKzziLxDZqNsJEXzEob5x", + "2zVV22uNWdJNmkXpj5vCrMzwHGBoJdsyV7qACh29sK1w", + "JUP6LkbZbjS1jKKwapdHNy74zcZ3tLUZoi5QNyVTaV4", + "CGQfgxbZtfzZ9nCDjmD98pjjy6anLEgt1Diehk1DXTqs", + "7EmsKh37f9XqrbCWJh6mWd45q445nEpvffxJWCeZATqa", + "8P6WDZ59qwMrTgXWaxFN5J19dGwvY2XEavCyRdA25PNm", + "So11111111111111111111111111111111111111112", + "FFrXjdxtx8hT4vnuD9KmmKBzD5GGuDnmu6kp6TiyUEbB", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "D1mpr8pWbZNLeLeW1tHLYdqToQjrBVFBef4fudA5UeHZ", + "LBUZKhRxPF3XUpBCjp4YzTKgLccjZhTSDM9YuVaPwxo", + "98Ti6U7v5WxuKc6jtz94WdnG4vQZXDs4tfVa8926BCYS", + "FN3onY1uJqEe6rZxasVr7YDmWSpNiizr957ucksxcDe4", + "2MjmSYy3oTT7uoiPzDtjWRu23yCBuVdvpt5AR6jbkSWK", + "D1ZN9Wj1fRSUQfCjhvnu1hqDMT7hzjzBBpi12nVniYD6", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "So11111111111111111111111111111111111111112", + "5XhS7ue422PeJoCeQYbxxuwKx5pc9awtpiQ3JaoYecfh", + "GFiyZ9NNjRwWmAzQQFwtmbjkjVGp6QAgDqwuZbxG6Y1N", + "JC6jf3UMSD4UbDAjsbkDB1n9vYxgZPyMurDJGyEU4oNW" + ], + "DBmHWCVEGCzZ3zDNr9WzRaMmSqCjcixMh78imXfno9qJ": [ + "EwfLtdve4ojEg8CD1Ezq4qQTxxq5uSZg5ptdPbQE6Czf", + "8HVcoWfV9kbEyDKker9DhDuw6pstx945nPu5pnzE8xcW", + "A8Mw1eJ58A97Z2DrU6grtDRSeHd53cBMMc5qJDREhNJu", + "MemoSq4gqABAXKb96qnH8TysNcWxMyWCqXgDLGmfcHr", + "714DNwePzuJhvptduRGp3gsY8UvnD6MSpQUYeNyf5soX", + "EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v", + "819s5UgM3Yy7qzdBufKrRWhs3wXNW4Q6H89AFFcCLiF6", + "3dgoWY5oP1Cye59dDtGBuCGudvtNNSNERwHE5tPpdwgr", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "Afmku88s5uARyS4J6oDNDkr8iXxrNwZiBVopzpfBk6nz", + "DTTMBBPrZqA1J7nGRLvZcND6YqQ9CR6nVhXBCq6UKsSL", + "9rPYyANsfQZw3DnDmKE3YCQF5E8oD89UXoHn9JFEhJUz", + "HA5pNHNqsbFyqxa8rUJMF9WC1SthWr9Di1UT7e88amc7", + "BmPaociTLsfQK6svEJ88jSnwwUSDFfbXCyfmFFXmRMyL", + "HH7Ke7iKeWXBkhyqvKGvzQ2Q1X268DYUcjKag9aE8iYr", + "C2aFPdENg4A2HQsmrd5rTw5TaYBX5Ku887cWjbFKtZpw", + "pAMMBay6oceH9fJKBRHGP5D4bD4sWpmSwMn52FMfXEA", + "ADyA8hdefvWN2dbGGWFotbzWxrAvLW83WG6QCVXvJKqw", + "So11111111111111111111111111111111111111112", + "GS4CU59F31iL7aR2Q8zVS8DRrcRnXX1yjQ66TqNVQnaR", + "HQDyAXZVEYBPDHmbnQHXCBTiMLDDd6xtj8J2rQq8bHBB", + "9Xvx4u5FnnnD8pV3RjVUD5t72hpcBS2ATbQSeErxdbuy", + "F8sWxTnNZKoCShC5ESggTbxfktQtR8wEyVnJWYFXxkzL", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "4zJD9oGbTVhW9DRLwZYxFA13FGcQoC1uCq42Kzdi8ZYG", + "CAMMCzo5YL8w4VFF8KVHrK22GGUsp5VTaW7grrKgrWqK", + "GFCJhyxL6BJ3SoxaaPh8jSSHRikg7xVJZWtNLzMWxiSJ", + "3HvobtjDeCVuPzC93DTqGzkPd9BCNuphaRiZri6b4P3a", + "So11111111111111111111111111111111111111112", + "9iFER3bpjf1PTTCQCfTRu17EJgvsxo9pVyA9QWwEuX4x", + "8a6xYTbiYGUsW5gCz3b4MmMuph9vNhs2ZBAngofmqRFG", + "Fu3b6TniEwrn33sHXbt6Up8pmG9sBSU5XSjdvioS22VL", + "7XVJAvnCUHYoYS22wprdXTcAdyB5bXXbJuT8aQFJhDty", + "G2AxjNpDSrgeuev5CUWJ46smx3FTQjjWd7rsWiuKRLrD", + "72c4bmpS6jgNAv5yXxW9uNacm1cgtYkrxxWHaNqXRVFk", + "BqaZiuYoDGSQKMVbn6ALSQzuNNNChKEeUR7HDpwqQNvu", + "ADyA8hdefvWN2dbGGWFotbzWxrAvLW83WG6QCVXvJKqw", + "CvAte6ktvCwxQoDKPvZwXuSCej3n6jjB6ePvmbSyoBiY", + "ByvJc1XfPaBQirdrz3XiTKqPUF688KVLBCYitehe3ieq", + "GS4CU59F31iL7aR2Q8zVS8DRrcRnXX1yjQ66TqNVQnaR", + "C2aFPdENg4A2HQsmrd5rTw5TaYBX5Ku887cWjbFKtZpw", + "5m4goH9C7iCsWfC7e1VKVvsn7UJ87esQQmkgomf6MiU", + "AVmoTthdrX6tKt4nDjco2D775W2YK3sDhxPcMmzUAmTY", + "5PHirr8joyTMp9JMm6nW7hNDVyEYdkzDqazxPD7RaTjx", + "EQgSe7ekYKr8PoP3eyxiGvno2SKUvhqfqP2am8Gpyqyu", + "2qe6UGWxM83DR9Fk4E1bgDHv9GcWQVzcTe9qGpUdBmwV", + "Sysvar1nstructions1111111111111111111111111", + "EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v", + "7vfCXTUXx5WJV5JADk17DUJ4ksgau7utNKj4b963voxs", + "BNrK9LpEn65QA4TyBLVSMdngW3XHj3xLfFPwGdCBv8wV", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "CAEyfBBxYa7AhLpg2YYrEVQkHjuzkgEibU2utzN84RNz", + "3rmHSu74h1ZcmAisVcWerTCiRDQbUrBKmcwptYGjHfet", + "So11111111111111111111111111111111111111112", + "75Z6DvynfYJ62iJGCJcy4JaoHrHezaGou7CrseLDA2Sa", + "5Yxq8Y6berWCfM2iwiDz9WxbPqBLUDHb9UjzxcKT3HsL", + "C67W7c3bWGoYWuoWrM7GnWzYa6tocQBGmtEFP4aPndex", + "cpamdpZCGKUy5JxQXB4dcpGPiikHawvSWAd6mEn1sGG", + "HLnpSz9h2S4hiLQ43rnSD9XkcUThA7B8hQMKmDaiTLcC", + "99idLdEwptVKN5hdJTL8WY4kvHDnDt8QAtwtgjo85ANx", + "ExwiWa6djHoEqbmnu5rmJ82EhLcxibSto4D1troyv5Kf", + "C2PyRMeAD3fGauCaMHzLCfbriCuFfjbkpCtZNkgc1qT4", + "CgUjUnVDsMxLUpjM3mZxJQYUd8vc4bB1RXAVDv18zbW2", + "LBUZKhRxPF3XUpBCjp4YzTKgLccjZhTSDM9YuVaPwxo", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "9jj5qmnczTu2SfoWUDLwgQdEDD4tKWignkRLYuVF73oz", + "D5iwBLtF99KgiPMoQj4c968JaKE4febcnGWg556YQGzW", + "H7qVQCPhtDgTtenFwFXU8fEnGsJJNwDzsUyaed1472Pi", + "4txG4r7heKtg5CMvigRdMfxxHgJkmGSxYfmiYUSoPbY8", + "pfeeUxB6jkeY1Hxd7CsFCAjcbHA9rWtchMGdZ6VojVZ", + "9HuUttNaz1YvtTCnjgxbvB2niMGnCgDhUMk4o5md9T9S", + "2MJBf3JxAUCJ6ZzNm6GhwgyCf6gWEb5ZrqyuXaES8DvE", + "So11111111111111111111111111111111111111112", + "8pzSDboMmNbfA9U7YcEGcxxaAfcNUAVmDCPzMw44iHfU", + "92ewgmNfy94LSbcNxq99LCP4F4tL82NSD6P4YL17FVXE", + "GS4CU59F31iL7aR2Q8zVS8DRrcRnXX1yjQ66TqNVQnaR", + "ADyA8hdefvWN2dbGGWFotbzWxrAvLW83WG6QCVXvJKqw", + "G5UZAVbAf46s7cKWoyKu8kYTip9DGTpbLZ2qa9Aq69dP", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL", + "pAMMBay6oceH9fJKBRHGP5D4bD4sWpmSwMn52FMfXEA", + "BqaZiuYoDGSQKMVbn6ALSQzuNNNChKEeUR7HDpwqQNvu", + "BzNgxjy4iFSxNqqWZxRFWSfPt5NabUWJBiBQcnNZF3WV", + "AVmoTthdrX6tKt4nDjco2D775W2YK3sDhxPcMmzUAmTY", + "5PHirr8joyTMp9JMm6nW7hNDVyEYdkzDqazxPD7RaTjx", + "C2aFPdENg4A2HQsmrd5rTw5TaYBX5Ku887cWjbFKtZpw", + "8N3GDaZ2iwN65oxVatKTLPNooAVUJTbfiVJ1ahyqwjSk", + "mjo6p32YCVV6m33sVwPBqEFjxMHkPEt1dxHeeSJcRgh", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "So11111111111111111111111111111111111111112", + "cpamdpZCGKUy5JxQXB4dcpGPiikHawvSWAd6mEn1sGG", + "3rmHSu74h1ZcmAisVcWerTCiRDQbUrBKmcwptYGjHfet", + "4UfGQT9jtGqn5yWYLAFtSN6HdN1obLdstysGNewPb2gL", + "HLnpSz9h2S4hiLQ43rnSD9XkcUThA7B8hQMKmDaiTLcC", + "Cvysb6LwJKGKX8F2RzBhk5Y2noQ1hZZNGNz7Gc7cnmjq", + "DBLhwrSc72t9MK8XmaHJQ9e5me8wGHXiKQpsTne4kbSZ", + "BqaZiuYoDGSQKMVbn6ALSQzuNNNChKEeUR7HDpwqQNvu", + "pfeeUxB6jkeY1Hxd7CsFCAjcbHA9rWtchMGdZ6VojVZ", + "7VtfL8fvgNfhz17qKRMjzQEXgbdpnHHHQRh54R9jP2RJ", + "7GFUN3bWzJMKMRZ34JLsvcqdssDbXnp589SiE33KVwcC", + "C2aFPdENg4A2HQsmrd5rTw5TaYBX5Ku887cWjbFKtZpw", + "5PHirr8joyTMp9JMm6nW7hNDVyEYdkzDqazxPD7RaTjx", + "ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL", + "pAMMBay6oceH9fJKBRHGP5D4bD4sWpmSwMn52FMfXEA", + "9rPYyANsfQZw3DnDmKE3YCQF5E8oD89UXoHn9JFEhJUz", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "cZxGS9D7QxB1f4a6fdkZcpDD1DHteDmg5deSuZ6tw1R", + "ADyA8hdefvWN2dbGGWFotbzWxrAvLW83WG6QCVXvJKqw", + "Cg4EEGHF8fFJxHM9zdTGdyqxaeSiejRSSMGhvEkojrHx", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "FA6HWKBNMP3SoKedc4n1YzjVPhsumWL7BtGAam4MeRGE", + "D1ZN9Wj1fRSUQfCjhvnu1hqDMT7hzjzBBpi12nVniYD6", + "4SoQ8UkWfeDH47T56PA53CZCeW4KytYCiU65CwBWoJUt", + "EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v", + "FGptqdxjahafaCzpZ1T6EDtCzYMv7Dyn5MgBLyB3VUFW", + "pAMMBay6oceH9fJKBRHGP5D4bD4sWpmSwMn52FMfXEA", + "C2aFPdENg4A2HQsmrd5rTw5TaYBX5Ku887cWjbFKtZpw", + "CNu9qgqDDECdghuaTjotHVmc4WPjxep8jjZut1yV2ABC", + "ADyA8hdefvWN2dbGGWFotbzWxrAvLW83WG6QCVXvJKqw", + "So11111111111111111111111111111111111111112", + "BqaZiuYoDGSQKMVbn6ALSQzuNNNChKEeUR7HDpwqQNvu", + "A8moRUVovgpfGoFgnpjRZeXL3i8Q5mKasL4WtvUtxRVZ", + "pfeeUxB6jkeY1Hxd7CsFCAjcbHA9rWtchMGdZ6VojVZ", + "4FatxW1Q7u24WkHEyT8uP778mNDtwkTaTuSVUCdqeWvL", + "HrnG1QTwTfNHm3S9GxqEdraboWMHxgfgQ5KiBN3bwgL", + "GS4CU59F31iL7aR2Q8zVS8DRrcRnXX1yjQ66TqNVQnaR", + "77abxVXsUcqxSvwwReK31fFa6zQgpv3dVT2mAfqaHJd2", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "AVmoTthdrX6tKt4nDjco2D775W2YK3sDhxPcMmzUAmTY", + "3HqUBR2mY6JV7PKk8S6oMACbx94KDGSuSa9k3wN3pump", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "5PHirr8joyTMp9JMm6nW7hNDVyEYdkzDqazxPD7RaTjx", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "goonuddtQRrWqqn5nFyczVKaie28f3kDkHWkHtURSLE", + "Eq4RXu6YteLSADZ7X3gxMimqGm98uUN9KMSzQjoUqSKP", + "5mHTXRU1bgBDo9vUoHKkCidbFRc2mLHSgTTdQyZBjMfg", + "GbdLNQd31S8Mjs7cTT2mkmd8j1rXTtuFi4dYFtAFXS6G", + "GS4CU59F31iL7aR2Q8zVS8DRrcRnXX1yjQ66TqNVQnaR", + "6fC29w55f26ne7aaTH3Apqu6DWrgKk3oHmaRnd5Hpump", + "So11111111111111111111111111111111111111112", + "Bvtgim23rfocUzxVX9j9QFxTbBnH8JZxnaGLCEkXvjKS", + "4msDBQ7yw89kTYtATfLZvtS8Rq8gcN46C8BsEqfV37XF", + "DpvyjUU5u2BZzJ7aQj8JqEyRinbtNo5e7HrSpWrysT7", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "J8Er5GwtsZVzoaA65YsMQok4oDub1oCXZyVQzNSL5khf", + "MemoSq4gqABAXKb96qnH8TysNcWxMyWCqXgDLGmfcHr", + "3mPsFQTT92pzZmmKQBK6NdaQBZntG2acgBBe3dMiWFL6", + "6SbJjNvXKsPcmFmQJzSuHqnj9WrvGMYho5TGR8oQdpci", + "EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v", + "CsjYSSthNMbyuku1a3ZCpDjrMNSAfCsRFh2fwFmMb4EV", + "8gvW2F1ydbTPRbHXUdJBhebCBDjbaai2Vvh4zioJV8uL", + "3zG7BwJG59XLc8AJDF49r3YuiR6U8qs1Un3Eb8C51q8P", + "5DuvV9Cvc7fAiR4f2u6B1WnAJprR26mrDhRaESmp1hAt", + "5UAyQSt7xjhKxHYB41MhZy95DMzfveTKQXBWDyuzJpgb", + "Gp4Mr9mmpf9WTJHRqceGLTkrRdmNBC5k7MwertGAC1AX", + "whirLbMiicVdio4qvUfM5KAg6Ct8VwpYzGff3uctyCc", + "9zdZ7goFEMDfUsvcQoSdYUw6bvjXaQiPJk6ggFT4ZnS3", + "ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL", + "pAMMBay6oceH9fJKBRHGP5D4bD4sWpmSwMn52FMfXEA", + "GS4CU59F31iL7aR2Q8zVS8DRrcRnXX1yjQ66TqNVQnaR", + "5PHirr8joyTMp9JMm6nW7hNDVyEYdkzDqazxPD7RaTjx", + "2Q6PkrrjfdphnwamRv4V54H5gMXMHaiMXPna91JJAF6p", + "7VtfL8fvgNfhz17qKRMjzQEXgbdpnHHHQRh54R9jP2RJ", + "ADyA8hdefvWN2dbGGWFotbzWxrAvLW83WG6QCVXvJKqw", + "So11111111111111111111111111111111111111112", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL", + "whirLbMiicVdio4qvUfM5KAg6Ct8VwpYzGff3uctyCc", + "2oJw26fusd2cdkrrc5T1pj8DSr2sjPGYwF4Kxj8Lt5fX", + "41TUKbKzGRaAbf7UeT8nQSWhKQMFoe6ZCm3utR3rHmZg", + "6wZ6dD4nibbnYaMNMvnMzvEe7mNEYptDoLHmjHHBYaV5", + "42pVVHB7ytXND35agEvwjmY13fRvMQ9FqBYm6jz5f1zR", + "BgWHbpp9gHWmbLnCWuA2bkguCySJZjmS7W9MyVCXEAGj", + "CnBAV4RuDWj3wGKSPWvDQAXTdTNRot5H1F1C1oZFudp4", + "MemoSq4gqABAXKb96qnH8TysNcWxMyWCqXgDLGmfcHr", + "5AEf22nbmZGMVJgpCnMFMewcjYDpmTXDq3hPJQsXVV85", + "BapqB1fUrhWDaRjaGjBdLzNAtnSEbseH6FTktKN5cUKs", + "GhfoJQTeMnLVBcrtYAjUGz44wrUeojS7vu9PhTHqYu6y", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v", + "pAMMBay6oceH9fJKBRHGP5D4bD4sWpmSwMn52FMfXEA", + "9S6Mox6AhSDpgD9r2xRofJrJwomQxWkpKXpYECtbZTKk", + "9DbfGFwGK5WfyUAS3rK4WhSFgxDeWFhPGHHompfjgPo8", + "6xhYXekYrRUTu1dFA4fjRdyYqQwjLtt3GfEShkWwk8no", + "8N3GDaZ2iwN65oxVatKTLPNooAVUJTbfiVJ1ahyqwjSk", + "pfeeUxB6jkeY1Hxd7CsFCAjcbHA9rWtchMGdZ6VojVZ", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL", + "FpPxsDo855wKU6CfkwWZXitKnxmWCkH8FqrZebAKLNcg", + "FWsW1xNtWscwNmKv6wVsU1iTzRN6wmmk3MjxRP5tT7hz", + "So11111111111111111111111111111111111111112", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "8N3GDaZ2iwN65oxVatKTLPNooAVUJTbfiVJ1ahyqwjSk", + "Eox9qsSH5ZcuccpwvZGL48YdtWe9sEBDtcKkLDhPnCHn", + "8yNiqDqbq5Un2EpYsBNuwUSnQG5hL5HZaEn4yfNxx2iE", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", + "8Yd3RCdVcproCXpWnmNeN9KxUGZqdpSJ4fHXLCXgYEU5", + "2ewXGeX3q6r8y5m9wQEDyXJmJZakB7ui4rwTgABVTRLU", + "C2aFPdENg4A2HQsmrd5rTw5TaYBX5Ku887cWjbFKtZpw", + "Afoz3CY3AAUgLPX3AewKnfJJPQ1euBPX3dxWeDS3Baiw", + "pfeeUxB6jkeY1Hxd7CsFCAjcbHA9rWtchMGdZ6VojVZ", + "BqaZiuYoDGSQKMVbn6ALSQzuNNNChKEeUR7HDpwqQNvu", + "pfeeUxB6jkeY1Hxd7CsFCAjcbHA9rWtchMGdZ6VojVZ", + "ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL", + "DhfYGCwKg8z7wE6fxC97srJYq3p7Jjy71LQbzoJcZan7", + "FWsW1xNtWscwNmKv6wVsU1iTzRN6wmmk3MjxRP5tT7hz", + "8N3GDaZ2iwN65oxVatKTLPNooAVUJTbfiVJ1ahyqwjSk", + "5PHirr8joyTMp9JMm6nW7hNDVyEYdkzDqazxPD7RaTjx", + "6sHgbnU4GpDJSX2ZaAsNTnY7DXjdXyYXGx5RDM5d4bvf", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "BqaZiuYoDGSQKMVbn6ALSQzuNNNChKEeUR7HDpwqQNvu", + "C8L87cpQxUP9Z8vZbuDDfYvcSfQMBEUSrxshPGSJtwLe", + "7SY3Xs5XrrzFff2Fm5iVhLPY8vcopVi9EkaKdMCkHXxN", + "GAo4GdZ8CHcYxNcddg5tFLQug266kf2FpBW67AmCBdW5", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "7M3RKkAoEDq2iMnX68orY17FE3oSimqQzPCsD8trjsjo", + "3nANU8YfAG77sXHiE4rYBaXWLTwDLaosqz7Bw6vHT9mX", + "36KE7jUtoR5pMq8oj5cQoWWz6eQcfGC916xyQg5T55fY", + "EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v", + "whirLbMiicVdio4qvUfM5KAg6Ct8VwpYzGff3uctyCc", + "5LeqXAWpH4urp7KXMpiqpKqKesv7HWxWSHJ1YB2PXxuW", + "CKJjGF4jW5TS2Uz3oxHZbQ4yP1FzyCba5x8Q9pieg6gD", + "CWHPUNsQEdVmM3CQ6e6VxynQQ4ePnu2qWnmgWhQBW88o", + "MemoSq4gqABAXKb96qnH8TysNcWxMyWCqXgDLGmfcHr", + "6fx1CR2hgDqvtFpYD5iNo9iePohZrKFMud9xvuCv4DT5", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "So11111111111111111111111111111111111111112", + "cpamdpZCGKUy5JxQXB4dcpGPiikHawvSWAd6mEn1sGG", + "3rmHSu74h1ZcmAisVcWerTCiRDQbUrBKmcwptYGjHfet", + "GBoTk1ZYtQLPrqHvnnMJ8SPhJhcAP9ge1uSVq83h5wVh", + "HLnpSz9h2S4hiLQ43rnSD9XkcUThA7B8hQMKmDaiTLcC", + "9sZ1YyPLj1K74YkMAoDavjR4gcBMgKLYSzebMctY1Rud", + "YTPL3EuAxjwdqYBXUofThM4zGmgrYBvrVy3Aybxprmg", + "BNrK9LpEn65QA4TyBLVSMdngW3XHj3xLfFPwGdCBv8wV", + "8ncU5YW1CQwvr4gs7buH57bW58e86TDau4STrCJBuz8z", + "So11111111111111111111111111111111111111112", + "EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v", + "Sysvar1nstructions1111111111111111111111111", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "EunHLeqeJKvxnCPQSytnBP63HJVk2fbHceiKKpngyAo8", + "goonuddtQRrWqqn5nFyczVKaie28f3kDkHWkHtURSLE", + "GMCJvYGf5Ex2ARiMquaBDqU6iKM8uiEQkB8jCnoNfHpC", + "7yecFG22heommABQ5svcbQLK1Ua4ZrJsHPiktZ17jfm3", + "C2NrdcR9SUxJ46Mgw3pfyPCBktL2PNMbNreeETzHGXq7", + "whirLbMiicVdio4qvUfM5KAg6Ct8VwpYzGff3uctyCc", + "BnyhAEp7nD4m7DTWrTVVmS714ZENidhYEygvyP38nCiq" + ] + } +} \ No newline at end of file diff --git a/tests/test_audit_watch.py b/tests/test_audit_watch.py new file mode 100644 index 0000000000000000000000000000000000000000..c451e9f0069ed94794af14dd3d6a2bc83085dd69 --- /dev/null +++ b/tests/test_audit_watch.py @@ -0,0 +1,18 @@ +"""The host's key-read watcher: which audit events are the signer's own and which are alerted.""" +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parents[1] / "deploy")) +import fly_audit_watch as aw # noqa: E402 + +RAW = """type=SYSCALL msg=audit(1790000000.100:5): arch=c000003e syscall=257 success=yes exit=3 a0=ffffff9c items=1 ppid=9 pid=10 auid=4294967295 uid=175538 gid=175536 euid=175538 comm="python" exe="/usr/local/bin/python3.13" key="fly_keys" +type=PATH msg=audit(1790000000.100:5): item=0 name="/run/secrets/bot_key" +type=SYSCALL msg=audit(1790000001.100:6): arch=c000003e syscall=257 success=yes exit=3 auid=1000 uid=0 gid=0 comm="cat" exe="/usr/bin/cat" key="fly_keys" +type=SYSCALL msg=audit(1790000002.100:7): arch=c000003e syscall=257 success=yes exit=3 auid=4294967295 uid=0 gid=0 comm="aide" exe="/usr/bin/aide" key="fly_keys" +type=SYSCALL msg=audit(1790000003.100:8): arch=c000003e syscall=257 success=yes exit=3 auid=4294967295 uid=175537 gid=175536 comm="python" exe="/usr/local/bin/python3.13" key="fly_keys" +""" + + +def test_only_the_signer_and_aide_are_quiet(): + bad = aw.suspicious(aw.events(RAW), signer_uid=165536 + 10002) + assert [(e["exe"], e["uid"]) for e in bad] == [("/usr/bin/cat", "0"), ("/usr/local/bin/python3.13", "175537")] # a shell, and the brain's uid diff --git a/tests/test_autoupdate.py b/tests/test_autoupdate.py index ef81ebfa95eee8a6d2111aa1cab6248c8ed0b139..6ca14056861b9222d74729176c364564c94dc3f2 100644 --- a/tests/test_autoupdate.py +++ b/tests/test_autoupdate.py @@ -1,5 +1,7 @@ """The self-updater: a new head that is neither the installed revision nor a failed one is an update; revisions and failures are remembered on the data volume; nothing else about the install is a backup's business.""" +from pathlib import Path + from fly_trader.ops import autoupdate as au @@ -31,3 +33,54 @@ def test_backup_and_restore_leave_the_installs_state_alone(tmp_path, monkeypatch au.restore() assert (app / "fly_trader" / "x.py").read_text() == "v1" and not (app / "fly_trader" / "new.py").exists() # code back, additions gone assert (app / "data" / "state.pt").read_text() == "learned more" and (app / ".env").read_text() == "SECRET=1" # state untouched + + +class FakeFU: + def __init__(self, seq): + self.seq = seq + + def fetch(self, c, sha): + assert c["principals"] == ["bryce", "bryce-recovery"] and c["namespace"] == "fly-trader-release" + return {"seq": self.seq}, Path(c["root"]), b"{}", "bryce" + + +def test_updates_need_release_keys_and_a_newer_signed_release(tmp_path, monkeypatch): + import pytest + monkeypatch.setattr(au, "APP", tmp_path); monkeypatch.setattr(au, "RELEASE_FILE", tmp_path / "data" / "hf_release.json") + monkeypatch.setattr(au, "STAGE", tmp_path / "data" / "updates") + (tmp_path / "release").mkdir(); (tmp_path / "release" / "allowed_signers").write_text("# no keys yet\n") + with pytest.raises(au.Refused, match="no release keys"): + au.fetch_verified("a" * 40) + (tmp_path / "release" / "allowed_signers").write_text('bryce namespaces="fly-trader-release" ssh-ed25519 AAAA\n') + monkeypatch.setattr(au, "_FU", FakeFU(3)) + assert au.fetch_verified("a" * 40)[0]["seq"] == 3 + (tmp_path / "data").mkdir(exist_ok=True); au.RELEASE_FILE.write_text('{"seq": 3}') + with pytest.raises(au.Refused, match="not newer"): + au.fetch_verified("b" * 40) + + +def test_a_refused_release_is_never_retried(tmp_path, monkeypatch): + monkeypatch.setattr(au, "BAD_FILE", tmp_path / "bad") + monkeypatch.setattr(au, "fetch_verified", lambda sha: (_ for _ in ()).throw(au.Refused("signature does NOT verify"))) + assert au.update("c" * 40, None) is None and "c" * 40 in au.bad() + + +def test_the_real_verifier_refuses_an_unsigned_release(tmp_path, monkeypatch): + """The installed deploy/fly_update.py checks the signature: a manifest signed by another key is refused.""" + import subprocess + import pytest + from pathlib import Path as P + app = tmp_path / "app"; (app / "deploy").mkdir(parents=True); (app / "release").mkdir() + (app / "deploy" / "fly_update.py").write_text((P(__file__).parents[1] / "deploy" / "fly_update.py").read_text()) + for k in ("good", "evil"): + subprocess.run(["ssh-keygen", "-q", "-t", "ed25519", "-N", "", "-f", str(tmp_path / k)], check=True) + (app / "release" / "allowed_signers").write_text('bryce namespaces="fly-trader-release" ' + (tmp_path / "good.pub").read_text()) + manifest = tmp_path / "release.json"; manifest.write_text('{"schema": 1, "seq": 9, "files": []}') + subprocess.run(["ssh-keygen", "-Y", "sign", "-f", str(tmp_path / "evil"), "-n", "fly-trader-release", str(manifest)], check=True, capture_output=True) + monkeypatch.setattr(au, "APP", app); monkeypatch.setattr(au, "STAGE", tmp_path / "stage"); monkeypatch.setattr(au, "_FU", None) + fu = au._verifier() + blobs = {"release.json": manifest.read_bytes(), "release.json.sig": (tmp_path / "release.json.sig").read_bytes()} + monkeypatch.setattr(fu, "http", lambda url, dest=None, timeout=0: blobs[url.rsplit("/", 1)[1]]) + monkeypatch.setattr(au, "_FU", fu) + with pytest.raises(au.Refused, match="does NOT verify"): + au.fetch_verified("d" * 40) diff --git a/tests/test_broker_live.py b/tests/test_broker_live.py index 56b9bfacdaaf8c55fb27842bb8d7f2ec82da8cdc..8afa2a2830f29e5f7325262653e44d58734d701b 100644 --- a/tests/test_broker_live.py +++ b/tests/test_broker_live.py @@ -126,11 +126,27 @@ def gate_open(monkeypatch): monkeypatch.setattr(broker_live.time, "sleep", lambda s: None) +class RawSigner: + """Stands in for fly_trader/signer here: signs our slot of whatever the fake /order built (the signer's own + policy has its own tests, tests/test_signer.py).""" + + def __init__(self, kp): + self.kp = kp + + def call(self, method, **p): + from fly_trader.chain.signing import sign_transaction_b64, transaction_id + if method == "pubkeys": + return {"trading": str(self.kp.pubkey())} + assert method == "sign_swap" and p["in_amount"] > 0 and p["min_out"] > 0 + signed, _ = sign_transaction_b64(p["tx"], self.kp) + return {"tx": signed, "signature": transaction_id(signed)} + + def _broker(outcomes, lamports=1_000_000_000, tokens=None): kp = Keypair() rpc = FakeRpc(lamports, tokens, {MINT: 6}) jup = FakeJup(kp, rpc, outcomes) - return LiveBroker(rpc=rpc, jup=jup, keypair=kp), rpc, jup + return LiveBroker(rpc=rpc, jup=jup, signer=RawSigner(kp)), rpc, jup def _swap(broker, conn, decision_id, side="buy", amount=BUY_LAMPORTS, max_slippage=300): @@ -153,7 +169,7 @@ def test_verified_buy_lands_orders_and_fills_rows(db_conn, gate_open): assert res.ok and res.attempts == 1 and res.status == "confirmed" and res.code == 0 and res.error is None assert res.token_delta == BUY_TOKENS and res.lamports_delta == -(BUY_LAMPORTS + BUY_FEES) assert res.price_sol == pytest.approx(((BUY_LAMPORTS + BUY_FEES) / 1e9) / (BUY_TOKENS / 1e6)) - assert jup.orders[0]["exclude_routers"] == "jupiterz" and jup.orders[0]["slippage_bps"] == 150 + assert jup.orders[0]["exclude_routers"] == "jupiterz,dflow,okx" and jup.orders[0]["slippage_bps"] == 150 assert jup.orders[0]["input_mint"] == config.WSOL_MINT and jup.orders[0]["output_mint"] == MINT orders = _orders(db_conn, did)