#!/usr/bin/env bash
# pearl-miner — one-command launcher for the PearlDaddy GEMM-transcript miner.
#
# Distribution mode (the published tarball): a `src/` directory with the full
# miner workspace sits next to this script; first run copies it into
# ~/.pearl-miner/src and builds it (CUDA kernel + Rust ext — takes a while,
# cached afterwards). No git access needed.
#
# Dev fallback (no bundled src/): clones the pearl_mining repo instead
# (private — set PEARL_GIT_TOKEN).
#
# Usage:
#   ./pearl-miner --user <prl1...> [--host <ip:port>] [--worker <name>] [--gpus <all|0,1>] [--tls]
#   ./pearl-miner selftest --user <prl1...>     # register + one job, no GPU needed
#
# Env overrides:
#   PEARL_POOL_HOST    pool host:port            (default: pool.pearl-daddy.com:9000)
#   PEARL_MINER_HOME   install/cache directory   (default: ~/.pearl-miner)
#   PEARL_REPO_URL     fallback source repo      (default: https://github.com/PearlDaddy/pearl_mining.git)
#   PEARL_BRANCH       fallback branch           (default: pool-client)
#   PEARL_GIT_TOKEN    GitHub token for the fallback clone
set -euo pipefail

ORIG_ARGS=("$@")

DEFAULT_HOST="pool.pearl-daddy.com:9000"
HOST="${PEARL_POOL_HOST:-$DEFAULT_HOST}"
WALLET=""
WORKER="${HOSTNAME:-rig}"
GPUS="all"
SUBCMD="pool"
TLS=""

REPO_URL="${PEARL_REPO_URL:-https://github.com/PearlDaddy/pearl_mining.git}"
BRANCH="${PEARL_BRANCH:-pool-client}"
HOME_DIR="${PEARL_MINER_HOME:-$HOME/.pearl-miner}"
SRC_DIR="$HOME_DIR/src"

# Some GPU-template images export UV_NO_CACHE=1 / odd UV_CACHE_DIRs, silently
# forcing every install to re-download ~5 GB of wheels and recompile the CUDA
# kernel. A mining rig WANTS the cache — own it under our home dir.
unset UV_NO_CACHE
export UV_CACHE_DIR="$HOME_DIR/uv-cache"
# Use the OS cert store for uv's HTTPS (the bundle download via curl already
# relies on it). Fixes boxes behind a TLS-intercepting proxy whose root CA is in
# the system store but NOT in uv's bundled roots ("invalid peer certificate:
# UnknownIssuer"). UV_SYSTEM_CERTS is the current uv flag; UV_NATIVE_TLS covers
# older uv. Harmless when unrecognized.
export UV_SYSTEM_CERTS=1 UV_NATIVE_TLS=1
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
BUNDLED_SRC="$SCRIPT_DIR/src"

SVC_ACTION=""

while [[ $# -gt 0 ]]; do
    case $1 in
        --host)    HOST="$2";   shift 2 ;;
        --user)    WALLET="$2"; shift 2 ;;
        --worker)  WORKER="$2"; shift 2 ;;
        --gpus)    GPUS="$2";   shift 2 ;;
        --tls)     TLS="--tls"; shift ;;
        selftest)  SUBCMD="selftest"; shift ;;
        mine|pool|mine-blocks) SUBCMD="pool"; shift ;;
        service)
            SUBCMD="service"; shift
            case "${1:-}" in
                stop|status|restart|update) SVC_ACTION="$1"; shift ;;
                *)                          SVC_ACTION="install" ;;
            esac ;;
        --help|-h)
            echo "Usage: $0 --user <wallet> [--host <ip:port>] [--worker <name>] [--gpus <indices>] [--tls]"
            echo "       $0 selftest --user <wallet>"
            echo "       $0 service --user <wallet> [...]   # install + start + run on boot"
            echo "       $0 service status|stop|restart|update"
            echo "Default pool: $DEFAULT_HOST (override with --host or PEARL_POOL_HOST)"
            exit 0 ;;
        *) echo "Unknown option: $1"; exit 1 ;;
    esac
done

log() { echo "[pearl-miner] $*"; }
die() { echo "[pearl-miner] ERROR: $*" >&2; exit 1; }

if [[ "$SUBCMD" != "service" || "$SVC_ACTION" == "install" ]]; then
    [[ -z "$WALLET" ]] && die "--user is required (your prl1... payout address)"
    [[ "$WALLET" != prl1* ]] && echo "WARNING: wallet address should start with prl1" >&2
fi

# ── Self-update (before anything heavy) ─────────────────────────────────────
# Compares the pool's published bundle commit against the bundled copy; on a
# mismatch downloads the new tarball, verifies its sha256, swaps src/ and the
# launcher in place, and re-execs once (PEARL_UPDATED reentry guard). Any
# failure → continue on the current version; updates never cost uptime.
self_update() {
    [[ -n "${PEARL_NO_AUTO_UPDATE:-}" || -n "${PEARL_UPDATED:-}" ]] && return 0
    [[ -f "$BUNDLED_SRC/BUNDLE_COMMIT" && -w "$SCRIPT_DIR" ]] || return 0
    # Trusted update-signing public key (ECDSA P-256), baked into the launcher.
    # An auto-update is applied ONLY if the downloaded tarball carries a valid
    # signature from the matching private key — closing the plaintext-HTTP RCE.
    local MINER_UPDATE_PUBKEY
    read -r -d '' MINER_UPDATE_PUBKEY <<'PUBKEY' || true
-----BEGIN PUBLIC KEY-----
MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEsPhtaqd5h8QH96SuIs9hgDhL92DC
8djGT+ysLrcSE6MQ+I+ADHmJrylbuSIrxsHUGtxJWiHWfLK3UIjSSgnhXA==
-----END PUBLIC KEY-----
PUBKEY
    local base="https://${HOST%%:*}/downloads"
    local remote local_c
    remote="$(curl -fsS --max-time 10 "$base/pearl-miner-src.commit" 2>/dev/null | tr -d '[:space:]')" || return 0
    [[ "$remote" =~ ^[0-9a-f]{40}$ ]] || return 0
    local_c="$(tr -d '[:space:]' < "$BUNDLED_SRC/BUNDLE_COMMIT")"
    [[ "$remote" == "$local_c" ]] && return 0

    log "newer bundle published (${remote:0:12} != ${local_c:0:12}) — self-updating"
    local tmp; tmp="$(mktemp -d)" || return 0
    if ! curl -fsSL --max-time 300 "$base/pearl-miner-linux-x64.tar.gz" -o "$tmp/bundle.tar.gz"; then
        log "WARNING: update download failed — continuing on current version"; rm -rf "$tmp"; return 0
    fi
    # Authenticity: verify a detached ECDSA-P256 signature over the tarball with
    # the baked-in public key. A sha256 fetched over the same channel proves
    # nothing — an attacker who can swap the bundle can swap the checksum too.
    # Fail CLOSED: any problem → stay on the current (trusted) version.
    if ! command -v openssl >/dev/null 2>&1; then
        log "WARNING: openssl missing — cannot verify update signature; staying on current version"; rm -rf "$tmp"; return 0
    fi
    if ! curl -fsSL --max-time 30 "$base/pearl-miner-linux-x64.tar.gz.sig" -o "$tmp/bundle.sig"; then
        log "WARNING: update signature download failed — staying on current version"; rm -rf "$tmp"; return 0
    fi
    printf '%s\n' "$MINER_UPDATE_PUBKEY" > "$tmp/pub.pem"
    # NOTE: the data file is POSITIONAL — `openssl dgst` has no `-in` flag, and
    # OpenSSL 3.x (Ubuntu) hard-errors on `-in <file>` ("Unknown option: in"),
    # which silently failed every fleet auto-update. Pass it positionally.
    if ! openssl dgst -sha256 -verify "$tmp/pub.pem" -signature "$tmp/bundle.sig" "$tmp/bundle.tar.gz" >/dev/null 2>&1; then
        log "WARNING: update SIGNATURE INVALID — refusing update (possible tampering)"; rm -rf "$tmp"; return 0
    fi
    log "update signature verified"
    if ! tar xzf "$tmp/bundle.tar.gz" -C "$tmp" 2>/dev/null; then
        log "WARNING: update unpack failed — continuing on current version"; rm -rf "$tmp"; return 0
    fi
    rm -rf "$SCRIPT_DIR/src"
    cp -a "$tmp/src" "$SCRIPT_DIR/src"
    cp -f "$tmp/pearl-miner" "$SCRIPT_DIR/$(basename "${BASH_SOURCE[0]}")" 2>/dev/null || true
    chmod +x "$SCRIPT_DIR/$(basename "${BASH_SOURCE[0]}")" 2>/dev/null || true
    rm -rf "$tmp"
    log "updated to ${remote:0:12} — restarting launcher"
    PEARL_UPDATED=1 exec "$SCRIPT_DIR/$(basename "${BASH_SOURCE[0]}")" "${ORIG_ARGS[@]}"
}

# ── Supervised service (install/status/stop/restart) ────────────────────────
# Stages the launcher + bundled src into a stable dir and registers it with
# whatever init the host actually has: systemd (real servers/VMs), supervisord
# (vast.ai-style containers), or a setsid restart-loop as the last resort.
SVC_NAME="pearl-miner"
SVC_DIST="$HOME_DIR/dist"
SVC_LOG="/var/log/pearl-miner.log"
SVC_PIDFILE="$HOME_DIR/service.pid"

svc_mechanism() {
    if [[ -d /run/systemd/system ]] && command -v systemctl >/dev/null 2>&1; then
        echo systemd
    elif command -v supervisorctl >/dev/null 2>&1 && [[ -d /etc/supervisor/conf.d ]]; then
        echo supervisord
    else
        echo loop
    fi
}

svc_install() {
    mkdir -p "$SVC_DIST"
    if [[ "$SCRIPT_DIR" != "$SVC_DIST" ]]; then
        cp -f "$SCRIPT_DIR/$(basename "${BASH_SOURCE[0]}")" "$SVC_DIST/pearl-miner" 2>/dev/null \
            || cp -f "${BASH_SOURCE[0]}" "$SVC_DIST/pearl-miner"
        chmod +x "$SVC_DIST/pearl-miner"
        if [[ -d "$BUNDLED_SRC" ]]; then
            rm -rf "$SVC_DIST/src"
            cp -a "$BUNDLED_SRC" "$SVC_DIST/src"
        fi
    fi
    local args="--user $WALLET --worker $WORKER --gpus $GPUS --host $HOST ${TLS:+--tls}"
    # Init daemons don't read shell profiles — bake the CUDA toolchain location
    # (detected from THIS shell, which passed the nvcc preflight) into the
    # service PATH, or the supervised miner dies on its own preflight.
    local nvcc_dir="/usr/local/cuda/bin"
    command -v nvcc >/dev/null 2>&1 && nvcc_dir="$(dirname "$(command -v nvcc)")"
    local svc_path="$HOME/.local/bin:$HOME/.cargo/bin:$nvcc_dir:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
    local mech; mech="$(svc_mechanism)"
    log "installing service via $mech (log: $SVC_LOG)"

    case "$mech" in
        systemd)
            cat > "/etc/systemd/system/$SVC_NAME.service" <<UNIT
[Unit]
Description=Pearl Miner (GEMM-transcript pool miner)
After=network-online.target
Wants=network-online.target

[Service]
Type=simple
Environment=HOME=$HOME
Environment=PEARL_MINER_HOME=$HOME_DIR
Environment=PATH=$svc_path
ExecStart=$SVC_DIST/pearl-miner $args
Restart=always
RestartSec=15
StandardOutput=append:$SVC_LOG
StandardError=append:$SVC_LOG

[Install]
WantedBy=multi-user.target
UNIT
            systemctl daemon-reload
            systemctl enable --now "$SVC_NAME"
            ;;
        supervisord)
            cat > "/etc/supervisor/conf.d/$SVC_NAME.conf" <<CONF
[program:$SVC_NAME]
command=$SVC_DIST/pearl-miner $args
directory=$SVC_DIST
autostart=true
autorestart=true
startsecs=10
stopwaitsecs=30
redirect_stderr=true
stdout_logfile=$SVC_LOG
stdout_logfile_maxbytes=50MB
environment=HOME="$HOME",PEARL_MINER_HOME="$HOME_DIR",PATH="$svc_path"
CONF
            supervisorctl reread >/dev/null
            supervisorctl update "$SVC_NAME" >/dev/null 2>&1 || supervisorctl update >/dev/null
            supervisorctl restart "$SVC_NAME" >/dev/null 2>&1 || true
            ;;
        loop)
            cat > "$SVC_DIST/run-loop.sh" <<LOOP
#!/usr/bin/env bash
# Auto-restart wrapper (no init system available). Survives disconnects;
# cannot survive a container reboot — re-run 'pearl-miner service' after one.
while true; do
    "$SVC_DIST/pearl-miner" $args
    echo "[pearl-miner-service] miner exited (\$?) — restarting in 15s" >> "$SVC_LOG"
    sleep 15
done
LOOP
            chmod +x "$SVC_DIST/run-loop.sh"
            svc_stop_loop quiet
            setsid nohup "$SVC_DIST/run-loop.sh" >> "$SVC_LOG" 2>&1 < /dev/null &
            echo $! > "$SVC_PIDFILE"
            ;;
    esac
    log "service installed and started — check: $0 service status"
}

svc_stop_loop() {
    if [[ -f "$SVC_PIDFILE" ]]; then
        local pid; pid="$(cat "$SVC_PIDFILE")"
        kill -- -"$pid" 2>/dev/null || kill "$pid" 2>/dev/null || true
        rm -f "$SVC_PIDFILE"
        [[ "${1:-}" == quiet ]] || log "stopped"
    else
        pkill -f "$SVC_DIST/run-loop.sh" 2>/dev/null || true
        [[ "${1:-}" == quiet ]] || log "no pidfile — best-effort stop done"
    fi
    pkill -f "$SVC_DIST/pearl-miner " 2>/dev/null || true
}

svc_action() {
    local mech; mech="$(svc_mechanism)"
    case "$mech:$SVC_ACTION" in
        systemd:status)      systemctl status "$SVC_NAME" --no-pager | head -12; tail -5 "$SVC_LOG" 2>/dev/null ;;
        systemd:stop)        systemctl disable --now "$SVC_NAME" ;;
        systemd:restart)     systemctl restart "$SVC_NAME" ;;
        supervisord:status)  supervisorctl status "$SVC_NAME"; tail -5 "$SVC_LOG" 2>/dev/null ;;
        supervisord:stop)    supervisorctl stop "$SVC_NAME" ;;
        supervisord:restart) supervisorctl restart "$SVC_NAME" ;;
        loop:status)
            if [[ -f "$SVC_PIDFILE" ]] && kill -0 "$(cat "$SVC_PIDFILE")" 2>/dev/null; then
                log "running (loop pid $(cat "$SVC_PIDFILE"))"
            else
                log "not running"
            fi
            tail -5 "$SVC_LOG" 2>/dev/null ;;
        loop:stop)           svc_stop_loop ;;
        systemd:update | supervisord:update | loop:update)
            log "forcing update of $SVC_DIST from the pool"
            local base="https://${HOST%%:*}/downloads"
            local tmp; tmp="$(mktemp -d)"
            curl -fsSL "$base/pearl-miner-linux-x64.tar.gz" -o "$tmp/b.tgz" \
                && tar xzf "$tmp/b.tgz" -C "$tmp" \
                && rm -rf "$SVC_DIST/src" && cp -a "$tmp/src" "$SVC_DIST/src" \
                && cp -f "$tmp/pearl-miner" "$SVC_DIST/pearl-miner" && chmod +x "$SVC_DIST/pearl-miner" \
                || { rm -rf "$tmp"; die "update failed"; }
            rm -rf "$tmp"
            case "$mech" in
                systemd)     systemctl restart "$SVC_NAME" ;;
                supervisord) supervisorctl restart "$SVC_NAME" ;;
                loop)        svc_stop_loop quiet
                             setsid nohup "$SVC_DIST/run-loop.sh" >> "$SVC_LOG" 2>&1 < /dev/null &
                             echo $! > "$SVC_PIDFILE" ;;
            esac
            log "updated and restarted" ;;
        loop:restart)        svc_stop_loop quiet
                             setsid nohup "$SVC_DIST/run-loop.sh" >> "$SVC_LOG" 2>&1 < /dev/null &
                             echo $! > "$SVC_PIDFILE"; log "restarted" ;;
    esac
}

if [[ "$SUBCMD" == "service" ]]; then
    if [[ "$SVC_ACTION" == "install" ]]; then
        svc_install
    else
        svc_action
    fi
    exit 0
fi

# ── Preflight ────────────────────────────────────────────────────────────────
command -v curl >/dev/null || die "curl is required (apt install curl)"
command -v nvcc >/dev/null || die "nvcc (CUDA toolkit required) to build the GPU kernel: 12.6+ for H100/Ada, 12.8+ for RTX 5090/Blackwell (13.x also supported).
  Use a CUDA -devel image (ships nvcc + math-library headers); on Ubuntu CUDA images it is preinstalled.
  torch is auto-matched to your CUDA version at build time. See https://developer.nvidia.com/cuda-downloads"
if command -v nvidia-smi >/dev/null; then
    GPU_NAME="$(nvidia-smi --query-gpu=name --format=csv,noheader 2>/dev/null | head -1 || true)"
    [[ -n "$GPU_NAME" ]] && log "GPU: $GPU_NAME (Hopper/H100, Blackwell/RTX 5090, and Ada/RTX 4090 are supported)"
fi

self_update

# ── Source install: bundled src/ first, git clone fallback ──────────────────
if [[ -f "$BUNDLED_SRC/BUNDLE_COMMIT" ]]; then
    WANT="$(cat "$BUNDLED_SRC/BUNDLE_COMMIT")"
    HAVE="$(cat "$SRC_DIR/BUNDLE_COMMIT" 2>/dev/null || true)"
    if [[ "$WANT" != "$HAVE" ]]; then
        log "installing bundled miner source (${WANT:0:12}) → $SRC_DIR"
        rm -rf "$SRC_DIR"
        mkdir -p "$SRC_DIR"
        cp -a "$BUNDLED_SRC/." "$SRC_DIR/"
    fi
elif [[ ! -d "$SRC_DIR/.git" && ! -f "$SRC_DIR/BUNDLE_COMMIT" ]]; then
    command -v git >/dev/null || die "no bundled src/ next to the script and git is missing"
    log "no bundled src/ — fetching miner source ($REPO_URL @ $BRANCH)"
    mkdir -p "$HOME_DIR"
    CLONE_URL="$REPO_URL"
    if [[ -n "${PEARL_GIT_TOKEN:-}" ]]; then
        CLONE_URL="${REPO_URL/https:\/\//https://x-access-token:${PEARL_GIT_TOKEN}@}"
    fi
    git clone --depth 1 --branch "$BRANCH" "$CLONE_URL" "$SRC_DIR"
    git -C "$SRC_DIR" remote set-url origin "$REPO_URL"   # never persist the token
    git -C "$SRC_DIR" submodule update --init miner/pearl-gemm/third_party/cutlass
elif [[ -d "$SRC_DIR/.git" ]]; then
    log "updating miner source (git dev mode)"
    git -C "$SRC_DIR" fetch --depth 1 origin "$BRANCH" || log "WARNING: fetch failed, using cached source"
    git -C "$SRC_DIR" reset --hard FETCH_HEAD 2>/dev/null || true
    git -C "$SRC_DIR" submodule update --init miner/pearl-gemm/third_party/cutlass || true
fi

# ── Toolchain: uv (Python) + rustup (py-pearl-mining ext) ────────────────────
export PATH="$HOME/.local/bin:$HOME/.cargo/bin:$PATH"
if ! command -v uv >/dev/null; then
    log "installing uv (Python package manager)"
    curl -LsSf https://astral.sh/uv/install.sh | sh
fi
if ! command -v cargo >/dev/null; then
    log "installing Rust toolchain (builds the pearl_mining extension)"
    curl -LsSf https://sh.rustup.rs | sh -s -- -y --profile minimal
    . "$HOME/.cargo/env"
fi

# ── Build (first run compiles the CUDA kernel — takes a while, then cached) ──
# CUDA-version-agnostic build (works on any CUDA 12.6–13.0 host):
#
# The kernel (pearl-gemm) compiles with the HOST's nvcc, and PyTorch hard-errors
# when the host CUDA *major* != torch's. So torch MUST match the host nvcc major.
# We do NOT use `uv sync --torch-backend=auto`: (a) `--torch-backend` is only a
# `uv pip` flag — `uv sync` rejects it; (b) `auto` detects the *driver*'s CUDA,
# not nvcc's — a 12.x-toolkit host with a 13-capable driver would get cu130 torch
# and fail the build; and (c) `uv sync` against the lock pulls in BOTH cu12 and
# cu13 nvidia runtime wheels, whose libcudart collide at import time. We instead
# drive the install with `uv pip` and a backend derived from `nvcc -V`, installing
# torch FIRST so the only nvidia-* runtime wheels in the venv are the host-matched
# ones:
#   1. fresh venv; install host-matched torch + a CUDA-major-matched CCCL wheel
#      (the kernel uses CCCL>=2.4 `cuda::maximum`, absent from toolkits < 12.5);
#   2. build pearl-gemm with --no-build-isolation (compiles against that torch);
#   3. install the scanner + its workspace members (editable) against that torch.
VENV_BIN="$SRC_DIR/.venv/bin/pure-gemm-scanner"
if [[ ! -x "$VENV_BIN" ]] || [[ "$SRC_DIR/uv.lock" -nt "$SRC_DIR/.venv" ]]; then
    # Loud, set-expectations notice — the first run is long and partly quiet
    # (kernel JIT compile), which otherwise looks hung and gets Ctrl-C'd.
    echo "" >&2
    echo "  ┌──────────────────────────────────────────────────────────────────┐" >&2
    echo "  │  First run: installing PyTorch + compiling the GPU kernel.          │" >&2
    echo "  │  This takes ~10–15 min (ONE TIME — cached for every run after).     │" >&2
    echo "  │  Some steps are quiet while the kernel compiles. This is normal —   │" >&2
    echo "  │  please leave it running and do NOT Ctrl-C.                         │" >&2
    echo "  └──────────────────────────────────────────────────────────────────┘" >&2
    echo "" >&2
    NVCC_VER="$(nvcc -V 2>/dev/null | grep -oE 'release [0-9]+\.[0-9]+' | grep -oE '[0-9]+\.[0-9]+' | head -1)"
    NVCC_MAJOR="${NVCC_VER%%.*}"; NVCC_MINOR="${NVCC_VER#*.}"
    # Map the host nvcc to a torch CUDA backend. Clamp ANY 13.x → cu130 (no cu131
    # wheels exist; cu130 runs on a 13.1 driver). Blackwell (RTX 5090, sm_120)
    # needs cu128+, so 12.8/12.9 → cu128, not cu126.
    # TORCH_MIN is a per-backend FLOOR (>=), not an exact pin: uv picks the latest
    # wheel ON that backend's index, which always ships the needed arch. An exact
    # pin is brittle — torch==2.11.0 isn't on cu130 (only cu126/cu128), and a cu130
    # index may carry 2.12.0 vs 2.12.1 (a caching proxy can even serve a stale
    # listing). A floor sidesteps all that: cu130 → >=2.12 (any 2.12.x+cu130 has
    # sm_120 AND sm_89), cu128 → >=2.11 (sm_120), cu126 → >=2.11 (sm_89; Blackwell
    # on cu126 is fail-fasted below).
    case "$NVCC_MAJOR" in
        13) TORCH_BACKEND="cu130"; TORCH_MIN="2.12"; CCCL_WHEEL="nvidia-cuda-cccl" ;;   # cu13 CCCL (unsuffixed)
        12) if [[ "${NVCC_MINOR:-0}" -ge 8 ]]; then
                TORCH_BACKEND="cu128"; TORCH_MIN="2.11"              # 12.8/12.9 → cu128 (has sm_120)
            else
                TORCH_BACKEND="cu126"; TORCH_MIN="2.11"             # 12.6/12.7 → cu126 (NO sm_120)
            fi
            CCCL_WHEEL="nvidia-cuda-cccl-cu12" ;;                          # cu12 CCCL with cuda::maximum (>=2.4)
        *)  die "unsupported CUDA toolkit '${NVCC_VER:-?}' — need 12.6+ (12.8+ for RTX 5090 / Blackwell)" ;;
    esac
    # Fail fast on the actual footgun: a Blackwell card (sm_120 consumer or sm_100
    # datacenter / B100) on a <12.8 toolkit would silently get a torch with no
    # matching kernels (the cryptic 'no kernel image' crash). Tell the user what to do.
    if command -v nvidia-smi >/dev/null; then
        GPU_CC="$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader 2>/dev/null | head -1 | tr -d ' ')"
        if [[ ( "$GPU_CC" == "12.0" || "$GPU_CC" == "10.0" ) && "$TORCH_BACKEND" == "cu126" ]]; then
            die "RTX 5090 / B100 / Blackwell (sm_120 / sm_100) needs CUDA >= 12.8, but this box has CUDA ${NVCC_VER}.
  Use a CUDA 12.8 or 13.0 image (the launcher then installs a torch with the right kernels)."
        fi
        # B300 / Blackwell Ultra (sm_103) JITs as sm_103a, a CUDA-13-only arch.
        if [[ "$GPU_CC" == "10.3" && "$TORCH_BACKEND" != "cu130" ]]; then
            die "B300 / Blackwell Ultra (sm_103) needs CUDA >= 13.0, but this box has CUDA ${NVCC_VER}.
  Use a CUDA 13.0 image (sm_103a is not available on older toolkits)."
        fi
    fi
    log "building miner (nvcc CUDA ${NVCC_VER:-?} → torch backend=$TORCH_BACKEND)"
    # Pin uv to the .venv we just created. Cloud GPU images (vast/RunPod) often
    # ship an already-activated venv ($VIRTUAL_ENV=/venv/main); without this,
    # `uv pip install` targets THAT, and the scanner never lands in .venv/bin.
    ( cd "$SRC_DIR" \
        && rm -rf .venv \
        && uv venv --python 3.12 .venv \
        && export VIRTUAL_ENV="$SRC_DIR/.venv" \
        && uv pip install --torch-backend="$TORCH_BACKEND" "torch>=$TORCH_MIN" "$CCCL_WHEEL" numpy \
        && uv pip install -e ./miner/pearl-gemm-build-utils \
        && uv pip install --no-build-isolation -e ./miner/pearl-gemm \
        && uv pip install \
               -e ./miner/miner-utils -e ./miner/miner-base -e ./miner/pearl-gateway \
               -e ./py-pearl-mining \
               -e ./miner/pure-gemm-scanner )
fi
# Notes on the install above:
#  - only pearl-gemm uses --no-build-isolation, so its CUDA kernel links against
#    the host-matched torch we just installed (an isolated env re-resolves cu130).
#  - the other members keep build isolation: py-pearl-mining builds with maturin
#    (its own backend), which an unisolated env would not provide.

# ── GPU selection ────────────────────────────────────────────────────────────
# Resolve the device list: "all" → every visible GPU; else a CSV like "0,1".
if [[ "$GPUS" == "all" ]]; then
    GPU_IDS=()   # portable (no bash-4 mapfile): one entry per visible GPU index
    while IFS= read -r _gid; do [[ -n "$_gid" ]] && GPU_IDS+=("$_gid"); done \
        < <(nvidia-smi --query-gpu=index --format=csv,noheader 2>/dev/null | tr -d ' ')
    [[ ${#GPU_IDS[@]} -eq 0 ]] && GPU_IDS=("")   # no nvidia-smi → one default device
else
    IFS=',' read -ra GPU_IDS <<< "$GPUS"
fi

# ── Run ──────────────────────────────────────────────────────────────────────
# Multi-GPU (T-Rex/lolMiner-style): on a box with >1 GPU, drive ONE scanner per
# GPU — each pinned via CUDA_VISIBLE_DEVICES, with an ISOLATED session file
# (per-GPU XDG_STATE_HOME — the session path is a single file, so processes
# sharing $HOME would clobber each other) and a per-GPU worker name, supervised
# with respawn. Python's GIL makes one-process-N-threads pointless here, so N
# processes is the idiomatic multi-GPU approach. The single-GPU path below is
# unchanged. Each card reports to the pool; the pool counts distinct GPU uuids.
if [[ "$SUBCMD" == "pool" && ${#GPU_IDS[@]} -gt 1 ]]; then
    log "→ multi-GPU: ${#GPU_IDS[@]} GPUs [${GPU_IDS[*]}] host=$HOST wallet=$WALLET worker=$WORKER"
    for gid in "${GPU_IDS[@]}"; do
        (
            export CUDA_VISIBLE_DEVICES="$gid"
            export XDG_STATE_HOME="$HOME_DIR/state/gpu$gid"
            mkdir -p "$XDG_STATE_HOME"
            while true; do
                "$VENV_BIN" "$SUBCMD" --host "$HOST" --wallet "$WALLET" \
                    --worker "$WORKER-gpu$gid" ${TLS:+$TLS}
                # 42 = a newer bundle is published; stop respawning so the outer
                # supervisor re-runs the launcher (which self-updates the bundle).
                [[ $? -eq 42 ]] && break
                log "gpu$gid scanner exited — respawning in 5s"; sleep 5
            done
        ) &
    done
    wait
    exit 0
fi

# Single GPU (or selftest/service): unchanged behavior.
if [[ "$GPUS" != "all" ]]; then
    export CUDA_VISIBLE_DEVICES="$GPUS"
fi
log "→ $SUBCMD host=$HOST wallet=$WALLET worker=$WORKER gpus=$GPUS"
exec "$VENV_BIN" "$SUBCMD" \
    --host "$HOST" \
    --wallet "$WALLET" \
    --worker "$WORKER" \
    ${TLS:+$TLS}
