#!/usr/bin/env bash
# DevGrail one-command installer.
#
#   curl -fsSL https://<devgrail-web>/install.sh | sudo bash
#
# Detects arch, installs Docker if missing, downloads the DevGrail image tarballs
# from devgrail-web (verified by SHA-256, loaded with `docker load` — no GitHub,
# no `docker pull`, no registry auth), lays down the compose + Traefik config,
# writes config + secrets, and brings the stack up behind Traefik + Let's Encrypt.
#
# Everything is resolved from one release manifest, so an upgrade cannot mix a
# new compose file with an old server image. A registry that publishes no release
# manifest falls back to resolving each artifact by slug, exactly as before.
#
# Re-running upgrades in place: only artifacts whose version changed are
# re-downloaded — images *and* assets; existing secrets and the data volume are
# preserved. Free space is checked against the declared artifact sizes before
# anything is fetched. An upgrade takes a snapshot of the database + config.yaml
# into /opt/devgrail/backups before the new server starts, then waits for the
# server's readiness probe and exits non-zero (with logs and the snapshot path)
# if it never becomes ready. Images older than :previous are reclaimed only once
# that probe is green — until then :previous is the rollback target.
#
# It also installs /usr/local/bin/devgrail (backup / restore / list) and offers
# to schedule a weekly backup.
#
# Prerequisites: DNS A records `DEVGRAIL_DOMAIN` and `*.DEVGRAIL_BASE_DOMAIN`
# pointing at this VPS. Inputs may be passed as env vars or answered at the prompt.
#
# Vars:
#   DEVGRAIL_WEB_URL      public site serving images + assets
#                         (default: the site this script was downloaded from)
#   DEVGRAIL_DOMAIN       dashboard/API/MCP hostname (prompted)
#   DEVGRAIL_BASE_DOMAIN  base domain for app subdomains (prompted)
#   ACME_EMAIL            Let's Encrypt contact email (prompted)
#   HTTP_PORT/HTTPS_PORT  host ports (default 80/443; real 80/443 needed for ACME)
#   DEVGRAIL_HEALTH_TIMEOUT   seconds to wait for readiness after start (default 180)
#   DEVGRAIL_SNAPSHOT_RETAIN  pre-upgrade snapshots to keep (default 5)
#   DEVGRAIL_VERSION      release to install, e.g. v0.3.0 (default: current)
#   DEVGRAIL_SCHEDULED_BACKUPS  yes|no — answer the weekly-backup prompt unattended
#   DEVGRAIL_SKIP_DNS_CHECK  1 — skip the DNS preflight (same as --skip-dns-check)
#   DEVGRAIL_LOCK_WAIT    seconds to wait for a concurrent run to finish (default 300)
#   DEVGRAIL_REQUIRE_SIGNATURE  1 — refuse a release whose signature cannot be
#                         verified, instead of warning
#   DEVGRAIL_RELEASE_PUBKEY_FILE  extra trusted release signing keys (PEM)
#
# Flags (pass with `| sudo bash -s -- <flag>`): --version=vX.Y.Z, --rollback,
# --auto-rollback, --dry-run, --yes, --skip-dns-check, --help. See usage() below.
set -euo pipefail

# __DEVGRAIL_WEB_URL__ is substituted with the serving origin by devgrail-web's
# /install.sh route handler (app/install.sh/route.ts) so the default always points
# at the site this script was actually downloaded from. An explicit env var still
# wins. This file is the single source of truth: `make publish` uploads it to the
# registry as the install.sh asset; devgrail-web serves it. Do not keep a second copy.
DEVGRAIL_WEB_URL="${DEVGRAIL_WEB_URL:-__DEVGRAIL_WEB_URL__}"
INSTALL_DIR="${DEVGRAIL_INSTALL_DIR:-/opt/devgrail}"
DEPLOY_DIR="$INSTALL_DIR/deploy"
CONFIG_DIR=/etc/devgrail
VERSIONS_FILE="$INSTALL_DIR/versions.env"
BACKUP_DIR="$INSTALL_DIR/backups"
# Verified copies of the installer assets, kept so an unchanged re-run
# re-downloads nothing. The rendered outputs under $DEPLOY_DIR are always
# regenerated from these — substitution is cheap, a download is not.
ASSET_DIR="$INSTALL_DIR/assets"
WEB="${DEVGRAIL_WEB_URL%/}"
# Set by snapshot_before_upgrade; reported in the success banner and on failure.
SNAPSHOT_ARCHIVE=""
# Staging directory for a snapshot in progress; removed by the EXIT trap so an
# aborted run leaves no half-written copy of the database lying around.
SNAPSHOT_STAGE=""

log()  { printf '\033[1;34m==>\033[0m %s\n' "$*"; }
warn() { printf '\033[1;33mWARNING:\033[0m %s\n' "$*" >&2; }
die()  { printf '\033[1;31merror:\033[0m %s\n' "$*" >&2; exit 1; }

# docker-compose.yml is product-owned: it is re-fetched and rewritten on every
# run, so an operator edit to it is lost at the next upgrade — silently, which is
# the part that makes it a trap. docker-compose.override.yml is the supported
# place for local changes; this installer never writes or reads it beyond adding
# it to the file list, and compose merges it over the base file.
#
# It has to be passed explicitly: compose only auto-discovers an override file
# when no -f is given at all, and -f is what pins this to $DEPLOY_DIR.
compose() {
  if [ -f "$DEPLOY_DIR/docker-compose.override.yml" ]; then
    docker compose --project-directory "$DEPLOY_DIR" \
      -f "$DEPLOY_DIR/docker-compose.yml" \
      -f "$DEPLOY_DIR/docker-compose.override.yml" "$@"
  else
    docker compose --project-directory "$DEPLOY_DIR" -f "$DEPLOY_DIR/docker-compose.yml" "$@"
  fi
}

usage() {
  cat <<'USAGE'
DevGrail installer.

  curl -fsSL https://<devgrail-web>/install.sh | sudo bash
  curl -fsSL https://<devgrail-web>/install.sh | sudo bash -s -- [options]

Options:
  --version=vX.Y.Z   install that published release instead of the current one
  --rollback         restore the previous images and the newest pre-upgrade
                     snapshot, then wait for the server to become ready
  --auto-rollback    if the server does not become ready after an upgrade, do
                     the above automatically instead of asking
  --dry-run          report what this run would change — release, image and
                     asset versions, download size, free disk, settings — and
                     exit without touching anything
  -y, --yes          never prompt; take the prior value (or the default) for
                     every question. For unattended runs.
  --skip-dns-check   do not verify that the domains resolve to this host
  -h, --help         show this and exit

Inputs may also be passed as environment variables; see the header of this file.
USAGE
}

DO_ROLLBACK=false
AUTO_ROLLBACK=false
DRY_RUN=false
ASSUME_YES=false
# The env var exists so an unattended install (cloud-init, a marketplace image)
# can opt out without rewriting the command line it was handed.
case "$(printf '%s' "${DEVGRAIL_SKIP_DNS_CHECK:-}" | tr '[:upper:]' '[:lower:]')" in
  1|y|yes|true) SKIP_DNS_CHECK=true ;;
  *)            SKIP_DNS_CHECK=false ;;
esac
while [ $# -gt 0 ]; do
  case "$1" in
    --version=*)     DEVGRAIL_VERSION="${1#*=}" ;;
    --version)       shift; DEVGRAIL_VERSION="${1:-}"
                     [ -n "$DEVGRAIL_VERSION" ] || die "--version needs a value, e.g. --version=v0.3.0" ;;
    --rollback)      DO_ROLLBACK=true ;;
    --auto-rollback) AUTO_ROLLBACK=true ;;
    --dry-run)       DRY_RUN=true ;;
    -y|--yes)        ASSUME_YES=true ;;
    --skip-dns-check) SKIP_DNS_CHECK=true ;;
    -h|--help)       usage; exit 0 ;;
    *) usage >&2; die "unknown option: $1" ;;
  esac
  shift
done

# --rollback restores images and a snapshot; there is nothing about it a dry run
# could report without doing it. Refuse the combination rather than pick one.
if [ "$DO_ROLLBACK" = true ] && [ "$DRY_RUN" = true ]; then
  die "--dry-run cannot be combined with --rollback."
fi

[ "$(id -u)" = 0 ] || die "run as root: curl -fsSL $WEB/install.sh | sudo bash"

case "$(uname -m)" in
  x86_64|amd64)   ARCH=amd64 ;;
  aarch64|arm64)  ARCH=arm64 ;;
  *) die "unsupported architecture: $(uname -m)" ;;
esac

# --- concurrency lock --------------------------------------------------------
# Two runs at once — an impatient operator re-running while the first is still
# downloading, or a scheduled upgrade landing on top of a manual one — race on
# .env, versions.env, config.yaml and `docker compose up`. The loser can leave a
# half-written .env (which holds the admin password) or a compose project
# pointing at an image the other run has already retagged. One flock closes it.
#
# Held on fd 9 for the lifetime of the process, so it is released on any exit,
# including a crash — there is no stale lock file to clean up by hand.
LOCK_WAIT="${DEVGRAIL_LOCK_WAIT:-300}"
acquire_lock() {
  if ! command -v flock >/dev/null 2>&1; then
    warn "flock is not available on this host — running without the install lock."
    echo "  Do not run two installers at the same time." >&2
    return 0
  fi
  # A dry run reads the same files, so it takes the lock shared: it must not
  # read a half-written .env, and it must not block a real install either.
  # On a host with no install there is nothing to race over and nothing to read,
  # so it does not create $INSTALL_DIR just to lock it.
  if [ "$DRY_RUN" = true ] && [ ! -d "$INSTALL_DIR" ]; then
    return 0
  fi
  mkdir -p "$INSTALL_DIR" || die "cannot create $INSTALL_DIR"
  exec 9>"$INSTALL_DIR/.install.lock" || die "cannot open $INSTALL_DIR/.install.lock"

  local mode=-x what=exclusive waited=0
  if [ "$DRY_RUN" = true ]; then mode=-s; what=shared; fi

  # Polled with -n rather than blocking with -w: busybox's flock (Alpine and
  # most minimal images) has no -w at all, and rejects it with a usage error
  # whose exit status is indistinguishable from "the lock is held" — which made
  # this refuse to run at all on those hosts. -n and -s are the forms busybox
  # and util-linux agree on.
  while ! flock "$mode" -n 9; do
    if [ "$waited" -ge "$LOCK_WAIT" ]; then
      die "another install.sh is already running on this host.
  Waited ${LOCK_WAIT}s for the $what lock on $INSTALL_DIR/.install.lock.
  Wait for that run to finish and try again. Nothing has been changed."
    fi
    if [ "$waited" -eq 0 ]; then
      log "Another install.sh holds the lock — waiting up to ${LOCK_WAIT}s..."
    fi
    sleep 1
    waited=$((waited + 1))
  done
}
acquire_lock

# --- detect a prior install (upgrade) --------------------------------------
# Read only now that the lock is held: this is the file a concurrent run would
# be rewriting, and adopting a half-written copy of it is precisely what the
# lock exists to prevent.
#
# On re-run the previous answers live in the deploy .env (rewritten every run).
# The WHOLE file is read into a PRIOR_* namespace, not a hand-maintained list of
# four keys: the rule is that an upgrade preserves everything unless the operator
# changes it. Reading only some keys meant HTTP_PORT/HTTPS_PORT were silently
# reset to 80/443 on every upgrade — best case Traefik then failed to bind, worst
# case it bound and hijacked another service's ports.
#
# Precedence, everywhere: explicit env var > prompt answer > prior value > default.
IS_UPGRADE=false
PRIOR_KEYS=""
if [ -f "$DEPLOY_DIR/.env" ]; then
  IS_UPGRADE=true
  while IFS= read -r _line || [ -n "$_line" ]; do
    # Skip blanks, comments, and anything without a '='.
    case "$_line" in ''|'#'*) continue ;; *=*) : ;; *) continue ;; esac
    _key="${_line%%=*}"
    _val="${_line#*=}"
    # Ignore anything that is not a plain shell identifier — this file is
    # written by us, but it is also the file an operator hand-edits.
    case "$_key" in ''|*[!A-Za-z0-9_]*) continue ;; esac
    # Strip one layer of surrounding quotes, matching how compose reads it.
    case "$_val" in
      \"*\") _val="${_val#\"}"; _val="${_val%\"}" ;;
      \'*\') _val="${_val#\'}"; _val="${_val%\'}" ;;
    esac
    eval "PRIOR_$_key=\$_val"
    PRIOR_KEYS="${PRIOR_KEYS:+$PRIOR_KEYS }$_key"
  done < "$DEPLOY_DIR/.env"
fi

# prior <KEY> — the prior install's value for KEY, or empty.
prior() { eval "printf '%s' \"\${PRIOR_$1:-}\""; }

# resolve <KEY> [default] — settle KEY per the precedence above. Prompted keys
# call prompt() instead, passing `prior KEY` as the offered default.
resolve() {
  local key="$1" def="${2:-}" cur
  eval "cur=\${$key:-}"
  [ -n "$cur" ] && return 0
  cur="$(prior "$key")"
  [ -n "$cur" ] || cur="$def"
  eval "$key=\$cur"
}

# --- TLS switch (default on) ------------------------------------------------
# DEVGRAIL_TLS=off serves plain HTTP on :80 — only for a private network or
# behind an external TLS terminator. Default on = HTTPS via Traefik + Let's
# Encrypt. The choice selects which Traefik router set to install and the scheme
# advertised in every server-generated URL (see internal/config Scheme()).
resolve DEVGRAIL_TLS on
case "$(printf '%s' "$DEVGRAIL_TLS" | tr '[:upper:]' '[:lower:]')" in
  off|false|0|no) TLS_ENABLED=false; SCHEME=http;  TRAEFIK_ASSET=traefik-dynamic-http.yml ;;
  *)              TLS_ENABLED=true;  SCHEME=https; TRAEFIK_ASSET=traefik-dynamic.yml ;;
esac

sha256_of() {
  if command -v sha256sum >/dev/null 2>&1; then sha256sum "$1" | awk '{print $1}'
  else shasum -a 256 "$1" | awk '{print $1}'; fi
}
gen_secret()   { openssl rand -hex 32 2>/dev/null || head -c 32 /dev/urandom | od -An -tx1 | tr -d ' \n'; }
gen_password() { (openssl rand -base64 18 2>/dev/null || head -c 15 /dev/urandom | base64) | tr -dc 'A-Za-z0-9' | cut -c1-20; }

# --- readiness ---------------------------------------------------------------
# `docker compose up -d` returning 0 only means containers were *created*. The
# failure that matters most — the server crash-looping on a migration — passes
# that check and hands the customer a URL that does not work. /api/v1/healthz
# answers the real question (database reachable, Docker reachable, schema
# matches the binary) and returns 503 until all three hold.
#
# Two probe targets: the public URL is what the customer will actually use, but
# it depends on DNS already pointing here and on Let's Encrypt having issued,
# neither of which is true on a first install. The container-local address
# verifies the server itself, so a not-yet-pointed DNS record does not masquerade
# as a broken server.
HEALTH_TIMEOUT="${DEVGRAIL_HEALTH_TIMEOUT:-180}"

probe_health() {
  # Public route through Traefik. -k because the certificate may still be the
  # ACME self-signed placeholder; this checks readiness, not trust.
  curl -fsSk --max-time 5 "$SCHEME://$DEVGRAIL_DOMAIN/api/v1/healthz" >/dev/null 2>&1 && return 0
  # The server directly, from inside its own container.
  compose exec -T devgrail-server /usr/local/bin/devgrail-server -healthcheck >/dev/null 2>&1
}

wait_for_health() {
  local waited=0
  log "Waiting for DevGrail to become ready (up to ${HEALTH_TIMEOUT}s)..."
  while [ "$waited" -lt "$HEALTH_TIMEOUT" ]; do
    if probe_health; then
      log "DevGrail is ready."
      return 0
    fi
    sleep 3
    waited=$((waited + 3))
  done
  return 1
}

# --- rollback ----------------------------------------------------------------
# data_volume prints the Docker volume backing /var/lib/devgrail. It is resolved
# from the running container rather than assumed from the Compose project name,
# which is derived from the deploy directory and so differs between installs.
data_volume() {
  local vol
  vol="$(docker inspect devgrail-server \
    --format '{{range .Mounts}}{{if eq .Destination "/var/lib/devgrail"}}{{.Name}}{{end}}{{end}}' 2>/dev/null)" || vol=""
  if [ -n "$vol" ]; then printf '%s' "$vol"; return 0; fi
  docker volume ls --format '{{.Name}}' 2>/dev/null | grep -E '(^|_)devgrail-data$' | head -n1 || true
}

newest_snapshot() {
  ls -1t "$BACKUP_DIR"/pre-upgrade-*.tgz 2>/dev/null | head -n1 || true
}

# Point :latest back at the image the previous run displaced. This is a tag move,
# not a copy — which is the whole reason the retag is worth doing on every
# upgrade: it turns rollback into a retag plus a compose recreate.
rollback_images() {
  local slug moved=0
  for slug in devgrail-server devgrail-container; do
    if docker image inspect "$slug:previous" >/dev/null 2>&1; then
      docker tag "$slug:previous" "$slug:latest"
      moved=$((moved + 1))
    else
      warn "no $slug:previous image on this host — leaving $slug:latest where it is"
    fi
  done
  [ "$moved" -gt 0 ]
}

# restore_snapshot <archive> — put the database and config.yaml back as they were
# before the upgrade that took this archive.
restore_snapshot() {
  local archive="$1" vol stage
  vol="$(data_volume)"
  [ -n "$vol" ] || die "cannot find the DevGrail data volume — refusing to guess which volume to overwrite."

  stage="$(mktemp -d)"
  tar xzf "$archive" -C "$stage" || { rm -rf "$stage"; die "could not read $archive"; }
  if [ ! -f "$stage/devgrail.db" ] || [ ! -f "$stage/config.yaml" ]; then
    rm -rf "$stage"
    die "$archive is missing devgrail.db or config.yaml — a database without the
  config.yaml that keys its encryption is not a usable restore."
  fi

  compose stop devgrail-server >/dev/null 2>&1 || true

  # config.yaml first, deliberately: the JWT secret in it decrypts everything in
  # the database, so a half-completed restore that stopped here is recoverable,
  # whereas one that restored only the database is not.
  install -m 600 "$stage/config.yaml" "$CONFIG_DIR/config.yaml" \
    || { rm -rf "$stage"; die "could not restore $CONFIG_DIR/config.yaml"; }

  # The -wal/-shm sidecars belong to the database being replaced. Leaving them
  # next to a restored file is how a "successful" restore produces a corrupt
  # database, so they go before the copy lands.
  docker run --rm --userns=host \
    -v "$vol":/var/lib/devgrail \
    -v "$stage":/restore:ro \
    --entrypoint /bin/sh devgrail-server:latest -c '
      rm -f /var/lib/devgrail/devgrail.db-wal /var/lib/devgrail/devgrail.db-shm &&
      cp /restore/devgrail.db /var/lib/devgrail/devgrail.db' \
    || { rm -rf "$stage"; die "could not restore the database into volume '$vol'"; }

  rm -rf "$stage"
  log "Restored database + config.yaml from $archive"
}

# do_rollback returns 0 when the rolled-back stack answers its readiness probe.
do_rollback() {
  [ -f "$DEPLOY_DIR/docker-compose.yml" ] \
    || die "no DevGrail install found at $INSTALL_DIR — there is nothing to roll back to."
  # Read from the prior .env rather than prompting: a rollback happens when
  # something is already wrong, and it should not stop to ask questions it can
  # answer itself.
  DEVGRAIL_DOMAIN="${DEVGRAIL_DOMAIN:-$(prior DEVGRAIL_DOMAIN)}"

  local archive
  archive="$(newest_snapshot)"
  log "Rolling back to the previous images and snapshot..."
  rollback_images || warn "no previous images were available to restore"

  if [ -n "$archive" ]; then
    restore_snapshot "$archive"
  else
    warn "No pre-upgrade snapshot in $BACKUP_DIR — restoring images only."
    echo "  The database keeps whatever schema the newer server migrated it to," >&2
    echo "  and the older binary will refuse to start against it (by design)." >&2
  fi

  compose up -d
  wait_for_health
}

# Prompt for a required input on /dev/tty (stdin is the piped script). An
# optional default ($3, e.g. the prior install's answer) is shown in brackets;
# pressing Enter keeps it. Without a terminal a default is used, else we die.
prompt() {
  local var="$1" msg="$2" def="${3:-}" val why; eval "val=\${$var:-}"
  [ -n "$val" ] && return
  # --yes is the same contract as having no terminal: take the offered default,
  # which on an upgrade is the prior install's answer.
  if [ "$ASSUME_YES" = true ] || [ ! -e /dev/tty ]; then
    [ -n "$def" ] && { eval "$var=\$def"; return; }
    if [ "$ASSUME_YES" = true ]; then why="--yes was given, so nothing is prompted for"
    else why="no terminal is available"; fi
    die "$var is not set and there is no prior value to fall back to ($why).
  Pass it as an environment variable."
  fi
  if [ -n "$def" ]; then printf '%s [%s]: ' "$msg" "$def" > /dev/tty
  else printf '%s: ' "$msg" > /dev/tty; fi
  read -r val < /dev/tty || true
  val="${val:-$def}"
  [ -n "$val" ] || die "$var is required"
  eval "$var=\$val"
}

# --rollback is a mode of its own: it touches no registry, asks nothing, and must
# not fall through into an install. Handled here, once the helpers above exist
# and before any prompt.
if [ "$DO_ROLLBACK" = true ]; then
  if do_rollback; then
    cat <<EOF

============================================================
  Rolled back. DevGrail is answering its readiness probe.

  Dashboard: $SCHEME://$DEVGRAIL_DOMAIN/
  Images:    devgrail-server:latest now points at the previous build.
  Logs:      docker compose --project-directory $DEPLOY_DIR logs -f
============================================================
EOF
    exit 0
  fi
  echo >&2
  printf '\033[1;31merror:\033[0m %s\n' "The rolled-back stack did not become ready either." >&2
  compose logs --tail=50 devgrail-server >&2 || true
  echo "  Snapshots available: $BACKUP_DIR" >&2
  exit 1
fi

if [ "$IS_UPGRADE" = true ]; then
  log "Existing DevGrail install detected — current answers shown as [defaults]; press Enter to keep, or type a new value to change."
fi
prompt DEVGRAIL_DOMAIN      "Public dashboard hostname (A record -> this VPS)"                "$(prior DEVGRAIL_DOMAIN)"
prompt DEVGRAIL_BASE_DOMAIN "Base domain for app subdomains (wildcard *.<base> -> this VPS)"  "$(prior DEVGRAIL_BASE_DOMAIN)"
# Only needed for Let's Encrypt; plain-HTTP installs skip it (ACME is never run).
if [ "$TLS_ENABLED" = true ]; then
  prompt ACME_EMAIL         "Email for Let's Encrypt certificates"                            "$(prior ACME_EMAIL)"
fi

# Never prompted, but just as destructive to reset: an operator running on
# non-standard ports had them silently rewritten to 80/443 by every upgrade.
resolve HTTP_PORT  80
resolve HTTPS_PORT 443
resolve DEVGRAIL_SERVER_IMAGE devgrail-server

# --- DNS preflight ----------------------------------------------------------
# "DNS is the only manual step" (spec §9) — and it was the only unverified one.
# The installer prompted for the domain, brought the stack up, and printed the
# required A records *after* it had finished. When they were not in place,
# Traefik's ACME TLS-ALPN-01 challenge failed and the customer got a browser TLS
# error with nothing connecting it to the cause. Let's Encrypt's failure rate
# limits then make retrying worse rather than better.
#
# So the names are resolved and compared against this host's addresses here,
# before Docker is installed and long before the stack is started. This is a
# warning, not a verdict: split-horizon DNS, a pre-cutover install and a record
# that has not propagated yet all look identical from inside the host. It asks
# before continuing when there is a terminal; --skip-dns-check turns it off.

# host_ips <name> — the IPv4 addresses <name> resolves to, one per line, empty
# when it does not resolve. Four resolvers because no single one is present
# everywhere: getent is glibc (absent on busybox), dig and host come from
# bind-utils which most VPS images do not install, nslookup is busybox's.
host_ips() {
  local name="$1" out=""
  if command -v getent >/dev/null 2>&1; then
    out="$(getent ahostsv4 "$name" 2>/dev/null | awk '{print $1}')" || out=""
  fi
  if [ -z "$out" ] && command -v dig >/dev/null 2>&1; then
    out="$(dig +short +time=3 +tries=1 A "$name" 2>/dev/null)" || out=""
  fi
  if [ -z "$out" ] && command -v host >/dev/null 2>&1; then
    out="$(host -W 3 -t A "$name" 2>/dev/null | sed -n 's/.*has address //p')" || out=""
  fi
  if [ -z "$out" ] && command -v nslookup >/dev/null 2>&1; then
    out="$(nslookup "$name" 2>/dev/null | sed -n '/^Name:/,$p' | sed -n 's/^Address: *//p')" || out=""
  fi
  # Whatever the tool printed, reduced to one de-duplicated IPv4 per line. The
  # grep also drops the resolver's own address, which nslookup prints first.
  # shellcheck disable=SC2086 # deliberate word splitting: one address per line
  printf '%s\n' $out | grep -E '^([0-9]{1,3}\.){3}[0-9]{1,3}$' | sort -u || true
}

have_resolver() {
  command -v getent   >/dev/null 2>&1 ||
  command -v dig      >/dev/null 2>&1 ||
  command -v host     >/dev/null 2>&1 ||
  command -v nslookup >/dev/null 2>&1
}

# this_host_ips — addresses that legitimately point at this machine. Both halves
# matter: a marketplace VPS carries its public address on an interface, but a
# NAT'd or floating-IP host does not, and there an echo service is the only way
# to learn what the world sees. Either matching is enough.
this_host_ips() {
  local locals="" public="" url
  if command -v ip >/dev/null 2>&1; then
    locals="$(ip -4 -o addr show scope global 2>/dev/null | awk '{print $4}' | cut -d/ -f1)" || locals=""
  fi
  if [ -z "$locals" ] && command -v hostname >/dev/null 2>&1; then
    locals="$(hostname -I 2>/dev/null)" || locals=""
  fi
  for url in https://api.ipify.org https://checkip.amazonaws.com https://ifconfig.me/ip; do
    public="$(curl -fsS --max-time 5 "$url" 2>/dev/null | tr -cd '0-9.')" || public=""
    [ -n "$public" ] && break
  done
  # shellcheck disable=SC2086 # deliberate word splitting: one address per line
  printf '%s\n' $locals $public | grep -E '^([0-9]{1,3}\.){3}[0-9]{1,3}$' | sort -u || true
}

# dns_problem <name> <this-host's-ips> <what> — one problem line, or nothing when
# <name> resolves to an address this host answers on.
dns_problem() {
  local name="$1" mine="$2" what="$3" got ip
  got="$(host_ips "$name")"
  if [ -z "$got" ]; then
    printf '  %s does not resolve — no %s found.\n' "$name" "$what"
    return 0
  fi
  for ip in $got; do
    case " $(printf '%s' "$mine" | tr '\n' ' ') " in *" $ip "*) return 0 ;; esac
  done
  printf '  %s resolves to %s, which is not this host.\n' \
    "$name" "$(printf '%s' "$got" | tr '\n' ' ')"
}

dns_preflight() {
  if [ "$SKIP_DNS_CHECK" = true ]; then
    log "Skipping the DNS preflight (--skip-dns-check)."
    return 0
  fi
  if ! have_resolver; then
    warn "no DNS lookup tool here (getent/dig/host/nslookup) — skipping the DNS preflight."
    return 0
  fi

  log "Checking DNS for $DEVGRAIL_DOMAIN and *.$DEVGRAIL_BASE_DOMAIN..."
  local mine problems probe
  mine="$(this_host_ips)"
  if [ -z "$mine" ]; then
    warn "could not determine this host's IP address — skipping the DNS preflight."
    return 0
  fi

  # A random label under the base domain can only resolve through the wildcard
  # record. Checking the base domain itself would pass on a bare A record and
  # then fail for every workspace subdomain, which is the case that matters.
  probe="devgrail-dns-probe-$$.$DEVGRAIL_BASE_DOMAIN"
  problems="$(dns_problem "$DEVGRAIL_DOMAIN" "$mine" "A record")"
  problems="$problems$(dns_problem "$probe" "$mine" "wildcard A record for *.$DEVGRAIL_BASE_DOMAIN")"

  if [ -z "$problems" ]; then
    log "DNS resolves to this host."
    return 0
  fi

  warn "DNS does not point at this host yet."
  printf '%s' "$problems" >&2
  echo "  This host answers on: $(printf '%s' "$mine" | tr '\n' ' ')" >&2
  echo >&2
  echo "  Required records:" >&2
  echo "    A  $DEVGRAIL_DOMAIN      -> this VPS" >&2
  echo "    A  *.$DEVGRAIL_BASE_DOMAIN  -> this VPS" >&2
  if [ "$TLS_ENABLED" = true ]; then
    echo >&2
    echo "  Continuing now means Let's Encrypt will attempt its TLS-ALPN-01" >&2
    echo "  challenge against a name that does not reach this host. It will fail," >&2
    echo "  and repeated failures hit rate limits that make retrying slower rather" >&2
    echo "  than faster — so it is usually cheaper to fix DNS first." >&2
  fi
  echo >&2
  echo "  Legitimate reasons to continue anyway: split-horizon DNS, a pre-cutover" >&2
  echo "  install, or a record that has not propagated yet. --skip-dns-check" >&2
  echo "  suppresses this check entirely." >&2
  echo >&2

  if [ "$ASSUME_YES" = true ]; then
    warn "--yes given — continuing with unverified DNS."
  elif [ -e /dev/tty ]; then
    printf 'Continue anyway? [y/N]: ' > /dev/tty
    read -r _answer < /dev/tty || _answer=""
    case "$_answer" in
      y|Y|yes|YES) log "Continuing with unverified DNS." ;;
      *) die "aborted before anything on this host was changed.
  Fix the records and re-run, or re-run with --skip-dns-check." ;;
    esac
  else
    # Unattended: an install driven by cloud-init frequently runs before the
    # records exist. Refusing would break that legitimate case, so it is said
    # loudly and the install proceeds.
    warn "no terminal to ask — continuing with unverified DNS."
  fi
}

dns_preflight

# --- Docker + tooling ------------------------------------------------------
# A dry run must not install anything, and it cannot read the registry without
# jq — so on a host missing either, the honest report is that they would be
# installed, and it stops there.
if [ "$DRY_RUN" = true ]; then
  MISSING_TOOLS=""
  command -v docker >/dev/null 2>&1 || MISSING_TOOLS="Docker"
  command -v jq >/dev/null 2>&1     || MISSING_TOOLS="${MISSING_TOOLS:+$MISSING_TOOLS and }jq"
  if [ -n "$MISSING_TOOLS" ]; then
    echo
    echo "============================================================"
    echo "  Dry run — nothing on this host has been changed."
    echo
    echo "  $MISSING_TOOLS would be installed first; this host does not have it yet."
    echo "  Nothing further can be reported without it: the release manifest,"
    echo "  the version diff and the disk figures all come from the registry."
    echo "============================================================"
    exit 0
  fi
fi

if ! command -v docker >/dev/null 2>&1; then
  log "Installing Docker..."
  curl -fsSL https://get.docker.com | sh
  systemctl enable --now docker 2>/dev/null || true
fi
docker compose version >/dev/null 2>&1 || die "the 'docker compose' plugin is required"

if ! command -v jq >/dev/null 2>&1; then
  log "Installing jq..."
  if   command -v apt-get >/dev/null 2>&1; then apt-get update -qq && apt-get install -y -qq jq
  elif command -v dnf >/dev/null 2>&1;     then dnf install -y jq
  elif command -v yum >/dev/null 2>&1;     then yum install -y jq
  elif command -v apk >/dev/null 2>&1;     then apk add --no-cache jq
  else die "please install 'jq' and re-run"; fi
fi

# openssl verifies the release signature (and generates secrets when present).
# Best-effort: a host without it falls back to /dev/urandom for secrets and to
# an unverified release, both of which say so rather than failing.
if ! command -v openssl >/dev/null 2>&1; then
  log "Installing openssl..."
  if   command -v apt-get >/dev/null 2>&1; then apt-get install -y -qq openssl || true
  elif command -v dnf >/dev/null 2>&1;     then dnf install -y openssl || true
  elif command -v yum >/dev/null 2>&1;     then yum install -y openssl || true
  elif command -v apk >/dev/null 2>&1;     then apk add --no-cache openssl || true
  fi
fi

# tar builds the pre-upgrade snapshot archive; df/awk size the free-space check.
for tool in tar df awk; do
  command -v "$tool" >/dev/null 2>&1 || die "'$tool' is required (used to take the pre-upgrade backup)"
done

# --- userns-remap preflight (workspaces run untrusted agent code) ----------
if ! docker info --format '{{.SecurityOptions}}' 2>/dev/null | grep -q 'name=userns'; then
  warn "Docker userns-remap is NOT enabled."
  echo "  Workspaces will run as host root inside the container — a container" >&2
  echo "  escape would be host root. To enable (one-time, needs root):" >&2
  echo "    1. Add to /etc/docker/daemon.json:  { \"userns-remap\": \"default\" }" >&2
  echo "    2. sudo systemctl restart docker" >&2
  echo "  Trusted infra (traefik, docker-socket-proxy, devgrail-server) opts out" >&2
  echo "  via userns_mode: host, so only workspaces are remapped. See" >&2
  echo "  DEVGRAIL_SYSTEM_SPEC.md §5. Continuing without remap..." >&2
  echo >&2
fi

# --- address-pool preflight (one Docker network per workspace) -------------
# Each workspace gets its own bridge network so untrusted agent code can't reach a
# peer workspace directly. Docker's default address pools only yield ~31 networks
# in total, so a busy host runs out and container creation starts failing with
# "could not find an available, non-overlapping IPv4 address pool". Widening the
# pools is a one-time daemon change; as with userns-remap we warn rather than
# auto-edit daemon.json and restart dockerd under the operator.
if ! docker info --format '{{json .}}' 2>/dev/null | grep -q 'DefaultAddressPools'; then
  warn "Docker default address pools are not customised."
  echo "  Each DevGrail workspace gets its own network; Docker's defaults run out" >&2
  echo "  at roughly 27 workspaces, after which container creation fails. To raise" >&2
  echo "  the ceiling (one-time, needs root):" >&2
  echo "    1. Add to /etc/docker/daemon.json:" >&2
  echo "       { \"default-address-pools\": [{\"base\": \"172.16.0.0/12\", \"size\": 24}] }" >&2
  echo "    2. sudo systemctl restart docker" >&2
  echo "  This restarts every container on the host. Safe to defer until you" >&2
  echo "  approach the limit. Continuing with the defaults..." >&2
  echo >&2
fi

# --- download + load images ------------------------------------------------
# TMP is chosen after the release is resolved, once the download size is known:
# /tmp is a tmpfs on many VPS images (sized from RAM, often well under 1 GiB)
# and a multi-hundred-megabyte image tarball does not fit. The trap is installed
# now regardless, so an abort at any later point still cleans up.
TMP=""
# `|| true` and the :+ guards because this runs on every exit path, including
# ones taken before TMP is chosen: `rm -rf` with no operands fails, and a failing
# EXIT trap is a confusing thing to leave behind after a clean run.
trap 'rm -rf ${TMP:+"$TMP"} ${SNAPSHOT_STAGE:+"$SNAPSHOT_STAGE"} ${SIG_DIR:+"$SIG_DIR"} 2>/dev/null || true' EXIT

# free_bytes <dir> — free space on the filesystem holding <dir>, in bytes. Prints
# 0 when it cannot be determined (df on an unwritable path, an exotic
# filesystem) — every caller treats 0 as "unknown, do not block on it", so a
# preflight that cannot measure never becomes a preflight that refuses.
free_bytes() {
  local v
  v="$(df -Pk "$1" 2>/dev/null | awk 'NR==2 {print $4 * 1024}')" || v=""
  case "$v" in ''|*[!0-9]*) v=0 ;; esac
  printf '%s' "$v"
}

# human <bytes> — MiB/GiB for a message an operator can act on. Integer shell
# arithmetic only: this runs before jq is guaranteed and awk is already required.
human() {
  local b="${1:-0}"
  if [ "$b" -ge 1073741824 ]; then printf '%s.%s GiB' "$((b / 1073741824))" "$(((b % 1073741824) * 10 / 1073741824))"
  elif [ "$b" -ge 1048576 ]; then printf '%s MiB' "$((b / 1048576))"
  else printf '%s bytes' "$b"; fi
}

# Prior install versions (for skip-if-unchanged upgrades). SERVER_RELEASE,
# CONTAINER_RELEASE and ASSET_VERSIONS were added later, so an install that
# predates them simply leaves these empty — reported as "unknown" or treated as
# "not yet fetched", never as a failure.
SERVER_VERSION=""; CONTAINER_VERSION=""
SERVER_RELEASE=""; CONTAINER_RELEASE=""
ASSET_VERSIONS=""
RELEASE_VERSION=""
# shellcheck source=/dev/null # written by a previous run of this script
[ -f "$VERSIONS_FILE" ] && . "$VERSIONS_FILE"
# RELEASE_VERSION is reused below for the release this run resolves, so the one
# the file just supplied — what is installed *now* — is kept under its own name.
PRIOR_RELEASE_VERSION="$RELEASE_VERSION"

# asset_version_of <slug> — the version recorded for this slug at the last run,
# or empty. ASSET_VERSIONS is a space-separated "slug=version" list, which keeps
# it to one shell variable regardless of how many assets a release carries (slugs
# are filenames, so they are not usable as variable names).
asset_version_of() {
  # shellcheck disable=SC2086 # deliberate word splitting into one entry per line
  printf '%s\n' ${ASSET_VERSIONS:-} | awk -F= -v s="$1" '$1==s {print $2; exit}'
}

# --- resolve the release ----------------------------------------------------
# A DevGrail release is seven artifacts, not one. Resolving them all from a
# single manifest is what stops an upgrade from mixing a new compose file with
# an old server image: before this, the compose and Traefik config were
# re-fetched unconditionally on every run because there was no asset version to
# compare against, so even a "nothing changed" re-run silently adopted whatever
# had been published since.
DEVGRAIL_VERSION="${DEVGRAIL_VERSION:-current}"
RELEASE_MANIFEST=""
RELEASE_VERSION=""

resolve_release() {
  local body
  body="$(curl -fsSL "$WEB/api/registry/releases/$DEVGRAIL_VERSION" 2>/dev/null)" || body=""
  if [ -z "$body" ]; then
    # An explicit version that does not resolve is an error — silently
    # installing something else is the one thing --version must never do.
    [ "$DEVGRAIL_VERSION" = current ] \
      || die "release '$DEVGRAIL_VERSION' is not published at $WEB.
  Nothing has been changed."
    # No release manifest at all means a registry older than this installer.
    # Fall back to resolving each artifact by slug, exactly as before.
    log "Registry publishes no release manifest — resolving each artifact individually."
    return 0
  fi
  RELEASE_MANIFEST="$body"
  RELEASE_VERSION="$(printf '%s' "$body" | jq -r '.release_version // empty')"
  log "Installing release ${RELEASE_VERSION:-unknown}"
}

resolve_release

# --- release signature -------------------------------------------------------
# Every artifact is verified against a SHA-256 the registry publishes — which
# catches a corrupted download but not a compromised registry, because whoever
# can serve the bytes can serve the matching digest. The same origin also serves
# this script, the compose file and the Traefik config, all executed as root.
#
# A detached signature over the release closes that, and only if the signing key
# never touches the web host. One signature covers all seven artifacts: that is
# the payoff of resolving a release from a single manifest.
#
# What is signed is a payload listing every artifact's kind, slug, version and
# digest — not the manifest itself, whose download URLs are rewritten per origin
# and so cannot be signed offline. Verification therefore proves the digests,
# and the digests prove the bytes. See docs/SECURITY.md.

# Public keys this installer trusts to sign a release. Generated by
# scripts/release-key.sh; the private half is never in this repo.
#
# Rotation: add the new key here and ship an installer carrying both, then start
# signing with the new one, then drop the old key a release later. Hosts that
# never upgraded the installer keep working throughout.
release_public_keys() {
  # key id f1f1eaf642454d65 — first embedded in v0.4.0
  cat <<'PUBKEYS'
-----BEGIN PUBLIC KEY-----
MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEK1NbYrxBM8Ia1iM4BcR0m/8DelN5
ZDvOwKmEOJA24dci3ftqHWwsxzbw8bACRps0rIOTqa45wblm6260fQVdIw==
-----END PUBLIC KEY-----
PUBKEYS
  # An operator running their own registry (or the test harness) can add keys
  # without editing this file. This grants trust rather than removing it, and
  # anyone able to set it already runs this script as root.
  if [ -n "${DEVGRAIL_RELEASE_PUBKEY_FILE:-}" ] && [ -f "$DEVGRAIL_RELEASE_PUBKEY_FILE" ]; then
    cat "$DEVGRAIL_RELEASE_PUBKEY_FILE"
  fi
}

# The verified payload, once there is one. Empty means nothing is enforced
# beyond the registry's own digests.
SIGNED_PAYLOAD_FILE=""
SIG_DIR=""

# unverified <why> — how a release that cannot be checked is treated. A warning
# by default: refusing would strand every host whose registry or installer
# predates signing, and an unsigned release is no worse than what existed
# before. DEVGRAIL_REQUIRE_SIGNATURE=1 turns it into a refusal for operators who
# want that guarantee.
unverified() {
  if [ "${DEVGRAIL_REQUIRE_SIGNATURE:-0}" = 1 ]; then
    die "refusing to install a release whose signature was not verified:
  $1
  DEVGRAIL_REQUIRE_SIGNATURE=1 is set. Nothing has been changed."
  fi
  warn "release signature not verified — $1"
  echo "  Artifacts are still checked against the digests this registry" >&2
  echo "  publishes, which detects a corrupted download but not a compromised" >&2
  echo "  registry. See docs/SECURITY.md." >&2
}

verify_release_signature() {
  local sig alg kid keys dir pem verified=false

  if [ -z "$RELEASE_MANIFEST" ]; then
    unverified "this registry publishes no release manifest"
    return 0
  fi
  sig="$(printf '%s' "$RELEASE_MANIFEST" | jq -c '.signature // empty' 2>/dev/null || true)"
  if [ -z "$sig" ] || [ "$sig" = null ]; then
    unverified "release ${RELEASE_VERSION:-current} was published without one"
    return 0
  fi

  keys="$(release_public_keys)"
  if [ -z "$keys" ]; then
    unverified "this installer carries no release signing key"
    return 0
  fi
  if ! command -v openssl >/dev/null 2>&1; then
    unverified "openssl is not installed here, so the signature cannot be checked"
    return 0
  fi

  alg="$(printf '%s' "$sig" | jq -r '.algorithm // empty')"
  kid="$(printf '%s' "$sig" | jq -r '.key_id // empty')"
  # An unknown algorithm is not the same as an unsigned release: the publisher
  # signed it with something this installer cannot check, and treating that as
  # "unsigned" would let an attacker downgrade verification by naming a scheme
  # we do not implement.
  [ "$alg" = "ecdsa-p256-sha256" ] \
    || die "release ${RELEASE_VERSION:-current} is signed with '$alg', which this
  installer cannot verify. Upgrade the installer. Nothing has been changed."

  dir="$(mktemp -d)"
  SIG_DIR="$dir"
  # -j, not -r: the signature is over exactly the published bytes, and -r adds a
  # trailing newline the signer never saw. That difference verifies as tampering.
  printf '%s' "$sig" | jq -j '.payload' > "$dir/payload"
  printf '%s' "$sig" | jq -r '.signature' | base64 -d > "$dir/sig" 2>/dev/null \
    || die "the signature on release ${RELEASE_VERSION:-current} is not valid base64.
  Nothing has been changed."

  # Each embedded key gets its own file; any one of them verifying is enough,
  # which is what makes rotation a matter of shipping two keys for a while.
  printf '%s\n' "$keys" > "$dir/keys.pem"
  awk -v d="$dir" '/-----BEGIN PUBLIC KEY-----/{n++} n>0 {print > (d "/key" n ".pem")}' "$dir/keys.pem"
  for pem in "$dir"/key*.pem; do
    [ -f "$pem" ] || continue
    if openssl dgst -sha256 -verify "$pem" -signature "$dir/sig" "$dir/payload" >/dev/null 2>&1; then
      verified=true
      break
    fi
  done
  if [ "$verified" != true ]; then
    die "the signature on release ${RELEASE_VERSION:-current} does not verify against
  any key this installer carries (signing key id: ${kid:-unknown}).
  That is either a corrupted download or a tampered release, and there is no
  way to tell which from here. Nothing has been changed."
  fi

  # A valid signature over somebody else's payload proves nothing about this
  # release, so the payload has to name it — otherwise an old signed release
  # could be replayed against a host asking for a newer one.
  if ! grep -qxF "devgrail-release-signature-v1" "$dir/payload"; then
    die "the release signature verifies but its payload is not in a format this
  installer understands. Nothing has been changed."
  fi
  if ! grep -qxF "release ${RELEASE_VERSION}" "$dir/payload"; then
    die "the signature verifies, but it was made over a different release than
  ${RELEASE_VERSION:-the one requested}. Nothing has been changed."
  fi

  SIGNED_PAYLOAD_FILE="$dir/payload"
  log "Release ${RELEASE_VERSION} signature verified (key ${kid:-unknown})."
}

# signed_digest_check <payload line> <what> — refuse anything the signed payload
# does not name. Called after the registry's own digest matched, so this is the
# check that survives a compromised registry: matching bytes are not enough if
# they are not the bytes the release was signed for.
signed_digest_check() {
  [ -n "$SIGNED_PAYLOAD_FILE" ] || return 0
  if grep -qxF "$1" "$SIGNED_PAYLOAD_FILE"; then
    return 0
  fi
  die "'$2' is not what release $RELEASE_VERSION was signed for.
  Its bytes match the digest this registry served, but that digest does not
  appear in the signed release payload:
    $1
  Refusing to install it. Nothing has been changed."
}

verify_release_signature

# release_field <slug> <jq filter> — a field of one artifact in the release
# manifest, or empty when there is no manifest or no such artifact.
release_field() {
  [ -n "$RELEASE_MANIFEST" ] || return 0
  printf '%s' "$RELEASE_MANIFEST" | jq -r --arg s "$1" --arg a "$ARCH" "$2" 2>/dev/null || true
}

# resolve_image <slug> <target_tag> <current_version> — settle which build of
# <slug> this run needs, and whether it has to be downloaded at all, without
# fetching a byte. Sets R_VER R_REL R_URL R_SHA R_SIZE R_SKIP.
#
# Split out from download_image so the disk preflight below can total up what is
# about to be written *before* the first download starts. Running out of space
# halfway through is precisely the condition that causes a mid-migration failure
# (INSTALL_UPGRADE_ANALYSIS M2), so discovering it after one image has already
# landed is not good enough.
R_VER=""; R_REL=""; R_URL=""; R_SHA=""; R_SIZE=0; R_SKIP=false
resolve_image() {
  local slug="$1" target="$2" cur="$3" manifest

  # Prefer the release manifest: it pins the exact version this release is, and
  # its download URLs carry that pin so they keep resolving to these bytes even
  # after a newer release is published.
  R_VER="$(release_field "$slug" '.images[]|select(.slug==$s)|.image_version')"
  R_URL="$(release_field "$slug" '.images[]|select(.slug==$s)|.artifacts[]|select(.arch==$a)|.download_url')"
  R_SHA="$(release_field "$slug" '.images[]|select(.slug==$s)|.artifacts[]|select(.arch==$a)|.sha256')"
  R_SIZE="$(release_field "$slug" '.images[]|select(.slug==$s)|.artifacts[]|select(.arch==$a)|.size')"
  R_REL="$RELEASE_VERSION"

  if [ -z "$R_VER" ]; then
    manifest="$(curl -fsSL "$WEB/api/registry/images/$slug")" \
      || die "cannot fetch image manifest for '$slug' from $WEB"
    R_VER="$(printf '%s' "$manifest" | jq -r '.image_version')"
    # Additive field: a registry that predates it, or a row uploaded before it
    # existed, yields null — which becomes an empty string, not an error.
    R_REL="$(printf '%s' "$manifest" | jq -r '.release_version // empty')"
    R_URL="$(printf '%s' "$manifest" | jq -r --arg a "$ARCH" '.artifacts[]|select(.arch==$a)|.download_url')"
    R_SHA="$(printf '%s' "$manifest" | jq -r --arg a "$ARCH" '.artifacts[]|select(.arch==$a)|.sha256')"
    R_SIZE="$(printf '%s' "$manifest" | jq -r --arg a "$ARCH" '.artifacts[]|select(.arch==$a)|.size')"
  fi
  [ -n "$R_URL" ] && [ "$R_URL" != "null" ] || die "no $ARCH build published for '$slug'"
  # A registry that does not declare sizes yields null; treat that as "unknown"
  # (0) rather than failing — the preflight then simply has less to check.
  case "$R_SIZE" in ''|null|*[!0-9]*) R_SIZE=0 ;; esac

  R_SKIP=false
  if [ -n "$cur" ] && [ "$cur" = "$R_VER" ] && docker image inspect "$target" >/dev/null 2>&1; then
    R_SKIP=true
  fi
}

LOADED_VERSION=""; LOADED_RELEASE=""
# download_image <slug> <target_tag> <current_version> [current_release]
# Uses the R_* values resolve_image already settled for this slug.
download_image() {
  local slug="$1" target="$2" cur="$3" cur_rel="${4:-}" loaded got tar
  local url="$R_URL" sha="$R_SHA" ver="$R_VER" rel="$R_REL"

  if [ "$R_SKIP" = true ]; then
    log "$slug already at ${rel:-v$ver} — skipping"
    LOADED_VERSION="$ver"
    # Prefer what the registry says now: it fills in the release for an install
    # that was made before the field existed, without re-downloading anything.
    LOADED_RELEASE="${rel:-$cur_rel}"
    return
  fi

  tar="$TMP/$slug.tar"
  log "Downloading $slug ${rel:-v$ver} ($ARCH)..."
  curl -fL --progress-bar -o "$tar" "$url" || die "download failed for '$slug'"
  log "Verifying checksum..."
  got="$(sha256_of "$tar")"
  [ "$got" = "$sha" ] || die "checksum mismatch for '$slug' (expected $sha, got $got)"
  signed_digest_check "image $slug $ver $ARCH $got" "$slug ($ARCH)"
  log "Loading image..."
  # Not `| head -n1`: head exits after one line, sed dies of SIGPIPE, and
  # `set -o pipefail` turns that into a fatal exit 141 — mid-upgrade, right after
  # the image was loaded, with no message. sed reads all of docker load's output
  # here; the first line is taken with parameter expansion afterwards.
  loaded="$(docker load -i "$tar" | sed -n 's/^Loaded image: //p')"
  loaded="${loaded%%$'\n'*}"
  [ -n "$loaded" ] || die "docker load produced no image for '$slug'"

  # Retagging to :latest moves that tag off the image currently running, which
  # then survives only as an untagged layer set until someone prunes it. Name it
  # :previous first — a tag, not a copy, so it costs nothing — and rollback
  # becomes a retag plus a compose recreate rather than a re-download.
  local prev_tag old_id new_id
  prev_tag="${target%:*}:previous"
  old_id="$(docker image inspect -f '{{.Id}}' "$target" 2>/dev/null || true)"
  new_id="$(docker image inspect -f '{{.Id}}' "$loaded" 2>/dev/null || true)"
  if [ -n "$old_id" ] && [ "$old_id" != "$new_id" ]; then
    docker tag "$old_id" "$prev_tag"
    log "Previous $slug kept as $prev_tag"
  fi

  docker tag "$loaded" "$target"
  rm -f "$tar"
  LOADED_VERSION="$ver"; LOADED_RELEASE="$rel"
}

# Resolve both images first, so the preflight sees the whole run.
resolve_image devgrail-server "devgrail-server:latest" "$SERVER_VERSION"
SERVER_SKIP="$R_SKIP"; SERVER_BYTES="$R_SIZE"
SERVER_R_VER="$R_VER"; SERVER_R_REL="$R_REL"; SERVER_R_URL="$R_URL"; SERVER_R_SHA="$R_SHA"

resolve_image devgrail-container "devgrail-container:latest" "$CONTAINER_VERSION"
CONTAINER_SKIP="$R_SKIP"; CONTAINER_BYTES="$R_SIZE"
CONTAINER_R_VER="$R_VER"; CONTAINER_R_REL="$R_REL"; CONTAINER_R_URL="$R_URL"; CONTAINER_R_SHA="$R_SHA"

# --- disk preflight ---------------------------------------------------------
# Every upgrade downloads a tarball and then needs the space again in the Docker
# data root when `docker load` unpacks it, and nothing prunes. Disk-full is not
# merely untidy here: it is the condition that produces a failure mid-migration,
# which is the worst state this product can be in. So the numbers are checked
# before the first byte is fetched, and reported when they do not add up.
#
# 2.5x the declared download size: 1x for the tarball, ~1x again for the
# unpacked layers, plus half for the compressed layers Docker keeps while
# loading and for headroom. A registry that declares no sizes yields 0, and the
# check for that filesystem is skipped rather than guessed at.
DOWNLOAD_BYTES=0
[ "$SERVER_SKIP" = true ]    || DOWNLOAD_BYTES=$((DOWNLOAD_BYTES + SERVER_BYTES))
[ "$CONTAINER_SKIP" = true ] || DOWNLOAD_BYTES=$((DOWNLOAD_BYTES + CONTAINER_BYTES))
NEED_BYTES=$((DOWNLOAD_BYTES * 5 / 2))

# --- dry run -----------------------------------------------------------------
# "Re-run the same script that did the first install" is a fine upgrade
# mechanism right up until the operator wants to know what it will do before it
# does it. Everything above this line is resolution — reading the registry and
# the prior install's own files — so the report can be produced with nothing
# downloaded, nothing written and nothing started.
#
# It sits here rather than earlier because the release has to be resolved for
# any of it to be true, and earlier than the next line because that is where the
# first byte gets staged on disk.

# dry_change <from> <to> — "a -> b", or "b (unchanged)" when they match.
dry_change() {
  local from="${1:-}" to="${2:-}"
  [ -n "$to" ] || to="unknown"
  if [ -z "$from" ]; then printf '%s (new)' "$to"
  elif [ "$from" = "$to" ]; then printf '%s (unchanged)' "$to"
  else printf '%s -> %s' "$from" "$to"; fi
}

dry_image_line() {
  local slug="$1" cur="$2" ver="$3" rel="$4" skip="$5" bytes="$6"
  if [ "$skip" = true ]; then
    printf '    %-20s v%s (unchanged)\n' "$slug" "$ver"
  else
    printf '    %-20s %s%s — would download %s\n' "$slug" \
      "$(dry_change "${cur:+v$cur}" "v$ver")" "${rel:+ $rel}" "$(human "$bytes")"
  fi
}

dry_asset_line() {
  local slug="$1" ver have
  ver="$(release_field "$slug" '.assets[]|select(.slug==$s)|.asset_version')"
  have="$(asset_version_of "$slug")"
  if [ -z "$ver" ]; then
    printf '    %-24s would be re-fetched (the registry publishes no version for it)\n' "$slug"
  elif [ "$ver" = "$have" ] && [ -f "$ASSET_DIR/$slug" ]; then
    printf '    %-24s v%s (unchanged)\n' "$slug" "$ver"
  else
    printf '    %-24s %s — would download\n' "$slug" "$(dry_change "${have:+v$have}" "v$ver")"
  fi
}

dry_run_report() {
  local tmpdir="${TMPDIR:-/tmp}" docker_root key was now changed=false health schema

  echo
  echo "============================================================"
  echo "  Dry run — nothing on this host has been changed."
  echo
  if [ "$IS_UPGRADE" = true ]; then
    echo "  Mode:     upgrade of the install at $INSTALL_DIR"
  else
    echo "  Mode:     first install into $INSTALL_DIR"
  fi
  echo "  Registry: $WEB"
  echo "  Release:  $(dry_change "$PRIOR_RELEASE_VERSION" "${RELEASE_VERSION:-not published as a release}")"
  if [ -n "$SIGNED_PAYLOAD_FILE" ]; then
    echo "  Signature: verified against a key embedded in this installer"
  else
    echo "  Signature: not verified (see the warning above)"
  fi
  echo
  echo "  Images"
  dry_image_line devgrail-server    "$SERVER_VERSION"    "$SERVER_R_VER" \
    "$SERVER_R_REL"    "$SERVER_SKIP"    "$SERVER_BYTES"
  dry_image_line devgrail-container "$CONTAINER_VERSION" "$CONTAINER_R_VER" \
    "$CONTAINER_R_REL" "$CONTAINER_SKIP" "$CONTAINER_BYTES"

  echo
  echo "  Assets"
  # Only the two this run would actually fetch. install.sh and uninstall.sh are
  # published as assets too, but they are served to `curl`, never downloaded
  # here, and listing them would suggest otherwise.
  dry_asset_line docker-compose.yml
  dry_asset_line "$TRAEFIK_ASSET"

  echo
  echo "  Disk"
  if [ "$NEED_BYTES" -gt 0 ]; then
    printf '    would need %s (2.5x the %s to download)\n' \
      "$(human "$NEED_BYTES")" "$(human "$DOWNLOAD_BYTES")"
  else
    echo "    nothing to download"
  fi
  printf '    free in %s: %s\n' "$tmpdir" "$(human "$(free_bytes "$tmpdir")")"
  docker_root="$(docker info --format '{{.DockerRootDir}}' 2>/dev/null || true)"
  [ -n "$docker_root" ] && [ -d "$docker_root" ] || docker_root=/var/lib/docker
  if [ -d "$docker_root" ]; then
    printf '    free in %s: %s\n' "$docker_root" "$(human "$(free_bytes "$docker_root")")"
  fi

  if [ "$IS_UPGRADE" = true ]; then
    echo
    echo "  Settings"
    for key in DEVGRAIL_DOMAIN DEVGRAIL_BASE_DOMAIN ACME_EMAIL DEVGRAIL_TLS HTTP_PORT HTTPS_PORT; do
      was="$(prior "$key")"
      eval "now=\${$key:-}"
      if [ "$was" != "$now" ]; then
        printf '    %-22s %s -> %s\n' "$key" "${was:-(unset)}" "${now:-(unset)}"
        changed=true
      fi
    done
    if [ "$changed" = false ]; then
      echo "    unchanged — every value in the prior .env would be preserved"
    fi

    echo
    echo "  Data"
    echo "    a snapshot of the database and config.yaml would be written to"
    echo "    $BACKUP_DIR before the new server starts (newest ${DEVGRAIL_SNAPSHOT_RETAIN:-5} kept)"
    health="$(curl -fsSk --max-time 5 "$SCHEME://$DEVGRAIL_DOMAIN/api/v1/healthz" 2>/dev/null || true)"
    schema="$(printf '%s' "$health" | jq -r '.schema_version.applied // empty' 2>/dev/null || true)"
    if [ -n "$schema" ]; then echo "    this database is at schema version $schema"; fi
    echo "    any pending migrations are applied by the new server when it starts;"
    echo "    which ones cannot be listed without loading that image, and a dry"
    echo "    run does not download it"
  fi

  echo
  echo "  Re-run without --dry-run to apply this."
  echo "============================================================"
  echo
  return 0
}

if [ "$DRY_RUN" = true ]; then
  dry_run_report
  exit 0
fi

# Where the tarballs are staged. /tmp is a tmpfs on many VPS images — sized from
# RAM and often well under a gigabyte — so falling back to $INSTALL_DIR/tmp when
# it does not fit turns "no space left on device" partway through a download
# into a directory that has the room.
choose_tmp() {
  local candidate="${TMPDIR:-/tmp}" fallback="$INSTALL_DIR/tmp" have have_fb
  have="$(free_bytes "$candidate")"
  # 0 means "could not measure", which is not the same as "no room". Nothing
  # here is worth refusing an upgrade over on a guess.
  if [ "$NEED_BYTES" -gt 0 ] && [ "$have" -gt 0 ] && [ "$have" -lt "$NEED_BYTES" ]; then
    mkdir -p "$fallback" || die "cannot create $fallback for the download"
    have_fb="$(free_bytes "$fallback")"
    if [ "$have_fb" -gt 0 ] && [ "$have_fb" -lt "$NEED_BYTES" ]; then
      die "not enough free space to download this release.
  Needed:  $(human "$NEED_BYTES") (2.5x the $(human "$DOWNLOAD_BYTES") of artifacts)
  Free in $candidate: $(human "$have")
  Free in $fallback: $(human "$have_fb")
  Free some space and re-run. Nothing has been changed."
    fi
    log "$candidate is too small for this download — staging in $fallback instead"
    TMP="$(mktemp -d "$fallback/install.XXXXXX")"
  else
    TMP="$(mktemp -d)"
  fi
}
choose_tmp

if [ "$NEED_BYTES" -gt 0 ]; then
  DOCKER_ROOT="$(docker info --format '{{.DockerRootDir}}' 2>/dev/null || true)"
  [ -n "$DOCKER_ROOT" ] && [ -d "$DOCKER_ROOT" ] || DOCKER_ROOT=/var/lib/docker
  if [ -d "$DOCKER_ROOT" ]; then
    DOCKER_FREE="$(free_bytes "$DOCKER_ROOT")"
    if [ "$DOCKER_FREE" -gt 0 ] && [ "$DOCKER_FREE" -lt "$NEED_BYTES" ]; then
      die "not enough free space in the Docker data root to load this release.
  Needed: $(human "$NEED_BYTES") (2.5x the $(human "$DOWNLOAD_BYTES") of artifacts)
  Free in $DOCKER_ROOT: $(human "$DOCKER_FREE")
  Reclaim space (\`docker image prune\`, \`docker system df\`) and re-run.
  Nothing has been changed."
    fi
  fi
fi

R_SKIP="$SERVER_SKIP"; R_VER="$SERVER_R_VER"; R_REL="$SERVER_R_REL"; R_URL="$SERVER_R_URL"; R_SHA="$SERVER_R_SHA"
download_image devgrail-server    "devgrail-server:latest"    "$SERVER_VERSION"    "$SERVER_RELEASE"
NEW_SERVER_VERSION="$LOADED_VERSION"; NEW_SERVER_RELEASE="$LOADED_RELEASE"
# The workspace image tag is hard-coded in internal/docker (DefaultImage =
# devgrail-container:latest), so retag the loaded release to :latest.
R_SKIP="$CONTAINER_SKIP"; R_VER="$CONTAINER_R_VER"; R_REL="$CONTAINER_R_REL"; R_URL="$CONTAINER_R_URL"; R_SHA="$CONTAINER_R_SHA"
download_image devgrail-container "devgrail-container:latest" "$CONTAINER_VERSION" "$CONTAINER_RELEASE"
NEW_CONTAINER_VERSION="$LOADED_VERSION"; NEW_CONTAINER_RELEASE="$LOADED_RELEASE"

# --- deploy assets ---------------------------------------------------------
log "Writing deploy config to $INSTALL_DIR..."
mkdir -p "$DEPLOY_DIR/traefik" "$DEPLOY_DIR/dynamic" "$CONFIG_DIR" "$ASSET_DIR"

NEW_ASSET_VERSIONS=""

# fetch_asset <slug> — leave the release's copy of <slug> at $ASSET_PATH,
# downloading and verifying it only if this run needs a version we do not
# already hold.
ASSET_PATH=""
fetch_asset() {
  local slug="$1" url ver sha cached got have
  cached="$ASSET_DIR/$slug"

  ver="$(release_field "$slug" '.assets[]|select(.slug==$s)|.asset_version')"
  url="$(release_field "$slug" '.assets[]|select(.slug==$s)|.download_url')"
  sha="$(release_field "$slug" '.assets[]|select(.slug==$s)|.sha256')"
  if [ -z "$url" ]; then
    # No release manifest (or an asset it does not carry): fetch whatever is
    # published, as installers did before releases existed. Unversioned, so it
    # cannot be skipped — that is the behaviour releases exist to fix.
    url="$WEB/install/$slug"; ver=""; sha=""
  fi

  have="$(asset_version_of "$slug")"
  if [ -n "$ver" ] && [ "$ver" = "$have" ] && [ -f "$cached" ]; then
    log "$slug already at v$ver — skipping"
  else
    curl -fsSL "$url" -o "$cached.tmp" || die "cannot fetch '$slug' from $WEB"
    got="$(sha256_of "$cached.tmp")"
    if [ -n "$sha" ]; then
      [ "$got" = "$sha" ] || die "checksum mismatch for '$slug' (expected $sha, got $got)"
    fi
    # Verified before the templating below rewrites it: what the release was
    # signed for is the published file, not what this host makes of it.
    if [ -n "$ver" ]; then
      signed_digest_check "asset $slug $ver $got" "$slug"
    elif [ -n "$SIGNED_PAYLOAD_FILE" ]; then
      # The release verified, but does not name this asset — so it came from the
      # unversioned fallback path and nothing signed it. Not fatal (that path
      # exists for registries older than releases), but it is a hole in an
      # otherwise verified install and has to be said out loud.
      warn "'$slug' is not part of signed release $RELEASE_VERSION — fetched unverified."
    fi
    mv -f "$cached.tmp" "$cached"
  fi

  if [ -n "$ver" ]; then
    NEW_ASSET_VERSIONS="${NEW_ASSET_VERSIONS:+$NEW_ASSET_VERSIONS }$slug=$ver"
  fi
  ASSET_PATH="$cached"
}

# The rendered outputs are always rewritten from the cached copies, even when
# nothing was downloaded: DEVGRAIL_DOMAIN or DEVGRAIL_TLS may have changed at
# the prompt, and a config that silently kept a stale domain would be worse than
# a redundant write.
fetch_asset docker-compose.yml
cp -f "$ASSET_PATH" "$DEPLOY_DIR/docker-compose.yml"

# The TLS or plain-HTTP router set per DEVGRAIL_TLS. Either is written to the
# same dynamic.yml the compose file mounts by default, so no compose var is needed.
fetch_asset "$TRAEFIK_ASSET"
sed "s|__DEVGRAIL_DOMAIN__|$DEVGRAIL_DOMAIN|g" "$ASSET_PATH" > "$DEPLOY_DIR/traefik/dynamic.yml"

# config.yaml holds the JWT secret (not passed via env). It used to be written
# only when absent, so registry_url/tls_enabled went stale as soon as the
# operator changed them at the prompt. The .env does override both at runtime
# (internal/config/config.go), so that was latent rather than active — but two
# files disagreeing about the domain is a trap to walk into during an incident.
# The derived keys are rewritten every run; the secret never is.
CONFIG_FILE="$CONFIG_DIR/config.yaml"
if [ -f "$CONFIG_FILE" ]; then
  # Same SIGPIPE-under-pipefail hazard as the docker load above. Aborting here
  # would be particularly bad: the operator would see a bare exit 141 while the
  # installer was reading the secret that decrypts their entire database.
  JWT_SECRET="$(sed -n 's/^jwt_secret:[[:space:]]*//p' "$CONFIG_FILE")"
  JWT_SECRET="$(printf '%s' "${JWT_SECRET%%$'\n'*}" | tr -d "\"' ")"
  # Generating a fresh secret here would orphan every encrypted value in the
  # database — every API key secret, user secret and workspace SSH password
  # (SECURITY_ANALYSIS.md #4). Refusing loudly is the only safe answer.
  [ -n "$JWT_SECRET" ] || die "$CONFIG_FILE exists but has no jwt_secret.
  That secret keys every encrypted value in the database; generating a new one
  would make all of them undecryptable. Restore the file from a backup
  (/opt/devgrail/backups holds config.yaml alongside the database) and re-run.
  Nothing has been changed."
else
  JWT_SECRET="$(gen_secret)"
fi

# Anything the operator added by hand is carried over — this installer owns the
# derived keys, not the whole file.
CONFIG_EXTRA=""
if [ -f "$CONFIG_FILE" ]; then
  CONFIG_EXTRA="$(grep -vE '^(jwt_secret|domain|base_domain|registry_url|traefik_dynamic_path|tls_enabled):' "$CONFIG_FILE" || true)"
fi

cat > "$CONFIG_FILE.tmp" <<EOF
jwt_secret: "$JWT_SECRET"
domain: "$DEVGRAIL_DOMAIN"
base_domain: "$DEVGRAIL_BASE_DOMAIN"
registry_url: "$WEB"
traefik_dynamic_path: /dynamic/managed.yml
tls_enabled: $TLS_ENABLED
EOF
if [ -n "$CONFIG_EXTRA" ]; then printf '%s\n' "$CONFIG_EXTRA" >> "$CONFIG_FILE.tmp"; fi
# Same reasoning as .env, more so: a truncated config.yaml loses the JWT secret,
# which is unrecoverable without a backup.
chmod 600 "$CONFIG_FILE.tmp"
mv -f "$CONFIG_FILE.tmp" "$CONFIG_FILE"

# Preserve the admin password across re-runs. It seeds the admin account on
# first boot only (internal/db/seed.go: SeedAdmin ignores it thereafter unless
# DEVGRAIL_FORCE_ADMIN_PASSWORD is set), so losing it here would leave the
# operator with an account whose password nobody knows.
ADMIN_PW="$(prior DEVGRAIL_ADMIN_PASSWORD)"
ADMIN_PW="${ADMIN_PW:-$(gen_password)}"

# Keys this installer owns. Everything else found in the prior .env is carried
# over verbatim below — an operator who added DEVGRAIL_FORCE_ADMIN_PASSWORD or
# any other compose variable must not lose it to an upgrade.
MANAGED_ENV_KEYS=" ACME_EMAIL DEVGRAIL_DOMAIN DEVGRAIL_BASE_DOMAIN DEVGRAIL_ADMIN_PASSWORD DEVGRAIL_SERVER_IMAGE DEVGRAIL_TLS TAG HTTP_PORT HTTPS_PORT "

cat > "$DEPLOY_DIR/.env.tmp" <<EOF
ACME_EMAIL=${ACME_EMAIL:-}
DEVGRAIL_DOMAIN=$DEVGRAIL_DOMAIN
DEVGRAIL_BASE_DOMAIN=$DEVGRAIL_BASE_DOMAIN
DEVGRAIL_ADMIN_PASSWORD=$ADMIN_PW
DEVGRAIL_SERVER_IMAGE=$DEVGRAIL_SERVER_IMAGE
DEVGRAIL_TLS=$DEVGRAIL_TLS
TAG=latest
HTTP_PORT=$HTTP_PORT
HTTPS_PORT=$HTTPS_PORT
EOF

for _key in $PRIOR_KEYS; do
  case "$MANAGED_ENV_KEYS" in *" $_key "*) continue ;; esac
  # Copied verbatim from the prior file rather than re-serialised from the
  # parsed value: quoting we did not write is not ours to normalise, and this
  # runs before the temp file replaces it. Keys are [A-Za-z0-9_], so the pattern
  # carries no regex metacharacters.
  grep -m1 "^$_key=" "$DEPLOY_DIR/.env" >> "$DEPLOY_DIR/.env.tmp" || true
done

# Written via a temp file: this holds the admin password, and a truncated write
# on a full disk would hand the operator a stack that cannot be logged into.
chmod 600 "$DEPLOY_DIR/.env.tmp"
mv -f "$DEPLOY_DIR/.env.tmp" "$DEPLOY_DIR/.env"

# The integers are the registry's ordering key (what the skip-if-unchanged check
# compares); the releases are the human-facing identity reported in the banner.
# ASSET_VERSIONS is quoted because it holds a space-separated list, and this file
# is sourced by the next run.
cat > "$VERSIONS_FILE" <<EOF
SERVER_VERSION=$NEW_SERVER_VERSION
CONTAINER_VERSION=$NEW_CONTAINER_VERSION
SERVER_RELEASE=$NEW_SERVER_RELEASE
CONTAINER_RELEASE=$NEW_CONTAINER_RELEASE
RELEASE_VERSION=$RELEASE_VERSION
ASSET_VERSIONS="$NEW_ASSET_VERSIONS"
EOF

# --- pre-upgrade snapshot ---------------------------------------------------
# Everything above rewrote config in place; the next line starts the new server,
# which migrates the database on boot. Until this ran there was no point in the
# upgrade where the pre-upgrade state was recoverable.
#
# The archive holds devgrail.db AND config.yaml. A database-only backup is
# worthless: the JWT secret in config.yaml keys every at-rest cipher
# (SECURITY_ANALYSIS.md #4), so without it every API key secret, user secret and
# workspace SSH password decrypts to garbage.
SNAPSHOT_RETAIN="${DEVGRAIL_SNAPSHOT_RETAIN:-5}"

snapshot_before_upgrade() {
  local vol db_bytes free_bytes need ts stage archive summary schema

  vol="$(data_volume)"
  if [ -z "$vol" ]; then
    log "No existing DevGrail data volume found — nothing to snapshot."
    return 0
  fi

  db_bytes="$(docker run --rm --userns=host -v "$vol":/var/lib/devgrail:ro \
    --entrypoint /bin/sh devgrail-server:latest \
    -c 'stat -c %s /var/lib/devgrail/devgrail.db 2>/dev/null || echo 0' 2>/dev/null)" || db_bytes=0
  case "$db_bytes" in ''|*[!0-9]*) db_bytes=0 ;; esac
  if [ "$db_bytes" -eq 0 ]; then
    log "Data volume '$vol' holds no database yet — nothing to snapshot."
    return 0
  fi

  mkdir -p "$BACKUP_DIR"
  chmod 700 "$BACKUP_DIR"

  # Room is needed twice: once for the vacuumed copy, once for the tarball built
  # from it. Refusing here is not just hygiene — a full disk is precisely the
  # condition that causes the mid-migration failure this snapshot exists for.
  need=$((db_bytes * 2))
  free_bytes="$(df -Pk "$BACKUP_DIR" | awk 'NR==2 {print $4 * 1024}')" || free_bytes=0
  if [ "${free_bytes:-0}" -lt "$need" ]; then
    die "not enough free space for the pre-upgrade snapshot in $BACKUP_DIR:
  need $need bytes, have ${free_bytes:-0}. Free some space, or move $INSTALL_DIR
  to a larger filesystem, then re-run. Nothing has been changed."
  fi

  ts="$(date -u +%Y%m%dT%H%M%SZ)"
  stage="$BACKUP_DIR/.stage-$ts"
  rm -rf "$stage"
  mkdir -p "$stage"
  SNAPSHOT_STAGE="$stage"

  log "Taking a pre-upgrade snapshot (data volume: $vol)..."
  # Stop only the server; Traefik stays up so the host keeps answering.
  compose stop devgrail-server >/dev/null 2>&1 || true

  # VACUUM INTO, never `cp`: in WAL mode the newest committed data lives in the
  # -wal sidecar, so copying devgrail.db alone yields a corrupt backup. It runs
  # in a one-shot container off the image just loaded, so the host needs no
  # sqlite3; /etc/devgrail is mounted so it reads the deployment's real db_path.
  # --userns=host mirrors the compose services, which opt out of userns-remap.
  summary="$(docker run --rm --userns=host \
    -v "$vol":/var/lib/devgrail \
    -v "$stage":/backup \
    -v "$CONFIG_DIR":/etc/devgrail:ro \
    -e DEVGRAIL_CONFIG=/etc/devgrail/config.yaml \
    --entrypoint /usr/local/bin/devgrail-server \
    devgrail-server:latest -vacuum-into /backup/devgrail.db)" \
    || die "pre-upgrade snapshot failed — refusing to start the new version.
  Your data is untouched and the previous version is still installed;
  re-run this installer once the cause is fixed."

  schema="$(printf '%s' "$summary" | jq -r '.schema_version // "unknown"')"

  cp -p "$CONFIG_DIR/config.yaml" "$stage/config.yaml" \
    || die "pre-upgrade snapshot: could not copy $CONFIG_DIR/config.yaml.
  A database-only backup cannot decrypt anything it contains, so this is not a
  usable snapshot — refusing to continue."

  cat > "$stage/manifest.json" <<EOF
{
  "created_at": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
  "domain": "$DEVGRAIL_DOMAIN",
  "schema_version": "$schema",
  "data_volume": "$vol",
  "prior_server_version": "${SERVER_VERSION:-unknown}",
  "prior_container_version": "${CONTAINER_VERSION:-unknown}",
  "prior_release_version": "${SERVER_RELEASE:-unknown}",
  "new_server_version": "${NEW_SERVER_VERSION:-unknown}",
  "new_container_version": "${NEW_CONTAINER_VERSION:-unknown}",
  "contents": ["devgrail.db", "config.yaml", "manifest.json"]
}
EOF

  archive="$BACKUP_DIR/pre-upgrade-$ts.tgz"
  tar czf "$archive" -C "$stage" devgrail.db config.yaml manifest.json \
    || die "pre-upgrade snapshot: could not write $archive"
  # The archive contains the JWT secret and every encrypted secret it protects.
  chmod 600 "$archive"
  rm -rf "$stage"
  SNAPSHOT_STAGE=""
  SNAPSHOT_ARCHIVE="$archive"
  log "Pre-upgrade snapshot written to $archive"

  # Keep the newest N. Unbounded snapshots eventually fill the disk, which is
  # the condition that causes the failure they protect against.
  # `|| true` because pipefail would otherwise abort the whole install on the
  # nothing-to-prune case, where ls has no matches.
  ls -1t "$BACKUP_DIR"/pre-upgrade-*.tgz 2>/dev/null | tail -n +$((SNAPSHOT_RETAIN + 1)) \
    | while read -r old; do rm -f "$old"; done || true
}

if [ "$IS_UPGRADE" = true ]; then
  snapshot_before_upgrade
fi

# --- devgrail maintenance command -------------------------------------------
# The pre-upgrade snapshot only exists when an upgrade happens to run. An
# operator also needs a backup they can take now — before editing something, or
# on a schedule — and a restore that does the steps in the one order that works.
# Both live in the server binary (`-backup` / `-restore`); this wrapper is what
# runs them as a one-shot container off the installed image, so the host needs
# neither sqlite3 nor a copy of the Go toolchain.
#
# Written from here rather than shipped as an eighth release artifact: it is
# entirely derived from paths this installer already owns, so it can never
# disagree with the install it was written for.
CLI_PATH=/usr/local/bin/devgrail

write_devgrail_cli() {
  cat > "$CLI_PATH.tmp" <<CLIEOF
#!/usr/bin/env bash
# DevGrail maintenance CLI — written by install.sh. Do not edit; re-running the
# installer overwrites it.
set -euo pipefail

INSTALL_DIR="$INSTALL_DIR"
DEPLOY_DIR="$DEPLOY_DIR"
CONFIG_DIR="$CONFIG_DIR"
BACKUP_DIR="$BACKUP_DIR"
IMAGE=devgrail-server:latest

log()  { printf '\033[1;34m==>\033[0m %s\n' "\$*"; }
die()  { printf '\033[1;31merror:\033[0m %s\n' "\$*" >&2; exit 1; }
# Mirrors install.sh: an operator's docker-compose.override.yml applies to the
# stop/start this does around a restore, or the restored server comes back
# without it.
compose() {
  if [ -f "\$DEPLOY_DIR/docker-compose.override.yml" ]; then
    docker compose --project-directory "\$DEPLOY_DIR" \\
      -f "\$DEPLOY_DIR/docker-compose.yml" \\
      -f "\$DEPLOY_DIR/docker-compose.override.yml" "\$@"
  else
    docker compose --project-directory "\$DEPLOY_DIR" -f "\$DEPLOY_DIR/docker-compose.yml" "\$@"
  fi
}

usage() {
  cat <<'USAGE'
DevGrail maintenance commands. Run as root.

  devgrail backup [--out PATH]   write a backup archive (database + config.yaml
                                 + manifest). Default: the backups directory,
                                 named backup-<timestamp>.tgz
  devgrail restore ARCHIVE       stop the server, restore the archive, start it
                                 again, and wait for the readiness probe
  devgrail list                  list the archives on this host
  devgrail help                  show this

A backup archive holds BOTH the database and config.yaml. The jwt_secret in
config.yaml keys every encrypted value in the database, so a database-only
backup restores rows that decrypt to garbage. Never separate them.

Workspace volumes are NOT included — they hold the customer's source code and
are far larger than the database. See docs/RUNBOOK.md for the export recipe.
USAGE
}

# data_volume prints the Docker volume backing /var/lib/devgrail, resolved from
# the running container rather than guessed from the Compose project name.
data_volume() {
  local vol
  vol="\$(docker inspect devgrail-server \\
    --format '{{range .Mounts}}{{if eq .Destination "/var/lib/devgrail"}}{{.Name}}{{end}}{{end}}' 2>/dev/null)" || vol=""
  if [ -n "\$vol" ]; then printf '%s' "\$vol"; return 0; fi
  docker volume ls --format '{{.Name}}' 2>/dev/null | grep -E '(^|_)devgrail-data\$' | head -n1 || true
}

# cmd_backup writes one archive via a one-shot container off the installed
# server image, with the data volume, /etc/devgrail and the output directory
# mounted. --userns=host mirrors the compose services, which opt out of
# userns-remap.
cmd_backup() {
  local out="" ts vol dir
  while [ \$# -gt 0 ]; do
    case "\$1" in
      --out=*) out="\${1#*=}" ;;
      --out)   shift; out="\${1:-}" ;;
      *) usage >&2; die "unknown option: \$1" ;;
    esac
    shift
  done
  ts="\$(date -u +%Y%m%dT%H%M%SZ)"
  [ -n "\$out" ] || out="\$BACKUP_DIR/backup-\$ts.tgz"
  dir="\$(cd "\$(dirname "\$out")" 2>/dev/null && pwd)" || die "no such directory: \$(dirname "\$out")"
  mkdir -p "\$BACKUP_DIR"; chmod 700 "\$BACKUP_DIR"

  vol="\$(data_volume)"
  [ -n "\$vol" ] || die "cannot find the DevGrail data volume — is DevGrail installed here?"

  # Taken against a running server on purpose: VACUUM INTO reads through a single
  # read transaction, so the copy is consistent without downtime. The volume is
  # mounted read-write even though nothing here modifies the data: opening a
  # WAL-mode SQLite database requires writing its -shm sidecar, so a read-only
  # mount fails outright with "attempt to write a readonly database".
  docker run --rm --userns=host \\
    -v "\$vol":/var/lib/devgrail \\
    -v "\$CONFIG_DIR":/etc/devgrail:ro \\
    -v "\$dir":/backup \\
    -e DEVGRAIL_CONFIG=/etc/devgrail/config.yaml \\
    --entrypoint /usr/local/bin/devgrail-server \\
    "\$IMAGE" -backup "/backup/\$(basename "\$out")" \\
    || die "backup failed — nothing was written"
  chmod 600 "\$out" 2>/dev/null || true
  log "Backup written to \$out"
}

cmd_restore() {
  local archive="\${1:-}" vol dir
  [ -n "\$archive" ] || { usage >&2; die "restore needs an archive path"; }
  [ -f "\$archive" ] || die "no such archive: \$archive"
  dir="\$(cd "\$(dirname "\$archive")" && pwd)"

  vol="\$(data_volume)"
  [ -n "\$vol" ] || die "cannot find the DevGrail data volume — refusing to guess which volume to overwrite."

  # The documented order, enforced: stop -> config.yaml -> database -> start ->
  # verify. Restoring under a running server would swap the database out from
  # under open connections.
  log "Stopping devgrail-server..."
  compose stop devgrail-server >/dev/null 2>&1 || true

  docker run --rm --userns=host \\
    -v "\$vol":/var/lib/devgrail \\
    -v "\$CONFIG_DIR":/etc/devgrail \\
    -v "\$dir":/restore:ro \\
    -e DEVGRAIL_CONFIG=/etc/devgrail/config.yaml \\
    --entrypoint /usr/local/bin/devgrail-server \\
    "\$IMAGE" -restore "/restore/\$(basename "\$archive")" \\
    || { compose up -d >/dev/null 2>&1 || true; die "restore failed — the previous state is unchanged and the server has been restarted"; }

  log "Starting devgrail-server..."
  compose up -d

  local waited=0
  while [ "\$waited" -lt 180 ]; do
    if compose exec -T devgrail-server /usr/local/bin/devgrail-server -healthcheck >/dev/null 2>&1; then
      log "DevGrail is ready. Restore complete."
      return 0
    fi
    sleep 3
    waited=\$((waited + 3))
  done
  compose logs --tail=50 devgrail-server >&2 || true
  die "the restored server did not become ready within 180s (logs above)"
}

case "\${1:-help}" in
  backup)  shift; cmd_backup "\$@" ;;
  restore) shift; cmd_restore "\$@" ;;
  list)    ls -lh "\$BACKUP_DIR" 2>/dev/null || echo "no archives in \$BACKUP_DIR" ;;
  help|-h|--help) usage ;;
  *) usage >&2; die "unknown command: \$1" ;;
esac
CLIEOF
  chmod 755 "$CLI_PATH.tmp"
  mv -f "$CLI_PATH.tmp" "$CLI_PATH"
}

if write_devgrail_cli; then
  log "Maintenance CLI installed: $CLI_PATH (backup / restore / list)"
else
  warn "could not write $CLI_PATH — backup and restore will need the docker run commands from docs/RUNBOOK.md"
fi

# --- scheduled backups -------------------------------------------------------
# Offered, never imposed: it writes to the customer's disk on a timer, and an
# operator who already has their own backup story should not get a second one
# they did not ask for. DEVGRAIL_SCHEDULED_BACKUPS=yes|no answers it unattended;
# an existing timer is left exactly as configured.
install_backup_timer() {
  command -v systemctl >/dev/null 2>&1 || return 1

  cat > /etc/systemd/system/devgrail-backup.service <<EOF
[Unit]
Description=DevGrail weekly backup
After=docker.service
Requires=docker.service

[Service]
Type=oneshot
ExecStart=$CLI_PATH backup
EOF

  # Randomised delay so a fleet of VPSes does not all wake at once; Persistent
  # so a machine that was off at the scheduled time still runs it on next boot.
  cat > /etc/systemd/system/devgrail-backup.timer <<EOF
[Unit]
Description=Weekly DevGrail backup

[Timer]
OnCalendar=weekly
RandomizedDelaySec=1h
Persistent=true

[Install]
WantedBy=timers.target
EOF

  systemctl daemon-reload >/dev/null 2>&1 || true
  systemctl enable --now devgrail-backup.timer >/dev/null 2>&1 || return 1
  return 0
}

BACKUP_TIMER_NOTE=""
if [ -f "$CLI_PATH" ] && command -v systemctl >/dev/null 2>&1; then
  if systemctl list-unit-files devgrail-backup.timer >/dev/null 2>&1 &&
     systemctl is-enabled devgrail-backup.timer >/dev/null 2>&1; then
    BACKUP_TIMER_NOTE="Weekly backups: already scheduled (systemctl status devgrail-backup.timer)"
  else
    WANT_TIMER="$(printf '%s' "${DEVGRAIL_SCHEDULED_BACKUPS:-}" | tr '[:upper:]' '[:lower:]')"
    # --yes takes the prompt's own default, which is yes: a machine nobody is
    # watching is exactly the one that benefits from a scheduled backup.
    # DEVGRAIL_SCHEDULED_BACKUPS still wins, so it stays possible to say no.
    if [ -z "$WANT_TIMER" ] && [ "$ASSUME_YES" = true ]; then WANT_TIMER=yes; fi
    if [ -z "$WANT_TIMER" ] && [ -e /dev/tty ]; then
      printf 'Schedule a weekly backup to %s? [Y/n]: ' "$BACKUP_DIR" > /dev/tty
      read -r _answer < /dev/tty || _answer=""
      case "$_answer" in n|N|no|NO) WANT_TIMER=no ;; *) WANT_TIMER=yes ;; esac
    fi
    case "$WANT_TIMER" in
      yes|y|true|1)
        if install_backup_timer; then
          BACKUP_TIMER_NOTE="Weekly backups: enabled (systemctl status devgrail-backup.timer)"
        else
          warn "could not enable the weekly backup timer — run '$CLI_PATH backup' yourself, or from your own scheduler"
        fi
        ;;
      *) BACKUP_TIMER_NOTE="Weekly backups: not scheduled. Enable later with: DEVGRAIL_SCHEDULED_BACKUPS=yes re-run, or run '$CLI_PATH backup' from your own scheduler." ;;
    esac
  fi
fi

# --- bring up --------------------------------------------------------------
docker network inspect devgrail >/dev/null 2>&1 || docker network create devgrail >/dev/null
log "Starting DevGrail..."
# Via compose(), so a docker-compose.override.yml is honoured here too — not
# just by the stop/logs calls above. An override that only applied to some of
# the installer's compose invocations would be worse than none.
compose up -d

# --- wait for readiness -----------------------------------------------------
# probe_health / wait_for_health are defined near the top, because --rollback
# needs them too.
if ! wait_for_health; then
  echo >&2
  printf '\033[1;31merror:\033[0m %s\n' "DevGrail did not become ready within ${HEALTH_TIMEOUT}s." >&2
  echo "  Last 50 lines from devgrail-server:" >&2
  compose logs --tail=50 devgrail-server >&2 || true
  echo >&2
  echo "  Health detail:" >&2
  curl -sSk --max-time 5 "$SCHEME://$DEVGRAIL_DOMAIN/api/v1/healthz" >&2 || true
  echo >&2

  # Automatic rollback, on an upgrade only. A first install has nothing to roll
  # back to: there are no :previous images and no snapshot, and "rolling back" to
  # an empty host would replace a diagnosable failure with a confusing one.
  ROLLED_BACK=false
  if [ "$IS_UPGRADE" = true ]; then
    # --yes deliberately does not imply yes here. Rolling back is a decision
    # about which of two states the host ends in, not a question in the way of
    # an unattended run; --auto-rollback is how that one is answered up front.
    DO_IT="$AUTO_ROLLBACK"
    if [ "$DO_IT" != true ] && [ "$ASSUME_YES" != true ] && [ -e /dev/tty ]; then
      printf 'Roll back to the previous version and snapshot? [y/N]: ' > /dev/tty
      read -r _answer < /dev/tty || _answer=""
      case "$_answer" in y|Y|yes|YES) DO_IT=true ;; esac
    fi
    if [ "$DO_IT" = true ]; then
      echo >&2
      if do_rollback; then
        ROLLED_BACK=true
      else
        printf '\033[1;31merror:\033[0m %s\n' "The rolled-back stack did not become ready either." >&2
      fi
    fi
  fi

  # Say plainly which state the host ended in. After a failed upgrade that is the
  # only thing the operator needs to know before deciding what to do next.
  if [ "$ROLLED_BACK" = true ]; then
    cat >&2 <<EOF

============================================================
  The upgrade failed and was ROLLED BACK.

  This host is running the previous version again and is answering
  its readiness probe at $SCHEME://$DEVGRAIL_DOMAIN/

  Nothing was lost. Report the log above before retrying the upgrade.
============================================================
EOF
    exit 1
  fi

  if [ -n "$SNAPSHOT_ARCHIVE" ]; then
    echo "  A pre-upgrade snapshot of the database and config.yaml was taken first:" >&2
    echo "    $SNAPSHOT_ARCHIVE" >&2
    echo "  Nothing was deleted, but the new server may already have migrated the" >&2
    echo "  database — that archive is the way back to the previous state." >&2
    echo "  To go back:  curl -fsSL $WEB/install.sh | sudo bash -s -- --rollback" >&2
  fi
  echo "  Logs: docker compose --project-directory $DEPLOY_DIR logs -f" >&2
  exit 1
fi

# --- reclaim superseded images ----------------------------------------------
# Only now, and never earlier: :previous is the rollback target, and until the
# new stack has answered its readiness probe it is the thing standing between a
# bad upgrade and a broken host. Once health is green, every DevGrail image older
# than :previous is dead weight — an upgrade loads a new one each time and
# nothing else removes them, so a host that upgrades for a year accumulates a
# year of unreferenced layers on the disk whose exhaustion causes the failure
# this whole sequence exists to survive.
reclaim_old_images() {
  local slug keep_latest keep_prev id removed=0
  for slug in devgrail-server devgrail-container; do
    keep_latest="$(docker image inspect -f '{{.Id}}' "$slug:latest"   2>/dev/null || true)"
    keep_prev="$(docker image inspect -f '{{.Id}}' "$slug:previous" 2>/dev/null || true)"
    [ -n "$keep_latest" ] || continue
    # Every image ID this repository still has, tagged or dangling. Untagged
    # ones are the displaced builds: `docker load` + retag leaves them with no
    # reference at all, which is exactly why nothing ever cleaned them up.
    for id in $(docker images --filter "reference=$slug" --format '{{.ID}}' 2>/dev/null | sort -u); do
      id="$(docker image inspect -f '{{.Id}}' "$id" 2>/dev/null || true)"
      [ -n "$id" ] || continue
      [ "$id" = "$keep_latest" ] && continue
      [ -n "$keep_prev" ] && [ "$id" = "$keep_prev" ] && continue
      # Not forced: an image a container is still using must stay, and that is
      # a refusal to respect rather than override. A workspace created before an
      # upgrade is running the old devgrail-container image until it is
      # recreated — removing it out from under the container is not reclamation,
      # it is breakage.
      if docker image rm "$id" >/dev/null 2>&1; then
        removed=$((removed + 1))
      fi
    done
  done
  [ "$removed" -gt 0 ] && log "Reclaimed $removed superseded image(s); :latest and :previous kept."
  return 0
}
reclaim_old_images || true

if [ "$TLS_ENABLED" = true ]; then
  TLS_NOTE="It may take a minute for Let's Encrypt to issue the TLS
  certificate on first start. Ensure DNS is pointed here:"
else
  TLS_NOTE="Serving plain HTTP (DEVGRAIL_TLS=off) — do not expose this directly to
  the public internet; run it on a private network or behind a TLS terminator.
  Ensure DNS is pointed here:"
fi

cat <<EOF

============================================================
  DevGrail is up and answering its readiness probe.

  Version:   ${NEW_SERVER_RELEASE:-unknown} (workspace ${NEW_CONTAINER_RELEASE:-unknown})
  Dashboard: $SCHEME://$DEVGRAIL_DOMAIN/
  Login:     admin / $ADMIN_PW

  $TLS_NOTE
    A  $DEVGRAIL_DOMAIN      -> this VPS
    A  *.$DEVGRAIL_BASE_DOMAIN  -> this VPS

  Logs:     docker compose --project-directory $DEPLOY_DIR logs -f
  Upgrade:  re-run this installer.
  Customise: docker-compose.yml and traefik/dynamic.yml under $DEPLOY_DIR are
            product-owned and rewritten on every run. Put local changes in
            $DEPLOY_DIR/docker-compose.override.yml, which is never touched.
  Pin:      | sudo bash -s -- --version=vX.Y.Z
  Roll back: | sudo bash -s -- --rollback
  Backup:   sudo devgrail backup      (restore: sudo devgrail restore <archive>)
  ${BACKUP_TIMER_NOTE:-}
============================================================
EOF

if [ -n "$SNAPSHOT_ARCHIVE" ]; then
  cat <<EOF
  Pre-upgrade snapshot (database + config.yaml + manifest):
    $SNAPSHOT_ARCHIVE
  Keep it until you have confirmed this upgrade. The newest $SNAPSHOT_RETAIN are retained.

EOF
fi