#!/usr/bin/env bash # DevGrail one-command installer. # # curl -fsSL https:///install.sh | sudo bash # # Detects arch, installs Docker if missing, downloads the DevGrail image tarballs # from devgrail-web (verified by SHA-256, loaded with `docker load` — no GitHub, # no `docker pull`, no registry auth), lays down the compose + Traefik config, # writes config + secrets, and brings the stack up behind Traefik + Let's Encrypt. # # Everything is resolved from one release manifest, so an upgrade cannot mix a # new compose file with an old server image. A registry that publishes no release # manifest falls back to resolving each artifact by slug, exactly as before. # # Re-running upgrades in place: only artifacts whose version changed are # re-downloaded — images *and* assets; existing secrets and the data volume are # preserved. Free space is checked against the declared artifact sizes before # anything is fetched. An upgrade takes a snapshot of the database + config.yaml # into /opt/devgrail/backups before the new server starts, then waits for the # server's readiness probe and exits non-zero (with logs and the snapshot path) # if it never becomes ready. Images older than :previous are reclaimed only once # that probe is green — until then :previous is the rollback target. # # It also installs /usr/local/bin/devgrail (backup / restore / list / passwd) # and offers to schedule a weekly backup. # # Prerequisites: DNS A records `DEVGRAIL_DOMAIN` and `*.DEVGRAIL_BASE_DOMAIN` # pointing at this VPS. Inputs may be passed as env vars or answered at the prompt. # # Vars: # DEVGRAIL_WEB_URL public site serving images + assets # (default: the site this script was downloaded from) # DEVGRAIL_DOMAIN dashboard/API/MCP hostname (prompted) # DEVGRAIL_BASE_DOMAIN base domain for app subdomains (prompted) # ACME_EMAIL Let's Encrypt contact email (prompted) # DEVGRAIL_ACME_CA which ACME CA to order certificates from: # production (default), staging, or a directory URL. # staging issues untrusted certs but is not meaningfully # rate limited — use it for repeated install testing on # one hostname. Same as --acme-ca=. # DEVGRAIL_ADMIN_USERNAME admin account name (prompted on a fresh install; # default admin). First boot only — it renames nobody # afterwards. # DEVGRAIL_ADMIN_PASSWORD admin password (prompted on a fresh install, or # generated and printed at the end). First boot only; # use `devgrail passwd` to change it later. # HTTP_PORT/HTTPS_PORT host ports (default 80/443; real 80/443 needed for ACME) # DEVGRAIL_HEALTH_TIMEOUT seconds to wait for readiness after start (default 180) # DEVGRAIL_SNAPSHOT_RETAIN pre-upgrade snapshots to keep (default 5) # DEVGRAIL_VERSION release to install, e.g. v0.3.0 (default: current) # DEVGRAIL_SCHEDULED_BACKUPS yes|no — answer the weekly-backup prompt unattended # DEVGRAIL_SKIP_DNS_CHECK 1 — skip the DNS preflight (same as --skip-dns-check) # DEVGRAIL_USERNS_REMAP yes|no — answer the userns-remap prompt unattended # DEVGRAIL_ADDRESS_POOLS yes|no — answer the address-pool prompt unattended # Both edit /etc/docker/daemon.json and restart Docker; # `no` also silences the explanation. See the "Docker # daemon preflight" section. # DEVGRAIL_LOCK_WAIT seconds to wait for a concurrent run to finish (default 300) # DEVGRAIL_REQUIRE_SIGNATURE 1 — refuse a release whose signature cannot be # verified, instead of warning # DEVGRAIL_RELEASE_PUBKEY_FILE extra trusted release signing keys (PEM) # # Flags (pass with `| sudo bash -s -- `): --version=vX.Y.Z, --rollback, # --auto-rollback, --dry-run, --yes, --skip-dns-check, --userns-remap=yes|no, # --address-pools=yes|no, --help. See usage() below. set -euo pipefail # __DEVGRAIL_WEB_URL__ is substituted with the serving origin by devgrail-web's # /install.sh route handler (app/install.sh/route.ts) so the default always points # at the site this script was actually downloaded from. An explicit env var still # wins. This file is the single source of truth: `make publish` uploads it to the # registry as the install.sh asset; devgrail-web serves it. Do not keep a second copy. DEVGRAIL_WEB_URL="${DEVGRAIL_WEB_URL:-__DEVGRAIL_WEB_URL__}" INSTALL_DIR="${DEVGRAIL_INSTALL_DIR:-/opt/devgrail}" DEPLOY_DIR="$INSTALL_DIR/deploy" CONFIG_DIR=/etc/devgrail VERSIONS_FILE="$INSTALL_DIR/versions.env" BACKUP_DIR="$INSTALL_DIR/backups" # Verified copies of the installer assets, kept so an unchanged re-run # re-downloads nothing. The rendered outputs under $DEPLOY_DIR are always # regenerated from these — substitution is cheap, a download is not. ASSET_DIR="$INSTALL_DIR/assets" WEB="${DEVGRAIL_WEB_URL%/}" # The concurrency lock (see acquire_lock) and the PID of whoever holds it. LOCK_FILE="$INSTALL_DIR/.install.lock" LOCK_OWNER_FILE="$INSTALL_DIR/.install.owner" # Set by snapshot_before_upgrade; reported in the success banner and on failure. SNAPSHOT_ARCHIVE="" # Staging directory for a snapshot in progress; removed by the EXIT trap so an # aborted run leaves no half-written copy of the database lying around. SNAPSHOT_STAGE="" # Scratch directories, declared here rather than where they are chosen so the # EXIT trap — installed as soon as the lock is taken, long before either has a # value — names only variables that exist. TMP="" SIG_DIR="" log() { printf '\033[1;34m==>\033[0m %s\n' "$*"; } warn() { printf '\033[1;33mWARNING:\033[0m %s\n' "$*" >&2; } die() { printf '\033[1;31merror:\033[0m %s\n' "$*" >&2; exit 1; } # docker-compose.yml is product-owned: it is re-fetched and rewritten on every # run, so an operator edit to it is lost at the next upgrade — silently, which is # the part that makes it a trap. docker-compose.override.yml is the supported # place for local changes; this installer never writes or reads it beyond adding # it to the file list, and compose merges it over the base file. # # It has to be passed explicitly: compose only auto-discovers an override file # when no -f is given at all, and -f is what pins this to $DEPLOY_DIR. compose() { if [ -f "$DEPLOY_DIR/docker-compose.override.yml" ]; then docker compose --project-directory "$DEPLOY_DIR" \ -f "$DEPLOY_DIR/docker-compose.yml" \ -f "$DEPLOY_DIR/docker-compose.override.yml" "$@" else docker compose --project-directory "$DEPLOY_DIR" -f "$DEPLOY_DIR/docker-compose.yml" "$@" fi } usage() { cat <<'USAGE' DevGrail installer. curl -fsSL https:///install.sh | sudo bash curl -fsSL https:///install.sh | sudo bash -s -- [options] Options: --version=vX.Y.Z install that published release instead of the current one --rollback restore the previous images and the newest pre-upgrade snapshot, then wait for the server to become ready --auto-rollback if the server does not become ready after an upgrade, do the above automatically instead of asking --dry-run report what this run would change — release, image and asset versions, download size, free disk, settings — and exit without touching anything -y, --yes never prompt; take the prior value (or the default) for every question. For unattended runs. --skip-dns-check do not verify that the domains resolve to this host --userns-remap=yes|no answer the userns-remap prompt without asking. yes edits /etc/docker/daemon.json and restarts Docker (only offered on a host with no Docker state); no skips the question --address-pools=yes|no answer the address-pool prompt without asking. yes widens Docker's default pools (raising the ~27-workspace ceiling) and restarts Docker; no skips the question --acme-ca=production|staging| which ACME CA to order certificates from (default production). staging issues browser-untrusted certificates but is not meaningfully rate limited, so repeated installs on one hostname do not exhaust Let's Encrypt's 5-per-week limit. Each CA keeps its own certificate store, so switching back to production restores the real certs -h, --help show this and exit A fresh interactive install asks for the admin username and password. Anything unattended (--yes, or no terminal) creates `admin` with a generated password, printed at the end. Both are first-boot seeds; afterwards, change the password in the dashboard under Settings, or with `sudo devgrail passwd`. Inputs may also be passed as environment variables; see the header of this file. USAGE } DO_ROLLBACK=false AUTO_ROLLBACK=false DRY_RUN=false ASSUME_YES=false # The env var exists so an unattended install (cloud-init, a marketplace image) # can opt out without rewriting the command line it was handed. case "$(printf '%s' "${DEVGRAIL_SKIP_DNS_CHECK:-}" | tr '[:upper:]' '[:lower:]')" in 1|y|yes|true) SKIP_DNS_CHECK=true ;; *) SKIP_DNS_CHECK=false ;; esac # Answers to the two daemon-hardening questions (see the "Docker daemon # preflight" section). Empty means "ask"; "no" means the operator has already # decided and does not want to be asked again on the next upgrade. Anything # unrecognised is normalised to empty rather than guessed at — both settings # restart dockerd, so a typo must not be read as consent. norm_yes_no() { case "$(printf '%s' "${1:-}" | tr '[:upper:]' '[:lower:]')" in 1|y|yes|true) echo yes ;; 0|n|no|false) echo no ;; *) echo "" ;; esac } USERNS_REMAP_ANSWER="$(norm_yes_no "${DEVGRAIL_USERNS_REMAP:-}")" ADDRESS_POOLS_ANSWER="$(norm_yes_no "${DEVGRAIL_ADDRESS_POOLS:-}")" while [ $# -gt 0 ]; do case "$1" in --version=*) DEVGRAIL_VERSION="${1#*=}" ;; --version) shift; DEVGRAIL_VERSION="${1:-}" [ -n "$DEVGRAIL_VERSION" ] || die "--version needs a value, e.g. --version=v0.3.0" ;; --rollback) DO_ROLLBACK=true ;; --auto-rollback) AUTO_ROLLBACK=true ;; --dry-run) DRY_RUN=true ;; -y|--yes) ASSUME_YES=true ;; --skip-dns-check) SKIP_DNS_CHECK=true ;; --userns-remap=*) USERNS_REMAP_ANSWER="$(norm_yes_no "${1#*=}")" [ -n "$USERNS_REMAP_ANSWER" ] || die "--userns-remap takes yes or no" ;; --address-pools=*) ADDRESS_POOLS_ANSWER="$(norm_yes_no "${1#*=}")" [ -n "$ADDRESS_POOLS_ANSWER" ] || die "--address-pools takes yes or no" ;; --acme-ca=*) DEVGRAIL_ACME_CA="${1#*=}" [ -n "$DEVGRAIL_ACME_CA" ] || die "--acme-ca takes production, staging or a directory URL" ;; -h|--help) usage; exit 0 ;; *) usage >&2; die "unknown option: $1" ;; esac shift done # --rollback restores images and a snapshot; there is nothing about it a dry run # could report without doing it. Refuse the combination rather than pick one. if [ "$DO_ROLLBACK" = true ] && [ "$DRY_RUN" = true ]; then die "--dry-run cannot be combined with --rollback." fi [ "$(id -u)" = 0 ] || die "run as root: curl -fsSL $WEB/install.sh | sudo bash" case "$(uname -m)" in x86_64|amd64) ARCH=amd64 ;; aarch64|arm64) ARCH=arm64 ;; *) die "unsupported architecture: $(uname -m)" ;; esac # --- concurrency lock -------------------------------------------------------- # Two runs at once — an impatient operator re-running while the first is still # downloading, or a scheduled upgrade landing on top of a manual one — race on # .env, versions.env, config.yaml and `docker compose up`. The loser can leave a # half-written .env (which holds the admin password) or a compose project # pointing at an image the other run has already retagged. One flock closes it. # # Held on fd 9 for the lifetime of the process, so the kernel drops it on any # exit, including a crash. That is not the whole story: fd 9 is not # close-on-exec, so anything the installer forks inherits it, and a *daemon* that # inherits it holds the lock for as long as it runs. `systemctl enable --now # docker` below is exactly that case on a host whose systemctl starts dockerd as # a plain child (containers, the local sandbox) — the installer exits, dockerd # keeps fd 9, and every later run waits out the timeout for a run that ended # minutes ago. Two defences, because neither alone is enough: # # 1. The commands that can fork a surviving daemon are run with 9>&- (see the # Docker install and the backup timer). Prevention, but only for the forks # this script knows about. # 2. The holder records its PID in $LOCK_OWNER_FILE, removed on exit. A waiter # that cannot take the lock and finds no live owner knows the lock leaked # into some other process and breaks it. Recovery, for everything else. LOCK_WAIT="${DEVGRAIL_LOCK_WAIT:-300}" # Consecutive owner-less polls before a held lock is declared leaked. There is a # sub-second window where a legitimate peer holds the flock but has not yet # written its owner file; requiring several *consecutive* misses closes it, # since a real installer writes the record microseconds after acquiring. LOCK_STALE_GRACE=5 # True when the lock is held by a process that is still running. Anything else — # no file, junk in the file, a PID that has gone — is "no live owner", which is # what makes a leaked lock recoverable. lock_owner_alive() { local pid [ -f "$LOCK_OWNER_FILE" ] || return 1 IFS= read -r pid < "$LOCK_OWNER_FILE" 2>/dev/null || return 1 case "$pid" in ''|*[!0-9]*) return 1 ;; esac kill -0 "$pid" 2>/dev/null } acquire_lock() { if ! command -v flock >/dev/null 2>&1; then warn "flock is not available on this host — running without the install lock." echo " Do not run two installers at the same time." >&2 return 0 fi # A dry run reads the same files, so it takes the lock shared: it must not # read a half-written .env, and it must not block a real install either. # On a host with no install there is nothing to race over and nothing to read, # so it does not create $INSTALL_DIR just to lock it. if [ "$DRY_RUN" = true ] && [ ! -d "$INSTALL_DIR" ]; then return 0 fi mkdir -p "$INSTALL_DIR" || die "cannot create $INSTALL_DIR" # Append, not truncate. The lock file itself carries no content, but `>` also # truncates, and keeping every open of this path non-destructive is what lets # the file be reasoned about at all while several runs hold it open. exec 9>>"$LOCK_FILE" || die "cannot open $LOCK_FILE" local mode=-x what=exclusive waited=0 stale_for=0 broke_once=false if [ "$DRY_RUN" = true ]; then mode=-s; what=shared; fi # Polled with -n rather than blocking with -w: busybox's flock (Alpine and # most minimal images) has no -w at all, and rejects it with a usage error # whose exit status is indistinguishable from "the lock is held" — which made # this refuse to run at all on those hosts. -n and -s are the forms busybox # and util-linux agree on. while ! flock "$mode" -n 9; do if lock_owner_alive; then stale_for=0 else stale_for=$((stale_for + 1)) fi # Break a leaked lock, at most once. Unlinking and reopening puts us on a # fresh inode that the leaking process does not hold, while it keeps its # lock on the orphaned one; a third run opening the path lands on the new # inode too, so mutual exclusion is restored rather than abandoned. Once # only: if something re-takes the lock afterwards, that is a real peer, and # the right answer is to wait for it and time out, not to keep unlinking. if [ "$broke_once" = false ] && [ "$stale_for" -ge "$LOCK_STALE_GRACE" ]; then warn "Breaking a lock left behind by an installer that has already exited." echo " $LOCK_FILE is held, but no running process claims it — usually a" >&2 echo " daemon (dockerd) that inherited the lock from an interrupted run." >&2 echo " Continuing." >&2 rm -f "$LOCK_FILE" exec 9>>"$LOCK_FILE" || die "cannot reopen $LOCK_FILE" broke_once=true continue fi if [ "$waited" -ge "$LOCK_WAIT" ]; then die "another install.sh is already running on this host. Waited ${LOCK_WAIT}s for the $what lock on $LOCK_FILE. Wait for that run to finish and try again. Nothing has been changed. If you are sure no installer is running, see who holds it: fuser -v $LOCK_FILE # or: lsof $LOCK_FILE and break the lock by hand with: rm -f $LOCK_FILE" fi if [ "$waited" -eq 0 ]; then log "Another install.sh holds the lock — waiting up to ${LOCK_WAIT}s..." fi sleep 1 waited=$((waited + 1)) done # Under the lock now, so this cannot race a peer. Written in both modes: a # leaked *shared* lock blocks a real install just as effectively as a leaked # exclusive one, and is just as invisible without the record. printf '%s\n' "$$" > "$LOCK_OWNER_FILE" 2>/dev/null || true } acquire_lock # Installed here, as soon as there is something to clean up, and never replaced: # a second `trap ... EXIT` would silently drop this one. Everything it removes is # guarded with :+ because it runs on every exit path, including ones taken before # TMP or the snapshot staging dir have been chosen — `rm -rf` with no operands # fails, and a failing EXIT trap is a confusing thing to leave behind after an # otherwise clean run. cleanup() { rm -rf ${TMP:+"$TMP"} ${SNAPSHOT_STAGE:+"$SNAPSHOT_STAGE"} ${SIG_DIR:+"$SIG_DIR"} 2>/dev/null || true # Only if it is still ours. Two dry runs can hold the shared lock at once, and # the loser of that race must not delete the winner's record — doing so would # make a live peer look like a leak to the next run. if [ -f "$LOCK_OWNER_FILE" ]; then local owner="" IFS= read -r owner < "$LOCK_OWNER_FILE" 2>/dev/null || owner="" [ "$owner" = "$$" ] && rm -f "$LOCK_OWNER_FILE" 2>/dev/null fi return 0 } trap cleanup EXIT # --- detect a prior install (upgrade) -------------------------------------- # Read only now that the lock is held: this is the file a concurrent run would # be rewriting, and adopting a half-written copy of it is precisely what the # lock exists to prevent. # # On re-run the previous answers live in the deploy .env (rewritten every run). # The WHOLE file is read into a PRIOR_* namespace, not a hand-maintained list of # four keys: the rule is that an upgrade preserves everything unless the operator # changes it. Reading only some keys meant HTTP_PORT/HTTPS_PORT were silently # reset to 80/443 on every upgrade — best case Traefik then failed to bind, worst # case it bound and hijacked another service's ports. # # Precedence, everywhere: explicit env var > prompt answer > prior value > default. IS_UPGRADE=false PRIOR_KEYS="" if [ -f "$DEPLOY_DIR/.env" ]; then IS_UPGRADE=true while IFS= read -r _line || [ -n "$_line" ]; do # Skip blanks, comments, and anything without a '='. case "$_line" in ''|'#'*) continue ;; *=*) : ;; *) continue ;; esac _key="${_line%%=*}" _val="${_line#*=}" # Ignore anything that is not a plain shell identifier — this file is # written by us, but it is also the file an operator hand-edits. case "$_key" in ''|*[!A-Za-z0-9_]*) continue ;; esac # Strip one layer of surrounding quotes, matching how compose reads it. case "$_val" in \"*\") _val="${_val#\"}"; _val="${_val%\"}" ;; \'*\') _val="${_val#\'}"; _val="${_val%\'}" ;; esac eval "PRIOR_$_key=\$_val" PRIOR_KEYS="${PRIOR_KEYS:+$PRIOR_KEYS }$_key" done < "$DEPLOY_DIR/.env" fi # prior — the prior install's value for KEY, or empty. prior() { eval "printf '%s' \"\${PRIOR_$1:-}\""; } # resolve [default] — settle KEY per the precedence above. Prompted keys # call prompt() instead, passing `prior KEY` as the offered default. resolve() { local key="$1" def="${2:-}" cur eval "cur=\${$key:-}" [ -n "$cur" ] && return 0 cur="$(prior "$key")" [ -n "$cur" ] || cur="$def" eval "$key=\$cur" } # --- TLS switch (default on) ------------------------------------------------ # DEVGRAIL_TLS=off serves plain HTTP on :80 — only for a private network or # behind an external TLS terminator. Default on = HTTPS via Traefik + Let's # Encrypt. The choice selects which Traefik router set to install and the scheme # advertised in every server-generated URL (see internal/config Scheme()). resolve DEVGRAIL_TLS on case "$(printf '%s' "$DEVGRAIL_TLS" | tr '[:upper:]' '[:lower:]')" in off|false|0|no) TLS_ENABLED=false; SCHEME=http; TRAEFIK_ASSET=traefik-dynamic-http.yml ;; *) TLS_ENABLED=true; SCHEME=https; TRAEFIK_ASSET=traefik-dynamic.yml ;; esac sha256_of() { if command -v sha256sum >/dev/null 2>&1; then sha256sum "$1" | awk '{print $1}' else shasum -a 256 "$1" | awk '{print $1}'; fi } # Same, over stdin rather than a file. sha256_str() { if command -v sha256sum >/dev/null 2>&1; then sha256sum | awk '{print $1}' else shasum -a 256 | awk '{print $1}'; fi } gen_secret() { openssl rand -hex 32 2>/dev/null || head -c 32 /dev/urandom | od -An -tx1 | tr -d ' \n'; } gen_password() { (openssl rand -base64 18 2>/dev/null || head -c 15 /dev/urandom | base64) | tr -dc 'A-Za-z0-9' | cut -c1-20; } # --- admin credential validation -------------------------------------------- # The password ends up in deploy/.env as a bare, unquoted value, read back by # docker compose, by this script's own PRIOR_* reader and by the test harnesses. # Rather than teach all of them to unquote, the charset an operator may type is # restricted to what survives that file verbatim: no whitespace, no quotes, and # none of # $ \ ` & | < > ( ) [ ] { } / which either end the value or mean # something to compose. A generated password (gen_password) is alphanumeric, so # it is inside this set by construction. Passwords set later through the app or # `devgrail passwd` never touch .env and are not restricted this way. ADMIN_PASSWORD_MIN=12 valid_admin_password() { local pw="$1" [ "${#pw}" -ge "$ADMIN_PASSWORD_MIN" ] || return 1 # bcrypt truncates at 72 bytes, so anything longer is only half-checked. [ "${#pw}" -le 72 ] || return 1 case "$pw" in *[!A-Za-z0-9!%*+,.:\;=?@^_~-]*) return 1 ;; esac return 0 } # 3-32 characters, starting alphanumeric. Matches what a login form and the # users table accept, and leaves no room for a leading dash or an empty name. valid_admin_username() { case "$1" in ''|[!A-Za-z0-9]*) return 1 ;; *[!A-Za-z0-9._-]*) return 1 ;; esac [ "${#1}" -ge 3 ] && [ "${#1}" -le 32 ] } # read_secret VAR PROMPT — read a line from the terminal without echoing it. # Same /dev/tty convention as prompt(): stdin is the piped script. read_secret() { local var="$1" msg="$2" val printf '%s: ' "$msg" > /dev/tty stty -echo 2>/dev/null || true read -r val < /dev/tty || true stty echo 2>/dev/null || true printf '\n' > /dev/tty eval "$var=\$val" } # --- readiness --------------------------------------------------------------- # `docker compose up -d` returning 0 only means containers were *created*. The # failure that matters most — the server crash-looping on a migration — passes # that check and hands the customer a URL that does not work. /api/v1/healthz # answers the real question (database reachable, Docker reachable, schema # matches the binary) and returns 503 until all three hold. # # Two probe targets: the public URL is what the customer will actually use, but # it depends on DNS already pointing here and on Let's Encrypt having issued, # neither of which is true on a first install. The container-local address # verifies the server itself, so a not-yet-pointed DNS record does not masquerade # as a broken server. HEALTH_TIMEOUT="${DEVGRAIL_HEALTH_TIMEOUT:-180}" probe_health() { # Public route through Traefik. -k because the certificate may still be the # ACME self-signed placeholder; this checks readiness, not trust. curl -fsSk --max-time 5 "$SCHEME://$DEVGRAIL_DOMAIN/api/v1/healthz" >/dev/null 2>&1 && return 0 # The server directly, from inside its own container. compose exec -T devgrail-server /usr/local/bin/devgrail-server -healthcheck >/dev/null 2>&1 } wait_for_health() { local waited=0 log "Waiting for DevGrail to become ready (up to ${HEALTH_TIMEOUT}s)..." while [ "$waited" -lt "$HEALTH_TIMEOUT" ]; do if probe_health; then log "DevGrail is ready." return 0 fi sleep 3 waited=$((waited + 3)) done return 1 } # --- rollback ---------------------------------------------------------------- # data_volume prints the Docker volume backing /var/lib/devgrail. It is resolved # from the running container rather than assumed from the Compose project name, # which is derived from the deploy directory and so differs between installs. data_volume() { local vol vol="$(docker inspect devgrail-server \ --format '{{range .Mounts}}{{if eq .Destination "/var/lib/devgrail"}}{{.Name}}{{end}}{{end}}' 2>/dev/null)" || vol="" if [ -n "$vol" ]; then printf '%s' "$vol"; return 0; fi docker volume ls --format '{{.Name}}' 2>/dev/null | grep -E '(^|_)devgrail-data$' | head -n1 || true } newest_snapshot() { ls -1t "$BACKUP_DIR"/pre-upgrade-*.tgz 2>/dev/null | head -n1 || true } # Point :latest back at the image the previous run displaced. This is a tag move, # not a copy — which is the whole reason the retag is worth doing on every # upgrade: it turns rollback into a retag plus a compose recreate. rollback_images() { local slug moved=0 for slug in devgrail-server devgrail-container; do if docker image inspect "$slug:previous" >/dev/null 2>&1; then docker tag "$slug:previous" "$slug:latest" moved=$((moved + 1)) else warn "no $slug:previous image on this host — leaving $slug:latest where it is" fi done [ "$moved" -gt 0 ] } # restore_snapshot — put the database and config.yaml back as they were # before the upgrade that took this archive. restore_snapshot() { local archive="$1" vol stage vol="$(data_volume)" [ -n "$vol" ] || die "cannot find the DevGrail data volume — refusing to guess which volume to overwrite." stage="$(mktemp -d)" tar xzf "$archive" -C "$stage" || { rm -rf "$stage"; die "could not read $archive"; } if [ ! -f "$stage/devgrail.db" ] || [ ! -f "$stage/config.yaml" ]; then rm -rf "$stage" die "$archive is missing devgrail.db or config.yaml — a database without the config.yaml that keys its encryption is not a usable restore." fi compose stop devgrail-server >/dev/null 2>&1 || true # config.yaml first, deliberately: the JWT secret in it decrypts everything in # the database, so a half-completed restore that stopped here is recoverable, # whereas one that restored only the database is not. install -m 600 "$stage/config.yaml" "$CONFIG_DIR/config.yaml" \ || { rm -rf "$stage"; die "could not restore $CONFIG_DIR/config.yaml"; } # The -wal/-shm sidecars belong to the database being replaced. Leaving them # next to a restored file is how a "successful" restore produces a corrupt # database, so they go before the copy lands. docker run --rm --userns=host \ -v "$vol":/var/lib/devgrail \ -v "$stage":/restore:ro \ --entrypoint /bin/sh devgrail-server:latest -c ' rm -f /var/lib/devgrail/devgrail.db-wal /var/lib/devgrail/devgrail.db-shm && cp /restore/devgrail.db /var/lib/devgrail/devgrail.db' \ || { rm -rf "$stage"; die "could not restore the database into volume '$vol'"; } rm -rf "$stage" log "Restored database + config.yaml from $archive" } # do_rollback returns 0 when the rolled-back stack answers its readiness probe. do_rollback() { [ -f "$DEPLOY_DIR/docker-compose.yml" ] \ || die "no DevGrail install found at $INSTALL_DIR — there is nothing to roll back to." # Read from the prior .env rather than prompting: a rollback happens when # something is already wrong, and it should not stop to ask questions it can # answer itself. DEVGRAIL_DOMAIN="${DEVGRAIL_DOMAIN:-$(prior DEVGRAIL_DOMAIN)}" local archive archive="$(newest_snapshot)" log "Rolling back to the previous images and snapshot..." rollback_images || warn "no previous images were available to restore" if [ -n "$archive" ]; then restore_snapshot "$archive" else warn "No pre-upgrade snapshot in $BACKUP_DIR — restoring images only." echo " The database keeps whatever schema the newer server migrated it to," >&2 echo " and the older binary will refuse to start against it (by design)." >&2 fi compose up -d wait_for_health } # Prompt for a required input on /dev/tty (stdin is the piped script). An # optional default ($3, e.g. the prior install's answer) is shown in brackets; # pressing Enter keeps it. Without a terminal a default is used, else we die. prompt() { local var="$1" msg="$2" def="${3:-}" val why; eval "val=\${$var:-}" [ -n "$val" ] && return # --yes is the same contract as having no terminal: take the offered default, # which on an upgrade is the prior install's answer. if [ "$ASSUME_YES" = true ] || [ ! -e /dev/tty ]; then [ -n "$def" ] && { eval "$var=\$def"; return; } if [ "$ASSUME_YES" = true ]; then why="--yes was given, so nothing is prompted for" else why="no terminal is available"; fi die "$var is not set and there is no prior value to fall back to ($why). Pass it as an environment variable." fi if [ -n "$def" ]; then printf '%s [%s]: ' "$msg" "$def" > /dev/tty else printf '%s: ' "$msg" > /dev/tty; fi read -r val < /dev/tty || true val="${val:-$def}" [ -n "$val" ] || die "$var is required" eval "$var=\$val" } # Ask a yes/no question on /dev/tty (stdin is the piped script). $2 is the answer # a bare Enter takes. Callers decide what --yes and "no terminal" mean before # getting here; this only runs when there is a terminal and a question to ask. ask_yn() { local msg="$1" def="${2:-n}" ans hint case "$def" in y|Y) hint='[Y/n]' ;; *) hint='[y/N]' ;; esac printf '%s %s: ' "$msg" "$hint" > /dev/tty read -r ans < /dev/tty || ans="" ans="${ans:-$def}" case "$ans" in y|Y|yes|YES|Yes) return 0 ;; *) return 1 ;; esac } # --rollback is a mode of its own: it touches no registry, asks nothing, and must # not fall through into an install. Handled here, once the helpers above exist # and before any prompt. if [ "$DO_ROLLBACK" = true ]; then if do_rollback; then cat <&2 printf '\033[1;31merror:\033[0m %s\n' "The rolled-back stack did not become ready either." >&2 compose logs --tail=50 devgrail-server >&2 || true echo " Snapshots available: $BACKUP_DIR" >&2 exit 1 fi if [ "$IS_UPGRADE" = true ]; then log "Existing DevGrail install detected — current answers shown as [defaults]; press Enter to keep, or type a new value to change." fi prompt DEVGRAIL_DOMAIN "Public dashboard hostname (A record -> this VPS)" "$(prior DEVGRAIL_DOMAIN)" prompt DEVGRAIL_BASE_DOMAIN "Base domain for app subdomains (wildcard *. -> this VPS)" "$(prior DEVGRAIL_BASE_DOMAIN)" # Only needed for Let's Encrypt; plain-HTTP installs skip it (ACME is never run). if [ "$TLS_ENABLED" = true ]; then prompt ACME_EMAIL "Email for Let's Encrypt certificates" "$(prior ACME_EMAIL)" fi # --- ACME CA (production / staging / custom) -------------------------------- # Let's Encrypt production issues at most 5 certificates per exact set of # identifiers per 168h. Any test cycle that starts from an empty certificate # store re-orders on every run — `uninstall.sh --purge` removes the traefik-acme # volume, and the first install after enabling userns-remap moves Docker's data # root to /var/lib/docker/., orphaning the old volume just the same — # so five reinstalls on one hostname exhaust the quota. Traefik then falls back # to its self-signed "TRAEFIK DEFAULT CERT", which is what an invalid-certificate # browser warning on a freshly installed host actually means. Ordering from the # staging CA avoids it: the certificates are untrusted, but nothing else about # the issuance path differs, so it still exercises DNS, :443 and TLS-ALPN-01. # # Not prompted: production is right for every real install, and this only exists # for repeated testing against one hostname. ACME_CA_PRODUCTION_URL=https://acme-v02.api.letsencrypt.org/directory ACME_CA_STAGING_URL=https://acme-staging-v02.api.letsencrypt.org/directory # set_acme_ca — settle DEVGRAIL_ACME_CASERVER and the # store file that goes with it. The full URLs are accepted as well as the # aliases so a prior .env, which records the URL, round-trips unchanged. # # The store is per-CA on purpose: acme.json holds the registered ACME account, # and an account issued by one CA is unknown to another, so a shared file makes # every order fail with "account does not exist". Separate files also mean a # staging run leaves the production certificates already on this host intact, # and switching back picks them up again instead of re-ordering. set_acme_ca() { case "$(printf '%s' "${1:-}" | tr '[:upper:]' '[:lower:]')" in ''|production|prod|live|"$ACME_CA_PRODUCTION_URL") DEVGRAIL_ACME_CASERVER="$ACME_CA_PRODUCTION_URL" DEVGRAIL_ACME_STORAGE=acme.json ;; staging|test|"$ACME_CA_STAGING_URL") DEVGRAIL_ACME_CASERVER="$ACME_CA_STAGING_URL" DEVGRAIL_ACME_STORAGE=acme-staging.json ;; http://*|https://*) # A custom directory (pebble, an internal CA). The store name is derived # from the URL so two custom CAs cannot share a store or clobber acme.json. DEVGRAIL_ACME_CASERVER="$1" DEVGRAIL_ACME_STORAGE="acme-$(printf '%s' "$1" | sha256_str | cut -c1-8).json" ;; *) die "the ACME CA must be production, staging, or a directory URL starting with https:// — got '$1'. Set it with --acme-ca=, DEVGRAIL_ACME_CA, or DEVGRAIL_ACME_CASERVER in $DEPLOY_DIR/.env." ;; esac } if [ -n "${DEVGRAIL_ACME_CA:-}" ]; then set_acme_ca "$DEVGRAIL_ACME_CA" else # No explicit choice: keep what the prior install used. An upgrade must not # silently move a staging test back onto production and spend its quota. set_acme_ca "$(prior DEVGRAIL_ACME_CASERVER)" fi ACME_CA_IS_PRODUCTION=true [ "$DEVGRAIL_ACME_CASERVER" = "$ACME_CA_PRODUCTION_URL" ] || ACME_CA_IS_PRODUCTION=false # --- admin credentials ------------------------------------------------------ # Both are first-boot seeds: the server creates the admin account from them the # first time it starts and ignores them ever after (internal/db/seed.go), so an # upgrade asks nothing and simply carries the prior .env values over. Changing # them later is `devgrail passwd` or Settings -> Password in the dashboard. # # Unattended (--yes, or no terminal) keeps what every install did before this # was askable: the user `admin`, and a generated password printed in the closing # banner — the only place it is ever shown. ADMIN_PW_GENERATED=false ADMIN_USER="${DEVGRAIL_ADMIN_USERNAME:-}" ADMIN_PW="${DEVGRAIL_ADMIN_PASSWORD:-}" # Validate what the environment supplied before it reaches deploy/.env: a value # this script cannot round-trip would produce a stack nobody can log into, and # the failure would surface as "wrong password", not as a bad install. if [ -n "$ADMIN_USER" ] && ! valid_admin_username "$ADMIN_USER"; then die "DEVGRAIL_ADMIN_USERNAME must be 3-32 characters — letters, digits, dot, dash or underscore — starting with a letter or digit." fi if [ -n "$ADMIN_PW" ] && ! valid_admin_password "$ADMIN_PW"; then die "DEVGRAIL_ADMIN_PASSWORD must be $ADMIN_PASSWORD_MIN-72 characters of letters, digits or the punctuation ! % * + , - . : ; = ? @ ^ _ ~ (it is stored verbatim in deploy/.env, which quotes nothing)." fi [ -n "$ADMIN_USER" ] || ADMIN_USER="$(prior DEVGRAIL_ADMIN_USERNAME)" [ -n "$ADMIN_PW" ] || ADMIN_PW="$(prior DEVGRAIL_ADMIN_PASSWORD)" if [ "$IS_UPGRADE" = false ] && [ -e /dev/tty ] && [ "$ASSUME_YES" != true ]; then while [ -z "$ADMIN_USER" ]; do prompt ADMIN_USER "Admin username" admin valid_admin_username "$ADMIN_USER" && break warn "3-32 characters: letters, digits, dot, dash or underscore, starting with a letter or digit." ADMIN_USER="" done if [ -z "$ADMIN_PW" ]; then if ask_yn "Generate a secure admin password automatically?" y; then ADMIN_PW="$(gen_password)" ADMIN_PW_GENERATED=true else _tries=0 _pw1="" _pw2="" while [ "$_tries" -lt 3 ]; do _tries=$((_tries + 1)) read_secret _pw1 "Admin password (at least $ADMIN_PASSWORD_MIN characters)" read_secret _pw2 "Repeat password" if [ "$_pw1" != "$_pw2" ]; then warn "The passwords do not match." elif ! valid_admin_password "$_pw1"; then warn "$ADMIN_PASSWORD_MIN-72 characters of letters, digits or ! % * + , - . : ; = ? @ ^ _ ~" else ADMIN_PW="$_pw1" break fi done unset _pw1 _pw2 _tries if [ -z "$ADMIN_PW" ]; then ADMIN_PW="$(gen_password)" ADMIN_PW_GENERATED=true warn "Using a generated password instead; it is printed at the end of this run." fi fi fi fi # Unattended, or an upgrade whose .env predates these keys. [ -n "$ADMIN_USER" ] || ADMIN_USER="admin" if [ -z "$ADMIN_PW" ]; then ADMIN_PW="$(gen_password)" # Only announce it on a fresh install. On an upgrade the admin row already # exists, so this value is never applied — an operator who blanked the # password in .env after logging in (which the compose file suggests) must not # be handed a banner naming a password that does not work. [ "$IS_UPGRADE" = true ] || ADMIN_PW_GENERATED=true fi # Never prompted, but just as destructive to reset: an operator running on # non-standard ports had them silently rewritten to 80/443 by every upgrade. resolve HTTP_PORT 80 resolve HTTPS_PORT 443 resolve DEVGRAIL_SERVER_IMAGE devgrail-server # --- DNS preflight ---------------------------------------------------------- # "DNS is the only manual step" (spec §9) — and it was the only unverified one. # The installer prompted for the domain, brought the stack up, and printed the # required A records *after* it had finished. When they were not in place, # Traefik's ACME TLS-ALPN-01 challenge failed and the customer got a browser TLS # error with nothing connecting it to the cause. Let's Encrypt's failure rate # limits then make retrying worse rather than better. # # So the names are resolved and compared against this host's addresses here, # before Docker is installed and long before the stack is started. This is a # warning, not a verdict: split-horizon DNS, a pre-cutover install and a record # that has not propagated yet all look identical from inside the host. It asks # before continuing when there is a terminal; --skip-dns-check turns it off. # host_ips — the IPv4 addresses resolves to, one per line, empty # when it does not resolve. Four resolvers because no single one is present # everywhere: getent is glibc (absent on busybox), dig and host come from # bind-utils which most VPS images do not install, nslookup is busybox's. host_ips() { local name="$1" out="" if command -v getent >/dev/null 2>&1; then out="$(getent ahostsv4 "$name" 2>/dev/null | awk '{print $1}')" || out="" fi if [ -z "$out" ] && command -v dig >/dev/null 2>&1; then out="$(dig +short +time=3 +tries=1 A "$name" 2>/dev/null)" || out="" fi if [ -z "$out" ] && command -v host >/dev/null 2>&1; then out="$(host -W 3 -t A "$name" 2>/dev/null | sed -n 's/.*has address //p')" || out="" fi if [ -z "$out" ] && command -v nslookup >/dev/null 2>&1; then out="$(nslookup "$name" 2>/dev/null | sed -n '/^Name:/,$p' | sed -n 's/^Address: *//p')" || out="" fi # Whatever the tool printed, reduced to one de-duplicated IPv4 per line. The # grep also drops the resolver's own address, which nslookup prints first. # shellcheck disable=SC2086 # deliberate word splitting: one address per line printf '%s\n' $out | grep -E '^([0-9]{1,3}\.){3}[0-9]{1,3}$' | sort -u || true } have_resolver() { command -v getent >/dev/null 2>&1 || command -v dig >/dev/null 2>&1 || command -v host >/dev/null 2>&1 || command -v nslookup >/dev/null 2>&1 } # this_host_ips — addresses that legitimately point at this machine. Both halves # matter: a marketplace VPS carries its public address on an interface, but a # NAT'd or floating-IP host does not, and there an echo service is the only way # to learn what the world sees. Either matching is enough. this_host_ips() { local locals="" public="" url if command -v ip >/dev/null 2>&1; then locals="$(ip -4 -o addr show scope global 2>/dev/null | awk '{print $4}' | cut -d/ -f1)" || locals="" fi if [ -z "$locals" ] && command -v hostname >/dev/null 2>&1; then locals="$(hostname -I 2>/dev/null)" || locals="" fi for url in https://api.ipify.org https://checkip.amazonaws.com https://ifconfig.me/ip; do public="$(curl -fsS --max-time 5 "$url" 2>/dev/null | tr -cd '0-9.')" || public="" [ -n "$public" ] && break done # shellcheck disable=SC2086 # deliberate word splitting: one address per line printf '%s\n' $locals $public | grep -E '^([0-9]{1,3}\.){3}[0-9]{1,3}$' | sort -u || true } # dns_problem — one problem line, or nothing when # resolves to an address this host answers on. dns_problem() { local name="$1" mine="$2" what="$3" got ip got="$(host_ips "$name")" if [ -z "$got" ]; then printf ' %s does not resolve — no %s found.\n' "$name" "$what" return 0 fi for ip in $got; do case " $(printf '%s' "$mine" | tr '\n' ' ') " in *" $ip "*) return 0 ;; esac done printf ' %s resolves to %s, which is not this host.\n' \ "$name" "$(printf '%s' "$got" | tr '\n' ' ')" } dns_preflight() { if [ "$SKIP_DNS_CHECK" = true ]; then log "Skipping the DNS preflight (--skip-dns-check)." return 0 fi if ! have_resolver; then warn "no DNS lookup tool here (getent/dig/host/nslookup) — skipping the DNS preflight." return 0 fi log "Checking DNS for $DEVGRAIL_DOMAIN and *.$DEVGRAIL_BASE_DOMAIN..." local mine problems probe mine="$(this_host_ips)" if [ -z "$mine" ]; then warn "could not determine this host's IP address — skipping the DNS preflight." return 0 fi # A random label under the base domain can only resolve through the wildcard # record. Checking the base domain itself would pass on a bare A record and # then fail for every workspace subdomain, which is the case that matters. probe="devgrail-dns-probe-$$.$DEVGRAIL_BASE_DOMAIN" problems="$(dns_problem "$DEVGRAIL_DOMAIN" "$mine" "A record")" problems="$problems$(dns_problem "$probe" "$mine" "wildcard A record for *.$DEVGRAIL_BASE_DOMAIN")" if [ -z "$problems" ]; then log "DNS resolves to this host." return 0 fi warn "DNS does not point at this host yet." printf '%s' "$problems" >&2 echo " This host answers on: $(printf '%s' "$mine" | tr '\n' ' ')" >&2 echo >&2 echo " Required records:" >&2 echo " A $DEVGRAIL_DOMAIN -> this VPS" >&2 echo " A *.$DEVGRAIL_BASE_DOMAIN -> this VPS" >&2 if [ "$TLS_ENABLED" = true ]; then echo >&2 echo " Continuing now means Let's Encrypt will attempt its TLS-ALPN-01" >&2 echo " challenge against a name that does not reach this host. It will fail," >&2 echo " and repeated failures hit rate limits that make retrying slower rather" >&2 echo " than faster — so it is usually cheaper to fix DNS first." >&2 fi echo >&2 echo " Legitimate reasons to continue anyway: split-horizon DNS, a pre-cutover" >&2 echo " install, or a record that has not propagated yet. --skip-dns-check" >&2 echo " suppresses this check entirely." >&2 echo >&2 if [ "$ASSUME_YES" = true ]; then warn "--yes given — continuing with unverified DNS." elif [ -e /dev/tty ]; then printf 'Continue anyway? [y/N]: ' > /dev/tty read -r _answer < /dev/tty || _answer="" case "$_answer" in y|Y|yes|YES) log "Continuing with unverified DNS." ;; *) die "aborted before anything on this host was changed. Fix the records and re-run, or re-run with --skip-dns-check." ;; esac else # Unattended: an install driven by cloud-init frequently runs before the # records exist. Refusing would break that legitimate case, so it is said # loudly and the install proceeds. warn "no terminal to ask — continuing with unverified DNS." fi } dns_preflight # --- Docker + tooling ------------------------------------------------------ # A dry run must not install anything, and it cannot read the registry without # jq — so on a host missing either, the honest report is that they would be # installed, and it stops there. if [ "$DRY_RUN" = true ]; then MISSING_TOOLS="" command -v docker >/dev/null 2>&1 || MISSING_TOOLS="Docker" command -v jq >/dev/null 2>&1 || MISSING_TOOLS="${MISSING_TOOLS:+$MISSING_TOOLS and }jq" if [ -n "$MISSING_TOOLS" ]; then echo echo "============================================================" echo " Dry run — nothing on this host has been changed." echo echo " $MISSING_TOOLS would be installed first; this host does not have it yet." echo " Nothing further can be reported without it: the release manifest," echo " the version diff and the disk figures all come from the registry." echo "============================================================" exit 0 fi fi if ! command -v docker >/dev/null 2>&1; then log "Installing Docker..." # 9>&- keeps the install lock out of dockerd. Where systemd starts the daemon # it never sees our fds anyway, but on a host whose systemctl forks dockerd # directly (containers, the local sandbox) the daemon would inherit fd 9 and # hold the lock forever. acquire_lock can recover from that; not causing it is # cheaper. Plain `docker`/`docker compose` calls need no such guard — they are # short-lived clients, and workspace containers are children of dockerd. curl -fsSL https://get.docker.com | sh 9>&- systemctl enable --now docker 2>/dev/null 9>&- || true fi docker compose version >/dev/null 2>&1 || die "the 'docker compose' plugin is required" if ! command -v jq >/dev/null 2>&1; then log "Installing jq..." if command -v apt-get >/dev/null 2>&1; then apt-get update -qq && apt-get install -y -qq jq elif command -v dnf >/dev/null 2>&1; then dnf install -y jq elif command -v yum >/dev/null 2>&1; then yum install -y jq elif command -v apk >/dev/null 2>&1; then apk add --no-cache jq else die "please install 'jq' and re-run"; fi fi # openssl verifies the release signature (and generates secrets when present). # Best-effort: a host without it falls back to /dev/urandom for secrets and to # an unverified release, both of which say so rather than failing. if ! command -v openssl >/dev/null 2>&1; then log "Installing openssl..." if command -v apt-get >/dev/null 2>&1; then apt-get install -y -qq openssl || true elif command -v dnf >/dev/null 2>&1; then dnf install -y openssl || true elif command -v yum >/dev/null 2>&1; then yum install -y openssl || true elif command -v apk >/dev/null 2>&1; then apk add --no-cache openssl || true fi fi # tar builds the pre-upgrade snapshot archive; df/awk size the free-space check. for tool in tar df awk; do command -v "$tool" >/dev/null 2>&1 || die "'$tool' is required (used to take the pre-upgrade backup)" done # --- Docker daemon preflight (userns-remap + default address pools) -------- # # Two daemon-level settings materially change how DevGrail runs, and neither can # be expressed in a container spec: both live in /etc/docker/daemon.json and take # effect only when dockerd restarts. The installer is the one moment an operator # is reliably at a terminal, as root, on a host where these are still cheap to # change — so this section detects, explains, offers and applies, rather than # printing a wall of manual steps and hoping. # # Nothing here is applied without an explicit yes, and --yes is deliberately not # that yes: restarting dockerd stops every container on the host, and enabling # remap moves the daemon's data root. Those are decisions, not defaults. # DEVGRAIL_USERNS_REMAP / DEVGRAIL_ADDRESS_POOLS (and the matching flags) are how # an unattended run — a marketplace image build, cloud-init — opts in on purpose. DAEMON_JSON="${DEVGRAIL_DAEMON_JSON:-/etc/docker/daemon.json}" # Restart dockerd across the init systems this installer supports: systemd on the # Debian/RHEL images, OpenRC on Alpine. Non-zero means no known mechanism worked, # which callers treat as "put the config back". restart_docker() { if command -v systemctl >/dev/null 2>&1; then systemctl restart docker >/dev/null 2>&1 && return 0 fi if command -v rc-service >/dev/null 2>&1; then rc-service docker restart >/dev/null 2>&1 && return 0 fi if command -v service >/dev/null 2>&1; then service docker restart >/dev/null 2>&1 && return 0 fi return 1 } # Wait for the daemon to answer again. dockerd is slow to come back when it has # images to re-index, and the first start under userns-remap is slower still — it # lays down a fresh data root — so this window is generous on purpose. wait_for_docker() { local i=0 while [ "$i" -lt 90 ]; do docker info >/dev/null 2>&1 && return 0 sleep 1 i=$((i + 1)) done return 1 } # Merge a jq expression into $DAEMON_JSON and restart the daemon. # $1 human label for the change, used in messages # $2 jq expression applied to the existing object (or {} when there is none) # # The prior file is kept as .devgrail-bak-. If dockerd does not # come back, the backup is restored and the daemon restarted again: leaving a # host without Docker is a worse outcome than either setting is worth. A # daemon.json that does not parse is never touched — that is an operator edit in # progress, and clobbering it would destroy work we cannot see. apply_daemon_json() { local label="$1" filter="$2" current='{}' backup="" merged stamp if [ -s "$DAEMON_JSON" ]; then if ! jq -e . "$DAEMON_JSON" >/dev/null 2>&1; then warn "$DAEMON_JSON exists but is not valid JSON — leaving it alone." echo " Fix the file (or move it aside) and re-run to apply $label." >&2 return 1 fi current="$(cat "$DAEMON_JSON")" stamp="$(date +%Y%m%d%H%M%S)" backup="$DAEMON_JSON.devgrail-bak-$stamp" cp -p "$DAEMON_JSON" "$backup" || { warn "could not back up $DAEMON_JSON — not changing it." return 1 } fi if ! merged="$(printf '%s' "$current" | jq "$filter" 2>/dev/null)" || [ -z "$merged" ]; then warn "could not compute the new $DAEMON_JSON — leaving it alone." [ -n "$backup" ] && rm -f "$backup" return 1 fi mkdir -p "$(dirname "$DAEMON_JSON")" if ! printf '%s\n' "$merged" > "$DAEMON_JSON.devgrail-new"; then warn "could not write $DAEMON_JSON — leaving it alone." [ -n "$backup" ] && rm -f "$backup" return 1 fi chmod 0644 "$DAEMON_JSON.devgrail-new" mv -f "$DAEMON_JSON.devgrail-new" "$DAEMON_JSON" log "Restarting Docker to apply $label (this stops running containers)..." if restart_docker && wait_for_docker; then log "Docker restarted; $label is in effect." [ -n "$backup" ] && log "Previous daemon config kept at $backup" return 0 fi warn "Docker did not come back after applying $label — rolling the change back." if [ -n "$backup" ]; then mv -f "$backup" "$DAEMON_JSON"; else rm -f "$DAEMON_JSON"; fi if restart_docker && wait_for_docker; then warn "Docker is back on its previous configuration; $label was NOT applied." return 1 fi die "Docker is not responding and restoring the previous configuration did not revive it. $DAEMON_JSON holds what it held before this run. Investigate with: journalctl -u docker --no-pager -n 50 Nothing else on this host has been changed." } # True when the daemon reports userns-remap as active. userns_remap_enabled() { docker info --format '{{.SecurityOptions}}' 2>/dev/null | grep -q 'name=userns' } # True when this host already has Docker state that enabling userns-remap would # orphan. Remap moves the daemon to a per-UID data root, so every image, # container and volume created before the switch becomes invisible — including, # on an upgrade, DevGrail's own data volume. This check is what keeps the offer # limited to "fresh host, nothing to lose". docker_has_state() { local what n for what in "ps -aq" "volume ls -q" "image ls -q"; do # shellcheck disable=SC2086 # grep -c '.' rather than wc -l: it counts a final line that has no trailing # newline, and skips blank ones. wc -l would report 0 for a one-line answer # that ended without a newline — i.e. read "this host is empty" off a host # that has exactly one image. n="$({ docker $what 2>/dev/null || true; } | grep -c '.' || true)" [ "${n:-0}" -gt 0 ] && return 0 done return 1 } # How many networks the daemon's configured pools can hand out. Docker carves # each pool into / blocks, so a pool contributes 2^(size - mask) networks. # Zero means no pools are configured, i.e. the stock defaults, which top out # near 31 — the number this whole section exists to raise. address_pool_capacity() { local pools cap pools="$(docker info --format '{{json .DefaultAddressPools}}' 2>/dev/null || true)" case "$pools" in ''|null) echo 0; return 0 ;; esac cap="$(printf '%s' "$pools" | jq -r ' [ .[]? | (.Size - (.Base | split("/")[1] | tonumber)) | select(. >= 0) | pow(2; .) ] | add // 0 | floor' 2>/dev/null || true)" case "$cap" in ''|null) echo 0 ;; *) echo "$cap" ;; esac } # True when no route on this host already lives inside the candidate range. # POSIX sh has no CIDR arithmetic and the candidates are fixed, so each carries a # regex matching an address inside it. Routes on docker0 / br-* / veth are # excluded: Docker's own bridges already sit in 172.17-172.31 and would otherwise # veto the very range we want to hand it. pool_base_free() { local re="$1" routes command -v ip >/dev/null 2>&1 || return 0 routes="$({ ip -4 route show 2>/dev/null || true; } \ | grep -Ev 'dev (docker[0-9]*|br-[0-9a-f]+|veth[0-9a-z]*)' || true)" if printf '%s\n' "$routes" | grep -Eq "$re"; then return 1; fi return 0 } running_container_count() { # See docker_has_state for why this counts with grep rather than wc. { docker ps -q 2>/dev/null || true; } | grep -c '.' || true } # --- userns-remap ---------------------------------------------------------- if userns_remap_enabled; then log "Docker userns-remap is enabled — workspace root maps to an unprivileged host UID." elif [ "$USERNS_REMAP_ANSWER" = no ]; then : # the operator has decided; do not re-ask on every upgrade else echo >&2 echo " A container escape would be host root" >&2 echo " ----------------------------------------------------------" >&2 echo " Workspaces run AI agents and whatever code those agents write." >&2 echo " Inside the container the agent is the unprivileged 'dev' user, but" >&2 echo " it has passwordless sudo — so it can become container root at will," >&2 echo " and without userns-remap container root IS host root. A bug in the" >&2 echo " kernel or the container runtime would then be a full host takeover." >&2 echo >&2 echo " userns-remap maps container root onto an unprivileged host UID, so" >&2 echo " the same escape lands as a nobody. DevGrail's own infra containers" >&2 echo " (traefik, docker-socket-proxy, devgrail-server) opt out via" >&2 echo " userns_mode: host, so only workspaces are remapped." >&2 echo " See DEVGRAIL_SYSTEM_SPEC.md section 5." >&2 echo >&2 if docker_has_state; then # Not offered, and not a warning the operator can act on in place: switching # now would hide everything already on this host behind a new data root. warn "userns-remap is NOT enabled, and cannot be turned on safely from here." echo " This host already has Docker images, containers or volumes. Enabling" >&2 echo " remap moves the daemon to a new data root, so all of them — including" >&2 echo " DevGrail's data volume — would become invisible. Migrating that is a" >&2 echo " deliberate, backed-up operation, not something an installer should do" >&2 echo " behind a prompt." >&2 echo >&2 echo " To do it by hand, with the stack down and a backup taken:" >&2 echo " 1. devgrail backup" >&2 echo " 2. Add to $DAEMON_JSON: { \"userns-remap\": \"default\" }" >&2 echo " 3. systemctl restart docker" >&2 echo " 4. Re-run this installer, then devgrail restore " >&2 echo " Continuing without remap..." >&2 echo >&2 elif [ "$DRY_RUN" = true ]; then echo " Dry run: this host has no Docker state yet, so a real run would offer" >&2 echo " to enable userns-remap and restart Docker. Nothing has been changed." >&2 echo >&2 else echo " This host has no Docker containers, images or volumes yet, so turning" >&2 echo " it on now is free — there is nothing for the data-root change to" >&2 echo " orphan. Once workspaces exist it stops being free." >&2 echo >&2 DO_USERNS=false if [ "$USERNS_REMAP_ANSWER" = yes ]; then DO_USERNS=true log "DEVGRAIL_USERNS_REMAP=yes — enabling userns-remap." elif [ -e /dev/tty ] && [ "$ASSUME_YES" != true ]; then if ask_yn " Enable userns-remap now?" y; then DO_USERNS=true; fi fi if [ "$DO_USERNS" = true ]; then # dockerd creates the "dockremap" user and its subordinate ranges itself, # but only if these files exist for it to append to. Creating them empty is # a no-op where they already work, and removes a known first-start failure # on minimal images. [ -f /etc/subuid ] || : > /etc/subuid [ -f /etc/subgid ] || : > /etc/subgid if apply_daemon_json "userns-remap" '. + {"userns-remap": "default"}'; then if userns_remap_enabled; then log "userns-remap is active — workspace root is now an unprivileged host UID." else warn "Docker restarted but does not report userns-remap as active." echo " Check 'docker info' and journalctl -u docker. Continuing." >&2 fi fi else warn "Continuing without userns-remap." echo " To enable it later (one-time, needs root):" >&2 echo " 1. Add to $DAEMON_JSON: { \"userns-remap\": \"default\" }" >&2 echo " 2. systemctl restart docker" >&2 echo " Do it before workspaces exist — afterwards the data-root change" >&2 echo " orphans them. Set DEVGRAIL_USERNS_REMAP=no to stop being asked." >&2 echo >&2 fi fi fi # --- default address pools ------------------------------------------------- # Each workspace gets its own bridge network so untrusted agent code cannot reach # a peer workspace directly. Docker's stock pools only yield ~31 networks in # total, so a busy host runs out and container creation starts failing with # "could not find an available, non-overlapping IPv4 address pool". POOL_CAP="$(address_pool_capacity)" if [ "${POOL_CAP:-0}" -ge 256 ]; then log "Docker address pools allow ~$POOL_CAP networks — no workspace ceiling to worry about." elif [ "$ADDRESS_POOLS_ANSWER" = no ]; then : # already decided else echo >&2 echo " Workspace ceiling: about 27" >&2 echo " ----------------------------------------------------------" >&2 echo " Every DevGrail workspace gets its own Docker network so one" >&2 echo " workspace cannot reach another. Docker's stock address pools yield" >&2 echo " only ~31 networks in total, and past that container creation fails" >&2 echo " with 'could not find an available, non-overlapping IPv4 address" >&2 echo " pool' — an error that shows up when you add a workspace, not now." >&2 echo >&2 if [ "${POOL_CAP:-0}" -gt 0 ]; then # Pools are configured, just narrow. That is an operator decision about this # host's addressing, and it is not one to overwrite from behind a prompt. warn "Docker address pools are configured but allow only ~$POOL_CAP networks." echo " They were set deliberately, so this installer will not rewrite them." >&2 echo " To widen them, edit default-address-pools in $DAEMON_JSON and" >&2 echo " restart Docker. Continuing..." >&2 echo >&2 else POOL_BASE=""; POOL_NEW_CAP=0 if pool_base_free '(^|[[:space:]])172\.(1[6-9]|2[0-9]|3[01])\.'; then POOL_BASE=172.16.0.0/12; POOL_NEW_CAP=4096 elif pool_base_free '(^|[[:space:]])10\.201\.'; then POOL_BASE=10.201.0.0/16; POOL_NEW_CAP=256 fi if [ -z "$POOL_BASE" ]; then warn "No safe range to offer — this host already routes both 172.16/12 and 10.201/16." echo " Handing Docker a pool that shadows one of your own routes is worse" >&2 echo " than the ceiling it would lift. Pick a free range yourself, add it" >&2 echo " as default-address-pools in $DAEMON_JSON, and restart Docker." >&2 echo " Continuing with the defaults..." >&2 echo >&2 elif [ "$DRY_RUN" = true ]; then echo " Dry run: a real run would offer to widen the pools to $POOL_BASE" >&2 echo " (a /24 per network, ~$POOL_NEW_CAP workspaces) and restart Docker." >&2 echo " Nothing has been changed." >&2 echo >&2 else RUNNING="$(running_container_count)" echo " Widening the pools to $POOL_BASE (a /24 per network) raises the" >&2 echo " ceiling to ~$POOL_NEW_CAP. Networks that already exist keep their" >&2 echo " current subnets — this only affects networks created from now on." >&2 echo >&2 if [ "${RUNNING:-0}" -gt 0 ]; then echo " Cost: applying it restarts the Docker daemon, which stops the" >&2 echo " $RUNNING container(s) running here. The DevGrail stack comes back on" >&2 echo " its own (restart: unless-stopped); workspaces do not — you start" >&2 echo " those again from the dashboard." >&2 else echo " Cost: it restarts the Docker daemon. Nothing is running on this" >&2 echo " host yet, so that costs nothing right now." >&2 fi echo >&2 DO_POOLS=false POOL_DEFAULT=y [ "${RUNNING:-0}" -gt 0 ] && POOL_DEFAULT=n if [ "$ADDRESS_POOLS_ANSWER" = yes ]; then DO_POOLS=true log "DEVGRAIL_ADDRESS_POOLS=yes — widening the default address pools." elif [ -e /dev/tty ] && [ "$ASSUME_YES" != true ]; then if ask_yn " Widen the address pools now?" "$POOL_DEFAULT"; then DO_POOLS=true; fi fi if [ "$DO_POOLS" = true ]; then apply_daemon_json "the wider address pools" \ ". + {\"default-address-pools\": [{\"base\": \"$POOL_BASE\", \"size\": 24}]}" || true else warn "Continuing with the default address pools (~27 workspaces)." echo " To raise the ceiling later (one-time, needs root):" >&2 echo " 1. Add to $DAEMON_JSON:" >&2 echo " { \"default-address-pools\": [{\"base\": \"$POOL_BASE\", \"size\": 24}] }" >&2 echo " 2. systemctl restart docker" >&2 echo " Safe to defer until you approach the limit. Set" >&2 echo " DEVGRAIL_ADDRESS_POOLS=no to stop being asked." >&2 echo >&2 fi fi fi fi # --- download + load images ------------------------------------------------ # TMP is chosen after the release is resolved, once the download size is known: # /tmp is a tmpfs on many VPS images (sized from RAM, often well under 1 GiB) # and a multi-hundred-megabyte image tarball does not fit. It is declared (and # the EXIT trap that removes it installed) up with the lock, so an abort at any # point between here and there still cleans up. # free_bytes — free space on the filesystem holding , in bytes. Prints # 0 when it cannot be determined (df on an unwritable path, an exotic # filesystem) — every caller treats 0 as "unknown, do not block on it", so a # preflight that cannot measure never becomes a preflight that refuses. free_bytes() { local v v="$(df -Pk "$1" 2>/dev/null | awk 'NR==2 {print $4 * 1024}')" || v="" case "$v" in ''|*[!0-9]*) v=0 ;; esac printf '%s' "$v" } # human — MiB/GiB for a message an operator can act on. Integer shell # arithmetic only: this runs before jq is guaranteed and awk is already required. human() { local b="${1:-0}" if [ "$b" -ge 1073741824 ]; then printf '%s.%s GiB' "$((b / 1073741824))" "$(((b % 1073741824) * 10 / 1073741824))" elif [ "$b" -ge 1048576 ]; then printf '%s MiB' "$((b / 1048576))" else printf '%s bytes' "$b"; fi } # Prior install versions (for skip-if-unchanged upgrades). SERVER_RELEASE, # CONTAINER_RELEASE and ASSET_VERSIONS were added later, so an install that # predates them simply leaves these empty — reported as "unknown" or treated as # "not yet fetched", never as a failure. SERVER_VERSION=""; CONTAINER_VERSION="" SERVER_RELEASE=""; CONTAINER_RELEASE="" ASSET_VERSIONS="" RELEASE_VERSION="" # shellcheck source=/dev/null # written by a previous run of this script [ -f "$VERSIONS_FILE" ] && . "$VERSIONS_FILE" # RELEASE_VERSION is reused below for the release this run resolves, so the one # the file just supplied — what is installed *now* — is kept under its own name. PRIOR_RELEASE_VERSION="$RELEASE_VERSION" # asset_version_of — the version recorded for this slug at the last run, # or empty. ASSET_VERSIONS is a space-separated "slug=version" list, which keeps # it to one shell variable regardless of how many assets a release carries (slugs # are filenames, so they are not usable as variable names). asset_version_of() { # shellcheck disable=SC2086 # deliberate word splitting into one entry per line printf '%s\n' ${ASSET_VERSIONS:-} | awk -F= -v s="$1" '$1==s {print $2; exit}' } # --- resolve the release ---------------------------------------------------- # A DevGrail release is seven artifacts, not one. Resolving them all from a # single manifest is what stops an upgrade from mixing a new compose file with # an old server image: before this, the compose and Traefik config were # re-fetched unconditionally on every run because there was no asset version to # compare against, so even a "nothing changed" re-run silently adopted whatever # had been published since. DEVGRAIL_VERSION="${DEVGRAIL_VERSION:-current}" RELEASE_MANIFEST="" RELEASE_VERSION="" resolve_release() { local body body="$(curl -fsSL "$WEB/api/registry/releases/$DEVGRAIL_VERSION" 2>/dev/null)" || body="" if [ -z "$body" ]; then # An explicit version that does not resolve is an error — silently # installing something else is the one thing --version must never do. [ "$DEVGRAIL_VERSION" = current ] \ || die "release '$DEVGRAIL_VERSION' is not published at $WEB. Nothing has been changed." # No release manifest at all means a registry older than this installer. # Fall back to resolving each artifact by slug, exactly as before. log "Registry publishes no release manifest — resolving each artifact individually." return 0 fi RELEASE_MANIFEST="$body" RELEASE_VERSION="$(printf '%s' "$body" | jq -r '.release_version // empty')" log "Installing release ${RELEASE_VERSION:-unknown}" } resolve_release # --- release signature ------------------------------------------------------- # Every artifact is verified against a SHA-256 the registry publishes — which # catches a corrupted download but not a compromised registry, because whoever # can serve the bytes can serve the matching digest. The same origin also serves # this script, the compose file and the Traefik config, all executed as root. # # A detached signature over the release closes that, and only if the signing key # never touches the web host. One signature covers all seven artifacts: that is # the payoff of resolving a release from a single manifest. # # What is signed is a payload listing every artifact's kind, slug, version and # digest — not the manifest itself, whose download URLs are rewritten per origin # and so cannot be signed offline. Verification therefore proves the digests, # and the digests prove the bytes. See docs/SECURITY.md. # Public keys this installer trusts to sign a release. Generated by # scripts/release-key.sh; the private half is never in this repo. # # Rotation: add the new key here and ship an installer carrying both, then start # signing with the new one, then drop the old key a release later. Hosts that # never upgraded the installer keep working throughout. release_public_keys() { # key id f1f1eaf642454d65 — first embedded in v0.4.0 cat <<'PUBKEYS' -----BEGIN PUBLIC KEY----- MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEK1NbYrxBM8Ia1iM4BcR0m/8DelN5 ZDvOwKmEOJA24dci3ftqHWwsxzbw8bACRps0rIOTqa45wblm6260fQVdIw== -----END PUBLIC KEY----- PUBKEYS # An operator running their own registry (or the test harness) can add keys # without editing this file. This grants trust rather than removing it, and # anyone able to set it already runs this script as root. if [ -n "${DEVGRAIL_RELEASE_PUBKEY_FILE:-}" ] && [ -f "$DEVGRAIL_RELEASE_PUBKEY_FILE" ]; then cat "$DEVGRAIL_RELEASE_PUBKEY_FILE" fi } # The verified payload, once there is one. Empty means nothing is enforced # beyond the registry's own digests. SIGNED_PAYLOAD_FILE="" SIG_DIR="" # unverified — how a release that cannot be checked is treated. A warning # by default: refusing would strand every host whose registry or installer # predates signing, and an unsigned release is no worse than what existed # before. DEVGRAIL_REQUIRE_SIGNATURE=1 turns it into a refusal for operators who # want that guarantee. unverified() { if [ "${DEVGRAIL_REQUIRE_SIGNATURE:-0}" = 1 ]; then die "refusing to install a release whose signature was not verified: $1 DEVGRAIL_REQUIRE_SIGNATURE=1 is set. Nothing has been changed." fi warn "release signature not verified — $1" echo " Artifacts are still checked against the digests this registry" >&2 echo " publishes, which detects a corrupted download but not a compromised" >&2 echo " registry. See docs/SECURITY.md." >&2 } verify_release_signature() { local sig alg kid keys dir pem verified=false if [ -z "$RELEASE_MANIFEST" ]; then unverified "this registry publishes no release manifest" return 0 fi sig="$(printf '%s' "$RELEASE_MANIFEST" | jq -c '.signature // empty' 2>/dev/null || true)" if [ -z "$sig" ] || [ "$sig" = null ]; then unverified "release ${RELEASE_VERSION:-current} was published without one" return 0 fi keys="$(release_public_keys)" if [ -z "$keys" ]; then unverified "this installer carries no release signing key" return 0 fi if ! command -v openssl >/dev/null 2>&1; then unverified "openssl is not installed here, so the signature cannot be checked" return 0 fi alg="$(printf '%s' "$sig" | jq -r '.algorithm // empty')" kid="$(printf '%s' "$sig" | jq -r '.key_id // empty')" # An unknown algorithm is not the same as an unsigned release: the publisher # signed it with something this installer cannot check, and treating that as # "unsigned" would let an attacker downgrade verification by naming a scheme # we do not implement. [ "$alg" = "ecdsa-p256-sha256" ] \ || die "release ${RELEASE_VERSION:-current} is signed with '$alg', which this installer cannot verify. Upgrade the installer. Nothing has been changed." dir="$(mktemp -d)" SIG_DIR="$dir" # -j, not -r: the signature is over exactly the published bytes, and -r adds a # trailing newline the signer never saw. That difference verifies as tampering. printf '%s' "$sig" | jq -j '.payload' > "$dir/payload" printf '%s' "$sig" | jq -r '.signature' | base64 -d > "$dir/sig" 2>/dev/null \ || die "the signature on release ${RELEASE_VERSION:-current} is not valid base64. Nothing has been changed." # Each embedded key gets its own file; any one of them verifying is enough, # which is what makes rotation a matter of shipping two keys for a while. printf '%s\n' "$keys" > "$dir/keys.pem" awk -v d="$dir" '/-----BEGIN PUBLIC KEY-----/{n++} n>0 {print > (d "/key" n ".pem")}' "$dir/keys.pem" for pem in "$dir"/key*.pem; do [ -f "$pem" ] || continue if openssl dgst -sha256 -verify "$pem" -signature "$dir/sig" "$dir/payload" >/dev/null 2>&1; then verified=true break fi done if [ "$verified" != true ]; then die "the signature on release ${RELEASE_VERSION:-current} does not verify against any key this installer carries (signing key id: ${kid:-unknown}). That is either a corrupted download or a tampered release, and there is no way to tell which from here. Nothing has been changed." fi # A valid signature over somebody else's payload proves nothing about this # release, so the payload has to name it — otherwise an old signed release # could be replayed against a host asking for a newer one. if ! grep -qxF "devgrail-release-signature-v1" "$dir/payload"; then die "the release signature verifies but its payload is not in a format this installer understands. Nothing has been changed." fi if ! grep -qxF "release ${RELEASE_VERSION}" "$dir/payload"; then die "the signature verifies, but it was made over a different release than ${RELEASE_VERSION:-the one requested}. Nothing has been changed." fi SIGNED_PAYLOAD_FILE="$dir/payload" log "Release ${RELEASE_VERSION} signature verified (key ${kid:-unknown})." } # signed_digest_check — refuse anything the signed payload # does not name. Called after the registry's own digest matched, so this is the # check that survives a compromised registry: matching bytes are not enough if # they are not the bytes the release was signed for. signed_digest_check() { [ -n "$SIGNED_PAYLOAD_FILE" ] || return 0 if grep -qxF "$1" "$SIGNED_PAYLOAD_FILE"; then return 0 fi die "'$2' is not what release $RELEASE_VERSION was signed for. Its bytes match the digest this registry served, but that digest does not appear in the signed release payload: $1 Refusing to install it. Nothing has been changed." } verify_release_signature # release_field — a field of one artifact in the release # manifest, or empty when there is no manifest or no such artifact. release_field() { [ -n "$RELEASE_MANIFEST" ] || return 0 printf '%s' "$RELEASE_MANIFEST" | jq -r --arg s "$1" --arg a "$ARCH" "$2" 2>/dev/null || true } # same_image — true when both names resolve to one image. Either # name missing is "not the same", which is the answer that makes callers do the # safe thing. same_image() { local a b a="$(docker image inspect -f '{{.Id}}' "$1" 2>/dev/null)" || return 1 b="$(docker image inspect -f '{{.Id}}' "$2" 2>/dev/null)" || return 1 [ -n "$a" ] && [ "$a" = "$b" ] } # resolve_image — settle which build of # this run needs, and whether it has to be downloaded at all, without # fetching a byte. Sets R_VER R_REL R_URL R_SHA R_SIZE R_TAG R_SKIP R_RETAG. # # Split out from download_image so the disk preflight below can total up what is # about to be written *before* the first download starts. Running out of space # halfway through is precisely the condition that causes a mid-migration failure # (INSTALL_UPGRADE_ANALYSIS M2), so discovering it after one image has already # landed is not good enough. R_VER=""; R_REL=""; R_URL=""; R_SHA=""; R_SIZE=0; R_TAG=""; R_SKIP=false; R_RETAG=false resolve_image() { local slug="$1" target="$2" cur="$3" manifest # Prefer the release manifest: it pins the exact version this release is, and # its download URLs carry that pin so they keep resolving to these bytes even # after a newer release is published. R_VER="$(release_field "$slug" '.images[]|select(.slug==$s)|.image_version')" R_URL="$(release_field "$slug" '.images[]|select(.slug==$s)|.artifacts[]|select(.arch==$a)|.download_url')" R_SHA="$(release_field "$slug" '.images[]|select(.slug==$s)|.artifacts[]|select(.arch==$a)|.sha256')" R_SIZE="$(release_field "$slug" '.images[]|select(.slug==$s)|.artifacts[]|select(.arch==$a)|.size')" R_REL="$RELEASE_VERSION" if [ -z "$R_VER" ]; then manifest="$(curl -fsSL "$WEB/api/registry/images/$slug")" \ || die "cannot fetch image manifest for '$slug' from $WEB" R_VER="$(printf '%s' "$manifest" | jq -r '.image_version')" # Additive field: a registry that predates it, or a row uploaded before it # existed, yields null — which becomes an empty string, not an error. R_REL="$(printf '%s' "$manifest" | jq -r '.release_version // empty')" R_URL="$(printf '%s' "$manifest" | jq -r --arg a "$ARCH" '.artifacts[]|select(.arch==$a)|.download_url')" R_SHA="$(printf '%s' "$manifest" | jq -r --arg a "$ARCH" '.artifacts[]|select(.arch==$a)|.sha256')" R_SIZE="$(printf '%s' "$manifest" | jq -r --arg a "$ARCH" '.artifacts[]|select(.arch==$a)|.size')" fi [ -n "$R_URL" ] && [ "$R_URL" != "null" ] || die "no $ARCH build published for '$slug'" # A registry that does not declare sizes yields null; treat that as "unknown" # (0) rather than failing — the preflight then simply has less to check. case "$R_SIZE" in ''|null|*[!0-9]*) R_SIZE=0 ;; esac # The tag `docker load` gave these exact bytes on the run that fetched them: # publish.sh builds every image as :. Empty for a registry old # enough to publish no release version, which leaves the checks below on their # pre-release behaviour rather than guessing at a tag. R_TAG="" case "$R_REL" in ''|null) ;; *) R_TAG="$slug:$R_REL" ;; esac # Skipping requires that $target *is* this version — not merely that something # is tagged $target. `--rollback` points :latest back at the displaced image # and leaves versions.env naming the new one (it is written before the health # gate, so the rollback cannot un-write it). Testing existence alone made every # later run report "already at vX — skipping" for a version the host had rolled # off: it stayed on the old binary while versions.env and this script's own # closing banner both claimed the new one, /healthz was the only thing telling # the truth, and there was no way forward short of deleting versions.env. R_SKIP=false R_RETAG=false if [ -n "$cur" ] && [ "$cur" = "$R_VER" ] && docker image inspect "$target" >/dev/null 2>&1; then if [ -z "$R_TAG" ] || same_image "$target" "$R_TAG"; then R_SKIP=true elif docker image inspect "$R_TAG" >/dev/null 2>&1; then # The bytes are still on this host under their release tag — downloaded, # checksum- and signature-checked by the run that loaded them. Moving the # tag back is the whole recovery; re-fetching 750 MiB to arrive at the # image already sitting there is not. R_RETAG=true fi fi } LOADED_VERSION=""; LOADED_RELEASE="" # retag_target — point at , naming # whatever it displaced :previous first. # # Retagging moves :latest off the image currently running, which then survives # only as an untagged layer set until someone prunes it. Naming it :previous is # a tag, not a copy — so it costs nothing, and rollback becomes a retag plus a # compose recreate rather than a re-download. retag_target() { local target="$1" src="$2" prev_tag old_id new_id prev_tag="${target%:*}:previous" old_id="$(docker image inspect -f '{{.Id}}' "$target" 2>/dev/null || true)" new_id="$(docker image inspect -f '{{.Id}}' "$src" 2>/dev/null || true)" if [ -n "$old_id" ] && [ "$old_id" != "$new_id" ]; then docker tag "$old_id" "$prev_tag" log "Previous ${target%:*} kept as $prev_tag" fi docker tag "$src" "$target" } # download_image [current_release] # Uses the R_* values resolve_image already settled for this slug. download_image() { local slug="$1" target="$2" cur="$3" cur_rel="${4:-}" loaded got tar local url="$R_URL" sha="$R_SHA" ver="$R_VER" rel="$R_REL" if [ "$R_SKIP" = true ]; then log "$slug already at ${rel:-v$ver} — skipping" LOADED_VERSION="$ver" # Prefer what the registry says now: it fills in the release for an install # that was made before the field existed, without re-downloading anything. LOADED_RELEASE="${rel:-$cur_rel}" return fi if [ "$R_RETAG" = true ]; then log "$slug ${rel:-v$ver} is loaded but $target points elsewhere — restoring it" retag_target "$target" "$R_TAG" LOADED_VERSION="$ver"; LOADED_RELEASE="$rel" return fi tar="$TMP/$slug.tar" log "Downloading $slug ${rel:-v$ver} ($ARCH)..." curl -fL --progress-bar -o "$tar" "$url" || die "download failed for '$slug'" log "Verifying checksum..." got="$(sha256_of "$tar")" [ "$got" = "$sha" ] || die "checksum mismatch for '$slug' (expected $sha, got $got)" signed_digest_check "image $slug $ver $ARCH $got" "$slug ($ARCH)" log "Loading image..." # Not `| head -n1`: head exits after one line, sed dies of SIGPIPE, and # `set -o pipefail` turns that into a fatal exit 141 — mid-upgrade, right after # the image was loaded, with no message. sed reads all of docker load's output # here; the first line is taken with parameter expansion afterwards. loaded="$(docker load -i "$tar" | sed -n 's/^Loaded image: //p')" loaded="${loaded%%$'\n'*}" [ -n "$loaded" ] || die "docker load produced no image for '$slug'" retag_target "$target" "$loaded" rm -f "$tar" LOADED_VERSION="$ver"; LOADED_RELEASE="$rel" } # Resolve both images first, so the preflight sees the whole run. resolve_image devgrail-server "devgrail-server:latest" "$SERVER_VERSION" SERVER_SKIP="$R_SKIP"; SERVER_RETAG="$R_RETAG"; SERVER_BYTES="$R_SIZE" SERVER_R_VER="$R_VER"; SERVER_R_REL="$R_REL"; SERVER_R_URL="$R_URL"; SERVER_R_SHA="$R_SHA"; SERVER_R_TAG="$R_TAG" resolve_image devgrail-container "devgrail-container:latest" "$CONTAINER_VERSION" CONTAINER_SKIP="$R_SKIP"; CONTAINER_RETAG="$R_RETAG"; CONTAINER_BYTES="$R_SIZE" CONTAINER_R_VER="$R_VER"; CONTAINER_R_REL="$R_REL"; CONTAINER_R_URL="$R_URL"; CONTAINER_R_SHA="$R_SHA"; CONTAINER_R_TAG="$R_TAG" # --- disk preflight --------------------------------------------------------- # Every upgrade downloads a tarball and then needs the space again in the Docker # data root when `docker load` unpacks it, and nothing prunes. Disk-full is not # merely untidy here: it is the condition that produces a failure mid-migration, # which is the worst state this product can be in. So the numbers are checked # before the first byte is fetched, and reported when they do not add up. # # 2.5x the declared download size: 1x for the tarball, ~1x again for the # unpacked layers, plus half for the compressed layers Docker keeps while # loading and for headroom. A registry that declares no sizes yields 0, and the # check for that filesystem is skipped rather than guessed at. DOWNLOAD_BYTES=0 # A retag fetches nothing either — the bytes are already in the Docker data root. [ "$SERVER_SKIP" = true ] || [ "$SERVER_RETAG" = true ] || DOWNLOAD_BYTES=$((DOWNLOAD_BYTES + SERVER_BYTES)) [ "$CONTAINER_SKIP" = true ] || [ "$CONTAINER_RETAG" = true ] || DOWNLOAD_BYTES=$((DOWNLOAD_BYTES + CONTAINER_BYTES)) NEED_BYTES=$((DOWNLOAD_BYTES * 5 / 2)) # --- dry run ----------------------------------------------------------------- # "Re-run the same script that did the first install" is a fine upgrade # mechanism right up until the operator wants to know what it will do before it # does it. Everything above this line is resolution — reading the registry and # the prior install's own files — so the report can be produced with nothing # downloaded, nothing written and nothing started. # # It sits here rather than earlier because the release has to be resolved for # any of it to be true, and earlier than the next line because that is where the # first byte gets staged on disk. # dry_change — "a -> b", or "b (unchanged)" when they match. dry_change() { local from="${1:-}" to="${2:-}" [ -n "$to" ] || to="unknown" if [ -z "$from" ]; then printf '%s (new)' "$to" elif [ "$from" = "$to" ]; then printf '%s (unchanged)' "$to" else printf '%s -> %s' "$from" "$to"; fi } dry_image_line() { local slug="$1" cur="$2" ver="$3" rel="$4" skip="$5" bytes="$6" retag="${7:-false}" if [ "$skip" = true ]; then printf ' %-20s v%s (unchanged)\n' "$slug" "$ver" elif [ "$retag" = true ]; then # Not "(unchanged)": the version is the one recorded, but :latest is not on # it — which is exactly the state an operator needs told, since it is what # a rollback leaves behind and what this run would undo. printf ' %-20s v%s%s already loaded — would point %s:latest back at it\n' \ "$slug" "$ver" "${rel:+ $rel}" "$slug" else printf ' %-20s %s%s — would download %s\n' "$slug" \ "$(dry_change "${cur:+v$cur}" "v$ver")" "${rel:+ $rel}" "$(human "$bytes")" fi } dry_asset_line() { local slug="$1" ver have ver="$(release_field "$slug" '.assets[]|select(.slug==$s)|.asset_version')" have="$(asset_version_of "$slug")" if [ -z "$ver" ]; then printf ' %-24s would be re-fetched (the registry publishes no version for it)\n' "$slug" elif [ "$ver" = "$have" ] && [ -f "$ASSET_DIR/$slug" ]; then printf ' %-24s v%s (unchanged)\n' "$slug" "$ver" else printf ' %-24s %s — would download\n' "$slug" "$(dry_change "${have:+v$have}" "v$ver")" fi } dry_run_report() { local tmpdir="${TMPDIR:-/tmp}" docker_root key was now changed=false health schema echo echo "============================================================" echo " Dry run — nothing on this host has been changed." echo if [ "$IS_UPGRADE" = true ]; then echo " Mode: upgrade of the install at $INSTALL_DIR" else echo " Mode: first install into $INSTALL_DIR" fi echo " Registry: $WEB" echo " Release: $(dry_change "$PRIOR_RELEASE_VERSION" "${RELEASE_VERSION:-not published as a release}")" if [ -n "$SIGNED_PAYLOAD_FILE" ]; then echo " Signature: verified against a key embedded in this installer" else echo " Signature: not verified (see the warning above)" fi echo echo " Images" dry_image_line devgrail-server "$SERVER_VERSION" "$SERVER_R_VER" \ "$SERVER_R_REL" "$SERVER_SKIP" "$SERVER_BYTES" "$SERVER_RETAG" dry_image_line devgrail-container "$CONTAINER_VERSION" "$CONTAINER_R_VER" \ "$CONTAINER_R_REL" "$CONTAINER_SKIP" "$CONTAINER_BYTES" "$CONTAINER_RETAG" echo echo " Assets" # Only the two this run would actually fetch. install.sh and uninstall.sh are # published as assets too, but they are served to `curl`, never downloaded # here, and listing them would suggest otherwise. dry_asset_line docker-compose.yml dry_asset_line "$TRAEFIK_ASSET" echo echo " Disk" if [ "$NEED_BYTES" -gt 0 ]; then printf ' would need %s (2.5x the %s to download)\n' \ "$(human "$NEED_BYTES")" "$(human "$DOWNLOAD_BYTES")" else echo " nothing to download" fi printf ' free in %s: %s\n' "$tmpdir" "$(human "$(free_bytes "$tmpdir")")" docker_root="$(docker info --format '{{.DockerRootDir}}' 2>/dev/null || true)" [ -n "$docker_root" ] && [ -d "$docker_root" ] || docker_root=/var/lib/docker if [ -d "$docker_root" ]; then printf ' free in %s: %s\n' "$docker_root" "$(human "$(free_bytes "$docker_root")")" fi if [ "$IS_UPGRADE" = true ]; then echo echo " Settings" # DEVGRAIL_ADMIN_USERNAME is listed; the password never is. for key in DEVGRAIL_DOMAIN DEVGRAIL_BASE_DOMAIN ACME_EMAIL DEVGRAIL_ACME_CASERVER DEVGRAIL_ADMIN_USERNAME DEVGRAIL_TLS HTTP_PORT HTTPS_PORT; do was="$(prior "$key")" eval "now=\${$key:-}" if [ "$was" != "$now" ]; then printf ' %-22s %s -> %s\n' "$key" "${was:-(unset)}" "${now:-(unset)}" changed=true fi done if [ "$changed" = false ]; then echo " unchanged — every value in the prior .env would be preserved" fi echo echo " Data" echo " a snapshot of the database and config.yaml would be written to" echo " $BACKUP_DIR before the new server starts (newest ${DEVGRAIL_SNAPSHOT_RETAIN:-5} kept)" health="$(curl -fsSk --max-time 5 "$SCHEME://$DEVGRAIL_DOMAIN/api/v1/healthz" 2>/dev/null || true)" schema="$(printf '%s' "$health" | jq -r '.schema_version.applied // empty' 2>/dev/null || true)" if [ -n "$schema" ]; then echo " this database is at schema version $schema"; fi echo " any pending migrations are applied by the new server when it starts;" echo " which ones cannot be listed without loading that image, and a dry" echo " run does not download it" fi echo echo " Re-run without --dry-run to apply this." echo "============================================================" echo return 0 } if [ "$DRY_RUN" = true ]; then dry_run_report exit 0 fi # Where the tarballs are staged. /tmp is a tmpfs on many VPS images — sized from # RAM and often well under a gigabyte — so falling back to $INSTALL_DIR/tmp when # it does not fit turns "no space left on device" partway through a download # into a directory that has the room. choose_tmp() { local candidate="${TMPDIR:-/tmp}" fallback="$INSTALL_DIR/tmp" have have_fb have="$(free_bytes "$candidate")" # 0 means "could not measure", which is not the same as "no room". Nothing # here is worth refusing an upgrade over on a guess. if [ "$NEED_BYTES" -gt 0 ] && [ "$have" -gt 0 ] && [ "$have" -lt "$NEED_BYTES" ]; then mkdir -p "$fallback" || die "cannot create $fallback for the download" have_fb="$(free_bytes "$fallback")" if [ "$have_fb" -gt 0 ] && [ "$have_fb" -lt "$NEED_BYTES" ]; then die "not enough free space to download this release. Needed: $(human "$NEED_BYTES") (2.5x the $(human "$DOWNLOAD_BYTES") of artifacts) Free in $candidate: $(human "$have") Free in $fallback: $(human "$have_fb") Free some space and re-run. Nothing has been changed." fi log "$candidate is too small for this download — staging in $fallback instead" TMP="$(mktemp -d "$fallback/install.XXXXXX")" else TMP="$(mktemp -d)" fi } choose_tmp if [ "$NEED_BYTES" -gt 0 ]; then DOCKER_ROOT="$(docker info --format '{{.DockerRootDir}}' 2>/dev/null || true)" [ -n "$DOCKER_ROOT" ] && [ -d "$DOCKER_ROOT" ] || DOCKER_ROOT=/var/lib/docker if [ -d "$DOCKER_ROOT" ]; then DOCKER_FREE="$(free_bytes "$DOCKER_ROOT")" if [ "$DOCKER_FREE" -gt 0 ] && [ "$DOCKER_FREE" -lt "$NEED_BYTES" ]; then die "not enough free space in the Docker data root to load this release. Needed: $(human "$NEED_BYTES") (2.5x the $(human "$DOWNLOAD_BYTES") of artifacts) Free in $DOCKER_ROOT: $(human "$DOCKER_FREE") Reclaim space (\`docker image prune\`, \`docker system df\`) and re-run. Nothing has been changed." fi fi fi R_SKIP="$SERVER_SKIP"; R_RETAG="$SERVER_RETAG"; R_TAG="$SERVER_R_TAG" R_VER="$SERVER_R_VER"; R_REL="$SERVER_R_REL"; R_URL="$SERVER_R_URL"; R_SHA="$SERVER_R_SHA" download_image devgrail-server "devgrail-server:latest" "$SERVER_VERSION" "$SERVER_RELEASE" NEW_SERVER_VERSION="$LOADED_VERSION"; NEW_SERVER_RELEASE="$LOADED_RELEASE" # The workspace image tag is hard-coded in internal/docker (DefaultImage = # devgrail-container:latest), so retag the loaded release to :latest. R_SKIP="$CONTAINER_SKIP"; R_RETAG="$CONTAINER_RETAG"; R_TAG="$CONTAINER_R_TAG" R_VER="$CONTAINER_R_VER"; R_REL="$CONTAINER_R_REL"; R_URL="$CONTAINER_R_URL"; R_SHA="$CONTAINER_R_SHA" download_image devgrail-container "devgrail-container:latest" "$CONTAINER_VERSION" "$CONTAINER_RELEASE" NEW_CONTAINER_VERSION="$LOADED_VERSION"; NEW_CONTAINER_RELEASE="$LOADED_RELEASE" # --- deploy assets --------------------------------------------------------- log "Writing deploy config to $INSTALL_DIR..." mkdir -p "$DEPLOY_DIR/traefik" "$DEPLOY_DIR/dynamic" "$CONFIG_DIR" "$ASSET_DIR" NEW_ASSET_VERSIONS="" # fetch_asset — leave the release's copy of at $ASSET_PATH, # downloading and verifying it only if this run needs a version we do not # already hold. ASSET_PATH="" fetch_asset() { local slug="$1" url ver sha cached got have cached="$ASSET_DIR/$slug" ver="$(release_field "$slug" '.assets[]|select(.slug==$s)|.asset_version')" url="$(release_field "$slug" '.assets[]|select(.slug==$s)|.download_url')" sha="$(release_field "$slug" '.assets[]|select(.slug==$s)|.sha256')" if [ -z "$url" ]; then # No release manifest (or an asset it does not carry): fetch whatever is # published, as installers did before releases existed. Unversioned, so it # cannot be skipped — that is the behaviour releases exist to fix. url="$WEB/install/$slug"; ver=""; sha="" fi have="$(asset_version_of "$slug")" if [ -n "$ver" ] && [ "$ver" = "$have" ] && [ -f "$cached" ]; then log "$slug already at v$ver — skipping" else curl -fsSL "$url" -o "$cached.tmp" || die "cannot fetch '$slug' from $WEB" got="$(sha256_of "$cached.tmp")" if [ -n "$sha" ]; then [ "$got" = "$sha" ] || die "checksum mismatch for '$slug' (expected $sha, got $got)" fi # Verified before the templating below rewrites it: what the release was # signed for is the published file, not what this host makes of it. if [ -n "$ver" ]; then signed_digest_check "asset $slug $ver $got" "$slug" elif [ -n "$SIGNED_PAYLOAD_FILE" ]; then # The release verified, but does not name this asset — so it came from the # unversioned fallback path and nothing signed it. Not fatal (that path # exists for registries older than releases), but it is a hole in an # otherwise verified install and has to be said out loud. warn "'$slug' is not part of signed release $RELEASE_VERSION — fetched unverified." fi mv -f "$cached.tmp" "$cached" fi if [ -n "$ver" ]; then NEW_ASSET_VERSIONS="${NEW_ASSET_VERSIONS:+$NEW_ASSET_VERSIONS }$slug=$ver" fi ASSET_PATH="$cached" } # The rendered outputs are always rewritten from the cached copies, even when # nothing was downloaded: DEVGRAIL_DOMAIN or DEVGRAIL_TLS may have changed at # the prompt, and a config that silently kept a stale domain would be worse than # a redundant write. fetch_asset docker-compose.yml cp -f "$ASSET_PATH" "$DEPLOY_DIR/docker-compose.yml" # This script is always served at latest, but --version can install an older # release whose compose file predates DEVGRAIL_ADMIN_USERNAME. That file would # simply drop the variable, the server would create `admin`, and the banner below # would name a user that does not exist. Say so and fall back instead. if [ "$ADMIN_USER" != admin ] && ! grep -q DEVGRAIL_ADMIN_USERNAME "$DEPLOY_DIR/docker-compose.yml"; then warn "release ${NEW_SERVER_RELEASE:-this one} cannot take a custom admin username; creating \`admin\` instead." ADMIN_USER="admin" fi # The TLS or plain-HTTP router set per DEVGRAIL_TLS. Either is written to the # same dynamic.yml the compose file mounts by default, so no compose var is needed. fetch_asset "$TRAEFIK_ASSET" sed "s|__DEVGRAIL_DOMAIN__|$DEVGRAIL_DOMAIN|g" "$ASSET_PATH" > "$DEPLOY_DIR/traefik/dynamic.yml" # config.yaml holds the JWT secret (not passed via env). It used to be written # only when absent, so registry_url/tls_enabled went stale as soon as the # operator changed them at the prompt. The .env does override both at runtime # (internal/config/config.go), so that was latent rather than active — but two # files disagreeing about the domain is a trap to walk into during an incident. # The derived keys are rewritten every run; the secret never is. CONFIG_FILE="$CONFIG_DIR/config.yaml" if [ -f "$CONFIG_FILE" ]; then # Same SIGPIPE-under-pipefail hazard as the docker load above. Aborting here # would be particularly bad: the operator would see a bare exit 141 while the # installer was reading the secret that decrypts their entire database. JWT_SECRET="$(sed -n 's/^jwt_secret:[[:space:]]*//p' "$CONFIG_FILE")" JWT_SECRET="$(printf '%s' "${JWT_SECRET%%$'\n'*}" | tr -d "\"' ")" # Generating a fresh secret here would orphan every encrypted value in the # database — every API key secret, user secret and workspace SSH password # (SECURITY_ANALYSIS.md #4). Refusing loudly is the only safe answer. [ -n "$JWT_SECRET" ] || die "$CONFIG_FILE exists but has no jwt_secret. That secret keys every encrypted value in the database; generating a new one would make all of them undecryptable. Restore the file from a backup (/opt/devgrail/backups holds config.yaml alongside the database) and re-run. Nothing has been changed." else JWT_SECRET="$(gen_secret)" fi # Anything the operator added by hand is carried over — this installer owns the # derived keys, not the whole file. CONFIG_EXTRA="" if [ -f "$CONFIG_FILE" ]; then CONFIG_EXTRA="$(grep -vE '^(jwt_secret|domain|base_domain|registry_url|traefik_dynamic_path|tls_enabled):' "$CONFIG_FILE" || true)" fi cat > "$CONFIG_FILE.tmp" <> "$CONFIG_FILE.tmp"; fi # Same reasoning as .env, more so: a truncated config.yaml loses the JWT secret, # which is unrecoverable without a backup. chmod 600 "$CONFIG_FILE.tmp" mv -f "$CONFIG_FILE.tmp" "$CONFIG_FILE" # ADMIN_USER / ADMIN_PW were settled with the other answers above, prior values # included: they seed the admin account on first boot only (internal/db/seed.go: # SeedAdmin ignores them thereafter unless DEVGRAIL_FORCE_ADMIN_PASSWORD is set), # so dropping the password here would leave the operator with an account whose # password nobody knows. # Keys this installer owns. Everything else found in the prior .env is carried # over verbatim below — an operator who added DEVGRAIL_FORCE_ADMIN_PASSWORD or # any other compose variable must not lose it to an upgrade. MANAGED_ENV_KEYS=" ACME_EMAIL DEVGRAIL_ACME_CASERVER DEVGRAIL_ACME_STORAGE DEVGRAIL_DOMAIN DEVGRAIL_BASE_DOMAIN DEVGRAIL_ADMIN_USERNAME DEVGRAIL_ADMIN_PASSWORD DEVGRAIL_SERVER_IMAGE DEVGRAIL_TLS TAG HTTP_PORT HTTPS_PORT " cat > "$DEPLOY_DIR/.env.tmp" <> "$DEPLOY_DIR/.env.tmp" || true done # Written via a temp file: this holds the admin password, and a truncated write # on a full disk would hand the operator a stack that cannot be logged into. chmod 600 "$DEPLOY_DIR/.env.tmp" mv -f "$DEPLOY_DIR/.env.tmp" "$DEPLOY_DIR/.env" # The integers are the registry's ordering key (what the skip-if-unchanged check # compares); the releases are the human-facing identity reported in the banner. # ASSET_VERSIONS is quoted because it holds a space-separated list, and this file # is sourced by the next run. cat > "$VERSIONS_FILE" </dev/null || echo 0' 2>/dev/null)" || db_bytes=0 case "$db_bytes" in ''|*[!0-9]*) db_bytes=0 ;; esac if [ "$db_bytes" -eq 0 ]; then log "Data volume '$vol' holds no database yet — nothing to snapshot." return 0 fi mkdir -p "$BACKUP_DIR" chmod 700 "$BACKUP_DIR" # Room is needed twice: once for the vacuumed copy, once for the tarball built # from it. Refusing here is not just hygiene — a full disk is precisely the # condition that causes the mid-migration failure this snapshot exists for. need=$((db_bytes * 2)) free_bytes="$(df -Pk "$BACKUP_DIR" | awk 'NR==2 {print $4 * 1024}')" || free_bytes=0 if [ "${free_bytes:-0}" -lt "$need" ]; then die "not enough free space for the pre-upgrade snapshot in $BACKUP_DIR: need $need bytes, have ${free_bytes:-0}. Free some space, or move $INSTALL_DIR to a larger filesystem, then re-run. Nothing has been changed." fi ts="$(date -u +%Y%m%dT%H%M%SZ)" stage="$BACKUP_DIR/.stage-$ts" rm -rf "$stage" mkdir -p "$stage" SNAPSHOT_STAGE="$stage" log "Taking a pre-upgrade snapshot (data volume: $vol)..." # Stop only the server; Traefik stays up so the host keeps answering. compose stop devgrail-server >/dev/null 2>&1 || true # VACUUM INTO, never `cp`: in WAL mode the newest committed data lives in the # -wal sidecar, so copying devgrail.db alone yields a corrupt backup. It runs # in a one-shot container off the image just loaded, so the host needs no # sqlite3; /etc/devgrail is mounted so it reads the deployment's real db_path. # --userns=host mirrors the compose services, which opt out of userns-remap. summary="$(docker run --rm --userns=host \ -v "$vol":/var/lib/devgrail \ -v "$stage":/backup \ -v "$CONFIG_DIR":/etc/devgrail:ro \ -e DEVGRAIL_CONFIG=/etc/devgrail/config.yaml \ --entrypoint /usr/local/bin/devgrail-server \ devgrail-server:latest -vacuum-into /backup/devgrail.db)" \ || die "pre-upgrade snapshot failed — refusing to start the new version. Your data is untouched and the previous version is still installed; re-run this installer once the cause is fixed." schema="$(printf '%s' "$summary" | jq -r '.schema_version // "unknown"')" cp -p "$CONFIG_DIR/config.yaml" "$stage/config.yaml" \ || die "pre-upgrade snapshot: could not copy $CONFIG_DIR/config.yaml. A database-only backup cannot decrypt anything it contains, so this is not a usable snapshot — refusing to continue." cat > "$stage/manifest.json" </dev/null | tail -n +$((SNAPSHOT_RETAIN + 1)) \ | while read -r old; do rm -f "$old"; done || true } if [ "$IS_UPGRADE" = true ]; then snapshot_before_upgrade fi # --- devgrail maintenance command ------------------------------------------- # The pre-upgrade snapshot only exists when an upgrade happens to run. An # operator also needs a backup they can take now — before editing something, or # on a schedule — and a restore that does the steps in the one order that works. # Both live in the server binary (`-backup` / `-restore`); this wrapper is what # runs them as a one-shot container off the installed image, so the host needs # neither sqlite3 nor a copy of the Go toolchain. # # Written from here rather than shipped as an eighth release artifact: it is # entirely derived from paths this installer already owns, so it can never # disagree with the install it was written for. CLI_PATH=/usr/local/bin/devgrail write_devgrail_cli() { cat > "$CLI_PATH.tmp" <\033[0m %s\n' "\$*"; } die() { printf '\033[1;31merror:\033[0m %s\n' "\$*" >&2; exit 1; } # Mirrors install.sh: an operator's docker-compose.override.yml applies to the # stop/start this does around a restore, or the restored server comes back # without it. compose() { if [ -f "\$DEPLOY_DIR/docker-compose.override.yml" ]; then docker compose --project-directory "\$DEPLOY_DIR" \\ -f "\$DEPLOY_DIR/docker-compose.yml" \\ -f "\$DEPLOY_DIR/docker-compose.override.yml" "\$@" else docker compose --project-directory "\$DEPLOY_DIR" -f "\$DEPLOY_DIR/docker-compose.yml" "\$@" fi } usage() { cat <<'USAGE' DevGrail maintenance commands. Run as root. devgrail backup [--out PATH] write a backup archive (database + config.yaml + manifest). Default: the backups directory, named backup-.tgz devgrail restore ARCHIVE stop the server, restore the archive, start it again, and wait for the readiness probe devgrail list list the archives on this host devgrail passwd [USERNAME] set a user's password (default: the admin this install created) and revoke their sessions. The way back in after a forgotten password. devgrail help show this A backup archive holds BOTH the database and config.yaml. The jwt_secret in config.yaml keys every encrypted value in the database, so a database-only backup restores rows that decrypt to garbage. Never separate them. Workspace volumes are NOT included — they hold the customer's source code and are far larger than the database. See docs/RUNBOOK.md for the export recipe. USAGE } # data_volume prints the Docker volume backing /var/lib/devgrail, resolved from # the running container rather than guessed from the Compose project name. data_volume() { local vol vol="\$(docker inspect devgrail-server \\ --format '{{range .Mounts}}{{if eq .Destination "/var/lib/devgrail"}}{{.Name}}{{end}}{{end}}' 2>/dev/null)" || vol="" if [ -n "\$vol" ]; then printf '%s' "\$vol"; return 0; fi docker volume ls --format '{{.Name}}' 2>/dev/null | grep -E '(^|_)devgrail-data\$' | head -n1 || true } # cmd_backup writes one archive via a one-shot container off the installed # server image, with the data volume, /etc/devgrail and the output directory # mounted. --userns=host mirrors the compose services, which opt out of # userns-remap. cmd_backup() { local out="" ts vol dir while [ \$# -gt 0 ]; do case "\$1" in --out=*) out="\${1#*=}" ;; --out) shift; out="\${1:-}" ;; *) usage >&2; die "unknown option: \$1" ;; esac shift done ts="\$(date -u +%Y%m%dT%H%M%SZ)" [ -n "\$out" ] || out="\$BACKUP_DIR/backup-\$ts.tgz" dir="\$(cd "\$(dirname "\$out")" 2>/dev/null && pwd)" || die "no such directory: \$(dirname "\$out")" mkdir -p "\$BACKUP_DIR"; chmod 700 "\$BACKUP_DIR" vol="\$(data_volume)" [ -n "\$vol" ] || die "cannot find the DevGrail data volume — is DevGrail installed here?" # Taken against a running server on purpose: VACUUM INTO reads through a single # read transaction, so the copy is consistent without downtime. The volume is # mounted read-write even though nothing here modifies the data: opening a # WAL-mode SQLite database requires writing its -shm sidecar, so a read-only # mount fails outright with "attempt to write a readonly database". docker run --rm --userns=host \\ -v "\$vol":/var/lib/devgrail \\ -v "\$CONFIG_DIR":/etc/devgrail:ro \\ -v "\$dir":/backup \\ -e DEVGRAIL_CONFIG=/etc/devgrail/config.yaml \\ --entrypoint /usr/local/bin/devgrail-server \\ "\$IMAGE" -backup "/backup/\$(basename "\$out")" \\ || die "backup failed — nothing was written" chmod 600 "\$out" 2>/dev/null || true log "Backup written to \$out" } cmd_restore() { local archive="\${1:-}" vol dir [ -n "\$archive" ] || { usage >&2; die "restore needs an archive path"; } [ -f "\$archive" ] || die "no such archive: \$archive" dir="\$(cd "\$(dirname "\$archive")" && pwd)" vol="\$(data_volume)" [ -n "\$vol" ] || die "cannot find the DevGrail data volume — refusing to guess which volume to overwrite." # The documented order, enforced: stop -> config.yaml -> database -> start -> # verify. Restoring under a running server would swap the database out from # under open connections. log "Stopping devgrail-server..." compose stop devgrail-server >/dev/null 2>&1 || true docker run --rm --userns=host \\ -v "\$vol":/var/lib/devgrail \\ -v "\$CONFIG_DIR":/etc/devgrail \\ -v "\$dir":/restore:ro \\ -e DEVGRAIL_CONFIG=/etc/devgrail/config.yaml \\ --entrypoint /usr/local/bin/devgrail-server \\ "\$IMAGE" -restore "/restore/\$(basename "\$archive")" \\ || { compose up -d >/dev/null 2>&1 || true; die "restore failed — the previous state is unchanged and the server has been restarted"; } log "Starting devgrail-server..." compose up -d local waited=0 while [ "\$waited" -lt 180 ]; do if compose exec -T devgrail-server /usr/local/bin/devgrail-server -healthcheck >/dev/null 2>&1; then log "DevGrail is ready. Restore complete." return 0 fi sleep 3 waited=\$((waited + 3)) done compose logs --tail=50 devgrail-server >&2 || true die "the restored server did not become ready within 180s (logs above)" } # cmd_passwd resets a password without a login — the way back in when nobody # knows the admin's. It runs the same one-shot container cmd_backup does; the new # password goes in on stdin, never as an argument or an environment variable, # because both are readable via \`ps\` and \`docker inspect\` while the process runs. # # No stop/start: SQLite is in WAL mode, so the write lands under the running # server, and the sessions it revokes are re-checked on the next request. cmd_passwd() { local user="\${1:-}" pw1 pw2 vol if [ -z "\$user" ]; then user="\$(sed -n 's/^DEVGRAIL_ADMIN_USERNAME=//p' "\$DEPLOY_DIR/.env" 2>/dev/null | tail -n1)" user="\${user:-admin}" fi [ -t 0 ] || die "passwd needs a terminal to read the new password from." printf 'New password for %s: ' "\$user" > /dev/tty stty -echo 2>/dev/null || true read -r pw1 < /dev/tty || true stty echo 2>/dev/null || true printf '\n' > /dev/tty printf 'Repeat password: ' > /dev/tty stty -echo 2>/dev/null || true read -r pw2 < /dev/tty || true stty echo 2>/dev/null || true printf '\n' > /dev/tty [ -n "\$pw1" ] || die "no password entered" [ "\$pw1" = "\$pw2" ] || die "the passwords do not match" vol="\$(data_volume)" [ -n "\$vol" ] || die "cannot find the DevGrail data volume — is DevGrail installed here?" printf '%s' "\$pw1" | docker run --rm -i --userns=host \\ -v "\$vol":/var/lib/devgrail \\ -v "\$CONFIG_DIR":/etc/devgrail:ro \\ -e DEVGRAIL_CONFIG=/etc/devgrail/config.yaml \\ --entrypoint /usr/local/bin/devgrail-server \\ "\$IMAGE" -set-password "\$user" \\ || die "password not changed" } case "\${1:-help}" in backup) shift; cmd_backup "\$@" ;; restore) shift; cmd_restore "\$@" ;; passwd) shift; cmd_passwd "\$@" ;; list) ls -lh "\$BACKUP_DIR" 2>/dev/null || echo "no archives in \$BACKUP_DIR" ;; help|-h|--help) usage ;; *) usage >&2; die "unknown command: \$1" ;; esac CLIEOF chmod 755 "$CLI_PATH.tmp" mv -f "$CLI_PATH.tmp" "$CLI_PATH" } if write_devgrail_cli; then log "Maintenance CLI installed: $CLI_PATH (backup / restore / list)" else warn "could not write $CLI_PATH — backup and restore will need the docker run commands from docs/RUNBOOK.md" fi # --- scheduled backups ------------------------------------------------------- # Offered, never imposed: it writes to the customer's disk on a timer, and an # operator who already has their own backup story should not get a second one # they did not ask for. DEVGRAIL_SCHEDULED_BACKUPS=yes|no answers it unattended; # an existing timer is left exactly as configured. install_backup_timer() { command -v systemctl >/dev/null 2>&1 || return 1 cat > /etc/systemd/system/devgrail-backup.service < /etc/systemd/system/devgrail-backup.timer <&- for the same reason as the Docker install above: whatever these start # must not inherit the install lock and outlive this run holding it. systemctl daemon-reload >/dev/null 2>&1 9>&- || true systemctl enable --now devgrail-backup.timer >/dev/null 2>&1 9>&- || return 1 return 0 } BACKUP_TIMER_NOTE="" if [ -f "$CLI_PATH" ] && command -v systemctl >/dev/null 2>&1; then if systemctl list-unit-files devgrail-backup.timer >/dev/null 2>&1 && systemctl is-enabled devgrail-backup.timer >/dev/null 2>&1; then BACKUP_TIMER_NOTE="Weekly backups: already scheduled (systemctl status devgrail-backup.timer)" else WANT_TIMER="$(printf '%s' "${DEVGRAIL_SCHEDULED_BACKUPS:-}" | tr '[:upper:]' '[:lower:]')" # --yes takes the prompt's own default, which is yes: a machine nobody is # watching is exactly the one that benefits from a scheduled backup. # DEVGRAIL_SCHEDULED_BACKUPS still wins, so it stays possible to say no. if [ -z "$WANT_TIMER" ] && [ "$ASSUME_YES" = true ]; then WANT_TIMER=yes; fi if [ -z "$WANT_TIMER" ] && [ -e /dev/tty ]; then printf 'Schedule a weekly backup to %s? [Y/n]: ' "$BACKUP_DIR" > /dev/tty read -r _answer < /dev/tty || _answer="" case "$_answer" in n|N|no|NO) WANT_TIMER=no ;; *) WANT_TIMER=yes ;; esac fi case "$WANT_TIMER" in yes|y|true|1) if install_backup_timer; then BACKUP_TIMER_NOTE="Weekly backups: enabled (systemctl status devgrail-backup.timer)" else warn "could not enable the weekly backup timer — run '$CLI_PATH backup' yourself, or from your own scheduler" fi ;; *) BACKUP_TIMER_NOTE="Weekly backups: not scheduled. Enable later with: DEVGRAIL_SCHEDULED_BACKUPS=yes re-run, or run '$CLI_PATH backup' from your own scheduler." ;; esac fi fi # --- bring up -------------------------------------------------------------- docker network inspect devgrail >/dev/null 2>&1 || docker network create devgrail >/dev/null log "Starting DevGrail..." # Via compose(), so a docker-compose.override.yml is honoured here too — not # just by the stop/logs calls above. An override that only applied to some of # the installer's compose invocations would be worse than none. compose up -d # --- wait for readiness ----------------------------------------------------- # probe_health / wait_for_health are defined near the top, because --rollback # needs them too. if ! wait_for_health; then echo >&2 printf '\033[1;31merror:\033[0m %s\n' "DevGrail did not become ready within ${HEALTH_TIMEOUT}s." >&2 echo " Last 50 lines from devgrail-server:" >&2 compose logs --tail=50 devgrail-server >&2 || true echo >&2 echo " Health detail:" >&2 curl -sSk --max-time 5 "$SCHEME://$DEVGRAIL_DOMAIN/api/v1/healthz" >&2 || true echo >&2 # Automatic rollback, on an upgrade only. A first install has nothing to roll # back to: there are no :previous images and no snapshot, and "rolling back" to # an empty host would replace a diagnosable failure with a confusing one. ROLLED_BACK=false if [ "$IS_UPGRADE" = true ]; then # --yes deliberately does not imply yes here. Rolling back is a decision # about which of two states the host ends in, not a question in the way of # an unattended run; --auto-rollback is how that one is answered up front. DO_IT="$AUTO_ROLLBACK" if [ "$DO_IT" != true ] && [ "$ASSUME_YES" != true ] && [ -e /dev/tty ]; then printf 'Roll back to the previous version and snapshot? [y/N]: ' > /dev/tty read -r _answer < /dev/tty || _answer="" case "$_answer" in y|Y|yes|YES) DO_IT=true ;; esac fi if [ "$DO_IT" = true ]; then echo >&2 if do_rollback; then ROLLED_BACK=true else printf '\033[1;31merror:\033[0m %s\n' "The rolled-back stack did not become ready either." >&2 fi fi fi # Say plainly which state the host ended in. After a failed upgrade that is the # only thing the operator needs to know before deciding what to do next. if [ "$ROLLED_BACK" = true ]; then cat >&2 <&2 echo " $SNAPSHOT_ARCHIVE" >&2 echo " Nothing was deleted, but the new server may already have migrated the" >&2 echo " database — that archive is the way back to the previous state." >&2 echo " To go back: curl -fsSL $WEB/install.sh | sudo bash -s -- --rollback" >&2 fi echo " Logs: docker compose --project-directory $DEPLOY_DIR logs -f" >&2 exit 1 fi # --- reclaim superseded images ---------------------------------------------- # Only now, and never earlier: :previous is the rollback target, and until the # new stack has answered its readiness probe it is the thing standing between a # bad upgrade and a broken host. Once health is green, every DevGrail image older # than :previous is dead weight — an upgrade loads a new one each time and # nothing else removes them, so a host that upgrades for a year accumulates a # year of unreferenced layers on the disk whose exhaustion causes the failure # this whole sequence exists to survive. reclaim_old_images() { local slug keep_latest keep_prev id removed=0 for slug in devgrail-server devgrail-container; do keep_latest="$(docker image inspect -f '{{.Id}}' "$slug:latest" 2>/dev/null || true)" keep_prev="$(docker image inspect -f '{{.Id}}' "$slug:previous" 2>/dev/null || true)" [ -n "$keep_latest" ] || continue # Every image ID this repository still has, tagged or dangling. Untagged # ones are the displaced builds: `docker load` + retag leaves them with no # reference at all, which is exactly why nothing ever cleaned them up. for id in $(docker images --filter "reference=$slug" --format '{{.ID}}' 2>/dev/null | sort -u); do id="$(docker image inspect -f '{{.Id}}' "$id" 2>/dev/null || true)" [ -n "$id" ] || continue [ "$id" = "$keep_latest" ] && continue [ -n "$keep_prev" ] && [ "$id" = "$keep_prev" ] && continue # Not forced: an image a container is still using must stay, and that is # a refusal to respect rather than override. A workspace created before an # upgrade is running the old devgrail-container image until it is # recreated — removing it out from under the container is not reclamation, # it is breakage. if docker image rm "$id" >/dev/null 2>&1; then removed=$((removed + 1)) fi done done [ "$removed" -gt 0 ] && log "Reclaimed $removed superseded image(s); :latest and :previous kept." return 0 } reclaim_old_images || true if [ "$ADMIN_PW_GENERATED" = true ]; then LOGIN_NOTE="$ADMIN_USER / $ADMIN_PW" else LOGIN_NOTE="$ADMIN_USER / (the password you set)" fi if [ "$TLS_ENABLED" = true ] && [ "$ACME_CA_IS_PRODUCTION" = false ]; then TLS_NOTE="Certificates come from a NON-PRODUCTION ACME CA ($DEVGRAIL_ACME_CASERVER), so browsers will show this site as untrusted — that is expected, and this install is for testing. Re-run with --acme-ca=production for real certificates. Ensure DNS is pointed here:" elif [ "$TLS_ENABLED" = true ]; then TLS_NOTE="It may take a minute for Let's Encrypt to issue the TLS certificate on first start. Ensure DNS is pointed here:" else TLS_NOTE="Serving plain HTTP (DEVGRAIL_TLS=off) — do not expose this directly to the public internet; run it on a private network or behind a TLS terminator. Ensure DNS is pointed here:" fi cat < this VPS A *.$DEVGRAIL_BASE_DOMAIN -> this VPS Logs: docker compose --project-directory $DEPLOY_DIR logs -f Upgrade: re-run this installer. Customise: docker-compose.yml and traefik/dynamic.yml under $DEPLOY_DIR are product-owned and rewritten on every run. Put local changes in $DEPLOY_DIR/docker-compose.override.yml, which is never touched. Pin: | sudo bash -s -- --version=vX.Y.Z Roll back: | sudo bash -s -- --rollback Backup: sudo devgrail backup (restore: sudo devgrail restore ) Password: change it under Settings in the dashboard, or with sudo devgrail passwd ${BACKUP_TIMER_NOTE:-} ============================================================ EOF if [ -n "$SNAPSHOT_ARCHIVE" ]; then cat <