threadcount-community/scripts/deploy.sh

#!/bin/bash
# Deploy ThreadCount: update the code and nothing else.
#
# This exists because /root/deploy-setup.sh is a PROVISIONING script — it rewrites the nginx site
# and re-points /opt/threadcount/.env at /etc/threadcount/secrets.env. Running it as a deploy on
# 2026-09-06 destroyed the live .env and dropped nginx's TLS block. Use this instead.
#
#   sudo /usr/local/sbin/threadcount-deploy.sh
set -euo pipefail

APP=/opt/threadcount
SEC=/etc/threadcount/secrets.env
USER=threadcount
# Hardlink copies of the build and the dependencies that were serving the linen room when this
# deploy started. They live inside $APP because hardlinks cannot cross filesystems, and a deploy
# that ends healthy deletes them. The leading dot is not decoration: tsconfig.json includes
# **/*.ts and excludes only the top-level node_modules, and TypeScript skips dot-directories when
# it expands a wildcard — a snapshot called anything else would drag a second copy of every .ts
# file in node_modules into the build's type check.
PREV=$APP/.deploy-prev
# Written last, removed last. Its presence is what makes $PREV a way back: the hardlink copies
# below take long enough to be interrupted halfway, and half a build put back over a deployment
# that still works is the one outcome worse than no rollback at all.
MARKER=$PREV/.complete
DUMP=""

log() { echo "==> $*"; }

# The secrets file is the one thing nothing here may touch, and the one thing everything needs.
[ -s "$SEC" ] || { echo "FATAL: $SEC is missing or empty — refusing to deploy."; exit 1; }
grep -q '^DATABASE_URL=' "$SEC" || { echo "FATAL: no DATABASE_URL in $SEC — refusing to deploy."; exit 1; }

# Turnstile, checked here for the same reason SESSION_SECRET is checked below: instrumentation.ts
# refuses to start production without both, so leaving them out turns a five-second grep into a
# build, a migration and a restart that ends in a failed health check with the app already down.
# NEXT_PUBLIC_TURNSTILE_SITEKEY in particular is compiled into the bundle, so it has to be present
# before the build rather than before the restart.
grep -q '^TURNSTILE_SECRET=' "$SEC" || { echo "FATAL: no TURNSTILE_SECRET in $SEC — the app refuses to start without it."; exit 1; }
grep -q '^NEXT_PUBLIC_TURNSTILE_SITEKEY=' "$SEC" || { echo "FATAL: no NEXT_PUBLIC_TURNSTILE_SITEKEY in $SEC — it is compiled into the build."; exit 1; }

# A missing or placeholder SESSION_SECRET is the failure this script used to sail straight past.
# `next build` needs no secrets, so the build succeeds; proxy.ts then treats every session cookie
# as invalid and redirects the whole app to /auth, and every sign-in throws out of lib/session.ts
# as an opaque 500. Nothing about that shows up in an HTTP status, so it is checked here instead.
# The values are read in a subshell: nothing after this line should carry the database password
# in its environment.
( set -a; . "$SEC"; set +a
  [ -n "${SESSION_SECRET:-}" ] || { echo "FATAL: no SESSION_SECRET in $SEC — nobody could sign in."; exit 1; }
  [ "$SESSION_SECRET" != "change-me" ] || { echo "FATAL: SESSION_SECRET is still the .env.example placeholder."; exit 1; }
  [ "${#SESSION_SECRET}" -ge 32 ] || { echo "FATAL: SESSION_SECRET is under 32 characters — generate a real one."; exit 1; }
  [ -n "${DATABASE_URL:-}" ] || { echo "FATAL: DATABASE_URL in $SEC is empty."; exit 1; }
  # Present but empty is the same as absent to the application, and the greps above only prove the
  # line exists.
  [ -n "${TURNSTILE_SECRET:-}" ] || { echo "FATAL: TURNSTILE_SECRET in $SEC is empty."; exit 1; }
  [ -n "${NEXT_PUBLIC_TURNSTILE_SITEKEY:-}" ] || { echo "FATAL: NEXT_PUBLIC_TURNSTILE_SITEKEY in $SEC is empty."; exit 1; }
  # The documented escape hatch for a local `next start`. On the production box it would switch bot
  # protection off across every unauthenticated form, silently.
  [ -z "${TURNSTILE_OPTIONAL:-}" ] || { echo "FATAL: TURNSTILE_OPTIONAL is set in $SEC — that is for local smoke tests only."; exit 1; }
) || exit 1

# The probe asks /api/health, which runs a SELECT 1 through Prisma and answers 503 when it cannot.
# The marketing home page was useless for this: it is prerendered at build time and answers 200 out
# of .next even when the server behind it is dead, so a deploy with an unreachable database still
# reported healthy. /app was better — it goes through proxy.ts inside the running server — but its
# redirect to /auth is produced without ever touching the database, so the one failure that takes
# the whole product down is still the one it cannot see. Asking the database is the only probe that
# distinguishes "serving" from "working". The rollback path below uses it too, so it is a function:
# a rollback nobody probed is only a guess that the old build came back.
code=""
body=""
healthy() {
  code=""
  for _ in $(seq 1 20); do
    sleep 1
    body=$(curl -s --max-time 5 -w '\n%{http_code}' http://127.0.0.1:3000/api/health || true)
    code=$(printf '%s' "$body" | tail -1)
    [ "$code" = "200" ] && return 0
  done
  return 1
}

# True only of a snapshot that finished being written. Anything else in $PREV is wreckage from a
# deploy that died while copying, and nothing may be restored out of it.
snapshot_is_complete() { [ -f "$MARKER" ]; }

# Called once the snapshot has given up everything it was holding. The marker goes first, so $PREV
# stops advertising itself as a way back the moment it stops being one; the rmdir is deliberately
# not an rm -rf, so anything unexpected in there survives for somebody to look at.
clear_snapshot() {
  rm -f "$MARKER"
  rmdir "$PREV" 2>/dev/null || true
}

# Put back the build and the node_modules that were serving before this deploy touched anything.
# The checkout is deliberately left at the new commit: `next start` reads nothing but .next and
# node_modules at runtime, so the restored build serves exactly what it served this morning
# whatever the source files now say, and an operator can still look at the failing commit in place.
# Answers false when there was no complete build to go back to, so nothing downstream claims a
# rollback that did not happen.
restore_previous_build() {
  # An unfinished snapshot is thrown away rather than served. Restoring a partly-linked .next would
  # replace a site that works with one that does not, and leave nothing to undo it with.
  if ! snapshot_is_complete; then
    rm -rf "$PREV"
    return 1
  fi
  # The dependencies go back whether or not there is a build to go with them: the alternative is
  # whatever half-installed tree an npm ci that died partway left behind.
  if [ -d "$PREV/node_modules" ]; then
    rm -rf "$APP/node_modules"
    mv "$PREV/node_modules" "$APP/node_modules"
  fi
  if [ ! -d "$PREV/.next" ]; then
    clear_snapshot
    return 1
  fi
  log "restoring the build that was running before this deploy"
  rm -rf "$APP/.next"
  mv "$PREV/.next" "$APP/.next"
  # Each mv above is a rename within one filesystem, so it either happened or it did not: a restore
  # cut short by a reboot leaves whole directories behind, still marked, and the next run carries on
  # from where this one stopped. That is why the marker is cleared here and not on the way in.
  clear_snapshot
  systemctl restart threadcount || true
}

# Every failure from the moment npm ci starts deleting node_modules comes through here. Before this
# existed, a build that failed on a clean .next (a type error that only appears without the cache,
# an OOM on this small box) aborted the script on `set -e` with .next already deleted: no restart,
# no rollback, and the rollback text at the bottom never printed, so the linen room stayed down
# until somebody worked out unaided that they had to rewind the checkout and deploy again.
fail() {
  # A Ctrl-C while this is putting the old build back would leave the linen room with neither.
  trap '' INT TERM
  echo "FATAL: $*"
  journalctl -u threadcount -n 30 --no-pager || true
  if ! restore_previous_build; then
    echo "There was no previous build to restore — the site stays down until a deploy succeeds."
  elif healthy; then
    echo "Rolled back: $BEFORE is serving again. The checkout is still at $AFTER."
  else
    echo "ROLLBACK DID NOT COME BACK HEALTHY (last status from /api/health was ${code:-none})."
    journalctl -u threadcount -n 30 --no-pager || true
  fi
  echo "To put the code back where the running build came from:"
  echo "  cd $APP && sudo -u $USER git reset --hard $BEFORE && sudo $0"
  if [ -n "$DUMP" ]; then
    # The restored build carries the Prisma client generated against the OLD schema. If a migration
    # dropped a column before the build failed, that client throws on every query naming it, and
    # only the dump puts the column back.
    echo "If a migration landed first, restore the schema from the dump taken above:"
    echo "  gzip -dc $DUMP | sudo -u $USER psql \"\$DATABASE_URL\""
  fi
  exit 1
}

cd "$APP"
BEFORE=$(runuser -u "$USER" -- git rev-parse --short HEAD)
log "at $BEFORE, pulling main"
runuser -u "$USER" -- git pull -q --ff-only origin main
AFTER=$(runuser -u "$USER" -- git rev-parse --short HEAD)
log "now at $AFTER"

# Take the way back BEFORE npm ci removes node_modules. `cp -al` links rather than copies, so it
# costs a couple of seconds and almost no disk, and the running server keeps reading the originals
# untouched: npm ci and next build both delete and recreate files rather than writing through them,
# so nothing below can reach back into the snapshot through a shared inode.
#
# A snapshot is used here instead of the tidier "build in a scratch directory and swap it in", which
# would also close the degraded window described below, because .next/required-server-files.json
# records the absolute path the build ran in — a build made anywhere but $APP is not safely
# relocatable. Closing that window properly means making $APP a symlink to a release directory,
# which is a change to the box, not to this script.
#
# `cp -a` also preserves ownership, which matters on the way back: next writes its cache inside
# .next as the service user, and a restored tree owned by root would fail at the first write.
log "snapshotting the running build so a failure can be undone"
# A deploy killed mid-run — Ctrl-C, a dropped SSH session, the OOM killer, a reboot — leaves last
# time's snapshot holding the only intact build. Put it back before wiping it, or this deploy
# deletes the one copy that still works and a second failure has nothing to fall back to.
#
# Being marked complete is the test, not being there. Every way this script can end takes $PREV
# with it: fail() moves the snapshot back out and clears it, and the healthy path rm -rf's it. So a
# marked snapshot still sitting here at the start of a run can only mean the last run died without
# unwinding — there is no "normal" state in which it is present. An unmarked one means the last run
# died while it was still copying: that is not a build, and it must not be restored as one. (A
# snapshot left by the version of this script that predates the marker looks exactly the same, so
# the first deploy after this change discards it and simply rebuilds.)
#
# It used to also require that $APP/.next be missing, which made the branch very nearly
# unreachable: `next build` recreates .next and starts writing trace and cache into it within
# seconds, so a build killed partway — the exact case this recovery exists for — leaves a
# half-written .next sitting there, the branch was skipped, and the rm -rf below then destroyed the
# only good build on the box. Restoring when $APP/.next happens to be fine costs a service restart
# and nothing else: the snapshot was hardlinked off that same build, so it is either byte-identical
# to it or one deploy older and known to have served the linen room, and either way this run
# rebuilds from the new checkout a couple of minutes later.
if [ -d "$PREV" ]; then
  if snapshot_is_complete; then
    log "an earlier deploy was interrupted before it finished; putting its build back first"
    restore_previous_build || true
  else
    log "an earlier deploy died while taking its snapshot; discarding it rather than restoring half a build"
    rm -rf "$PREV"
  fi
fi

# Armed before the first write to $PREV, because the hardlink copies below are thousands of links
# and take real time to make. Nothing in $APP has been touched yet, so there is nothing to roll back
# and fail() would only announce a rollback that never happened: clearing away the half-made
# snapshot and stopping is the whole job here. The marker written afterwards covers what no trap
# can — a SIGKILL, the OOM killer, the power going out.
trap 'trap "" INT TERM; rm -rf "$PREV"; echo "FATAL: interrupted while snapshotting — nothing on this box was changed."; exit 1' INT TERM

rm -rf "$PREV"
mkdir -p "$PREV"
# A box with nothing built yet (a rebuild, a fresh provision) has nothing to roll back to. That is
# worth saying out loud, but it is not a reason to refuse: the site is already not serving.
if [ -d "$APP/.next" ]; then
  cp -al "$APP/.next" "$PREV/.next" || { rm -rf "$PREV"; echo "FATAL: could not snapshot .next — refusing to deploy with no way back."; exit 1; }
else
  log "no existing build to snapshot — this deploy has nothing to roll back to"
fi
if [ -d "$APP/node_modules" ]; then
  cp -al "$APP/node_modules" "$PREV/node_modules" || { rm -rf "$PREV"; echo "FATAL: could not snapshot node_modules — refusing to deploy with no way back."; exit 1; }
fi
# Both copies have returned, and only now is $PREV a build anyone may go back to. A marker that
# cannot be written means the disk is full, which is a reason to stop before npm ci starts deleting
# things rather than after.
touch "$MARKER" || { rm -rf "$PREV"; echo "FATAL: could not mark the snapshot complete — refusing to deploy with no way back."; exit 1; }

# From here on an interrupt has to unwind rather than just stop, for the same reason a failed build
# does: node_modules and .next are about to stop being a working install. There is a complete
# snapshot behind us now, so fail() has something real to put back.
trap 'fail "interrupted."' INT TERM

# From here until the restart the site is degraded, not down: npm ci removes the node_modules the
# running server loads lazily from, and the build below removes the .next it serves page bundles and
# /_next/static chunks out of. Expect 500s and unstyled pages for the two to four minutes that takes,
# which is why deploys belong outside stocktake hours.
log "installing dependencies"
runuser -u "$USER" -- npm ci --no-audit --no-fund || fail "npm ci failed."

# A dump before anything can touch the schema. `prisma migrate deploy` runs below, the migration
# history already contains column drops, and the build after it can still fail — so a deploy can
# leave a changed schema behind a rolled-back binary. The rollback printed at the bottom only
# rewinds the code, which is no use without this file. A dump that fails stops the deploy: there
# is no version of "migrate anyway" that is worth the alternative.
command -v pg_dump >/dev/null || fail "pg_dump is not installed — refusing to migrate with no way back."
DUMPS=/var/backups/threadcount
DUMP="$DUMPS/pre-deploy-$(date +%Y%m%d-%H%M%S)-$AFTER.sql.gz"
install -d -m 700 "$DUMPS"
log "dumping the database to $DUMP"
if ! runuser -u "$USER" -- bash -c "set -a; . '$SEC'; set +a; exec pg_dump --no-owner --clean --if-exists \"\$DATABASE_URL\"" | gzip -9 > "$DUMP"; then
  rm -f "$DUMP"
  # Cleared so the rollback advice cannot point an operator at a dump that was never written.
  DUMP=""
  fail "pg_dump failed — refusing to migrate."
fi
# A gzip of nothing is still ~20 bytes, so the floor goes on the uncompressed size: anything under
# a kilobyte is a dump that connected and then produced nothing worth restoring.
if [ "$(gzip -dc "$DUMP" | wc -c)" -lt 1024 ]; then
  echo "The dump at $DUMP is empty."
  DUMP=""
  fail "the database dump is empty — refusing to migrate."
fi
log "dump is $(du -h "$DUMP" | cut -f1)"
# Ten deploys' worth is plenty of history for a rollback, and this is a disk nobody watches.
ls -1t "$DUMPS"/pre-deploy-*.sql.gz 2>/dev/null | tail -n +11 | xargs -r rm -f || true

# Migrate before build, so a schema the new code needs is there when the build type-checks it.
# A destructive migration can therefore land while a build later fails: the rollback below puts the
# old build back, but only the dump above undoes the migration. The three steps run separately so
# the failure names the one an operator has to go and read.
log "migrating, generating, building"
runuser -u "$USER" -- bash -c "set -a; . '$SEC'; set +a; npx prisma migrate deploy" || fail "prisma migrate deploy failed."
runuser -u "$USER" -- bash -c "set -a; . '$SEC'; set +a; npx prisma generate" || fail "prisma generate failed."
# The cd is spelled out because this is the one line here that deletes a directory by relative path,
# and a deploy run from anywhere but $APP must not be able to aim it somewhere else.
# The deployed version, written down before the build so the build can carry it: .release.json is
# what the operations console reads (lib/ops/version.ts), and NEXT_PUBLIC_RELEASE is compiled into
# the bundle so error reports (lib/glitchtip.ts) name the release instead of leaving it blank.
# Not in secrets.env — that file is the operator's, and this script never edits it.
DEPLOYED_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ)
printf '{"sha":"%s","from":"%s","at":"%s"}\n' "$AFTER" "$BEFORE" "$DEPLOYED_AT" | runuser -u "$USER" -- tee "$APP/.release.json" > /dev/null || fail "could not write .release.json"
runuser -u "$USER" -- bash -c "set -a; . '$SEC'; set +a; export NEXT_PUBLIC_RELEASE='$AFTER'; cd '$APP' && rm -rf .next && npm run build" || fail "npm run build failed."

log "restarting"
systemctl restart threadcount || fail "systemctl restart threadcount failed."

if healthy; then
  log "healthy on $AFTER (was $BEFORE)"
  # Only now does the old build stop being the way back. The dump stays either way: it is the only
  # way back for a schema change, and a deploy can look healthy and still turn out to be wrong.
  trap - INT TERM
  # Unmark before deleting, the same order clear_snapshot uses and for the same reason: rm -rf
  # walks the tree as it pleases, so an interrupt partway through this must not be able to leave
  # the marker standing over a half-deleted build for the next run to trust and restore.
  rm -f "$MARKER"
  rm -rf "$PREV"
  exit 0
fi

fail "not healthy after 20s (last status from /api/health was ${code:-none})."