ThreadCount Community edition
Uniform stock management for healthcare linen rooms. Licensed under the GNU AGPL v3.
This commit is contained in:
ThreadCount
Executable
+304
@@ -0,0 +1,304 @@
|
||||
#!/bin/bash
|
||||
# Deploy ThreadCount: update the code and nothing else.
|
||||
#
|
||||
# This exists because /root/deploy-setup.sh is a PROVISIONING script — it rewrites the nginx site
|
||||
# and re-points /opt/threadcount/.env at /etc/threadcount/secrets.env. Running it as a deploy on
|
||||
# 2026-09-06 destroyed the live .env and dropped nginx's TLS block. Use this instead.
|
||||
#
|
||||
# sudo /usr/local/sbin/threadcount-deploy.sh
|
||||
set -euo pipefail
|
||||
|
||||
APP=/opt/threadcount
|
||||
SEC=/etc/threadcount/secrets.env
|
||||
USER=threadcount
|
||||
# Hardlink copies of the build and the dependencies that were serving the linen room when this
|
||||
# deploy started. They live inside $APP because hardlinks cannot cross filesystems, and a deploy
|
||||
# that ends healthy deletes them. The leading dot is not decoration: tsconfig.json includes
|
||||
# **/*.ts and excludes only the top-level node_modules, and TypeScript skips dot-directories when
|
||||
# it expands a wildcard — a snapshot called anything else would drag a second copy of every .ts
|
||||
# file in node_modules into the build's type check.
|
||||
PREV=$APP/.deploy-prev
|
||||
# Written last, removed last. Its presence is what makes $PREV a way back: the hardlink copies
|
||||
# below take long enough to be interrupted halfway, and half a build put back over a deployment
|
||||
# that still works is the one outcome worse than no rollback at all.
|
||||
MARKER=$PREV/.complete
|
||||
DUMP=""
|
||||
|
||||
log() { echo "==> $*"; }
|
||||
|
||||
# The secrets file is the one thing nothing here may touch, and the one thing everything needs.
|
||||
[ -s "$SEC" ] || { echo "FATAL: $SEC is missing or empty — refusing to deploy."; exit 1; }
|
||||
grep -q '^DATABASE_URL=' "$SEC" || { echo "FATAL: no DATABASE_URL in $SEC — refusing to deploy."; exit 1; }
|
||||
|
||||
# Turnstile, checked here for the same reason SESSION_SECRET is checked below: instrumentation.ts
|
||||
# refuses to start production without both, so leaving them out turns a five-second grep into a
|
||||
# build, a migration and a restart that ends in a failed health check with the app already down.
|
||||
# NEXT_PUBLIC_TURNSTILE_SITEKEY in particular is compiled into the bundle, so it has to be present
|
||||
# before the build rather than before the restart.
|
||||
grep -q '^TURNSTILE_SECRET=' "$SEC" || { echo "FATAL: no TURNSTILE_SECRET in $SEC — the app refuses to start without it."; exit 1; }
|
||||
grep -q '^NEXT_PUBLIC_TURNSTILE_SITEKEY=' "$SEC" || { echo "FATAL: no NEXT_PUBLIC_TURNSTILE_SITEKEY in $SEC — it is compiled into the build."; exit 1; }
|
||||
|
||||
# A missing or placeholder SESSION_SECRET is the failure this script used to sail straight past.
|
||||
# `next build` needs no secrets, so the build succeeds; proxy.ts then treats every session cookie
|
||||
# as invalid and redirects the whole app to /auth, and every sign-in throws out of lib/session.ts
|
||||
# as an opaque 500. Nothing about that shows up in an HTTP status, so it is checked here instead.
|
||||
# The values are read in a subshell: nothing after this line should carry the database password
|
||||
# in its environment.
|
||||
( set -a; . "$SEC"; set +a
|
||||
[ -n "${SESSION_SECRET:-}" ] || { echo "FATAL: no SESSION_SECRET in $SEC — nobody could sign in."; exit 1; }
|
||||
[ "$SESSION_SECRET" != "change-me" ] || { echo "FATAL: SESSION_SECRET is still the .env.example placeholder."; exit 1; }
|
||||
[ "${#SESSION_SECRET}" -ge 32 ] || { echo "FATAL: SESSION_SECRET is under 32 characters — generate a real one."; exit 1; }
|
||||
[ -n "${DATABASE_URL:-}" ] || { echo "FATAL: DATABASE_URL in $SEC is empty."; exit 1; }
|
||||
# Present but empty is the same as absent to the application, and the greps above only prove the
|
||||
# line exists.
|
||||
[ -n "${TURNSTILE_SECRET:-}" ] || { echo "FATAL: TURNSTILE_SECRET in $SEC is empty."; exit 1; }
|
||||
[ -n "${NEXT_PUBLIC_TURNSTILE_SITEKEY:-}" ] || { echo "FATAL: NEXT_PUBLIC_TURNSTILE_SITEKEY in $SEC is empty."; exit 1; }
|
||||
# The documented escape hatch for a local `next start`. On the production box it would switch bot
|
||||
# protection off across every unauthenticated form, silently.
|
||||
[ -z "${TURNSTILE_OPTIONAL:-}" ] || { echo "FATAL: TURNSTILE_OPTIONAL is set in $SEC — that is for local smoke tests only."; exit 1; }
|
||||
) || exit 1
|
||||
|
||||
# The probe asks /api/health, which runs a SELECT 1 through Prisma and answers 503 when it cannot.
|
||||
# The marketing home page was useless for this: it is prerendered at build time and answers 200 out
|
||||
# of .next even when the server behind it is dead, so a deploy with an unreachable database still
|
||||
# reported healthy. /app was better — it goes through proxy.ts inside the running server — but its
|
||||
# redirect to /auth is produced without ever touching the database, so the one failure that takes
|
||||
# the whole product down is still the one it cannot see. Asking the database is the only probe that
|
||||
# distinguishes "serving" from "working". The rollback path below uses it too, so it is a function:
|
||||
# a rollback nobody probed is only a guess that the old build came back.
|
||||
code=""
|
||||
body=""
|
||||
healthy() {
|
||||
code=""
|
||||
for _ in $(seq 1 20); do
|
||||
sleep 1
|
||||
body=$(curl -s --max-time 5 -w '\n%{http_code}' http://127.0.0.1:3000/api/health || true)
|
||||
code=$(printf '%s' "$body" | tail -1)
|
||||
[ "$code" = "200" ] && return 0
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
# True only of a snapshot that finished being written. Anything else in $PREV is wreckage from a
|
||||
# deploy that died while copying, and nothing may be restored out of it.
|
||||
snapshot_is_complete() { [ -f "$MARKER" ]; }
|
||||
|
||||
# Called once the snapshot has given up everything it was holding. The marker goes first, so $PREV
|
||||
# stops advertising itself as a way back the moment it stops being one; the rmdir is deliberately
|
||||
# not an rm -rf, so anything unexpected in there survives for somebody to look at.
|
||||
clear_snapshot() {
|
||||
rm -f "$MARKER"
|
||||
rmdir "$PREV" 2>/dev/null || true
|
||||
}
|
||||
|
||||
# Put back the build and the node_modules that were serving before this deploy touched anything.
|
||||
# The checkout is deliberately left at the new commit: `next start` reads nothing but .next and
|
||||
# node_modules at runtime, so the restored build serves exactly what it served this morning
|
||||
# whatever the source files now say, and an operator can still look at the failing commit in place.
|
||||
# Answers false when there was no complete build to go back to, so nothing downstream claims a
|
||||
# rollback that did not happen.
|
||||
restore_previous_build() {
|
||||
# An unfinished snapshot is thrown away rather than served. Restoring a partly-linked .next would
|
||||
# replace a site that works with one that does not, and leave nothing to undo it with.
|
||||
if ! snapshot_is_complete; then
|
||||
rm -rf "$PREV"
|
||||
return 1
|
||||
fi
|
||||
# The dependencies go back whether or not there is a build to go with them: the alternative is
|
||||
# whatever half-installed tree an npm ci that died partway left behind.
|
||||
if [ -d "$PREV/node_modules" ]; then
|
||||
rm -rf "$APP/node_modules"
|
||||
mv "$PREV/node_modules" "$APP/node_modules"
|
||||
fi
|
||||
if [ ! -d "$PREV/.next" ]; then
|
||||
clear_snapshot
|
||||
return 1
|
||||
fi
|
||||
log "restoring the build that was running before this deploy"
|
||||
rm -rf "$APP/.next"
|
||||
mv "$PREV/.next" "$APP/.next"
|
||||
# Each mv above is a rename within one filesystem, so it either happened or it did not: a restore
|
||||
# cut short by a reboot leaves whole directories behind, still marked, and the next run carries on
|
||||
# from where this one stopped. That is why the marker is cleared here and not on the way in.
|
||||
clear_snapshot
|
||||
systemctl restart threadcount || true
|
||||
}
|
||||
|
||||
# Every failure from the moment npm ci starts deleting node_modules comes through here. Before this
|
||||
# existed, a build that failed on a clean .next (a type error that only appears without the cache,
|
||||
# an OOM on this small box) aborted the script on `set -e` with .next already deleted: no restart,
|
||||
# no rollback, and the rollback text at the bottom never printed, so the linen room stayed down
|
||||
# until somebody worked out unaided that they had to rewind the checkout and deploy again.
|
||||
fail() {
|
||||
# A Ctrl-C while this is putting the old build back would leave the linen room with neither.
|
||||
trap '' INT TERM
|
||||
echo "FATAL: $*"
|
||||
journalctl -u threadcount -n 30 --no-pager || true
|
||||
if ! restore_previous_build; then
|
||||
echo "There was no previous build to restore — the site stays down until a deploy succeeds."
|
||||
elif healthy; then
|
||||
echo "Rolled back: $BEFORE is serving again. The checkout is still at $AFTER."
|
||||
else
|
||||
echo "ROLLBACK DID NOT COME BACK HEALTHY (last status from /api/health was ${code:-none})."
|
||||
journalctl -u threadcount -n 30 --no-pager || true
|
||||
fi
|
||||
echo "To put the code back where the running build came from:"
|
||||
echo " cd $APP && sudo -u $USER git reset --hard $BEFORE && sudo $0"
|
||||
if [ -n "$DUMP" ]; then
|
||||
# The restored build carries the Prisma client generated against the OLD schema. If a migration
|
||||
# dropped a column before the build failed, that client throws on every query naming it, and
|
||||
# only the dump puts the column back.
|
||||
echo "If a migration landed first, restore the schema from the dump taken above:"
|
||||
echo " gzip -dc $DUMP | sudo -u $USER psql \"\$DATABASE_URL\""
|
||||
fi
|
||||
exit 1
|
||||
}
|
||||
|
||||
cd "$APP"
|
||||
BEFORE=$(runuser -u "$USER" -- git rev-parse --short HEAD)
|
||||
log "at $BEFORE, pulling main"
|
||||
runuser -u "$USER" -- git pull -q --ff-only origin main
|
||||
AFTER=$(runuser -u "$USER" -- git rev-parse --short HEAD)
|
||||
log "now at $AFTER"
|
||||
|
||||
# Take the way back BEFORE npm ci removes node_modules. `cp -al` links rather than copies, so it
|
||||
# costs a couple of seconds and almost no disk, and the running server keeps reading the originals
|
||||
# untouched: npm ci and next build both delete and recreate files rather than writing through them,
|
||||
# so nothing below can reach back into the snapshot through a shared inode.
|
||||
#
|
||||
# A snapshot is used here instead of the tidier "build in a scratch directory and swap it in", which
|
||||
# would also close the degraded window described below, because .next/required-server-files.json
|
||||
# records the absolute path the build ran in — a build made anywhere but $APP is not safely
|
||||
# relocatable. Closing that window properly means making $APP a symlink to a release directory,
|
||||
# which is a change to the box, not to this script.
|
||||
#
|
||||
# `cp -a` also preserves ownership, which matters on the way back: next writes its cache inside
|
||||
# .next as the service user, and a restored tree owned by root would fail at the first write.
|
||||
log "snapshotting the running build so a failure can be undone"
|
||||
# A deploy killed mid-run — Ctrl-C, a dropped SSH session, the OOM killer, a reboot — leaves last
|
||||
# time's snapshot holding the only intact build. Put it back before wiping it, or this deploy
|
||||
# deletes the one copy that still works and a second failure has nothing to fall back to.
|
||||
#
|
||||
# Being marked complete is the test, not being there. Every way this script can end takes $PREV
|
||||
# with it: fail() moves the snapshot back out and clears it, and the healthy path rm -rf's it. So a
|
||||
# marked snapshot still sitting here at the start of a run can only mean the last run died without
|
||||
# unwinding — there is no "normal" state in which it is present. An unmarked one means the last run
|
||||
# died while it was still copying: that is not a build, and it must not be restored as one. (A
|
||||
# snapshot left by the version of this script that predates the marker looks exactly the same, so
|
||||
# the first deploy after this change discards it and simply rebuilds.)
|
||||
#
|
||||
# It used to also require that $APP/.next be missing, which made the branch very nearly
|
||||
# unreachable: `next build` recreates .next and starts writing trace and cache into it within
|
||||
# seconds, so a build killed partway — the exact case this recovery exists for — leaves a
|
||||
# half-written .next sitting there, the branch was skipped, and the rm -rf below then destroyed the
|
||||
# only good build on the box. Restoring when $APP/.next happens to be fine costs a service restart
|
||||
# and nothing else: the snapshot was hardlinked off that same build, so it is either byte-identical
|
||||
# to it or one deploy older and known to have served the linen room, and either way this run
|
||||
# rebuilds from the new checkout a couple of minutes later.
|
||||
if [ -d "$PREV" ]; then
|
||||
if snapshot_is_complete; then
|
||||
log "an earlier deploy was interrupted before it finished; putting its build back first"
|
||||
restore_previous_build || true
|
||||
else
|
||||
log "an earlier deploy died while taking its snapshot; discarding it rather than restoring half a build"
|
||||
rm -rf "$PREV"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Armed before the first write to $PREV, because the hardlink copies below are thousands of links
|
||||
# and take real time to make. Nothing in $APP has been touched yet, so there is nothing to roll back
|
||||
# and fail() would only announce a rollback that never happened: clearing away the half-made
|
||||
# snapshot and stopping is the whole job here. The marker written afterwards covers what no trap
|
||||
# can — a SIGKILL, the OOM killer, the power going out.
|
||||
trap 'trap "" INT TERM; rm -rf "$PREV"; echo "FATAL: interrupted while snapshotting — nothing on this box was changed."; exit 1' INT TERM
|
||||
|
||||
rm -rf "$PREV"
|
||||
mkdir -p "$PREV"
|
||||
# A box with nothing built yet (a rebuild, a fresh provision) has nothing to roll back to. That is
|
||||
# worth saying out loud, but it is not a reason to refuse: the site is already not serving.
|
||||
if [ -d "$APP/.next" ]; then
|
||||
cp -al "$APP/.next" "$PREV/.next" || { rm -rf "$PREV"; echo "FATAL: could not snapshot .next — refusing to deploy with no way back."; exit 1; }
|
||||
else
|
||||
log "no existing build to snapshot — this deploy has nothing to roll back to"
|
||||
fi
|
||||
if [ -d "$APP/node_modules" ]; then
|
||||
cp -al "$APP/node_modules" "$PREV/node_modules" || { rm -rf "$PREV"; echo "FATAL: could not snapshot node_modules — refusing to deploy with no way back."; exit 1; }
|
||||
fi
|
||||
# Both copies have returned, and only now is $PREV a build anyone may go back to. A marker that
|
||||
# cannot be written means the disk is full, which is a reason to stop before npm ci starts deleting
|
||||
# things rather than after.
|
||||
touch "$MARKER" || { rm -rf "$PREV"; echo "FATAL: could not mark the snapshot complete — refusing to deploy with no way back."; exit 1; }
|
||||
|
||||
# From here on an interrupt has to unwind rather than just stop, for the same reason a failed build
|
||||
# does: node_modules and .next are about to stop being a working install. There is a complete
|
||||
# snapshot behind us now, so fail() has something real to put back.
|
||||
trap 'fail "interrupted."' INT TERM
|
||||
|
||||
# From here until the restart the site is degraded, not down: npm ci removes the node_modules the
|
||||
# running server loads lazily from, and the build below removes the .next it serves page bundles and
|
||||
# /_next/static chunks out of. Expect 500s and unstyled pages for the two to four minutes that takes,
|
||||
# which is why deploys belong outside stocktake hours.
|
||||
log "installing dependencies"
|
||||
runuser -u "$USER" -- npm ci --no-audit --no-fund || fail "npm ci failed."
|
||||
|
||||
# A dump before anything can touch the schema. `prisma migrate deploy` runs below, the migration
|
||||
# history already contains column drops, and the build after it can still fail — so a deploy can
|
||||
# leave a changed schema behind a rolled-back binary. The rollback printed at the bottom only
|
||||
# rewinds the code, which is no use without this file. A dump that fails stops the deploy: there
|
||||
# is no version of "migrate anyway" that is worth the alternative.
|
||||
command -v pg_dump >/dev/null || fail "pg_dump is not installed — refusing to migrate with no way back."
|
||||
DUMPS=/var/backups/threadcount
|
||||
DUMP="$DUMPS/pre-deploy-$(date +%Y%m%d-%H%M%S)-$AFTER.sql.gz"
|
||||
install -d -m 700 "$DUMPS"
|
||||
log "dumping the database to $DUMP"
|
||||
if ! runuser -u "$USER" -- bash -c "set -a; . '$SEC'; set +a; exec pg_dump --no-owner --clean --if-exists \"\$DATABASE_URL\"" | gzip -9 > "$DUMP"; then
|
||||
rm -f "$DUMP"
|
||||
# Cleared so the rollback advice cannot point an operator at a dump that was never written.
|
||||
DUMP=""
|
||||
fail "pg_dump failed — refusing to migrate."
|
||||
fi
|
||||
# A gzip of nothing is still ~20 bytes, so the floor goes on the uncompressed size: anything under
|
||||
# a kilobyte is a dump that connected and then produced nothing worth restoring.
|
||||
if [ "$(gzip -dc "$DUMP" | wc -c)" -lt 1024 ]; then
|
||||
echo "The dump at $DUMP is empty."
|
||||
DUMP=""
|
||||
fail "the database dump is empty — refusing to migrate."
|
||||
fi
|
||||
log "dump is $(du -h "$DUMP" | cut -f1)"
|
||||
# Ten deploys' worth is plenty of history for a rollback, and this is a disk nobody watches.
|
||||
ls -1t "$DUMPS"/pre-deploy-*.sql.gz 2>/dev/null | tail -n +11 | xargs -r rm -f || true
|
||||
|
||||
# Migrate before build, so a schema the new code needs is there when the build type-checks it.
|
||||
# A destructive migration can therefore land while a build later fails: the rollback below puts the
|
||||
# old build back, but only the dump above undoes the migration. The three steps run separately so
|
||||
# the failure names the one an operator has to go and read.
|
||||
log "migrating, generating, building"
|
||||
runuser -u "$USER" -- bash -c "set -a; . '$SEC'; set +a; npx prisma migrate deploy" || fail "prisma migrate deploy failed."
|
||||
runuser -u "$USER" -- bash -c "set -a; . '$SEC'; set +a; npx prisma generate" || fail "prisma generate failed."
|
||||
# The cd is spelled out because this is the one line here that deletes a directory by relative path,
|
||||
# and a deploy run from anywhere but $APP must not be able to aim it somewhere else.
|
||||
# The deployed version, written down before the build so the build can carry it: .release.json is
|
||||
# what the operations console reads (lib/ops/version.ts), and NEXT_PUBLIC_RELEASE is compiled into
|
||||
# the bundle so error reports (lib/glitchtip.ts) name the release instead of leaving it blank.
|
||||
# Not in secrets.env — that file is the operator's, and this script never edits it.
|
||||
DEPLOYED_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
printf '{"sha":"%s","from":"%s","at":"%s"}\n' "$AFTER" "$BEFORE" "$DEPLOYED_AT" | runuser -u "$USER" -- tee "$APP/.release.json" > /dev/null || fail "could not write .release.json"
|
||||
runuser -u "$USER" -- bash -c "set -a; . '$SEC'; set +a; export NEXT_PUBLIC_RELEASE='$AFTER'; cd '$APP' && rm -rf .next && npm run build" || fail "npm run build failed."
|
||||
|
||||
log "restarting"
|
||||
systemctl restart threadcount || fail "systemctl restart threadcount failed."
|
||||
|
||||
if healthy; then
|
||||
log "healthy on $AFTER (was $BEFORE)"
|
||||
# Only now does the old build stop being the way back. The dump stays either way: it is the only
|
||||
# way back for a schema change, and a deploy can look healthy and still turn out to be wrong.
|
||||
trap - INT TERM
|
||||
# Unmark before deleting, the same order clear_snapshot uses and for the same reason: rm -rf
|
||||
# walks the tree as it pleases, so an interrupt partway through this must not be able to leave
|
||||
# the marker standing over a half-deleted build for the next run to trust and restore.
|
||||
rm -f "$MARKER"
|
||||
rm -rf "$PREV"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
fail "not healthy after 20s (last status from /api/health was ${code:-none})."
|
||||
Reference in New Issue
Block a user