Daniele
67fc1455c5
Nightly Build / build (push) Successful in 4m5s
An MCP server whose `initialize` answer is an error — a broken or version- mismatched connector — starts fine and then never exits. `McpServer::start` returned `Err` correctly, but the `Child` lives in the read-loop task rather than in the returned value, so `kill_on_drop` followed a task nothing ever drops. Every retry therefore left a live process holding three pipes and a pidfd, and the supervisor's retry ceiling is deliberately not permanent. The end state was not a dead connector but a dead instance: the process hit its 1024-descriptor limit, `accept()` began failing with EMFILE, and incoming connections queued on a socket nobody could accept from — while the process, the port and every other connector still looked healthy. Observed in production at ~5h from the first bad handshake to unreachable, with 229 orphaned interpreters. `stop_server`/`stop_all` had the same hole from the other side: they document the dropped handle as killing the process, but the task holds its end of stdin, so the child stayed blocked on a read that would never return. Both close with one seam. `McpServer` now owns a oneshot sender whose receiver the read-loop selects on; nothing ever sends, so the drop is the message. That covers a deliberate stop, the last `Arc` going away, and a `?` in `start()` unwinding past the local binding before it was ever returned — including the caller's `timeout`, which drops the same future. The loop then kills and, as importantly, reaps: an unreaped child trades the orphan for a zombie holding the same pipes. Both leaks are covered by tests that fail without the fix. Also raise LimitNOFILE to 65536: the installers write it, and update.sh heals an existing unit additively, leaving an admin's own value alone. The leak is the bug, but 1024 for a process sharing descriptors between the listener, every user's SQLite handles and three pipes per connector is thin regardless.
422 lines
19 KiB
Bash
Executable File
422 lines
19 KiB
Bash
Executable File
#!/usr/bin/env sh
|
|
# update.sh — update Skald Circle to the latest version on the same channel
|
|
#
|
|
# Usage:
|
|
# ~/.local/share/skald-circle/update.sh
|
|
#
|
|
# Reads the .release-channel file written by the installer to determine
|
|
# whether to pull from the release or nightly channel. On release, checks
|
|
# the remote LATEST version first and skips the download if already current.
|
|
#
|
|
# Robustness notes (why the flow looks the way it does):
|
|
# * The tarball is downloaded and validated in a *staging* directory BEFORE
|
|
# the running service is touched — a broken download never takes the app
|
|
# down.
|
|
# * The service is stopped and we WAIT until the process is actually gone
|
|
# before extracting: overwriting a live binary in place fails with ETXTBSY
|
|
# (Linux) or on a running Mach-O (macOS), which would abort the update with
|
|
# the service left down.
|
|
# * The whole flow runs inside main(), invoked on the very last line, so the
|
|
# shell has parsed the entire script into memory before `tar` overwrites
|
|
# update.sh with its own new copy (the tarball ships this script). Without
|
|
# this, the shell would read garbage for the tail and never restart.
|
|
# * A trap restarts the service if the update fails after the stop, so a
|
|
# failed update never leaves the box down.
|
|
|
|
set -eu
|
|
|
|
# ── Determine install directory ───────────────────────────────────────────────
|
|
if [ -n "${SKALD_DIR:-}" ]; then
|
|
INSTALL_DIR="$SKALD_DIR"
|
|
else
|
|
INSTALL_DIR="$(cd "$(dirname "$0")" && pwd)"
|
|
fi
|
|
|
|
CHANNEL_FILE="${INSTALL_DIR}/.release-channel"
|
|
if [ ! -f "$CHANNEL_FILE" ]; then
|
|
echo "✖ .release-channel not found in ${INSTALL_DIR}" >&2
|
|
echo " This installation was not created by an installer or is too old." >&2
|
|
echo " Please reinstall with:" >&2
|
|
echo " curl -fsSL https://builds.skaldagent.net/install.sh | bash" >&2
|
|
exit 1
|
|
fi
|
|
|
|
CHANNEL="$(tr -d '[:space:]' < "$CHANNEL_FILE")"
|
|
|
|
# ── Detect interactive stdin ──────────────────────────────────────────────────
|
|
if [ -t 0 ]; then
|
|
IS_INTERACTIVE=true
|
|
else
|
|
IS_INTERACTIVE=false
|
|
fi
|
|
|
|
# ── Colours (if terminal) ─────────────────────────────────────────────────────
|
|
if [ -t 1 ]; then
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
CYAN='\033[0;36m'
|
|
BOLD='\033[1m'
|
|
NC='\033[0m'
|
|
else
|
|
RED=''; GREEN=''; YELLOW=''; CYAN=''; BOLD=''; NC=''
|
|
fi
|
|
|
|
info() { printf "${GREEN}%s${NC}\n" "$*"; }
|
|
warn() { printf "${YELLOW}⚠ %s${NC}\n" "$*"; }
|
|
err() { printf "${RED}✖ %s${NC}\n" "$*"; }
|
|
header(){ printf "\n${BOLD}%s${NC}\n" "$*"; }
|
|
banner(){ printf "\n${CYAN}${BOLD}%s${NC}\n" "$*"; }
|
|
|
|
# ── Platform detection ────────────────────────────────────────────────────────
|
|
OS="$(uname -s)"
|
|
ARCH="$(uname -m)"
|
|
|
|
case "$OS" in
|
|
Linux) OS="linux" ;;
|
|
Darwin) OS="darwin" ;;
|
|
*) err "Unsupported OS: $OS"; exit 1 ;;
|
|
esac
|
|
|
|
case "$ARCH" in
|
|
x86_64)
|
|
ARCH="amd64"
|
|
if [ "$OS" = "darwin" ]; then
|
|
err "Intel Macs are not supported. Apple Silicon (M1+) only."
|
|
exit 1
|
|
fi
|
|
;;
|
|
aarch64|arm64)
|
|
ARCH="arm64"
|
|
;;
|
|
*) err "Unsupported architecture: $ARCH"; exit 1 ;;
|
|
esac
|
|
|
|
command -v curl >/dev/null 2>&1 || { err "curl is required but not installed."; exit 1; }
|
|
|
|
BASE_URL="https://builds.skaldagent.net"
|
|
|
|
# ── Global state (referenced by the EXIT trap) ────────────────────────────────
|
|
TMP_TARBALL=""
|
|
STAGING=""
|
|
STOPPED=0 # set once the service has been stopped
|
|
STARTED=0 # set once it has been (re)started
|
|
|
|
# ── Stop service ──────────────────────────────────────────────────────────────
|
|
# NOTE: match the *normalized* OS values set above (linux/darwin), not uname's
|
|
# capitalized output. Getting this wrong turns both stop_service and
|
|
# start_service into silent no-ops, and then none of the ordering this file
|
|
# documents at the top actually happens: the tarball is extracted over the
|
|
# running binary (ETXTBSY on Linux, aborting the update mid-way), and the
|
|
# safety-net restart in cleanup() is a no-op too, so the box stays down.
|
|
stop_service() {
|
|
case "$OS" in
|
|
linux)
|
|
if command -v systemctl >/dev/null 2>&1; then
|
|
if systemctl --user is-active skald-circle.service >/dev/null 2>&1; then
|
|
info "⏹️ Stopping service …"
|
|
systemctl --user stop skald-circle.service
|
|
fi
|
|
fi
|
|
;;
|
|
darwin)
|
|
if command -v launchctl >/dev/null 2>&1; then
|
|
if launchctl list com.skald.circle >/dev/null 2>&1; then
|
|
info "⏹️ Stopping agent …"
|
|
launchctl unload "$HOME/Library/LaunchAgents/com.skald.circle.plist" 2>/dev/null || true
|
|
fi
|
|
fi
|
|
;;
|
|
esac
|
|
}
|
|
|
|
# ── Wait until the server process is actually gone ────────────────────────────
|
|
# systemctl stop is synchronous, but launchctl unload kills the process group
|
|
# asynchronously — so we poll for the real binary before overwriting it.
|
|
wait_until_stopped() {
|
|
# Match the server binary by path prefix. Deliberately unanchored: the
|
|
# server runs with no args today, but we'd rather over-match (a harmless
|
|
# extra wait) than miss a still-running process and race the extraction.
|
|
# `skald-setup` only runs at first-run setup, never during an update.
|
|
pat="${INSTALL_DIR}/bin/skald"
|
|
if command -v pgrep >/dev/null 2>&1; then
|
|
i=0
|
|
while pgrep -f "$pat" >/dev/null 2>&1; do
|
|
i=$((i + 1))
|
|
if [ "$i" -gt 20 ]; then
|
|
warn "Server still running after ~20s; proceeding anyway."
|
|
return 0
|
|
fi
|
|
sleep 1
|
|
done
|
|
else
|
|
# No pgrep: give the service manager a moment to tear the process down.
|
|
sleep 3
|
|
fi
|
|
}
|
|
|
|
# ── Start service ─────────────────────────────────────────────────────────────
|
|
start_service() {
|
|
case "$OS" in
|
|
linux)
|
|
if command -v systemctl >/dev/null 2>&1; then
|
|
info "▶ Starting service …"
|
|
systemctl --user start skald-circle.service
|
|
fi
|
|
;;
|
|
darwin)
|
|
if command -v launchctl >/dev/null 2>&1; then
|
|
info "▶ Starting agent …"
|
|
launchctl load "$HOME/Library/LaunchAgents/com.skald.circle.plist" 2>/dev/null || true
|
|
fi
|
|
;;
|
|
esac
|
|
}
|
|
|
|
# ── systemd user lingering ────────────────────────────────────────────────────
|
|
# Same helper the installers run, repeated here so an install predating it gets
|
|
# healed by an ordinary update: a `systemctl --user` unit lives under the
|
|
# per-user manager, which systemd stops when the user's last session ends —
|
|
# so without lingering the server dies at logout and never starts at boot.
|
|
# Idempotent, and a failure is only ever a warning: the update itself is fine.
|
|
ensure_linger() {
|
|
[ "$OS" = "linux" ] || return 0
|
|
|
|
local target="${USER:-$(id -un)}"
|
|
|
|
command -v loginctl >/dev/null 2>&1 || return 0
|
|
|
|
case "$(loginctl show-user "$target" --property=Linger 2>/dev/null || true)" in
|
|
*=yes) return 0 ;;
|
|
esac
|
|
|
|
if loginctl enable-linger "$target" 2>/dev/null \
|
|
|| sudo -n loginctl enable-linger "$target" 2>/dev/null \
|
|
|| { [ "$IS_INTERACTIVE" = true ] && sudo loginctl enable-linger "$target"; }; then
|
|
info "✔ Lingering enabled — the server now survives logout and starts at boot"
|
|
else
|
|
warn "Lingering is not enabled for ${target}."
|
|
echo " The server stops when your last session ends. Run this once:"
|
|
echo " sudo loginctl enable-linger ${target}"
|
|
fi
|
|
}
|
|
|
|
# ── File-descriptor limit ─────────────────────────────────────────────────────
|
|
# Same reasoning as ensure_linger: the installers now write LimitNOFILE into the
|
|
# unit, and this heals an install that predates them, since an update never
|
|
# rewrites the unit file.
|
|
#
|
|
# Worth the repair rather than leaving it to the next reinstall, because running
|
|
# out of descriptors does not degrade gracefully. One process shares the default
|
|
# 1024 between the HTTP listener, every user's SQLite handles and three pipes per
|
|
# connector; past the ceiling accept() fails with EMFILE and the app stops
|
|
# answering while the process, the port and the health of every connector all
|
|
# still look fine.
|
|
#
|
|
# Strictly additive: it appends one line to [Service] and touches nothing else,
|
|
# so a hand-customized unit survives. Skipped entirely if the admin already set
|
|
# any LimitNOFILE of their own.
|
|
ensure_fd_limit() {
|
|
[ "$OS" = "linux" ] || return 0
|
|
|
|
local unit="$HOME/.config/systemd/user/skald-circle.service"
|
|
|
|
[ -f "$unit" ] || return 0
|
|
command -v systemctl >/dev/null 2>&1 || return 0
|
|
grep -q '^[[:space:]]*LimitNOFILE=' "$unit" && return 0
|
|
grep -q '^\[Service\]' "$unit" || return 0
|
|
|
|
# Write through a temp file so an interrupted update can never leave a
|
|
# half-written unit behind.
|
|
local tmp="${unit}.tmp.$$"
|
|
if awk '/^\[Service\]/ && !done { print; print "LimitNOFILE=65536"; done=1; next } { print }' \
|
|
"$unit" > "$tmp" 2>/dev/null && [ -s "$tmp" ]; then
|
|
mv "$tmp" "$unit" \
|
|
&& systemctl --user daemon-reload 2>/dev/null \
|
|
&& info "✔ Raised the file-descriptor limit to 65536"
|
|
else
|
|
rm -f "$tmp"
|
|
warn "Could not raise the file-descriptor limit; the default 1024 still applies."
|
|
echo " Add this under [Service] in ${unit}:"
|
|
echo " LimitNOFILE=65536"
|
|
fi
|
|
}
|
|
|
|
# ── Cleanup + safety net ──────────────────────────────────────────────────────
|
|
# Runs on every exit. Removes temp files and, if the update died after the
|
|
# service was stopped but before it came back up, makes a best-effort restart so
|
|
# the box is never left down.
|
|
cleanup() {
|
|
[ -n "${TMP_TARBALL:-}" ] && rm -f "$TMP_TARBALL" 2>/dev/null || true
|
|
[ -n "${STAGING:-}" ] && rm -rf "$STAGING" 2>/dev/null || true
|
|
if [ "$STOPPED" = "1" ] && [ "$STARTED" != "1" ]; then
|
|
warn "Update failed after the service was stopped — attempting to restart it …"
|
|
start_service || true
|
|
fi
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
# ── Main ──────────────────────────────────────────────────────────────────────
|
|
main() {
|
|
# ── Resolve download URL + version (may exit early if already current) ─────
|
|
case "$CHANNEL" in
|
|
release)
|
|
LATEST="$(curl -fsSL "${BASE_URL}/releases/LATEST" | head -1 | tr -d '[:space:]')"
|
|
if [ -z "$LATEST" ]; then
|
|
err "Could not fetch latest release version from ${BASE_URL}/releases/LATEST"
|
|
exit 1
|
|
fi
|
|
|
|
VERSION_FILE="${INSTALL_DIR}/.release-version"
|
|
if [ -f "$VERSION_FILE" ]; then
|
|
CURRENT="$(tr -d '[:space:]' < "$VERSION_FILE")"
|
|
if [ "$CURRENT" = "$LATEST" ]; then
|
|
info "✔ Already up to date (${CURRENT})."
|
|
exit 0
|
|
fi
|
|
info "🚀 Update available: ${CURRENT} → ${LATEST}"
|
|
else
|
|
info "🚀 Installing latest release: ${LATEST}"
|
|
fi
|
|
|
|
VERSION="$LATEST"
|
|
TARBALL_URL="${BASE_URL}/releases/${VERSION}/skald-circle-${VERSION}-${OS}-${ARCH}.tar.gz"
|
|
DISPLAY_VERSION="$VERSION"
|
|
;;
|
|
|
|
nightly)
|
|
TARBALL_URL="${BASE_URL}/nightly/skald-circle-nightly-${OS}-${ARCH}.tar.gz"
|
|
DISPLAY_VERSION="nightly"
|
|
info "🚀 Updating to latest nightly build …"
|
|
;;
|
|
|
|
*)
|
|
err "Unknown channel in ${CHANNEL_FILE}: '${CHANNEL}'"
|
|
err "Expected 'release' or 'nightly'."
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
banner "╔══════════════════════════════════════════╗"
|
|
banner "║ Skald Circle — Updater (${DISPLAY_VERSION}) ║"
|
|
banner "╚══════════════════════════════════════════╝"
|
|
echo ""
|
|
echo " Channel : ${CHANNEL}"
|
|
echo " Platform : ${OS}/${ARCH}"
|
|
echo " Install : ${INSTALL_DIR}"
|
|
echo ""
|
|
|
|
# ── Download + validate BEFORE touching the running service ────────────────
|
|
# A broken download or a bad archive must never take the app down: we only
|
|
# stop the service once we hold a known-good tarball.
|
|
TMP_TARBALL="$(mktemp -t skald-update.XXXXXX.tar.gz)"
|
|
|
|
info "↓ Downloading Skald Circle (${DISPLAY_VERSION}) …"
|
|
curl -fsSL -o "$TMP_TARBALL" "$TARBALL_URL"
|
|
|
|
info "🔎 Verifying archive …"
|
|
STAGING="$(mktemp -d -t skald-update-staging.XXXXXX)"
|
|
tar xzf "$TMP_TARBALL" -C "$STAGING" --strip-components=1
|
|
if [ ! -x "$STAGING/bin/skald" ]; then
|
|
err "Downloaded archive is invalid — skald binary not found."
|
|
err "The running server was left untouched."
|
|
exit 1
|
|
fi
|
|
|
|
# ── Stop the service and wait for the process to actually exit ─────────────
|
|
stop_service
|
|
STOPPED=1
|
|
wait_until_stopped
|
|
|
|
# ── Install (binary is no longer busy) ─────────────────────────────────────
|
|
info "📦 Installing update …"
|
|
tar xzf "$TMP_TARBALL" -C "$INSTALL_DIR" --strip-components=1
|
|
|
|
if [ ! -x "$INSTALL_DIR/bin/skald" ]; then
|
|
err "Extraction failed — skald binary not found."
|
|
exit 1
|
|
fi
|
|
|
|
# ── Drop files the new build no longer ships ───────────────────────────────
|
|
# Extracting over the install dir only ever adds and overwrites, so anything
|
|
# deleted upstream survives forever: a renamed doc page keeps being mounted
|
|
# read-only into every container for the assistant to read, a removed command
|
|
# keeps being discovered. For the directories the tarball owns end to end we
|
|
# therefore prune whatever the (already verified) staging copy does not have.
|
|
#
|
|
# Pruned AFTER extracting rather than by replacing the directory, so every
|
|
# intermediate state is a complete install and the only files removed are
|
|
# ones the new build has verifiably dropped.
|
|
#
|
|
# agents/ is deliberately not in this list: adding an agent is a documented
|
|
# extension point (agents/<id>/meta.json + AGENT.md), so the directory is not
|
|
# ours alone and pruning it would delete somebody's work — at the price of an
|
|
# upstream-deleted agent lingering. skills/ is out for a stronger version of the
|
|
# same reason: the tarball ships no skills at all, so that directory holds only
|
|
# instance data — every skill in it was registered by a member — and pruning it
|
|
# would delete their work at every update. bin/ is out too: two files, both
|
|
# overwritten every time, nothing to reclaim.
|
|
for owned in web commands docs; do
|
|
[ -d "$STAGING/$owned" ] && [ -d "${INSTALL_DIR}/$owned" ] || continue
|
|
( cd "${INSTALL_DIR}/$owned" && find . -type f ) | while IFS= read -r rel; do
|
|
rel="${rel#./}"
|
|
[ -e "${STAGING}/${owned}/${rel}" ] || rm -f "${INSTALL_DIR}/${owned}/${rel}"
|
|
done
|
|
find "${INSTALL_DIR}/$owned" -mindepth 1 -type d -empty -delete 2>/dev/null || true
|
|
done
|
|
|
|
# Update version file for release channel
|
|
if [ "$CHANNEL" = "release" ]; then
|
|
echo "$VERSION" > "$INSTALL_DIR/.release-version"
|
|
fi
|
|
|
|
# ── Rebuild Python venv (best effort — new deps may have appeared) ─────────
|
|
info "🔧 Rebuilding Python virtual environment …"
|
|
VENV_DIR="${INSTALL_DIR}/.venv"
|
|
REQUIREMENTS="${INSTALL_DIR}/requirements.txt"
|
|
|
|
rm -rf "$VENV_DIR"
|
|
if command -v uv >/dev/null 2>&1; then
|
|
uv venv --seed "$VENV_DIR" && uv pip install -r "$REQUIREMENTS" \
|
|
&& info "✔ Python venv ready (uv)" \
|
|
|| warn "Python venv setup failed — the TTS plugins and host-run connectors will be unavailable."
|
|
elif command -v python3 >/dev/null 2>&1; then
|
|
python3 -m venv "$VENV_DIR" && "$VENV_DIR/bin/pip" install -r "$REQUIREMENTS" \
|
|
&& info "✔ Python venv ready (pip)" \
|
|
|| warn "Python venv setup failed — the TTS plugins and host-run connectors will be unavailable."
|
|
else
|
|
warn "python3 not found — the TTS plugins and host-run connectors will be unavailable."
|
|
fi
|
|
|
|
# ── Restart ────────────────────────────────────────────────────────────────
|
|
ensure_linger
|
|
# Before the start, so the new limit applies to the process we are about to
|
|
# bring up rather than to the one after it.
|
|
ensure_fd_limit
|
|
start_service
|
|
STARTED=1
|
|
|
|
# ── Hint about optional deps ───────────────────────────────────────────────
|
|
if [ -f "${INSTALL_DIR}/requirements-optional.txt" ]; then
|
|
info "💡 Optional GPU/ML dependencies available:"
|
|
info " ${INSTALL_DIR}/requirements-optional.txt"
|
|
echo " Install them manually if you use the Orpheus TTS plugin:"
|
|
echo " cd ${INSTALL_DIR} && .venv/bin/pip install -r requirements-optional.txt"
|
|
echo ""
|
|
fi
|
|
|
|
# ── Done ───────────────────────────────────────────────────────────────────
|
|
echo ""
|
|
info "✅ Skald Circle updated to ${DISPLAY_VERSION}!"
|
|
echo ""
|
|
echo " Status: $( [ "$OS" = "linux" ] && echo "systemctl --user status skald-circle" || echo "launchctl list com.skald.circle" )"
|
|
echo " Logs: $( [ "$OS" = "linux" ] && echo "journalctl --user -u skald-circle -f" || echo "tail -f ${INSTALL_DIR}/logs/stdout.log" )"
|
|
echo " Update: ${INSTALL_DIR}/update.sh"
|
|
echo ""
|
|
}
|
|
|
|
# Invoked on the very last line: by the time `tar` overwrites this file on disk
|
|
# (the tarball ships update.sh), the shell has already parsed the whole script,
|
|
# so the restart tail always runs. Do not add executable code below this line.
|
|
main "$@"
|