From 905fc5477567cedfbe2740e47c4a6fb9ee3bf58a Mon Sep 17 00:00:00 2001 From: Daniele Date: Mon, 10 Aug 2026 18:27:21 +0100 Subject: [PATCH] ci: build from a persistent tree instead of the runner workspace MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous commit tried to fix the mtime invalidation with `git restore-mtime`. It does not work on this box, in the worst way: the packaged version (2022.12) drives `git whatchanged`, which git 2.53 refuses to run without --i-still-use-this — and the tool swallows that failure and exits 0 having updated nothing. Verified on the runner: "0 commits evaluated, 675 files missing, 0 files updated", while the job happily went on to rebuild everything. Fix the cause instead of the symptom. Both building workflows now sync a tree that survives between runs and build there. `git checkout` only rewrites files whose content actually changed, so mtimes are correct as a consequence rather than as a reconstruction — and no external tool is involved. Gitea serves this repo from the same machine the runner runs on, so the sync reads the bare repo directly: no network, no token. Two properties this buys that restore-mtime did not: - The absolute source path is pinned. The runner derives its workspace path from the job definition, so editing a workflow moved it and invalidated every workspace crate by itself — the previous commit paid that cost without knowing it. - It cannot fail towards staleness. Checking out an older commit stamps those files newer, which costs an extra rebuild; restore-mtime moved mtimes backwards, which could have let cargo reuse artifacts built from newer code. Each workflow gets its own tree, for the same reason they already have their own CARGO_TARGET_DIR: they track different branches, and one shared tree would rewrite half the files on every switch. --- .gitea/workflows/nightly.yml | 76 ++++++++++++++++++++++++------------ .gitea/workflows/release.yml | 57 ++++++++++++++++++--------- 2 files changed, 91 insertions(+), 42 deletions(-) diff --git a/.gitea/workflows/nightly.yml b/.gitea/workflows/nightly.yml index 8cd262f..1307090 100644 --- a/.gitea/workflows/nightly.yml +++ b/.gitea/workflows/nightly.yml @@ -20,6 +20,11 @@ jobs: env: CARGO_TARGET_DIR: /home/dguiducci/.cache/skald-ci/target + # The persistent build tree — see the sync step. Kept separate from the + # release workflow's: the two track different branches, and one shared + # tree would rewrite half the files on every switch, which is exactly the + # mtime churn this whole arrangement removes. + SRC: /home/dguiducci/.cache/skald-ci/src-nightly # Release builds have incremental compilation OFF by default, which is the # worst case for this tree: skald-core is 51k lines in one crate, so a # one-line change recodegens all of it. The nightly trades a marginally @@ -28,30 +33,52 @@ jobs: CARGO_INCREMENTAL: 1 steps: - - uses: actions/checkout@v4 - with: - # `git restore-mtime` dates each file from the last commit that - # touched it, so it needs the full history. A shallow clone would date - # every file at HEAD and defeat the whole point. Cheap here: ~170 - # commits, and the Gitea instance runs on this same machine. - fetch-depth: 0 - - # Cargo decides what to recompile by mtime, and the runner deletes the job - # workspace after every run — so a fresh clone stamps every source file - # with "now" and all 20 workspace crates rebuild even on a commit that - # only touched web/*.js. Measured: 20 of 722 rlibs rebuilt on a JS-only - # commit, i.e. the ~700 third-party deps stay cached and our own code - # never does. Restoring mtimes from git history is what makes - # CARGO_TARGET_DIR actually cache the workspace and not just the deps. + # Deliberately not actions/checkout. Cargo decides what to recompile by + # mtime, and the runner deletes its own workspace after every job — so a + # fresh clone stamps every source file with "now" and all 20 workspace + # crates rebuilt on every run whatever the commit touched. Measured on a + # commit that only changed web/*.js: 20 of 722 rlibs rebuilt, i.e. the + # ~700 third-party deps stayed cached (their sources live in + # ~/.cargo/registry, with stable mtimes) and our own code never did. # - # Consequence to respect: do NOT reuse this target dir to rebuild an older - # commit. Mtimes would go backwards and cargo could treat artifacts built - # from newer code as fresh. Use a separate CARGO_TARGET_DIR for that. - - name: Restore source mtimes from git history - run: git restore-mtime + # A tree that survives between runs fixes it at the source: `git checkout` + # only rewrites files whose content actually changed, so everything else + # keeps its mtime and cargo skips it. No external tool is involved — note + # that the obvious alternative, `git restore-mtime`, is a trap here: the + # packaged version drives the deprecated `git whatchanged`, which git 2.53 + # refuses to run, and it reports that failure by exiting 0 having updated + # nothing. + # + # This also pins the absolute source path, which the runner's workspace + # does not: that path is derived from the job definition, so every edit to + # this file moved it and invalidated every workspace crate on its own. + # + # Note which way this fails: checking out an older commit stamps those + # files *newer*, which can only cost an extra rebuild — it can never let + # cargo reuse an artifact built from newer code. + - name: Sync the persistent build tree + run: | + set -eu + # Gitea serves this repo from the same machine the runner runs on, so + # the tree syncs straight off the bare repo: no network, no token. + ORIGIN=/home/dguiducci/skald/gitea/data/git/repositories/dguiducci/skald-circle.git + if [ ! -d "$SRC/.git" ]; then + mkdir -p "$(dirname "$SRC")" + git clone --no-checkout "$ORIGIN" "$SRC" + fi + cd "$SRC" + git remote set-url origin "$ORIGIN" + git fetch --prune --force origin + git checkout -f --detach "$GITHUB_SHA" + # Clear leftovers from the previous run (dist/ above all) so nothing + # stale can be packaged or deployed. Tracked files are untouched, and + # CARGO_TARGET_DIR lives outside this tree. + git clean -ffdxq + echo "[sync] $(git log --oneline -1)" - name: Build native (linux/amd64) run: | + cd "$SRC" RUSTFLAGS="-A warnings" cargo build --release --no-default-features RUSTFLAGS="-A warnings" cargo build --release --no-default-features -p skald-setup @@ -61,12 +88,13 @@ jobs: AR_aarch64_unknown_linux_gnu: aarch64-linux-gnu-ar CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER: aarch64-linux-gnu-gcc run: | + cd "$SRC" RUSTFLAGS="-A warnings" cargo build --release --no-default-features --target aarch64-unknown-linux-gnu RUSTFLAGS="-A warnings" cargo build --release --no-default-features -p skald-setup --target aarch64-unknown-linux-gnu - name: Package amd64 run: | - cd "${GITHUB_WORKSPACE:-.}" + cd "$SRC" ./ci/package.sh \ --version nightly \ --os linux \ @@ -76,7 +104,7 @@ jobs: - name: Package arm64 run: | - cd "${GITHUB_WORKSPACE:-.}" + cd "$SRC" ./ci/package.sh \ --version nightly \ --os linux \ @@ -86,7 +114,7 @@ jobs: - name: Deploy to builds.skaldagent.net run: | - cd "${GITHUB_WORKSPACE:-.}" + cd "$SRC" DEST=/var/www/builds.skaldagent.net/nightly mkdir -p "$DEST" # Nightly reuses a fixed filename, so publish atomically: copy to a @@ -102,7 +130,7 @@ jobs: - name: Publish the nightly installer run: | - cd "${GITHUB_WORKSPACE:-.}" + cd "$SRC" # install-nightly.sh is served straight from the web root # (curl -fsSL https://builds.skaldagent.net/install-nightly.sh | bash), # so without this it stays whatever was copied there by hand and drifts diff --git a/.gitea/workflows/release.yml b/.gitea/workflows/release.yml index 5fe6d1f..07c20db 100644 --- a/.gitea/workflows/release.yml +++ b/.gitea/workflows/release.yml @@ -36,35 +36,55 @@ jobs: # and one that doesn't would make each run invalidate the other's # workspace crates, which is exactly the cost this whole change removes. CARGO_TARGET_DIR: /home/dguiducci/.cache/skald-ci/target-release + # The persistent build tree. Separate from the nightly's for the same + # reason as the target dir: this one tracks `release`, that one tracks + # `main`, and a shared tree would rewrite half the files on every switch — + # reintroducing precisely the mtime churn the arrangement removes. + SRC: /home/dguiducci/.cache/skald-ci/src-release steps: - - uses: actions/checkout@v4 - with: - # Full history: `git restore-mtime` below dates each file from the - # last commit that touched it. - fetch-depth: 0 - - # See the long note in nightly.yml: the runner wipes the job workspace - # after every run, so without this every source file is stamped "now" on - # checkout and all 20 workspace crates rebuild from scratch regardless of - # what the commit changed. This is what lets CARGO_TARGET_DIR cache our - # own crates and not just the ~700 third-party deps. - - name: Restore source mtimes from git history - run: git restore-mtime + # Deliberately not actions/checkout — see the long note in nightly.yml. + # Short version: the runner deletes its workspace after every job, so a + # fresh clone stamps every source file "now" and cargo, which decides + # freshness by mtime, rebuilt all 20 workspace crates on every run + # whatever the commit touched. A tree that survives makes `git checkout` + # rewrite only the files that actually changed. + - name: Sync the persistent build tree + run: | + set -eu + # Gitea serves this repo from the same machine the runner runs on, so + # the tree syncs straight off the bare repo: no network, no token. + ORIGIN=/home/dguiducci/skald/gitea/data/git/repositories/dguiducci/skald-circle.git + if [ ! -d "$SRC/.git" ]; then + mkdir -p "$(dirname "$SRC")" + git clone --no-checkout "$ORIGIN" "$SRC" + fi + cd "$SRC" + git remote set-url origin "$ORIGIN" + git fetch --prune --force origin + git checkout -f --detach "$GITHUB_SHA" + # Clear leftovers from the previous run (dist/ above all) so a stale + # tarball can never be published as this version. + git clean -ffdxq + echo "[sync] $(git log --oneline -1)" - name: Extract version from Cargo.toml id: extract-version run: | + cd "$SRC" VER="v$(grep '^version' Cargo.toml | head -1 | sed 's/.*"\(.*\)"/\1/')" echo "version=$VER" >> "$GITHUB_OUTPUT" echo "[release] Building version $VER" # Also run verify-version on push to catch any race (belt-and-suspenders) - name: Verify version is new - run: ./ci/verify-version.sh --builds-dir /var/www/builds.skaldagent.net + run: | + cd "$SRC" + ./ci/verify-version.sh --builds-dir /var/www/builds.skaldagent.net - name: Build native (linux/amd64) run: | + cd "$SRC" RUSTFLAGS="-A warnings" cargo build --release --no-default-features RUSTFLAGS="-A warnings" cargo build --release --no-default-features -p skald-setup @@ -74,12 +94,13 @@ jobs: AR_aarch64_unknown_linux_gnu: aarch64-linux-gnu-ar CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER: aarch64-linux-gnu-gcc run: | + cd "$SRC" RUSTFLAGS="-A warnings" cargo build --release --no-default-features --target aarch64-unknown-linux-gnu RUSTFLAGS="-A warnings" cargo build --release --no-default-features -p skald-setup --target aarch64-unknown-linux-gnu - name: Package amd64 run: | - cd "${GITHUB_WORKSPACE:-.}" + cd "$SRC" ./ci/package.sh \ --version "${{ steps.extract-version.outputs.version }}" \ --os linux \ @@ -89,7 +110,7 @@ jobs: - name: Package arm64 run: | - cd "${GITHUB_WORKSPACE:-.}" + cd "$SRC" ./ci/package.sh \ --version "${{ steps.extract-version.outputs.version }}" \ --os linux \ @@ -99,7 +120,7 @@ jobs: - name: Deploy to builds.skaldagent.net run: | - cd "${GITHUB_WORKSPACE:-.}" + cd "$SRC" VERSION="${{ steps.extract-version.outputs.version }}" TARGET="/var/www/builds.skaldagent.net/releases/${VERSION}" mkdir -p "$TARGET" @@ -125,7 +146,7 @@ jobs: - name: Publish the release installer run: | - cd "${GITHUB_WORKSPACE:-.}" + cd "$SRC" # install.sh is served straight from the web root # (curl -fsSL https://builds.skaldagent.net/install.sh | bash), so # without this it stays whatever was copied there by hand and drifts