From f3ca6be69838a2a32bddb23e6b1475f5876a643e Mon Sep 17 00:00:00 2001 From: vickydotbat Date: Wed, 29 Jul 2026 09:51:10 +0200 Subject: [PATCH] feat(ci): prune release assets outside the newest two tags (#52) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Gitea has no release-asset retention, so every v* tag kept its full cross-platform binary set forever — 31 tags had accumulated 1.8 GB on the production host disk. Nothing consumes an old asset: CI takes the Crucible binary from the Nix flake, and every deleted file is reproducible from its tag. scripts/prune-release-assets.sh lists every release page, sorts by creation date, skips the newest two published releases, and deletes the remaining releases' attachments. Two rather than one so a release in flight never leaves the previous download broken. Drafts never consume a keep slot. Releases, tags, source archives and release notes are untouched. Nothing is excluded — SHA256SUMS and the wrappers go with the binaries: the checksums are only meaningful next to the files they cover, and the consumer drift-check reads the wrappers from the latest release, which stays complete. The step is best-effort: a stale asset is cheaper than a blocked publish, so API failures warn and the step carries continue-on-error. Co-Authored-By: Claude Opus 5 (1M context) --- .gitea/workflows/build-binaries.yml | 14 ++++++ Makefile | 1 + README.md | 4 +- scripts/prune-release-assets.sh | 67 +++++++++++++++++++++++++ tests/prune-release-assets.sh | 78 +++++++++++++++++++++++++++++ tests/workflow-contract.sh | 2 + 6 files changed, 165 insertions(+), 1 deletion(-) create mode 100755 scripts/prune-release-assets.sh create mode 100755 tests/prune-release-assets.sh mode change 100644 => 100755 tests/workflow-contract.sh diff --git a/.gitea/workflows/build-binaries.yml b/.gitea/workflows/build-binaries.yml index 15d3efa..28c4162 100644 --- a/.gitea/workflows/build-binaries.yml +++ b/.gitea/workflows/build-binaries.yml @@ -65,3 +65,17 @@ jobs: "${api}/releases/${id}/assets?name=$(basename "$f")" done ' + + # Gitea has no asset retention (#52): without this every tag keeps its + # ~57 MB binary set forever. Best-effort — a stale asset is cheaper than + # a blocked publish, so a failure here never fails the release. + - name: Prune assets of older releases + continue-on-error: true + env: + TOKEN: ${{ secrets.GITEA_TOKEN }} + REPO: ${{ github.repository }} + SERVER: ${{ github.server_url }} + run: | + nix develop --command bash -c ' + API="${SERVER}/api/v1/repos/${REPO}" scripts/prune-release-assets.sh + ' diff --git a/Makefile b/Makefile index 96fed53..49ed1bc 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,7 @@ check: vet test shellcheck scripts/*.sh yamllint .gitea bash tests/workflow-contract.sh + bash tests/prune-release-assets.sh vet: go vet ./... diff --git a/README.md b/README.md index aba79e8..c931e1a 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,9 @@ https://git.westgate.pw/ShadowsOverWestgate/sow-docs). - `ci.yml` — vet, test, shellcheck, yamllint, binary smoke, and cross-build all targets once per pull request. - `build-binaries.yml` — on a `v*` tag, cross-build and upload the binaries, - `SHA256SUMS`, and the wrappers to the Gitea release. + `SHA256SUMS`, and the wrappers to the Gitea release, then delete the assets + of every release except the newest two — Gitea keeps them forever otherwise, + and every binary is reproducible from its tag. - `sync-wrappers.yml` — on a `main` push that touches `wrappers/`, auto-PR the canonical wrappers to the consumer repos in `wrappers/consumers.txt`. Consumer drift checks run after those PRs merge to `main`, not on the PRs diff --git a/scripts/prune-release-assets.sh b/scripts/prune-release-assets.sh new file mode 100755 index 0000000..1644af4 --- /dev/null +++ b/scripts/prune-release-assets.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# Delete the binary assets of every release except the newest KEEP ones. +# +# Gitea has no asset retention, so 9 assets x ~57 MB per tag pile up forever on +# the host disk (sow-tools #52). Nothing consumes an old asset: CI takes the +# Crucible binary from the Nix flake, and every deleted file is reproducible +# from its tag. Tags, source archives, release notes and the releases +# themselves are never touched — only attachments. +# +# Nothing is excluded, SHA256SUMS and the wrappers included: the checksums are +# only meaningful next to the binaries they cover, and the drift-check reads +# the wrappers from the latest release, which always stays complete. +# +# Best-effort by design: a stale asset is cheaper than a blocked release, so +# every API failure is a warning, never a non-zero exit. +# +# Env: +# API repo API base, e.g. https://git.example/api/v1/repos/owner/repo [required] +# TOKEN Gitea token with releases:write [required] +# KEEP how many newest releases stay complete (default 2) +set -uo pipefail + +: "${API:?set API}" +: "${TOKEN:?set TOKEN}" +keep="${KEEP:-2}" +auth="Authorization: token ${TOKEN}" +per_page=50 + +warn() { echo "prune-release-assets: $*" >&2; } + +# Walk every page so an old backlog is cleared, not only the tag that fell out +# of the window on this run. Each page already embeds its releases' assets, so +# no follow-up request per release is needed. +releases='[]' +page=1 +while :; do + body="$(curl -fsS -H "$auth" "${API}/releases?limit=${per_page}&page=${page}&draft=false")" || { + warn "listing releases failed on page ${page}; pruning what was listed so far" + break + } + count="$(jq 'length' <<<"$body")" || { warn "unreadable release page ${page}"; break; } + releases="$(jq -s 'add' <(printf '%s' "$releases") <(printf '%s' "$body"))" + # A short page is the last one; this also stops a server that ignores `page`. + (( count < per_page )) && break + page=$(( page + 1 )) +done + +# Sort here rather than trusting the response order, and skip drafts in case +# the server ignored `draft=false` — a draft must not consume a keep slot and +# strip the binaries off the newest real release. +mapfile -t stale < <(jq -r --argjson keep "$keep" ' + [.[] | select(.draft != true)] + | sort_by(.created_at) | reverse | .[$keep:] + | .[] | "\(.id) \(.assets // [] | map(.id) | join(","))" +' <<<"$releases") + +(( ${#stale[@]} )) || { echo "prune-release-assets: nothing older than the newest ${keep} releases"; exit 0; } + +for line in "${stale[@]}"; do + id="${line%% *}" + assets="${line#* }" + for asset in ${assets//,/ }; do + echo "deleting asset ${asset} of release ${id}" + curl -fsS -X DELETE -H "$auth" "${API}/releases/${id}/assets/${asset}" >/dev/null \ + || warn "deleting asset ${asset} of release ${id} failed" + done +done diff --git a/tests/prune-release-assets.sh b/tests/prune-release-assets.sh new file mode 100755 index 0000000..8e6869c --- /dev/null +++ b/tests/prune-release-assets.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# Contract for scripts/prune-release-assets.sh, driven by a stub curl on PATH: +# only assets of releases outside the keep window go, the window is decided by +# release date rather than response order, drafts never consume a keep slot, +# releases and tags stay, and an API failure never fails the run. +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +script="$repo_root/scripts/prune-release-assets.sh" +tmp="$(mktemp -d)" +trap 'rm -rf "$tmp"' EXIT + +# Stub curl: one short page of releases, deliberately out of date order and +# with a draft that is newer than every published release. Each release embeds +# two assets. Every request is appended to $CALLS for the assertions below. +mkdir -p "$tmp/bin" +cat >"$tmp/bin/curl" <<'STUB' +#!/usr/bin/env bash +url="${!#}" +printf '%s\n' "$*" >>"$CALLS" +rel() { printf '{"id":%s,"created_at":"%s","draft":%s,"assets":[{"id":%s01},{"id":%s02}]}' "$1" "$2" "$3" "$1" "$1"; } +case "$url" in + *releases\?*) + [[ "${FAIL_LIST:-0}" == 1 ]] && exit 22 + printf '[%s,%s,%s,%s,%s]\n' \ + "$(rel 12 2024-01-02T00:00:00Z false)" \ + "$(rel 10 2024-01-04T00:00:00Z false)" \ + "$(rel 9 2024-06-01T00:00:00Z true)" \ + "$(rel 13 2024-01-01T00:00:00Z false)" \ + "$(rel 11 2024-01-03T00:00:00Z false)" + ;; + */assets/*) [[ "${FAIL_DELETE:-0}" == 1 ]] && exit 22 ;; +esac +exit 0 +STUB +chmod +x "$tmp/bin/curl" +export PATH="$tmp/bin:$PATH" API=https://git.example/api/v1/repos/o/r TOKEN=t + +run() { CALLS="$tmp/calls" bash "$script" >"$tmp/out" 2>"$tmp/err"; } + +# Default keep=2: the two newest published releases (10, 11) stay whole, the +# older two (12, 13) lose their assets, and the newer draft is ignored. +: >"$tmp/calls"; run +deletes="$(grep -c -- '-X DELETE' "$tmp/calls" || true)" +[[ "$deletes" == 4 ]] || { echo "expected 4 asset deletes, got $deletes" >&2; exit 1; } +grep -q 'assets/1201' "$tmp/calls" && grep -q 'assets/1302' "$tmp/calls" || { + echo "expected assets of releases 12 and 13 to be deleted" >&2; exit 1; } +if grep -q 'releases/10/assets/\|releases/11/assets/' "$tmp/calls"; then + echo "kept releases lost assets" >&2; exit 1 +fi +if grep -q 'releases/9/assets/' "$tmp/calls"; then + echo "a draft release was pruned" >&2; exit 1 +fi +# A release or tag must never be deleted, only attachments under /assets/. +if grep -- '-X DELETE' "$tmp/calls" | grep -qv '/assets/'; then + echo "a non-asset DELETE was issued" >&2; exit 1 +fi +# A short page ends the sweep: no second page, so a server ignoring `page` +# cannot spin the job until the job timeout. +[[ "$(grep -c 'releases?' "$tmp/calls")" == 1 ]] || { + echo "a short release page did not end the sweep" >&2; exit 1; } + +# KEEP=4 covers every published release: nothing to delete. +: >"$tmp/calls"; KEEP=4 run +if grep -q -- '-X DELETE' "$tmp/calls"; then echo "KEEP=4 still deleted" >&2; exit 1; fi + +# Failures are warnings, never a non-zero exit. +: >"$tmp/calls"; FAIL_DELETE=1 run || { echo "delete failure failed the run" >&2; exit 1; } +grep -q 'failed' "$tmp/err" || { echo "delete failure was not warned about" >&2; exit 1; } + +: >"$tmp/calls"; FAIL_LIST=1 run || { echo "listing failure failed the run" >&2; exit 1; } +grep -q 'listing releases failed' "$tmp/err" || { + echo "listing failure was not warned about" >&2; exit 1; } +if grep -q -- '-X DELETE' "$tmp/calls"; then + echo "deleted assets despite an unreadable release list" >&2; exit 1 +fi + +echo "prune-release-assets: prunes by release date outside the keep window, ignores drafts, never deletes a release, never fails the build" diff --git a/tests/workflow-contract.sh b/tests/workflow-contract.sh old mode 100644 new mode 100755 index f520728..b8c9662 --- a/tests/workflow-contract.sh +++ b/tests/workflow-contract.sh @@ -46,6 +46,8 @@ grep -Fqx ' releases: write' "$release" grep -Fqx ' timeout-minutes: 30' "$release" [[ "$(grep -Fc "$checkout" "$release")" -eq 1 ]] grep -Fq 'secrets.GITEA_TOKEN' "$release" +grep -Fq 'scripts/prune-release-assets.sh' "$release" +grep -Fq 'continue-on-error: true' "$release" if grep -Fq 'secrets.GITHUB_TOKEN' "$release"; then echo "workflow-contract: release uses the GitHub token alias" >&2 exit 1