Compare commits

..
1 Commits
Author SHA1 Message Date
archvillainette f34c25afb0 custom palette taxonomy
test / test (pull_request) Canceled after 0s
build-binaries / build-binaries (pull_request) Canceled after 12s
2026-07-22 17:04:13 +02:00
50 changed files with 164 additions and 3448 deletions
+13 -23
View File
@@ -1,22 +1,25 @@
# Cross-platform Crucible release binaries: a v* tag builds all targets,
# then uploads them, SHA256SUMS, and the canonical wrappers to its Gitea release.
# Cross-platform Crucible binaries (D7 trigger standard):
# PR / push main -> cross-build ALL targets to prove they compile (no publish)
# tag v* -> build all targets + SHA256SUMS, upload to the Gitea release
# Crucible is pure Go (CGO_ENABLED=0), so cross-compiling is a fast loop.
name: build-binaries
on:
pull_request:
push:
branches: [main]
tags: ['v*']
permissions:
code: read
releases: write
paths-ignore:
- "docs/**"
- "README.md"
- "AGENTS.md"
- "LICENSE"
jobs:
build-binaries:
runs-on: nix-docker
timeout-minutes: 30
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- uses: actions/checkout@v4
with: { fetch-depth: 0 }
- name: Cross-build all targets
@@ -42,8 +45,9 @@ jobs:
'
- name: Upload to Gitea release
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
env:
TOKEN: ${{ secrets.GITEA_TOKEN }}
TOKEN: ${{ secrets.GITHUB_TOKEN }}
TAG: ${{ github.ref_name }}
REPO: ${{ github.repository }}
SERVER: ${{ github.server_url }}
@@ -65,17 +69,3 @@ jobs:
"${api}/releases/${id}/assets?name=$(basename "$f")"
done
'
# Gitea has no asset retention (#52): without this every tag keeps its
# ~57 MB binary set forever. Best-effort — a stale asset is cheaper than
# a blocked publish, so a failure here never fails the release.
- name: Prune assets of older releases
continue-on-error: true
env:
TOKEN: ${{ secrets.GITEA_TOKEN }}
REPO: ${{ github.repository }}
SERVER: ${{ github.server_url }}
run: |
nix develop --command bash -c '
API="${SERVER}/api/v1/repos/${REPO}" scripts/prune-release-assets.sh
'
-47
View File
@@ -1,47 +0,0 @@
# Pull-request validation for Crucible. Releases and wrapper synchronization
# have their own narrow workflows because they need tag/main events.
name: ci
on:
pull_request:
permissions: read-all
jobs:
ci:
runs-on: nix-docker
timeout-minutes: 60
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
fetch-depth: 0
- name: vet + test + lint
run: |
nix develop --command bash -c '
set -euo pipefail
go vet ./...
go test ./...
shellcheck scripts/*.sh
yamllint .gitea
'
- name: binary smoke (fail-closed contract)
run: nix develop --command make smoke
- name: Cross-build all targets
run: |
nix develop --command bash -c '
set -euo pipefail
sha="$(git rev-parse --short=12 HEAD)"
ldflags="-s -w -X git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/buildinfo.Version=${sha}"
rm -rf dist && mkdir -p dist
export CGO_ENABLED=0
for target in linux/amd64 linux/arm64 darwin/amd64 darwin/arm64 windows/amd64 windows/arm64; do
os="${target%/*}"; arch="${target#*/}"
ext=""; [ "$os" = windows ] && ext=".exe"
echo "building crucible-${os}-${arch}${ext}"
GOOS="$os" GOARCH="$arch" go build -trimpath -ldflags "$ldflags" \
-o "dist/crucible-${os}-${arch}${ext}" ./cmd/crucible
done
'
+1 -4
View File
@@ -14,14 +14,11 @@ on:
- 'wrappers/crucible.sh'
- 'wrappers/crucible.ps1'
permissions: read-all
jobs:
sync:
runs-on: nix-docker
timeout-minutes: 30
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- uses: actions/checkout@v4
- name: Open sync PRs to consumers
env:
+30
View File
@@ -0,0 +1,30 @@
# Lint + unit tests for the Crucible suite. PR-first: PRs + push to main (D7).
name: test
on:
push:
branches: [main]
paths-ignore:
- "docs/**"
- "README.md"
- "AGENTS.md"
- "LICENSE"
pull_request:
jobs:
test:
runs-on: nix-docker
steps:
- uses: actions/checkout@v4
with: { fetch-depth: 0 }
- name: vet + test + lint
run: |
nix develop --command bash -c '
set -euo pipefail
go vet ./...
go test ./...
shellcheck scripts/*.sh
yamllint .gitea
'
- name: binary smoke (fail-closed contract)
run: nix develop --command make smoke
-18
View File
@@ -87,21 +87,3 @@ make smoke # assert fail-closed contract
## Tests
Tests must survive harmless changes to constants, defaults, wording, ordering, fixture data, and internal implementation details. A test that fails merely because a basic value changed is usually a bad test. Only assert exact values when the value is part of a documented public contract, external protocol, compatibility requirement, security rule, migration, or business rule.
## Agent skills
### Issue tracker
Issues live in Gitea at git.westgate.pw (`ShadowsOverWestgate/sow-tools`), managed with the `tea` CLI. Issues follow ownership — file work in the repo that owns it, not the one you happen to be standing in. See `docs/agents/issue-tracker.md`.
### Triage labels
Default label vocabulary (`needs-triage`, `needs-info`, `ready-for-agent`, `ready-for-human`, `wontfix`). See `docs/agents/triage-labels.md`.
### Domain docs
Single-context: `CONTEXT.md` at the repo root plus `docs/adr/`. See `docs/agents/domain.md`.
### Where work lives
Markdown here is **reference, law, or an ADR — nothing else** (`sow-codebase` ADR-0001). Live work -> wayfinder maps + Gitea issues (closeable, assignable, queryable). Settled decisions -> ADR files in `docs/adr/` (immutable, never closed, only superseded). Standing law -> `DOCTRINE.md` / `AGENTS.md` / `CONTEXT.md`. Current-state reference -> docs describing what the code does now. Everything else — plans, specs, concepts, handoffs, trackers — is process: it belongs in a Gitea issue, not a file. Harvest unfinished intent to an issue before deleting a process doc. `sow-docs` is deprecated and read-only.
-2
View File
@@ -6,8 +6,6 @@ SHELL := bash
check: vet test
shellcheck scripts/*.sh
yamllint .gitea
bash tests/workflow-contract.sh
bash tests/prune-release-assets.sh
vet:
go vet ./...
+5 -9
View File
@@ -19,7 +19,6 @@ Crucible is how the artifact repos turn source into artifacts.
| `crucible-depot` | `crucible depot` | content-addressed depot blob verify/move |
| `crucible-hak` | `crucible hak` | ERF/HAK pack/unpack + hak manifests |
| `crucible-module` | `crucible module` | build/extract/validate/compare the `.mod` |
| `crucible-nwsync` | `crucible nwsync` | NWSync blob emit + manifest assemble |
| `crucible-topdata` | `crucible topdata` | compile 2da/tlk topdata + packages |
| `crucible-wiki` | `crucible wiki` | render + deploy mechanical wiki pages |
@@ -79,16 +78,13 @@ This retires the old habit of checking in `nwn-tool` / `sow-toolkit`.
## CI
PR-first (D7): checks run once on pull requests; the only publish event is a
`v*` tag (see `runbooks/ci-trigger-standard.md` in sow-docs,
PR-first (D7): checks run on pull requests and on push to `main`; the only
publish event is a `v*` tag (see `runbooks/ci-trigger-standard.md` in sow-docs,
https://git.westgate.pw/ShadowsOverWestgate/sow-docs).
- `ci.yml` — vet, test, shellcheck, yamllint, binary smoke, and cross-build all
targets once per pull request.
- `build-binaries.yml` — on a `v*` tag, cross-build and upload the binaries,
`SHA256SUMS`, and the wrappers to the Gitea release, then delete the assets
of every release except the newest two — Gitea keeps them forever otherwise,
and every binary is reproducible from its tag.
- `test.yml` — vet, test, shellcheck, yamllint, binary smoke (PR + main).
- `build-binaries.yml` — cross-build all targets (PR + main); on a `v*` tag, upload
the binaries, `SHA256SUMS`, and the wrappers to the Gitea release.
- `sync-wrappers.yml` — on a `main` push that touches `wrappers/`, auto-PR the
canonical wrappers to the consumer repos in `wrappers/consumers.txt`.
Consumer drift checks run after those PRs merge to `main`, not on the PRs
-11
View File
@@ -1,11 +0,0 @@
// Command crucible-nwsync is the standalone nwsync builder (equivalent to
// `crucible nwsync`). A single-token binary keeps consumer wrapper scripts simple.
package main
import (
"os"
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/dispatch"
)
func main() { os.Exit(dispatch.RunBuilder("nwsync", os.Args[1:])) }
@@ -1,55 +0,0 @@
# ADR-0001: Markdown in this repo is reference, law, or an ADR — nothing else
- **Status:** Accepted
- **Date:** 2026-07-24
- **Origin:** Adopted workspace-wide from `sow-codebase` ADR-0001, which records
the full context (a `docs/` tree that had grown to 434 files, ~27MB, most of
it dead process artifacts). This repo adopts the same law so the rule is
local, not a cross-repo reference.
## Context
Markdown that *claims to describe current state or a plan* rots, because
reality diverges and nobody edits the file. Markdown that claims neither — a
dated, immutable decision record, or a standing rule — does not rot.
Two different things get conflated:
- **Rationale** — why a system is shaped the way it is. Worth keeping.
- **Process artifacts** — plans, specs, concepts, handoffs, trackers. A record
of how work happened, not what is true now.
Left unchecked, every completed effort leaves its planning behind and the repo
accumulates a "historical" pile that agents and humans must route around to
find the few live docs.
## Decision
Markdown may exist in this repo only as one of three genres:
1. **Current-state reference** — describes what the code does now, kept honest
by code review touching it.
2. **Standing law**`DOCTRINE.md`, root and folder-scoped `AGENTS.md`,
`CONTEXT.md`. States rules and vocabulary, not plans.
3. **Immutable decision record** — ADRs under `docs/adr/`. Append-only; never
edited, only superseded by a later ADR that points back.
Anything else — implementation plans, design specs, concepts, handoffs,
progress trackers, scratch — is **process** and does not live in the repo. It
lives in Gitea issues and wayfinder maps, where it can be assigned, closed, and
superseded. Small tasks need no written plan at all.
Deleting a process document is not destroying history: `git log -- <path>`
recovers it. The exception is *unfinished intent* (a design never built, an
open question still wanted) — that is live, not history, and must be harvested
to a Gitea ticket before its file is deleted. A citation from a living doc or
from code must be resolved before the cited file is deleted.
## Consequences
- The documentation map lists only current truth; there is no "historical" pile
to route around.
- No agent, under any plugin or skill, writes process-markdown into the repo.
The rule is plugin-agnostic on purpose — it binds the agent, not a named tool.
- History of deleted process docs is in git; unfinished intent is in the issue
tracker; rationale is in ADRs and law. Each thing has exactly one home.
@@ -1,12 +0,0 @@
# ADR-0002: `sow-tools` becomes the Crucible suite
- **Status:** Accepted
- **Date:** 2026-06-11
- **Migrated from:** `sow-docs` `07-decisions-log.md` entry **D11**, on the
retirement of `sow-docs`. Content is the original decision, unchanged.
- **Decision:** Rename the future toolchain surface to Crucible: one Go module,
multiple `cmd/` binaries, plus a dispatcher so wrapper commands can stay
stable.
- **Rationale:** Depot, HAK, module, topdata, and wiki tooling share internals
but have different command surfaces. One module avoids premature repo splits.
@@ -1,29 +0,0 @@
# ADR-0003: Crucible binary contract: standalone shims, fail-closed scaffold, no committed binaries
- **Status:** Accepted
- **Date:** 2026-06-11
- **Migrated from:** `sow-docs` `07-decisions-log.md` entry **D19**, on the
retirement of `sow-docs`. Content is the original decision, unchanged.
- **Decision:** Phase 5 builds `migration/sow-tools` (Crucible, D11) as a
multi-binary Go scaffold:
- One `crucible` dispatcher plus standalone `crucible-{depot,hak,module,topdata,wiki}`
binaries sharing one registry (`internal/dispatch`). The dispatcher is for
humans/CI; the **standalone shims are canonical for consumers** so wrapper
scripts resolve a single-token command and `"$builder" args` quoting stays
correct.
- Consumer resolution order: explicit env override
(`$SOW_MODULE_BUILD`/`$SOW_TOPDATA_BUILD`/`$CRUCIBLE`) → `crucible-<name>`
legacy `sow-<name>-build`. CI runs inside the pinned `crucible:<sha>` image.
- Every builder is **unwired** in the scaffold and fails closed (exit 70);
it never fakes an artifact. The `internal/` logic is migrated from
`gitea/sow-tools` by the operator at cutover (hard rule: no source
transplant by tooling).
- **Binaries are never committed** — they are CI artifacts / image layers.
Retires the committed `nwn-tool` / `tools/sow-toolkit`.
- Builders take `NWN_ROOT` only via explicit env/flag; no `$HOME` defaulting.
- **Rationale:** Locks the command surface, container, and CI shape so migrated
logic drops into a stable frame; aligns the three artifact repos onto one
Crucible contract while keeping back-compat with the pre-D11 skeletons.
- **Image name:** `registry.westgate.pw/deployment/crucible:<git-sha>` (the bare
`crucible:<sha>` is the local/dev tag).
@@ -1,24 +0,0 @@
# ADR-0004: Nix-built OCI images start server-side with Crucible; local Anvil images deferred
- **Status:** Accepted
- **Date:** 2026-06-12
- **Migrated from:** `sow-docs` `07-decisions-log.md` entry **D29**, on the
retirement of `sow-docs`. Content is the original decision, unchanged.
- **Decision:** Introduce Nix-built OCI images as an artifact-production option,
starting with `sow-tools`/Crucible in server-side CI. `sow-tools` will grow a
Nix-built `crucible` package plus `crucible-image`; PR CI builds/smokes it and
`main` publishes the normal pinned OCI image
`registry.westgate.pw/deployment/crucible:<sha>`. `docker/Dockerfile` stays during
parity and as the client-compatible fallback.
- **Local rule:** Nix users may get optional wrappers that provide tools and
call the existing Dockerfiles for local Anvil/NWServer testing. A true
`nix build .#nwserver-test-image` is explicitly deferred until real
`sow-codebase/src` exists and the Dockerfile image is proven.
- **Rationale:** Server-side image build shape is harder to change once deploy
promotion and registry gates are active, so prove Nix image builds before
cutover. Crucible is the lowest-risk pilot because it is a Go toolchain image;
NodeBB and Anvil/NWServer depend more heavily on upstream container filesystem
semantics.
- **Non-goals:** no source builds in `sow-platform`; no `nix-sidecar` or
Kubernetes-style runtime cache layer; no removal of client Dockerfiles.
-51
View File
@@ -1,51 +0,0 @@
# Domain Docs
How the engineering skills should consume this repo's domain documentation when exploring the codebase.
## Before exploring, read these
- **`CONTEXT.md`** at the repo root, or
- **`CONTEXT-MAP.md`** at the repo root if it exists — it points at one `CONTEXT.md` per context. Read each one relevant to the topic.
- **`docs/adr/`** — read ADRs that touch the area you're about to work in. In multi-context repos, also check `src/<context>/docs/adr/` for context-scoped decisions.
If any of these files don't exist, **proceed silently**. Don't flag their absence; don't suggest creating them upfront. The `/domain-modeling` skill (reached via `/grill-with-docs` and `/improve-codebase-architecture`) creates them lazily when terms or decisions actually get resolved.
## File structure
Single-context repo (most repos):
```
/
├── CONTEXT.md
├── docs/adr/
│ ├── 0001-event-sourced-orders.md
│ └── 0002-postgres-for-write-model.md
└── src/
```
Multi-context repo (presence of `CONTEXT-MAP.md` at the root):
```
/
├── CONTEXT-MAP.md
├── docs/adr/ ← system-wide decisions
└── src/
├── ordering/
│ ├── CONTEXT.md
│ └── docs/adr/ ← context-specific decisions
└── billing/
├── CONTEXT.md
└── docs/adr/
```
## Use the glossary's vocabulary
When your output names a domain concept (in an issue title, a refactor proposal, a hypothesis, a test name), use the term as defined in `CONTEXT.md`. Don't drift to synonyms the glossary explicitly avoids.
If the concept you need isn't in the glossary yet, that's a signal — either you're inventing language the project doesn't use (reconsider) or there's a real gap (note it for `/domain-modeling`).
## Flag ADR conflicts
If your output contradicts an existing ADR, surface it explicitly rather than silently overriding:
> _Contradicts ADR-0007 (event-sourced orders) — but worth reopening because…_
-88
View File
@@ -1,88 +0,0 @@
# Issue tracker: Gitea (via tea)
Issues for this repo live in Gitea at `git.westgate.pw`, repo
`ShadowsOverWestgate/sow-tools`. Use the `tea` CLI for all operations —
`gh` does not work here. For anything `tea` lacks a subcommand for, use
`tea api <endpoint>` (Gitea's API mirrors GitHub's closely).
Authenticate with your own `tea` login (`tea login add`); never commit tokens
or tea config into this repo. Note Gitea blocks self-review, so approving a PR
needs a different account than the one that opened it.
## Where work lives
Markdown in this repo is **reference, law, or an ADR — nothing else**
(`sow-codebase` ADR-0001). Four homes, no overlap:
- **Live work** → wayfinder maps + Gitea issues. Closeable, assignable,
queryable. Never a markdown file.
- **Settled decisions** → ADR files in `docs/adr/`. Immutable, findable, never
closed, never edited — only superseded by a later ADR pointing back. When an
issue ends in a durable decision, write the ADR, then close the issue
pointing at it. Decisions spanning repos go to `sow-platform/docs/adr/`.
- **Standing law** → `DOCTRINE.md`, `AGENTS.md`, `CONTEXT.md`. Rules that are
always true and vocabulary everyone shares — not the record of one decision.
- **Current-state reference** → docs describing what the code does now, kept
honest by review touching them.
Anything else — implementation plans, design specs, concepts, handoffs,
progress trackers, scratch — is **process**. It does not live in this repo. It
lives in a Gitea issue or a wayfinder map, where it can be assigned, closed,
and superseded. Small tasks need no written plan at all.
Deleting a process doc is not destroying history — `git log -- <path>` recovers
it. But *unfinished intent* (a design never built, an open question still
wanted) is live, not history: harvest it to a Gitea issue before deleting.
`sow-docs` is deprecated and read-only. Never add to it, never send work there.
## Which repo gets the issue
Issues follow ownership. File the issue in the repo that **owns the work**
see the Repo/Owns/Produces table in the workspace root `AGENTS.md`. Standing
in one repo is not a reason to file there.
If work spans repos, file it in the repo that owns the *outcome* and reference
the others from it. A wrong-repo issue is a routing bug, not a filing
preference — move it.
## Conventions
- **Create an issue**: `tea issues create --title "..." --description "..."`
- **Read an issue**: `tea issues <number>` and
`tea api repos/ShadowsOverWestgate/sow-tools/issues/<number>/comments` for comments.
- **List issues**: `tea issues list --state open` (add `--labels ...` to filter).
- **Comment**: `tea comment <number> "..." </dev/null`
Always redirect stdin. `tea` reads stdin to EOF and appends it to the body,
so any non-interactive shell (every agent) hangs forever without
`</dev/null`. Same trap on `tea issues create --description` and
`tea pr create`.
- **Apply / remove labels**: `tea api --method PATCH` on the issue, or
`tea api repos/ShadowsOverWestgate/sow-tools/issues/<number>/labels` endpoints.
- **Close**: `tea issues close <number>`
`tea` infers the repo from the git remote when run inside the clone.
Gitea shares one number space across issues and PRs.
## Pull requests as a triage surface
**PRs as a request surface: no.**
## When a skill says "publish to the issue tracker"
Create a Gitea issue with `tea issues create`.
## When a skill says "fetch the relevant ticket"
Run `tea issues <number>` plus the comments API call above.
## Wayfinding operations
Used by `/wayfinder`. The **map** is a single issue with **child** issues as tickets.
- **Map**: a single issue labelled `wayfinder:map`, holding the Notes / Decisions-so-far / Fog body. `tea issues create --title "..." --description "..." --labels wayfinder:map`.
- **Child ticket**: this Gitea instance (v1.27) has no native sub-issue hierarchy, so a child issue carries `Part of #<map>` at the top of its description, and is also added to a task list in the map body. Labels: `wayfinder:<type>` (`research`/`prototype`/`grilling`/`task`). Once claimed, the ticket is assigned to the driving dev.
- **Blocking**: Gitea's **native dependencies API** — the canonical, UI-visible representation (shows as "Depends on" / "Blocks" on the issue page). Add an edge with `tea api -X POST repos/ShadowsOverWestgate/sow-tools/issues/<child>/dependencies -f owner=ShadowsOverWestgate -f repo=sow-tools -F index=<blocker>`, where `<blocker>` is the blocker's issue **index** (its `#number` — Gitea's dependency API takes the index directly, unlike GitHub's numeric database id). Check status with `tea api repos/ShadowsOverWestgate/sow-tools/issues/<child>/dependencies` (GET) — a ticket is unblocked when every returned issue's `state` is `closed`.
- **Frontier query**: list the map's open children (`tea issues list --state open`, keep the ones whose description contains `Part of #<map>`), drop any with an open dependency (per the GET above) or an assignee; first in map order wins.
- **Claim**: `tea issues edit <n> --add-assignees <username>` — the session's first write.
- **Resolve**: `tea comment <n> "<answer>" </dev/null`, then `tea issues close <n>`, then append a context pointer (gist + link) to the map's Decisions-so-far.
-15
View File
@@ -1,15 +0,0 @@
# Triage Labels
The skills speak in terms of five canonical triage roles. This file maps those roles to the actual label strings used in this repo's issue tracker (Gitea — see `issue-tracker.md` for how to apply labels with `tea`).
| Label in mattpocock/skills | Label in our tracker | Meaning |
| -------------------------- | -------------------- | ---------------------------------------- |
| `needs-triage` | `needs-triage` | Maintainer needs to evaluate this issue |
| `needs-info` | `needs-info` | Waiting on reporter for more information |
| `ready-for-agent` | `ready-for-agent` | Fully specified, ready for an AFK agent |
| `ready-for-human` | `ready-for-human` | Requires human implementation |
| `wontfix` | `wontfix` | Will not be actioned |
When a skill mentions a role (e.g. "apply the AFK-ready triage label"), use the corresponding label string from this table.
Edit the right-hand column to match whatever vocabulary you actually use.
+2 -40
View File
@@ -29,46 +29,9 @@ aliases.
| `assets` | `fix-mdl` | Lowercase `.mdl` names and rewrite model identity to match. |
| `assets` | `check-dupes` | Report runtime-name (basename) collisions across dirs. |
| `assets` | `clean-dupes` | Delete clean-tree files whose basename collides with primary. |
| `depot` | `status` | Report referenced-vs-present drift against a backend. |
| `depot` | `push` | Upload referenced-but-absent blobs from a local depot. |
| `depot` | `verify` | Existence sweep plus sampled download re-hash. |
| `depot` | `get` | Fetch one blob with sha re-verify. |
| `depot` | `pull` | Incremental verified pull of every referenced blob. |
| `nwsync` | `emit` | Explode one artifact into NWSync blobs plus its own NSYM manifest. |
| `nwsync` | `assemble` | Merge per-artifact NSYM manifests into one merged manifest. |
`depot status` and `depot get` pick their backend either with `--out DIR`, a
depot tree on disk, or with `--target bunny|cdn`, a remote backend. The two
flags are mutually exclusive.
`nwsync emit` runs where an artifact is born (a `.hak`/`.erf`, or a loose file
such as the TLK); `nwsync assemble` runs at module release and reads only the
small per-artifact indexes. Both take **depot keys**: an artifact's index lives
beside the artifact itself with the extension replaced, so `emit` and
`assemble` agree on where it is without being told.
```
nwsync emit [--as NAME] [--out DIR] <artifact-key> <file>
nwsync assemble --group-id N [--tlk-key KEY] [--out DIR] <artifact-key>...
```
Both verbs upload by default; nothing bulky is ever written to the runner's
disk. `--out DIR` writes a local repository tree instead, which is the
conformance path against upstream `nwn_nwsync_write`. The zone comes from
`NWSYNC_STORAGE_ZONE` and `NWSYNC_STORAGE_PASSWORD`, with the host from
`BUNNY_STORAGE_HOST` — NWSync data is a separate zone from the asset depot.
`assemble`'s artifact keys are in `Mod_HakList` order, highest priority first: a
resref in more than one artifact resolves to the earliest one, the way the game
resolves it. `--tlk-key` has its own slot because the TLK shadows nothing.
`--group-id` is per channel — 1 is current, 2 is testing, and 0 leaves the field
out of the sidecar.
`emit` uploads blobs first and the index last, so the presence of an index is
the publication marker: an artifact whose emit died halfway leaves real blobs in
the zone and no index. Blob names are content hashes, so re-running skips
whatever already landed, and `assemble` fails closed on an artifact with no
index rather than publishing a manifest that is missing a hak.
`crucible-depot` remains registered but unwired. It fails closed with exit `70`
and never emits placeholder artifacts.
## Hidden compatibility aliases
@@ -86,7 +49,6 @@ but omitted from routine help and the interactive menu:
| `topdata` | `build-top-package` | `build-top-package` |
| `topdata` | `compare-topdata` | `compare-topdata` |
| `topdata` | `convert-topdata` | `convert-topdata` |
| `depot` | `--target local` | `--out $DEPOT_DIR` on `status`/`get` |
| `wiki` | `build-wiki` | `build-wiki` |
| `wiki` | `deploy-wiki` | `deploy-wiki` |
@@ -126,9 +126,8 @@ exactly as the reference does:
- Beamdog install dirs.
- `--nwn <install>` overrides (path to the install root or directly to the
binary).
- **Headless.** The engine is a GUI binary. Require `xvfb-run` and wrap the call
regardless of the caller's `DISPLAY` so compilation never opens the client
UI; fail before invoking the engine when `xvfb-run` is unavailable:
- **Headless.** The engine is a GUI binary. If there is no `DISPLAY` and
`xvfb-run` is present, wrap the call:
`xvfb-run -a --server-args=-screen 0 1024x768x24 nwmain-linux compilemodel <stem>`.
- **Per model (one at a time — the engine's `development/` and `modelcompiler/`
folders are flat and single-slot):**
+1 -2
View File
@@ -22,13 +22,12 @@
pname = "crucible";
inherit version;
src = ./.;
vendorHash = "sha256-0I8j7On9YGD2GK9xbj/KkgBrlkMJ6Y6XQv+KCLTgBBU=";
vendorHash = "sha256-hm6mrNAtXv0LidzHUfz4eukTFZouizGtxkZ8gKJFUVI=";
subPackages = [
"cmd/crucible"
"cmd/crucible-depot"
"cmd/crucible-hak"
"cmd/crucible-module"
"cmd/crucible-nwsync"
"cmd/crucible-topdata"
"cmd/crucible-wiki"
];
-2
View File
@@ -6,5 +6,3 @@ require (
golang.org/x/text v0.35.0
gopkg.in/yaml.v3 v3.0.1
)
require github.com/klauspost/compress v1.19.1
-2
View File
@@ -1,5 +1,3 @@
github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk=
github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8=
golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+6 -6
View File
@@ -40,13 +40,13 @@ func runCompile(args []string, stdout, stderr io.Writer, getenv func(string) str
}
binDir := filepath.Dir(nwmain)
// Always use a virtual X so compilation never opens the client UI.
xvfb := look("xvfb-run")
if xvfb == "" {
fmt.Fprintln(stderr, "assets compile: xvfb-run not found — install it to run the NWN model compiler headlessly")
return exitTool
// Headless wrap: no DISPLAY + xvfb-run present -> run under a virtual X.
var wrap []string
if getenv("DISPLAY") == "" {
if xvfb := look("xvfb-run"); xvfb != "" {
wrap = []string{xvfb, "-a", "--server-args=-screen 0 1024x768x24"}
}
}
wrap := []string{xvfb, "-a", "--server-args=-screen 0 1024x768x24"}
files, err := walk(dirs, mdlExt, !*nonRecursive)
if err != nil {
+2 -63
View File
@@ -4,7 +4,6 @@ import (
"bytes"
"os"
"path/filepath"
"strings"
"testing"
)
@@ -24,19 +23,13 @@ func TestCompileDrivesEngineAndReplacesInPlace(t *testing.T) {
if err := os.WriteFile(nwmain, []byte("#!/bin/sh\n"), 0o755); err != nil {
t.Fatal(err)
}
xvfbDir := t.TempDir()
xvfb := filepath.Join(xvfbDir, "xvfb-run")
if err := os.WriteFile(xvfb, []byte("#!/bin/sh\n"), 0o755); err != nil {
t.Fatal(err)
}
t.Setenv("PATH", xvfbDir+string(os.PathListSeparator)+os.Getenv("PATH"))
getenv := func(k string) string {
switch k {
case "HOME":
return home
case "DISPLAY":
return ":0" // a desktop display must not make compilation interactive
return ":0" // pretend a display exists so no xvfb wrap is needed
}
return ""
}
@@ -45,11 +38,7 @@ func TestCompileDrivesEngineAndReplacesInPlace(t *testing.T) {
orig := runner
defer func() { runner = orig }()
runner = func(dir string, env []string, name string, args ...string) ([]byte, error) {
if name != xvfb || len(args) != 5 || args[0] != "-a" ||
args[1] != "--server-args=-screen 0 1024x768x24" || args[2] != nwmain ||
args[3] != "compilemodel" {
t.Fatalf("engine command = %q %q, want xvfb-run wrapping nwmain", name, args)
}
// args: compilemodel <stem>
stem := args[len(args)-1]
compiled := filepath.Join(mc, stem+".mdl")
return nil, os.WriteFile(compiled, []byte("\x00\x00compiled"), 0o644)
@@ -88,11 +77,6 @@ func TestCompileAbortsOnNameMismatch(t *testing.T) {
if err := os.WriteFile(nwmain, []byte("#!/bin/sh\n"), 0o755); err != nil {
t.Fatal(err)
}
xvfbDir := t.TempDir()
if err := os.WriteFile(filepath.Join(xvfbDir, "xvfb-run"), []byte("#!/bin/sh\n"), 0o755); err != nil {
t.Fatal(err)
}
t.Setenv("PATH", xvfbDir+string(os.PathListSeparator)+os.Getenv("PATH"))
getenv := func(k string) string {
if k == "HOME" {
return home
@@ -124,48 +108,3 @@ func TestCompileAbortsOnNameMismatch(t *testing.T) {
t.Fatal("engine should not run for a name-mismatched model")
}
}
func TestCompileFailsClosedWithoutXvfb(t *testing.T) {
home := t.TempDir()
userData := filepath.Join(home, ".local", "share", "Neverwinter Nights")
for _, d := range []string{filepath.Join(userData, "development"), filepath.Join(userData, "modelcompiler")} {
if err := os.MkdirAll(d, 0o755); err != nil {
t.Fatal(err)
}
}
nwmain := filepath.Join(t.TempDir(), "nwmain-linux")
if err := os.WriteFile(nwmain, []byte("#!/bin/sh\n"), 0o755); err != nil {
t.Fatal(err)
}
t.Setenv("PATH", t.TempDir())
getenv := func(k string) string {
if k == "HOME" {
return home
}
return ""
}
orig := runner
defer func() { runner = orig }()
engineCalled := false
runner = func(string, []string, string, ...string) ([]byte, error) {
engineCalled = true
return nil, nil
}
srcDir := t.TempDir()
if err := os.WriteFile(filepath.Join(srcDir, "foo.mdl"),
[]byte("newmodel foo\nbeginmodelgeom foo\n node dummy foo\n parent null\n endnode\nendmodelgeom foo\ndonemodel foo\n"), 0o644); err != nil {
t.Fatal(err)
}
var stdout, stderr bytes.Buffer
if code := runCompile([]string{"--nwn", nwmain, srcDir}, &stdout, &stderr, getenv); code != exitTool {
t.Fatalf("compile exit = %d, want %d\n%s", code, exitTool, stderr.String())
}
if engineCalled {
t.Fatal("engine must not run without xvfb-run")
}
if !strings.Contains(stderr.String(), "xvfb-run") {
t.Fatalf("missing actionable xvfb-run error: %s", stderr.String())
}
}
-121
View File
@@ -1,121 +0,0 @@
package depot
import (
"context"
"errors"
"fmt"
"io"
"net/http"
"os"
"strings"
)
// KeyStore is the zone addressed by object key rather than by depot sha. The
// depot names every object after the sha256 of its contents; NWSync does not —
// a blob is named after the sha1 of its *uncompressed* bytes while the body
// uploaded is the compressed form, and a per-artifact index is named after its
// artifact. Both addressing modes want the same transport, retry and probe
// discipline, so the sha-addressed Backend rides on this rather than the other
// way round.
type KeyStore interface {
// ProbeKey returns the existence state of one key. transient=true means a
// retry might change the answer — never read it as "missing, re-upload".
ProbeKey(ctx context.Context, key string) (state ProbeState, transient bool, err error)
// PutReader uploads size bytes read from r to key. checksum is the
// uppercase hex sha256 of those bytes, which Bunny verifies server-side.
PutReader(ctx context.Context, key string, r io.Reader, size int64, checksum string) error
// GetKey fetches the whole object at key. Small objects only — it holds
// the body in memory and does no hash check, because a key is not always
// a content hash.
GetKey(ctx context.Context, key string) ([]byte, error)
}
// NewKeyStore returns a KeyStore for cfg's storage zone. Fails closed on a
// missing host or read key, matching NewBackend.
func NewKeyStore(cfg Config) (KeyStore, error) {
if cfg.StorageHost == "" {
return nil, errors.New("storage backend requires a storage host")
}
if cfg.StorageZone == "" {
return nil, errors.New("storage backend requires a storage zone")
}
if cfg.ReadKey == "" {
return nil, errors.New("storage backend requires a read key")
}
return &httpBackend{name: "bunny", client: newHTTPClient(cfg), cfg: cfg}, nil
}
// keyURL is the storage URL of one object key.
func (b *httpBackend) keyURL(key string) string {
host := b.cfg.StorageHost
// StorageHost is normally a bare host ("storage.bunnycdn.com"); allow a
// full scheme (used by tests against httptest.NewServer) to pass through
// unchanged.
if !strings.Contains(host, "://") {
host = "https://" + host
}
return fmt.Sprintf("%s/%s/%s", strings.TrimSuffix(host, "/"), b.cfg.StorageZone, key)
}
func (b *httpBackend) ProbeKey(ctx context.Context, key string) (ProbeState, bool, error) {
return b.rangeProbe(ctx, b.keyURL(key), map[string]string{"AccessKey": b.cfg.ReadKey})
}
func (b *httpBackend) PutReader(ctx context.Context, key string, r io.Reader, size int64, checksum string) error {
if b.name == "cdn" {
return errors.New("cdn backend is read-only")
}
if b.cfg.WriteKey == "" {
return errors.New("storage backend requires a write key to write")
}
req, err := http.NewRequestWithContext(ctx, http.MethodPut, b.keyURL(key), r)
if err != nil {
return err
}
req.ContentLength = size
req.Header.Set("AccessKey", b.cfg.WriteKey)
// Bunny defines Checksum as sha256 of the body and rejects a mismatch, so
// this is server-side integrity checking, not decoration.
req.Header.Set("Checksum", strings.ToUpper(checksum))
resp, err := b.client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
_, _ = io.Copy(io.Discard, resp.Body)
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return fmt.Errorf("put %s: unexpected status %d", key, resp.StatusCode)
}
return nil
}
func (b *httpBackend) GetKey(ctx context.Context, key string) ([]byte, error) {
resp, err := b.get(ctx, b.keyURL(key), map[string]string{"AccessKey": b.cfg.ReadKey})
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
_, _ = io.Copy(io.Discard, resp.Body)
return nil, fmt.Errorf("get %s: unexpected status %d", key, resp.StatusCode)
}
return io.ReadAll(resp.Body)
}
// putFile uploads the file at src to key, streaming it. checksum is the
// uppercase hex sha256 of the file's bytes.
func (b *httpBackend) putFile(ctx context.Context, key, src, checksum string) error {
f, err := os.Open(src)
if err != nil {
return err
}
defer f.Close()
info, err := f.Stat()
if err != nil {
return err
}
return b.PutReader(ctx, key, f, info.Size(), checksum)
}
+36 -4
View File
@@ -10,6 +10,7 @@ import (
"net/http"
"os"
"path/filepath"
"strings"
"time"
)
@@ -70,7 +71,16 @@ type httpBackend struct {
func (b *httpBackend) Name() string { return b.name }
func (b *httpBackend) storageURL(sha string) string { return b.keyURL(BlobKey(sha)) }
func (b *httpBackend) storageURL(sha string) string {
host := b.cfg.StorageHost
// StorageHost is normally a bare host ("storage.bunnycdn.com"); allow a
// full scheme (used by tests against httptest.NewServer) to pass through
// unchanged.
if strings.Contains(host, "://") {
return fmt.Sprintf("%s/%s/%s", strings.TrimSuffix(host, "/"), b.cfg.StorageZone, BlobKey(sha))
}
return fmt.Sprintf("https://%s/%s/%s", host, b.cfg.StorageZone, BlobKey(sha))
}
func (b *httpBackend) cdnURL(sha string) string {
return fmt.Sprintf("%s/%s", b.cfg.CDNBase, BlobKey(sha))
@@ -130,8 +140,7 @@ func (b *httpBackend) Probe(ctx context.Context, sha string) (ProbeState, bool,
return b.rangeProbe(ctx, b.storageURL(sha), map[string]string{"AccessKey": b.cfg.ReadKey})
}
// Put uploads src for sha. cdn is read-only. A depot object is named after the
// sha256 of its own bytes, so the key's sha doubles as the Checksum header.
// Put uploads src for sha. cdn is read-only.
func (b *httpBackend) Put(ctx context.Context, sha, src string) error {
if b.name == "cdn" {
return errors.New("cdn backend is read-only")
@@ -148,7 +157,30 @@ func (b *httpBackend) Put(ctx context.Context, sha, src string) error {
return nil
}
return b.putFile(ctx, BlobKey(sha), src, sha)
f, err := os.Open(src)
if err != nil {
return err
}
defer f.Close()
req, err := http.NewRequestWithContext(ctx, http.MethodPut, b.storageURL(sha), f)
if err != nil {
return err
}
req.Header.Set("AccessKey", b.cfg.WriteKey)
req.Header.Set("Checksum", strings.ToUpper(sha))
resp, err := b.client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
_, _ = io.Copy(io.Discard, resp.Body)
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return fmt.Errorf("bunny put %s: unexpected status %d", sha, resp.StatusCode)
}
return nil
}
// Get fetches sha into dest via temp file + rename, re-hashing and deleting
+28 -46
View File
@@ -11,7 +11,6 @@ import (
"os"
"path/filepath"
"sort"
"strings"
"sync"
"time"
)
@@ -52,15 +51,13 @@ func Run(args []string, stdout, stderr io.Writer, getenv func(string) string) in
func printRunUsage(w io.Writer) {
fmt.Fprint(w, `usage:
depot status [--manifests DIR] --target bunny|cdn | --out DIR
depot push [--manifests DIR] --source LOCAL_DEPOT --target bunny
depot verify [--manifests DIR] --target bunny|cdn [--sample N]
depot get <sha> <dest> --target cdn|bunny | --out DIR
depot pull [--manifests DIR] --dest DIR --target cdn|bunny
depot status [--manifests DIR] [--source LOCAL_DEPOT] --target bunny|cdn|local
depot push [--manifests DIR] --source LOCAL_DEPOT --target bunny
depot verify [--manifests DIR] --target bunny|cdn [--sample N]
depot get <sha> <dest> --target cdn|bunny|local
depot pull [--manifests DIR] --dest DIR --target cdn|bunny
--out DIR reads and writes a depot tree on disk at DIR instead of a remote
backend. --target names a remote backend only; the two are mutually exclusive.
--target local is a deprecated alias for --out $DEPOT_DIR.
--target local uses the DEPOT_DIR environment variable as the local root.
`)
}
@@ -68,27 +65,16 @@ backend. --target names a remote backend only; the two are mutually exclusive.
// than internal/backend failures (exit 70).
var errUsage = errors.New("usage error")
// resolveBackend picks the backend for a command that can work either against a
// depot tree on disk (--out DIR) or a remote backend (--target bunny|cdn).
// "--target local" stays as a deprecated alias for --out $DEPOT_DIR so older
// callers keep working.
func resolveBackend(out, target string, getenv func(string) string, cfg Config) (Backend, error) {
switch {
case out != "" && target != "":
return nil, fmt.Errorf("--out and --target are mutually exclusive: %w", errUsage)
case out != "":
return NewBackend("local", out, cfg)
case target == "local":
root := getenv("DEPOT_DIR")
// resolveBackend builds the named backend, resolving "local" against DEPOT_DIR.
func resolveBackend(target string, getenv func(string) string, cfg Config) (Backend, error) {
root := ""
if target == "local" {
root = getenv("DEPOT_DIR")
if root == "" {
return nil, fmt.Errorf("--target local requires DEPOT_DIR to be set (prefer --out DIR): %w", errUsage)
return nil, fmt.Errorf("--target local requires DEPOT_DIR to be set: %w", errUsage)
}
return NewBackend("local", root, cfg)
case target == "":
return nil, fmt.Errorf("--out DIR or --target bunny|cdn is required: %w", errUsage)
default:
return NewBackend(target, "", cfg)
}
return NewBackend(target, root, cfg)
}
// backendErrExit maps a resolveBackend error onto the exit contract.
@@ -133,18 +119,18 @@ func runStatus(args []string, stdout, stderr io.Writer, getenv func(string) stri
fs := flag.NewFlagSet("status", flag.ContinueOnError)
fs.SetOutput(stderr)
manifests := fs.String("manifests", "assets", "directory of *.yml manifests")
source := fs.String("source", "", "deprecated and ignored (use --out DIR for a depot tree on disk)")
out := fs.String("out", "", "depot tree on disk to read instead of a remote backend")
target := fs.String("target", "", "bunny|cdn")
_ = fs.String("source", "", "local depot root (unused for status target=local; see DEPOT_DIR)")
target := fs.String("target", "", "bunny|cdn|local")
if err := fs.Parse(args); err != nil {
return exitUsage
}
if *source != "" {
fmt.Fprintln(stderr, "depot status: --source is ignored; use --out DIR")
if *target == "" {
fmt.Fprintln(stderr, "depot status: --target is required")
return exitUsage
}
cfg := LoadConfig(getenv)
backend, err := resolveBackend(*out, *target, getenv, cfg)
backend, err := resolveBackend(*target, getenv, cfg)
if err != nil {
fmt.Fprintln(stderr, "depot status:", err)
return backendErrExit(err)
@@ -323,21 +309,13 @@ func runVerify(args []string, stdout, stderr io.Writer, getenv func(string) stri
func runGet(args []string, stdout, stderr io.Writer, getenv func(string) string) int {
fs := flag.NewFlagSet("get", flag.ContinueOnError)
fs.SetOutput(stderr)
out := fs.String("out", "", "depot tree on disk to read instead of a remote backend")
target := fs.String("target", "", "cdn|bunny")
// Positionals come first in the documented usage, and Go's flag package
// stops parsing at the first one, so split them off by hand.
split := 0
for split < len(args) && !strings.HasPrefix(args[split], "-") {
split++
}
positional := args[:split]
if err := fs.Parse(args[split:]); err != nil {
target := fs.String("target", "", "cdn|bunny|local")
if err := fs.Parse(args); err != nil {
return exitUsage
}
positional = append(positional, fs.Args()...)
positional := fs.Args()
if len(positional) != 2 {
fmt.Fprintln(stderr, "depot get: usage: depot get <sha> <dest> --target cdn|bunny | --out DIR")
fmt.Fprintln(stderr, "depot get: usage: depot get <sha> <dest> --target cdn|bunny|local")
return exitUsage
}
sha, dest := positional[0], positional[1]
@@ -345,9 +323,13 @@ func runGet(args []string, stdout, stderr io.Writer, getenv func(string) string)
fmt.Fprintf(stderr, "depot get: invalid sha %q\n", sha)
return exitUsage
}
if *target == "" {
fmt.Fprintln(stderr, "depot get: --target is required")
return exitUsage
}
cfg := LoadConfig(getenv)
backend, err := resolveBackend(*out, *target, getenv, cfg)
backend, err := resolveBackend(*target, getenv, cfg)
if err != nil {
fmt.Fprintln(stderr, "depot get:", err)
return backendErrExit(err)
-96
View File
@@ -54,91 +54,6 @@ func TestRunUsage(t *testing.T) {
})
}
func TestOutFlag(t *testing.T) {
t.Run("status --out reads the given tree", func(t *testing.T) {
sha := shaOf("out-blob")
manifests := t.TempDir()
writeManifest(t, manifests, sha, 8)
depotDir := t.TempDir()
writeBlob(t, depotDir, sha, "out-blob")
var out, errb bytes.Buffer
code := Run([]string{"status", "--manifests", manifests, "--out", depotDir}, &out, &errb, testGetenv(nil))
if code != 0 {
t.Fatalf("expected 0, got %d (stdout=%s stderr=%s)", code, out.String(), errb.String())
}
if !bytesContains(out.String(), "present=1") {
t.Fatalf("expected present=1, got %s", out.String())
}
})
t.Run("get --out fetches from the given tree", func(t *testing.T) {
sha := shaOf("get-blob")
depotDir := t.TempDir()
writeBlob(t, depotDir, sha, "get-blob")
dest := filepath.Join(t.TempDir(), "fetched")
var out, errb bytes.Buffer
code := Run([]string{"get", sha, dest, "--out", depotDir}, &out, &errb, testGetenv(nil))
if code != 0 {
t.Fatalf("expected 0, got %d (stderr=%s)", code, errb.String())
}
got, err := os.ReadFile(dest)
if err != nil {
t.Fatal(err)
}
if string(got) != "get-blob" {
t.Fatalf("got %q", got)
}
})
t.Run("--out with any --target is rejected", func(t *testing.T) {
for _, target := range []string{"bunny", "cdn", "local"} {
var out, errb bytes.Buffer
code := Run([]string{"status", "--manifests", t.TempDir(), "--out", t.TempDir(), "--target", target}, &out, &errb, testGetenv(nil))
if code != 64 {
t.Fatalf("--target %s: expected 64, got %d (stderr=%s)", target, code, errb.String())
}
}
})
t.Run("get accepts flags before the positionals", func(t *testing.T) {
sha := shaOf("flags-first-blob")
depotDir := t.TempDir()
writeBlob(t, depotDir, sha, "flags-first-blob")
dest := filepath.Join(t.TempDir(), "fetched")
var out, errb bytes.Buffer
code := Run([]string{"get", "--out", depotDir, sha, dest}, &out, &errb, testGetenv(nil))
if code != 0 {
t.Fatalf("expected 0, got %d (stderr=%s)", code, errb.String())
}
})
t.Run("neither --out nor --target is rejected", func(t *testing.T) {
var out, errb bytes.Buffer
code := Run([]string{"status", "--manifests", t.TempDir()}, &out, &errb, testGetenv(nil))
if code != 64 {
t.Fatalf("expected 64, got %d (stderr=%s)", code, errb.String())
}
})
t.Run("--target local still works as a hidden alias", func(t *testing.T) {
sha := shaOf("alias-blob")
manifests := t.TempDir()
writeManifest(t, manifests, sha, 11)
depotDir := t.TempDir()
writeBlob(t, depotDir, sha, "alias-blob")
var out, errb bytes.Buffer
code := Run([]string{"status", "--manifests", manifests, "--target", "local"}, &out, &errb,
testGetenv(map[string]string{"DEPOT_DIR": depotDir}))
if code != 0 {
t.Fatalf("expected 0, got %d (stdout=%s stderr=%s)", code, out.String(), errb.String())
}
})
}
func TestGetInvalidSHA(t *testing.T) {
var out, errb bytes.Buffer
dir := t.TempDir()
@@ -257,17 +172,6 @@ func bytesContains(s, substr string) bool {
return bytes.Contains([]byte(s), []byte(substr))
}
func writeBlob(t *testing.T, root, sha, content string) {
t.Helper()
path := filepath.Join(root, BlobKey(sha))
if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, []byte(content), 0644); err != nil {
t.Fatal(err)
}
}
func writeManifest(t *testing.T, dir, sha string, size int64) {
t.Helper()
content := fmt.Sprintf("assets:\n - path: foo\n sha256: %s\n size: %d\n", sha, size)
+5 -18
View File
@@ -21,7 +21,6 @@ import (
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/buildinfo"
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/depot"
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/menu"
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/nwsync"
)
// Exit codes follow the sysexits(3) convention so CI can distinguish
@@ -95,26 +94,16 @@ var Registry = []Builder{
{
Name: "depot",
Bin: "crucible-depot",
Summary: "content-addressed asset depot (disk/cdn/bunny)",
Summary: "content-addressed asset depot (local/cdn/bunny)",
Commands: []Command{
{Name: "status", Summary: "report referenced-vs-present drift against a backend", Usage: "crucible depot status [--manifests DIR] --target bunny|cdn | --out DIR"},
{Name: "status", Summary: "report referenced-vs-present drift against a backend", Usage: "crucible depot status [--manifests DIR] [--source DIR] --target bunny|cdn|local"},
{Name: "push", Summary: "upload referenced-but-absent blobs from a local depot", Usage: "crucible depot push [--manifests DIR] --source DIR --target bunny"},
{Name: "verify", Summary: "existence sweep plus sampled download re-hash", Usage: "crucible depot verify [--manifests DIR] --target bunny|cdn [--sample N]"},
{Name: "get", Summary: "fetch one blob with sha re-verify", Usage: "crucible depot get <sha> <dest> --target cdn|bunny | --out DIR"},
{Name: "get", Summary: "fetch one blob with sha re-verify", Usage: "crucible depot get <sha> <dest> --target cdn|bunny|local"},
{Name: "pull", Summary: "incremental verified pull of every referenced blob", Usage: "crucible depot pull [--manifests DIR] --dest DIR --target cdn|bunny"},
},
Wired: true,
},
{
Name: "nwsync",
Bin: "crucible-nwsync",
Summary: "publish NWSync blobs and manifests (emit/assemble)",
Commands: []Command{
{Name: "emit", Summary: "explode one artifact into blobs plus its own NSYM manifest", Usage: "crucible nwsync emit <artifact> --out DIR"},
{Name: "assemble", Summary: "merge per-artifact NSYM manifests into one", Usage: "crucible nwsync assemble --order NAMES --entries DIR --out DIR [--group-id N]"},
},
Wired: true,
},
{
Name: "assets",
Bin: "crucible-assets",
@@ -252,7 +241,7 @@ var Registry = []Builder{
"2da-to-module [flags] <input.2da> [output.json]",
"json-to-2da <input.json> <output.2da>",
},
Aliases: []CommandAlias{{Name: "convert-topdata"}},
Aliases: []CommandAlias{{Name: "convert-topdata"}},
},
},
Wired: true,
@@ -403,7 +392,7 @@ func runBuilder(name string, args []string, out, errw io.Writer) int {
}
}
if b.Wired {
// depot, assets and nwsync are self-contained builders: they parse their own
// depot and assets are self-contained builders: they parse their own
// subcommands and own their exit contract, bypassing the
// b.Commands/delegateLegacy legacy routing entirely.
switch b.Name {
@@ -411,8 +400,6 @@ func runBuilder(name string, args []string, out, errw io.Writer) int {
return depot.Run(args, out, errw, os.Getenv)
case "assets":
return assets.Run(args, out, errw, os.Getenv)
case "nwsync":
return nwsync.Run(args, out, errw)
}
}
if !b.Wired {
+4 -9
View File
@@ -148,10 +148,6 @@ func TestBuilderHelpIsOK(t *testing.T) {
}
}
// selfContained builders parse their own subcommands instead of delegating to
// the legacy internal/app surface.
var selfContained = map[string]bool{"depot": true, "assets": true, "nwsync": true}
func TestCanonicalCommandSurface(t *testing.T) {
want := map[string][]string{
"depot": {"status", "push", "verify", "get", "pull"},
@@ -160,7 +156,6 @@ func TestCanonicalCommandSurface(t *testing.T) {
"module": {"build", "extract", "validate", "compare", "manifest"},
"topdata": {"validate", "build", "package", "compare", "convert"},
"wiki": {"build", "deploy"},
"nwsync": {"emit", "assemble"},
}
for _, builder := range Registry {
got := builder.subcommands()
@@ -178,10 +173,10 @@ func TestRegistryCommandNamesAndAliasesAreUnambiguous(t *testing.T) {
for _, builder := range Registry {
seen := map[string]bool{}
for _, command := range builder.Commands {
// depot, assets and nwsync parse their own subcommands and bypass
// AppCommand routing entirely (see the self-contained-builder branch
// in runBuilder), so their Commands carry no AppCommand.
requireAppCommand := !selfContained[builder.Name]
// depot and assets parse their own subcommands and bypass AppCommand
// routing entirely (see the self-contained-builder branch in
// runBuilder), so their Commands carry no AppCommand.
requireAppCommand := builder.Name != "depot" && builder.Name != "assets"
if command.Name == "" || command.Summary == "" || command.Usage == "" || (requireAppCommand && command.AppCommand == "") {
t.Errorf("%s has incomplete command metadata: %#v", builder.Name, command)
}
+18 -28
View File
@@ -67,19 +67,6 @@ type resourceEntry struct {
Size uint32
}
// extensionTypes and typeExtensions must stay exact inverses of each other;
// init() panics if they drift apart.
//
// Numbers below 0x0BB8 follow upstream neverwinter.nim
// (neverwinter/restype.nim). The 0x0BB8 and up entries are lyt/vis/mdx: real
// Aurora archive types that NWN1 ships in its own data/*.bif but upstream
// happens not to register. xoreos corroborates those three numbers
// (src/aurora/types.h).
//
// This table is NWN:EE only. Do not add NWN2 formats (mdb, gr2, wlk, xml):
// NWN:EE either gives that number to something else (2070 is xbc here and mdb
// in NWN2) or has no number for it at all, so packing one into a HAK writes a
// resource the game misreads. See the reserved list in erf_test.go.
var extensionTypes = map[string]uint16{
"res": 0x0000,
"bmp": 0x0001,
@@ -109,13 +96,12 @@ var extensionTypes = map[string]uint16{
"utt": 0x07F0,
"dds": 0x07F1,
"uts": 0x07F3,
"ltr": 0x07F4,
"gff": 0x07F5,
"fac": 0x07F6,
"gff": 0x07F7,
"ute": 0x07F8,
"utd": 0x07FA,
"utp": 0x07FC,
"dft": 0x07FD,
"dfa": 0x07FD,
"gic": 0x07FE,
"gui": 0x07FF,
"utm": 0x0803,
@@ -131,15 +117,20 @@ var extensionTypes = map[string]uint16{
"ndb": 0x0810,
"ptm": 0x0811,
"ptt": 0x0812,
"ltr": 0x0813,
"shd": 0x0815,
"mdb": 0x0816,
"mtr": 0x0818,
"jpg": 0x081C,
"lod": 0x081E,
"gif": 0x081F,
"png": 0x0820,
"jpg": 0x0821,
"lyt": 0x0BB8,
"vis": 0x0BB9,
"mdx": 0x0BC0,
"wlk": 0x0BCC,
"xml": 0x0BCD,
"gr2": 0x0FA3,
}
var typeExtensions = map[uint16]string{
@@ -171,13 +162,12 @@ var typeExtensions = map[uint16]string{
0x07F0: "utt",
0x07F1: "dds",
0x07F3: "uts",
0x07F4: "ltr",
0x07F5: "gff",
0x07F6: "fac",
0x07F7: "gff",
0x07F8: "ute",
0x07FA: "utd",
0x07FC: "utp",
0x07FD: "dft",
0x07FD: "dfa",
0x07FE: "gic",
0x07FF: "gui",
0x0803: "utm",
@@ -193,15 +183,20 @@ var typeExtensions = map[uint16]string{
0x0810: "ndb",
0x0811: "ptm",
0x0812: "ptt",
0x0813: "ltr",
0x0815: "shd",
0x0816: "mdb",
0x0818: "mtr",
0x081C: "jpg",
0x081E: "lod",
0x081F: "gif",
0x0820: "png",
0x0821: "jpg",
0x0BB8: "lyt",
0x0BB9: "vis",
0x0BC0: "mdx",
0x0BCC: "wlk",
0x0BCD: "xml",
0x0FA3: "gr2",
}
func init() {
@@ -210,13 +205,8 @@ func init() {
if !ok {
panic(fmt.Sprintf("missing canonical extension for resource type 0x%04X", resourceType))
}
if canonicalExt != ext {
panic(fmt.Sprintf("resource type 0x%04X is %q in extensionTypes but %q in typeExtensions", resourceType, ext, canonicalExt))
}
}
for resourceType, ext := range typeExtensions {
if registered, ok := extensionTypes[ext]; !ok || registered != resourceType {
panic(fmt.Sprintf("extension %q is 0x%04X in typeExtensions but 0x%04X in extensionTypes (registered=%v)", ext, resourceType, registered, ok))
if canonicalExt == ext {
continue
}
}
}
+5 -69
View File
@@ -134,10 +134,14 @@ func TestExtensionMappingsSupportModernAssetTypes(t *testing.T) {
"mtr": 0x0818,
"shd": 0x0815,
"txi": 0x07E6,
"jpg": 0x0821,
"jpg": 0x081C,
"mdb": 0x0816,
"lyt": 0x0BB8,
"vis": 0x0BB9,
"mdx": 0x0BC0,
"xml": 0x0BCD,
"wlk": 0x0BCC,
"gr2": 0x0FA3,
}
for ext, wantType := range cases {
gotType, ok := ResourceTypeForExtension(ext)
@@ -187,71 +191,3 @@ func TestExtensionMappingsSupportAllUTBlueprintTypes(t *testing.T) {
}
}
}
// TestRestypeTableMatchesUpstream pins the restype numbers Crucible shares with
// neverwinter.nim's restype.nim. The values below are upstream's; a mismatch
// means a HAK we write is mislabelled for the game.
func TestRestypeTableMatchesUpstream(t *testing.T) {
upstream := map[string]uint16{
"res": 0, "bmp": 1, "mve": 2, "tga": 3, "wav": 4, "plt": 6, "ini": 7,
"bmu": 8, "txt": 10, "mdl": 2002, "nss": 2009, "ncs": 2010, "are": 2012,
"set": 2013, "ifo": 2014, "bic": 2015, "wok": 2016, "2da": 2017,
"tlk": 2018, "txi": 2022, "git": 2023, "uti": 2025, "utc": 2027,
"dlg": 2029, "itp": 2030, "utt": 2032, "dds": 2033, "uts": 2035,
"ltr": 2036, "gff": 2037, "fac": 2038, "ute": 2040, "utd": 2042,
"utp": 2044, "dft": 2045, "gic": 2046, "gui": 2047, "utm": 2051,
"dwk": 2052, "pwk": 2053, "utg": 2055, "jrl": 2056, "utw": 2058,
"ssf": 2060, "hak": 2061, "nwm": 2062, "bik": 2063, "ndb": 2064,
"ptm": 2065, "ptt": 2066, "shd": 2069, "mtr": 2072, "lod": 2078,
"gif": 2079, "png": 2080, "jpg": 2081,
}
for ext, want := range upstream {
got, ok := ResourceTypeForExtension(ext)
if !ok {
t.Errorf("%s: not registered, upstream has %d", ext, want)
continue
}
if got != want {
t.Errorf("%s: registered as %d, upstream has %d", ext, got, want)
}
}
// Extensions upstream registers that Crucible must not reuse for anything else.
reserved := map[uint16]string{2039: "bte", 2067: "bak", 2070: "xbc", 2076: "tml"}
for number, upstreamExt := range reserved {
if ext, ok := ExtensionForResourceType(number); ok {
t.Errorf("%d: registered as %s, upstream reserves it for %s", number, ext, upstreamExt)
}
}
}
// TestNWN2FormatsAreNotRegistered keeps NWN2 file formats out of an NWN:EE
// table. mdb and gr2 have Aurora numbers that mean something else (or nothing)
// in NWN:EE; wlk and xml have no archive number in any Aurora game, so the
// values Crucible used for them were invented. Packing any of these into a HAK
// writes a resource the game misreads.
func TestNWN2FormatsAreNotRegistered(t *testing.T) {
for _, ext := range []string{"mdb", "gr2", "wlk", "xml"} {
if number, ok := ResourceTypeForExtension(ext); ok {
t.Errorf("%s is an NWN2 format but is registered as 0x%04X", ext, number)
}
}
}
// TestRestypeTablesAreMutualInverses is the check init() is meant to enforce.
func TestRestypeTablesAreMutualInverses(t *testing.T) {
for ext, number := range extensionTypes {
canonical, ok := typeExtensions[number]
if !ok {
t.Errorf("%s: number 0x%04X missing from typeExtensions", ext, number)
continue
}
if canonical != ext {
t.Errorf("%s: maps to 0x%04X, which maps back to %s", ext, number, canonical)
}
}
for number, ext := range typeExtensions {
if got, ok := extensionTypes[ext]; !ok || got != number {
t.Errorf("0x%04X: maps to %s, which maps back to 0x%04X (ok=%v)", number, ext, got, ok)
}
}
}
-128
View File
@@ -1,128 +0,0 @@
package nwsync
import (
"crypto/sha1"
"encoding/json"
"fmt"
"path"
)
// AssembleOptions describes one merged manifest.
type AssembleOptions struct {
// ArtifactKeys are the depot keys of the artifacts to merge, in
// Mod_HakList order — highest priority first. Each one's index is read
// from the key beside it.
ArtifactKeys []string
TLKKey string // the TLK's key, if the manifest carries one
OutDir string // write locally instead of uploading — the conformance path
GroupID int // 1 = current, 2 = testing; 0 means absent
ModuleName string
Description string
Sink sink // test seam; nil means OutDir or the zone
}
// AssembleResult reports what one assemble run produced.
type AssembleResult struct {
SHA1 string
ManifestPath string
Entries int
}
// Assemble merges the per-artifact NSYM manifests named by Order into one
// manifest. It reads no bulk data at all — only the small index files.
//
// Merge rule is resref shadowing, not concatenation: a resref present in more
// than one artifact resolves to the earliest artifact in Order, which is how
// the game resolves it (upstream's resman adds haks in reverse and lets the
// last one win). Get this backwards and the wrong texture ships silently.
func Assemble(options AssembleOptions) (AssembleResult, error) {
if len(options.ArtifactKeys) == 0 {
return AssembleResult{}, fmt.Errorf("assemble: no artifact keys given")
}
target, err := openSink(options.OutDir, options.Sink)
if err != nil {
return AssembleResult{}, err
}
// The TLK carries no precedence — it is not a hak and shadows nothing —
// so it merges last, after every hak has had its say.
keys := append([]string{}, options.ArtifactKeys...)
if options.TLKKey != "" {
keys = append(keys, options.TLKKey)
}
merged := make([]Entry, 0, 1024)
winner := make(map[Identity]bool, 1024)
var onDiskBytes int64
for _, artifactKey := range keys {
key, err := resolveIndexKey(artifactKey, options.OutDir)
if err != nil {
return AssembleResult{}, err
}
data, sidecarBody, err := target.getIndex(key)
if err != nil {
// An artifact with no index is an artifact whose emit never
// finished. Publishing a manifest without it would ship a release
// missing a hak, so this fails closed.
return AssembleResult{}, fmt.Errorf("assemble: no index for %s: %w", artifactKey, err)
}
entries, err := readManifest(data)
if err != nil {
return AssembleResult{}, fmt.Errorf("%s: %w", target.describe(key), err)
}
sidecar, err := parseSidecar(target.describe(key), sidecarBody)
if err != nil {
return AssembleResult{}, err
}
// Two producers of blobs means a skewed emitter can write blobs the
// merged manifest quietly disagrees with. Refuse to merge across
// mismatched emitter versions.
if sidecar.EmitterVersion != emitterVersion {
return AssembleResult{}, fmt.Errorf(
"assemble: emitter version mismatch: %s was emitted by emitter %q, this is emitter %q",
artifactKey, sidecar.EmitterVersion, emitterVersion)
}
// on_disk_bytes overcounts by the handful of cross-artifact
// duplicates. It is a display statistic; no dedupe pass for it.
onDiskBytes += sidecar.OnDiskBytes
for _, entry := range entries {
identity := entry.identity()
if winner[identity] {
continue
}
winner[identity] = true
merged = append(merged, entry)
}
}
if len(merged) == 0 {
return AssembleResult{}, fmt.Errorf("assemble: merged manifest is empty")
}
data, err := writeManifest(merged)
if err != nil {
return AssembleResult{}, err
}
sha1Hex := fmt.Sprintf("%x", sha1.Sum(data))
manifestKey := path.Join("manifests", sha1Hex)
sidecar := Sidecar{
ModuleName: options.ModuleName,
Description: options.Description,
GroupID: options.GroupID,
}
if err := putManifestPair(target, manifestKey, data, merged, onDiskBytes, sidecar); err != nil {
return AssembleResult{}, err
}
return AssembleResult{SHA1: sha1Hex, ManifestPath: target.describe(manifestKey), Entries: len(merged)}, nil
}
func parseSidecar(where string, data []byte) (Sidecar, error) {
var sidecar Sidecar
if err := json.Unmarshal(data, &sidecar); err != nil {
return Sidecar{}, fmt.Errorf("%s: %w", where, err)
}
return sidecar, nil
}
-76
View File
@@ -1,76 +0,0 @@
package nwsync
import (
"bytes"
"encoding/binary"
"fmt"
"github.com/klauspost/compress/zstd"
)
// NWCompressedBuffer framing, as upstream's neverwinter/compressedbuf.nim
// writes it for NWSync blobs. All fields are little-endian uint32:
//
// magic "NSYC", version 3, algorithm 2 (zstd), uncompressed size,
// zstd header version 1, dictionary 0, then the raw zstd frame.
const (
blobMagic = 0x4359534E // "NSYC" little-endian
blobVersion = 3
algorithmZstd = 2
zstdHeaderVer = 1
zstdDictionary = 0
blobHeaderBytes = 24
)
var (
blobEncoder, _ = zstd.NewWriter(nil)
blobDecoder, _ = zstd.NewReader(nil)
)
// compressBlob wraps data in NWCompressedBuffer framing.
func compressBlob(data []byte) []byte {
var out bytes.Buffer
header := []uint32{blobMagic, blobVersion, algorithmZstd, uint32(len(data)), zstdHeaderVer, zstdDictionary}
for _, field := range header {
_ = binary.Write(&out, binary.LittleEndian, field)
}
out.Write(blobEncoder.EncodeAll(data, nil))
return out.Bytes()
}
// decompressBlob unwraps NWCompressedBuffer framing. It exists so a blob this
// package wrote — or one upstream wrote — can be compared by its uncompressed
// bytes, which is the only comparison that is meaningful across zstd
// implementations.
func decompressBlob(blob []byte) ([]byte, error) {
if len(blob) < blobHeaderBytes {
return nil, fmt.Errorf("blob too small: %d bytes", len(blob))
}
header := make([]uint32, 6)
if err := binary.Read(bytes.NewReader(blob[:blobHeaderBytes]), binary.LittleEndian, header); err != nil {
return nil, fmt.Errorf("decode blob header: %w", err)
}
switch {
case header[0] != blobMagic:
return nil, fmt.Errorf("invalid blob magic: %#x", header[0])
case header[1] != blobVersion:
return nil, fmt.Errorf("unsupported blob version: %d", header[1])
case header[2] != algorithmZstd:
return nil, fmt.Errorf("unsupported compression algorithm: %d", header[2])
case header[4] != zstdHeaderVer:
return nil, fmt.Errorf("unsupported zstd header version: %d", header[4])
case header[5] != zstdDictionary:
return nil, fmt.Errorf("zstd dictionaries are not supported")
}
if header[3] == 0 {
return nil, nil
}
data, err := blobDecoder.DecodeAll(blob[blobHeaderBytes:], nil)
if err != nil {
return nil, fmt.Errorf("decompress blob: %w", err)
}
if uint32(len(data)) != header[3] {
return nil, fmt.Errorf("blob size mismatch: header says %d, got %d", header[3], len(data))
}
return data, nil
}
-264
View File
@@ -1,264 +0,0 @@
package nwsync
import (
"bytes"
"context"
"crypto/sha1"
"fmt"
"os"
"path"
"path/filepath"
"slices"
"sort"
"strconv"
"strings"
"time"
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/buildinfo"
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf"
)
// fileSizeLimit matches upstream's --limit-file-size default of 15 MB. A
// resource over it is a hard failure, not a skip: upstream quit(1)s and so do
// we. Our largest resource today is 13.66 MiB, so the headroom is thin.
const fileSizeLimit = 15 * 1024 * 1024
// skippedTypes are never published, matching upstream's GobalResTypeSkipList.
var skippedTypes = resTypes("nss", "ndb", "gic")
// emitterVersion identifies the blob/manifest byte format this package
// produces. assemble refuses to merge indexes that disagree on it, because two
// producers of blobs mean a skewed emitter can otherwise write blobs the merged
// manifest quietly disagrees with. Bump it only when emitted bytes change — it
// is deliberately not the build revision, which would invalidate every
// published index on every unrelated commit.
const emitterVersion = "1"
// serverTypes are loaded only server-side; a manifest holding nothing else
// has no client contents. Mirrors upstream's GlobalResTypeServerList, whose
// trailing 0 is RESTYPE_INVALID.
var serverTypes = append(resTypes(
"are", "dlg", "fac", "gic", "git", "ifo", "itp", "jrl", "ncs", "ndb",
"nss", "ptm", "utc", "utd", "ute", "uti", "utm", "utp", "uts", "utt", "utw",
), 0)
func resTypes(extensions ...string) []uint16 {
types := make([]uint16, 0, len(extensions))
for _, extension := range extensions {
restype, ok := erf.ResourceTypeForExtension(extension)
if !ok {
panic("nwsync: unknown restype " + extension)
}
types = append(types, restype)
}
return types
}
// EmitResult reports what one emit run produced.
type EmitResult struct {
Name string // artifact name, without extension
ManifestPath string
Entries int
BlobsWritten int
}
// EmitOptions describes one emit run.
type EmitOptions struct {
ArtifactKey string // depot key of the artifact; the NSYM key is derived from it
ArtifactPath string // the file on disk
As string // name override, for a TLK whose filename is not its published name
OutDir string // write locally instead of uploading — the conformance path
Sink sink // test seam; nil means OutDir or the zone
}
// Emit explodes one artifact — a .hak/.erf or a loose file such as the TLK —
// into NWSync blobs plus a NSYM manifest describing only that artifact.
//
// Blobs go up as they are produced and the index lands last, so the presence of
// an index is the publication marker: an artifact whose emit died halfway has
// real blobs in the zone and no index, which is unambiguous. Blob names are
// content hashes, so re-running skips whatever already landed.
func Emit(options EmitOptions) (EmitResult, error) {
artifact, err := os.ReadFile(options.ArtifactPath)
if err != nil {
return EmitResult{}, fmt.Errorf("read artifact: %w", err)
}
if err := checkArtifactKey(options.ArtifactKey, artifact); err != nil {
return EmitResult{}, err
}
name := options.As
if name == "" {
name = path.Base(options.ArtifactKey)
}
extension := path.Ext(name)
name = strings.TrimSuffix(name, extension)
key, err := resolveIndexKey(options.ArtifactKey, options.OutDir)
if err != nil {
return EmitResult{}, err
}
resources, err := readArtifact(options.ArtifactPath, artifact, name)
if err != nil {
return EmitResult{}, err
}
target, err := openSink(options.OutDir, options.Sink)
if err != nil {
return EmitResult{}, err
}
entries, blobs, onDiskBytes, err := emitResources(resources, target)
if err != nil {
return EmitResult{}, err
}
if len(entries) == 0 {
return EmitResult{}, fmt.Errorf("%s: nothing to index (no publishable resources)", options.ArtifactPath)
}
data, err := writeManifest(entries)
if err != nil {
return EmitResult{}, err
}
if err := putManifestPair(target, key, data, entries, onDiskBytes, Sidecar{ModuleName: name}); err != nil {
return EmitResult{}, err
}
return EmitResult{Name: name, ManifestPath: target.describe(key), Entries: len(entries), BlobsWritten: blobs}, nil
}
// openSink returns the zone sink, or a local directory when outDir is set.
func openSink(outDir string, injected sink) (sink, error) {
if injected != nil {
return injected, nil
}
if outDir != "" {
return dirSink{root: outDir}, nil
}
return newZoneSink(context.Background(), os.Getenv)
}
// readArtifact returns the resources of an ERF/HAK/MOD, or the single resource
// a loose file represents. Upstream's resman does the same dispatch on the
// file's first three bytes. name is the artifact's published name, which for a
// loose file is also its resref.
func readArtifact(path string, data []byte, name string) ([]erf.Resource, error) {
if len(data) >= 3 {
switch string(data[:3]) {
case "ERF", "HAK":
archive, err := erf.Read(bytes.NewReader(data))
if err != nil {
return nil, fmt.Errorf("%s: %w", path, err)
}
return archive.Resources, nil
case "MOD":
// A persistent world never publishes module contents, so the .mod
// contributes no bytes to a manifest — it only says which haks and
// which TLK the manifest covers.
return nil, fmt.Errorf("%s: a module is never emitted; a manifest is haks plus the TLK", path)
}
}
extension := filepath.Ext(filepath.Base(path))
restype, ok := erf.ResourceTypeForExtension(extension)
if !ok {
return nil, fmt.Errorf("%s: unknown resource type %q", path, extension)
}
return []erf.Resource{{
Name: name,
Type: restype,
Data: data,
}}, nil
}
func emitResources(resources []erf.Resource, target sink) ([]Entry, int, int64, error) {
// A resref appearing twice inside one artifact resolves to the last one,
// the way resman lets the last container added win.
order := make([]Identity, 0, len(resources))
latest := make(map[Identity]erf.Resource, len(resources))
var tooBig []string
for _, resource := range resources {
if _, ok := erf.ExtensionForResourceType(resource.Type); !ok {
return nil, 0, 0, fmt.Errorf("resref %s is not resolvable (unknown restype %d)", resource.Name, resource.Type)
}
if slices.Contains(skippedTypes, resource.Type) {
continue
}
if len(resource.Data) > fileSizeLimit {
tooBig = append(tooBig, fmt.Sprintf("%s: %d bytes > %d", resource.Name, len(resource.Data), fileSizeLimit))
continue
}
identity := Identity{ResRef: strings.ToLower(resource.Name), ResType: resource.Type}
if _, seen := latest[identity]; !seen {
order = append(order, identity)
}
latest[identity] = resource
}
if len(tooBig) > 0 {
sort.Strings(tooBig)
return nil, 0, 0, fmt.Errorf("resources exceed the file size limit:\n %s", strings.Join(tooBig, "\n "))
}
entries := make([]Entry, 0, len(order))
var blobs int
var onDiskBytes int64
for _, identity := range order {
resource := latest[identity]
sum := sha1.Sum(resource.Data)
entries = append(entries, Entry{
SHA1: sum,
Size: uint32(len(resource.Data)),
ResRef: identity.ResRef,
ResType: identity.ResType,
})
written, err := target.putBlob(fmt.Sprintf("%x", sum), func() []byte { return compressBlob(resource.Data) })
if err != nil {
return nil, 0, 0, err
}
if written > 0 {
blobs++
onDiskBytes += written
}
}
return entries, blobs, onDiskBytes, nil
}
// created is the sidecar timestamp. SOURCE_DATE_EPOCH pins it so a build can
// be reproduced byte for byte; the manifest itself is deterministic already.
func created() int64 {
if raw := os.Getenv("SOURCE_DATE_EPOCH"); raw != "" {
if seconds, err := strconv.ParseInt(raw, 10, 64); err == nil {
return seconds
}
}
return time.Now().Unix()
}
// putManifestPair stores a NSYM manifest and its .json sidecar at key. The
// caller supplies the sidecar fields it knows; the rest are derived from the
// entries. data must be the serialised form of entries.
func putManifestPair(target sink, key string, data []byte, entries []Entry, onDiskBytes int64, sidecar Sidecar) error {
var totalBytes int64
clientContents := false
for _, entry := range entries {
totalBytes += int64(entry.Size)
if !slices.Contains(serverTypes, entry.ResType) {
clientContents = true
}
}
sidecar.Version = manifestVersion
sidecar.SHA1 = fmt.Sprintf("%x", sha1.Sum(data))
sidecar.HashTreeDepth = hashTreeDepth
sidecar.IncludesModuleContents = false
sidecar.IncludesClientContents = clientContents
sidecar.TotalFiles = len(entries)
sidecar.TotalBytes = totalBytes
sidecar.OnDiskBytes = onDiskBytes
sidecar.Created = created()
sidecar.CreatedWith = buildinfo.String()
sidecar.EmitterVersion = emitterVersion
body, err := marshalSidecar(sidecar)
if err != nil {
return err
}
return target.putIndex(key, data, body)
}
-200
View File
@@ -1,200 +0,0 @@
package nwsync
import (
"bytes"
"encoding/binary"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"path/filepath"
"sort"
"strings"
)
// NSYM manifest, version 3, exactly as upstream neverwinter/nwsync.nim writes
// it. Everything is little-endian:
//
// "NSYM", uint32 version, uint32 entry count, uint32 mapping count,
// entries: byte[20] raw sha1, uint32 size, char[16] resref, uint16 restype
// mappings: uint32 entry index, char[16] resref, uint16 restype
//
// Entries are sorted by lowercase sha1 hex then resref, and a resource whose
// sha1 was already written becomes a mapping instead of a second entry.
const (
manifestVersion = 3
hashTreeDepth = 2
resRefBytes = 16
)
// Entry is one resource in a manifest.
type Entry struct {
SHA1 [20]byte
Size uint32
ResRef string // lowercase, no extension, at most 16 characters
ResType uint16
}
func (e Entry) sha1Hex() string { return hex.EncodeToString(e.SHA1[:]) }
// Identity is what a resref resolves by: name plus type. It is the merge key
// inside one artifact and across artifacts alike.
type Identity struct {
ResRef string
ResType uint16
}
func (e Entry) identity() Identity { return Identity{ResRef: e.ResRef, ResType: e.ResType} }
// writeManifest serialises entries into NSYM v3 bytes.
func writeManifest(entries []Entry) ([]byte, error) {
sorted := make([]Entry, len(entries))
copy(sorted, entries)
sort.SliceStable(sorted, func(i, j int) bool {
left, right := sorted[i].sha1Hex(), sorted[j].sha1Hex()
if left != right {
return left < right
}
return sorted[i].ResRef < sorted[j].ResRef
})
var body, mappings bytes.Buffer
seen := make(map[string]uint32, len(sorted))
var entryCount, mappingCount uint32
for _, entry := range sorted {
padded, err := padResRef(entry.ResRef)
if err != nil {
return nil, err
}
if index, ok := seen[entry.sha1Hex()]; ok {
_ = binary.Write(&mappings, binary.LittleEndian, index)
mappings.Write(padded)
_ = binary.Write(&mappings, binary.LittleEndian, entry.ResType)
mappingCount++
continue
}
seen[entry.sha1Hex()] = entryCount
entryCount++
body.Write(entry.SHA1[:])
_ = binary.Write(&body, binary.LittleEndian, entry.Size)
body.Write(padded)
_ = binary.Write(&body, binary.LittleEndian, entry.ResType)
}
var out bytes.Buffer
out.WriteString("NSYM")
for _, field := range []uint32{manifestVersion, entryCount, mappingCount} {
_ = binary.Write(&out, binary.LittleEndian, field)
}
out.Write(body.Bytes())
out.Write(mappings.Bytes())
return out.Bytes(), nil
}
// readManifest parses NSYM v3 bytes. Mappings are expanded back into entries,
// the way upstream's reader does, so a caller sees one entry per resref.
func readManifest(data []byte) ([]Entry, error) {
reader := bytes.NewReader(data)
magic := make([]byte, 4)
if _, err := io.ReadFull(reader, magic); err != nil || string(magic) != "NSYM" {
return nil, fmt.Errorf("not a manifest (invalid magic bytes)")
}
var version, entryCount, mappingCount uint32
for _, field := range []*uint32{&version, &entryCount, &mappingCount} {
if err := binary.Read(reader, binary.LittleEndian, field); err != nil {
return nil, fmt.Errorf("truncated manifest header: %w", err)
}
}
if version != manifestVersion {
return nil, fmt.Errorf("unsupported manifest version %d", version)
}
entries := make([]Entry, 0, entryCount+mappingCount)
for i := uint32(0); i < entryCount; i++ {
var entry Entry
if _, err := io.ReadFull(reader, entry.SHA1[:]); err != nil {
return nil, fmt.Errorf("truncated entry %d: %w", i, err)
}
if err := binary.Read(reader, binary.LittleEndian, &entry.Size); err != nil {
return nil, fmt.Errorf("truncated entry %d: %w", i, err)
}
resref, restype, err := readResRef(reader)
if err != nil {
return nil, fmt.Errorf("truncated entry %d: %w", i, err)
}
entry.ResRef, entry.ResType = resref, restype
entries = append(entries, entry)
}
for i := uint32(0); i < mappingCount; i++ {
var index uint32
if err := binary.Read(reader, binary.LittleEndian, &index); err != nil {
return nil, fmt.Errorf("truncated mapping %d: %w", i, err)
}
if index >= entryCount {
return nil, fmt.Errorf("mapping %d references non-existent entry %d", i, index)
}
resref, restype, err := readResRef(reader)
if err != nil {
return nil, fmt.Errorf("truncated mapping %d: %w", i, err)
}
target := entries[index]
entries = append(entries, Entry{SHA1: target.SHA1, Size: target.Size, ResRef: resref, ResType: restype})
}
return entries, nil
}
func readResRef(reader *bytes.Reader) (string, uint16, error) {
raw := make([]byte, resRefBytes)
if _, err := io.ReadFull(reader, raw); err != nil {
return "", 0, err
}
var restype uint16
if err := binary.Read(reader, binary.LittleEndian, &restype); err != nil {
return "", 0, err
}
return strings.ToLower(string(bytes.TrimRight(raw, "\x00"))), restype, nil
}
func padResRef(resref string) ([]byte, error) {
if len(resref) > resRefBytes {
return nil, fmt.Errorf("resref %q exceeds %d characters", resref, resRefBytes)
}
padded := make([]byte, resRefBytes)
copy(padded, strings.ToLower(resref))
return padded, nil
}
// Sidecar is the .json file written next to every manifest. Clients fetch it
// (nwn_nwsync_fetch.nim), so the field names and order match upstream.
// Upstream omits an integer meta field whose value is 0, hence group_id's
// omitempty: 0 means absent, not "group zero".
type Sidecar struct {
Version int `json:"version"`
SHA1 string `json:"sha1"`
HashTreeDepth int `json:"hash_tree_depth"`
ModuleName string `json:"module_name"`
Description string `json:"description"`
IncludesModuleContents bool `json:"includes_module_contents"`
IncludesClientContents bool `json:"includes_client_contents"`
TotalFiles int `json:"total_files"`
TotalBytes int64 `json:"total_bytes"`
OnDiskBytes int64 `json:"on_disk_bytes"`
Created int64 `json:"created"`
CreatedWith string `json:"created_with"`
EmitterVersion string `json:"emitter_version,omitempty"`
GroupID int `json:"group_id,omitempty"`
}
func marshalSidecar(sidecar Sidecar) ([]byte, error) {
body, err := json.MarshalIndent(sidecar, "", " ")
if err != nil {
return nil, err
}
// Upstream terminates the file with CRLF; match it.
return append(body, '\r', '\n'), nil
}
// blobPath is the data store path for a blob, hash tree depth 2.
func blobPath(root, sha1Hex string) string {
return filepath.Join(root, "data", "sha1", sha1Hex[0:2], sha1Hex[2:4], sha1Hex)
}
-476
View File
@@ -1,476 +0,0 @@
package nwsync
import (
"bytes"
"crypto/sha1"
"crypto/sha256"
"encoding/binary"
"encoding/hex"
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf"
)
func restype(t *testing.T, extension string) uint16 {
t.Helper()
value, ok := erf.ResourceTypeForExtension(extension)
if !ok {
t.Fatalf("unknown restype %q", extension)
}
return value
}
// writeHak builds a HAK fixture from resref.ext => body pairs.
func writeHak(t *testing.T, path string, contents map[string][]byte) {
t.Helper()
resources := make([]erf.Resource, 0, len(contents))
for name, body := range contents {
stem, extension, _ := strings.Cut(name, ".")
resources = append(resources, erf.Resource{
Name: stem,
Type: restype(t, extension),
Data: body,
Size: int64(len(body)),
})
}
var out bytes.Buffer
if err := erf.Write(&out, erf.New("HAK", resources)); err != nil {
t.Fatalf("write hak: %v", err)
}
if err := os.WriteFile(path, out.Bytes(), 0o644); err != nil {
t.Fatalf("write hak file: %v", err)
}
}
// artifactKey is the depot key a file would be published under: the sha256 of
// its bytes, hash-tree depth 2, keeping the extension.
func artifactKey(t *testing.T, path string) string {
t.Helper()
body, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
sum := sha256.Sum256(body)
digest := hex.EncodeToString(sum[:])
return "artifacts/haks/sha256/" + digest[0:2] + "/" + digest[2:4] + "/" + digest + filepath.Ext(path)
}
// emitLocal emits one artifact into a local tree, the conformance path.
func emitLocal(t *testing.T, path, out string) (EmitResult, error) {
t.Helper()
return Emit(EmitOptions{
ArtifactKey: artifactKey(t, path),
ArtifactPath: path,
As: filepath.Base(path),
OutDir: out,
})
}
func TestBlobFramingRoundTrips(t *testing.T) {
data := []byte("the quick brown fox jumps over the lazy dog, repeatedly and at length")
blob := compressBlob(data)
header := make([]uint32, 6)
if err := binary.Read(bytes.NewReader(blob[:blobHeaderBytes]), binary.LittleEndian, header); err != nil {
t.Fatalf("read header: %v", err)
}
want := []uint32{blobMagic, 3, 2, uint32(len(data)), 1, 0}
for i := range want {
if header[i] != want[i] {
t.Errorf("header field %d = %d, want %d", i, header[i], want[i])
}
}
if string(blob[:4]) != "NSYC" {
t.Errorf("magic bytes = %q, want NSYC", blob[:4])
}
got, err := decompressBlob(blob)
if err != nil {
t.Fatalf("decompress: %v", err)
}
if !bytes.Equal(got, data) {
t.Errorf("round trip mismatch: %q", got)
}
}
func TestManifestBytesMatchUpstreamLayout(t *testing.T) {
shared := sha1.Sum([]byte("shared"))
other := sha1.Sum([]byte("other"))
// Deliberately out of order, and with two resrefs sharing one sha1: the
// second one must become a mapping, not a second entry.
entries := []Entry{
{SHA1: other, Size: 5, ResRef: "zzz", ResType: 1},
{SHA1: shared, Size: 6, ResRef: "bbb", ResType: 2},
{SHA1: shared, Size: 6, ResRef: "aaa", ResType: 3},
}
data, err := writeManifest(entries)
if err != nil {
t.Fatalf("write manifest: %v", err)
}
if string(data[:4]) != "NSYM" {
t.Fatalf("magic = %q", data[:4])
}
var version, entryCount, mappingCount uint32
reader := bytes.NewReader(data[4:16])
for _, field := range []*uint32{&version, &entryCount, &mappingCount} {
_ = binary.Read(reader, binary.LittleEndian, field)
}
if version != 3 || entryCount != 2 || mappingCount != 1 {
t.Fatalf("header = version %d, %d entries, %d mappings; want 3/2/1", version, entryCount, mappingCount)
}
wantSize := 16 + int(entryCount)*(20+4+16+2) + int(mappingCount)*(4+16+2)
if len(data) != wantSize {
t.Fatalf("manifest is %d bytes, want %d", len(data), wantSize)
}
// Sorted by sha1 hex then resref, so the shared hash's "aaa" is the entry
// and "bbb" is demoted to a mapping.
round, err := readManifest(data)
if err != nil {
t.Fatalf("read manifest: %v", err)
}
if len(round) != 3 {
t.Fatalf("round trip returned %d entries, want 3", len(round))
}
byResRef := map[string]Entry{}
for _, entry := range round {
byResRef[entry.ResRef] = entry
}
for _, entry := range entries {
got, ok := byResRef[entry.ResRef]
if !ok {
t.Fatalf("resref %q lost in round trip", entry.ResRef)
}
if got != entry {
t.Errorf("resref %q = %+v, want %+v", entry.ResRef, got, entry)
}
}
if round[0].ResRef != "aaa" && round[1].ResRef != "aaa" {
t.Errorf("entries are not sorted by sha1 then resref: %+v", round)
}
}
func TestEmitWritesBlobsAndManifest(t *testing.T) {
dir := t.TempDir()
hak := filepath.Join(dir, "sow_test_01.hak")
body := []byte("texture bytes")
writeHak(t, hak, map[string][]byte{
"bloodstain1.tga": body,
"copy1.txi": body, // same content, different resref: one blob
"script1.nss": []byte("void main() {}"),
"debug1.ndb": []byte("debug"),
"comment1.gic": []byte("comment"),
})
out := filepath.Join(dir, "out")
result, err := emitLocal(t, hak, out)
if err != nil {
t.Fatalf("emit: %v", err)
}
if result.Entries != 2 {
t.Errorf("emitted %d entries, want 2 (nss/ndb/gic are always skipped)", result.Entries)
}
if result.BlobsWritten != 1 {
t.Errorf("wrote %d blobs, want 1 (identical content shares a blob)", result.BlobsWritten)
}
// The blob is named by the sha1 of the uncompressed bytes, under a depth-2
// hash tree, and decompresses back to exactly those bytes.
sum := sha1.Sum(body)
name := hex.EncodeToString(sum[:])
path := filepath.Join(out, "data", "sha1", name[0:2], name[2:4], name)
blob, err := os.ReadFile(path)
if err != nil {
t.Fatalf("blob missing at %s: %v", path, err)
}
got, err := decompressBlob(blob)
if err != nil {
t.Fatalf("decompress blob: %v", err)
}
if !bytes.Equal(got, body) {
t.Errorf("blob decompressed to %q, want %q", got, body)
}
entries := readEmitted(t, result.ManifestPath)
for _, entry := range entries {
if entry.ResType == restype(t, "nss") || entry.ResType == restype(t, "ndb") || entry.ResType == restype(t, "gic") {
t.Errorf("skipped restype leaked into the manifest: %+v", entry)
}
}
var sidecar Sidecar
body2, err := os.ReadFile(result.ManifestPath + ".json")
if err != nil {
t.Fatalf("sidecar missing: %v", err)
}
if err := json.Unmarshal(body2, &sidecar); err != nil {
t.Fatalf("sidecar json: %v", err)
}
if sidecar.TotalFiles != 2 || sidecar.HashTreeDepth != 2 || sidecar.Version != 3 {
t.Errorf("sidecar = %+v", sidecar)
}
if sidecar.IncludesModuleContents {
t.Error("sidecar claims module contents; a published manifest never has them")
}
if strings.Contains(string(body2), "group_id") {
t.Error("per-artifact sidecar should omit group_id (0 means absent)")
}
if _, err := os.Stat(filepath.Join(out, "latest")); err == nil {
t.Error("a latest file was written; there must never be one")
}
}
func readEmitted(t *testing.T, indexPath string) []Entry {
t.Helper()
data, err := os.ReadFile(indexPath)
if err != nil {
t.Fatalf("read emitted manifest: %v", err)
}
entries, err := readManifest(data)
if err != nil {
t.Fatalf("parse emitted manifest: %v", err)
}
return entries
}
func TestEmitFailsClosedOnOversizeResource(t *testing.T) {
dir := t.TempDir()
hak := filepath.Join(dir, "big.hak")
writeHak(t, hak, map[string][]byte{"huge1.tga": make([]byte, fileSizeLimit+1)})
if _, err := emitLocal(t, hak, filepath.Join(dir, "out")); err == nil {
t.Fatal("emit accepted a resource over the 15 MB limit")
}
}
func TestEmitLooseFile(t *testing.T) {
dir := t.TempDir()
tlk := filepath.Join(dir, "sow_tlk.tlk")
if err := os.WriteFile(tlk, []byte("TLK V3.0 payload"), 0o644); err != nil {
t.Fatal(err)
}
out := filepath.Join(dir, "out")
result, err := emitLocal(t, tlk, out)
if err != nil {
t.Fatalf("emit tlk: %v", err)
}
entries := readEmitted(t, result.ManifestPath)
if len(entries) != 1 || entries[0].ResRef != "sow_tlk" || entries[0].ResType != restype(t, "tlk") {
t.Fatalf("tlk emitted as %+v", entries)
}
if result.BlobsWritten != 1 {
t.Errorf("wrote %d blobs, want 1", result.BlobsWritten)
}
}
// emitFixture emits two haks that share a resref, so the merge rule is
// observable: "top" holds the winning body, "assets" the shadowed one.
func emitFixture(t *testing.T) (out string, keys map[string]string, topBody, assetBody []byte) {
t.Helper()
dir := t.TempDir()
topBody = []byte("2da from sow_top")
assetBody = []byte("2da from the asset hak")
writeHak(t, filepath.Join(dir, "sow_top.hak"), map[string][]byte{"appearance.2da": topBody})
writeHak(t, filepath.Join(dir, "sow_core_01.hak"), map[string][]byte{
"appearance.2da": assetBody,
"bloodstain1.tga": []byte("blood"),
})
out = filepath.Join(dir, "out")
keys = map[string]string{}
for _, name := range []string{"sow_top", "sow_core_01"} {
path := filepath.Join(dir, name+".hak")
keys[name] = artifactKey(t, path)
if _, err := emitLocal(t, path, out); err != nil {
t.Fatalf("emit %s: %v", name, err)
}
}
return out, keys, topBody, assetBody
}
func TestAssembleShadowsByOrder(t *testing.T) {
entriesDir, keys, topBody, assetBody := emitFixture(t)
result, err := Assemble(AssembleOptions{
ArtifactKeys: []string{keys["sow_top"], keys["sow_core_01"]},
OutDir: entriesDir,
GroupID: 2,
})
if err != nil {
t.Fatalf("assemble: %v", err)
}
if result.Entries != 2 {
t.Fatalf("merged %d entries, want 2 (appearance.2da is shadowed, not duplicated)", result.Entries)
}
data, err := os.ReadFile(result.ManifestPath)
if err != nil {
t.Fatalf("read merged manifest: %v", err)
}
merged := sha1.Sum(data)
if hex.EncodeToString(merged[:]) != result.SHA1 || filepath.Base(result.ManifestPath) != result.SHA1 {
t.Errorf("manifest is not named by its own sha1: %s", result.ManifestPath)
}
entries, err := readManifest(data)
if err != nil {
t.Fatalf("parse merged manifest: %v", err)
}
for _, entry := range entries {
if entry.ResRef != "appearance" {
continue
}
if entry.SHA1 != sha1.Sum(topBody) {
t.Errorf("appearance.2da resolved to the wrong hak; want the earliest in --order")
}
if entry.SHA1 == sha1.Sum(assetBody) {
t.Error("appearance.2da resolved to the shadowed hak")
}
}
sidecar := Sidecar{}
body, err := os.ReadFile(result.ManifestPath + ".json")
if err != nil {
t.Fatalf("merged sidecar missing: %v", err)
}
if err := json.Unmarshal(body, &sidecar); err != nil {
t.Fatalf("merged sidecar json: %v", err)
}
if sidecar.GroupID != 2 {
t.Errorf("group_id = %d, want 2 (testing)", sidecar.GroupID)
}
if sidecar.SHA1 != result.SHA1 {
t.Errorf("sidecar sha1 = %s, want %s", sidecar.SHA1, result.SHA1)
}
if sidecar.TotalFiles != 2 {
t.Errorf("total_files = %d, want 2", sidecar.TotalFiles)
}
}
func TestAssembleReversedOrderPicksTheOtherHak(t *testing.T) {
entriesDir, keys, topBody, assetBody := emitFixture(t)
result, err := Assemble(AssembleOptions{
ArtifactKeys: []string{keys["sow_core_01"], keys["sow_top"]},
OutDir: entriesDir,
})
if err != nil {
t.Fatalf("assemble: %v", err)
}
data, _ := os.ReadFile(result.ManifestPath)
entries, err := readManifest(data)
if err != nil {
t.Fatalf("parse merged manifest: %v", err)
}
for _, entry := range entries {
if entry.ResRef == "appearance" && entry.SHA1 != sha1.Sum(assetBody) {
t.Errorf("appearance.2da did not follow --order; still resolves to %x", entry.SHA1)
}
}
_ = topBody
}
func TestAssembleRefusesMismatchedEmitterVersions(t *testing.T) {
entriesDir, keys, _, _ := emitFixture(t)
index, err := resolveIndexKey(keys["sow_core_01"], entriesDir)
if err != nil {
t.Fatal(err)
}
path := filepath.Join(entriesDir, index+".json")
var sidecar Sidecar
body, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
if err := json.Unmarshal(body, &sidecar); err != nil {
t.Fatal(err)
}
sidecar.EmitterVersion = "0"
patched, err := marshalSidecar(sidecar)
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, patched, 0o644); err != nil {
t.Fatal(err)
}
_, err = Assemble(AssembleOptions{
ArtifactKeys: []string{keys["sow_top"], keys["sow_core_01"]},
OutDir: entriesDir,
})
if err == nil || !strings.Contains(err.Error(), "emitter version mismatch") {
t.Fatalf("assemble merged across emitter versions: %v", err)
}
}
func TestEmitHonoursSourceDateEpoch(t *testing.T) {
t.Setenv("SOURCE_DATE_EPOCH", "1700000000")
dir := t.TempDir()
hak := filepath.Join(dir, "pinned.hak")
writeHak(t, hak, map[string][]byte{"one1.tga": []byte("body")})
result, err := emitLocal(t, hak, filepath.Join(dir, "out"))
if err != nil {
t.Fatalf("emit: %v", err)
}
body, err := os.ReadFile(result.ManifestPath + ".json")
if err != nil {
t.Fatal(err)
}
var sidecar Sidecar
if err := json.Unmarshal(body, &sidecar); err != nil {
t.Fatal(err)
}
if sidecar.Created != 1700000000 {
t.Errorf("created = %d, want the pinned SOURCE_DATE_EPOCH", sidecar.Created)
}
}
func TestEmitRejectsAModule(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "sow.mod")
var out bytes.Buffer
if err := erf.Write(&out, erf.New("MOD", []erf.Resource{
{Name: "module", Type: restype(t, "ifo"), Data: []byte("ifo"), Size: 3},
})); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, out.Bytes(), 0o644); err != nil {
t.Fatal(err)
}
if _, err := emitLocal(t, path, filepath.Join(dir, "out")); err == nil {
t.Fatal("emit accepted a .mod; a manifest never carries module contents")
}
}
func TestAssembleFailsClosedOnMissingIndex(t *testing.T) {
entriesDir, keys, _, _ := emitFixture(t)
missing := "artifacts/haks/sha256/00/11/" + strings.Repeat("0", 64) + ".hak"
_, err := Assemble(AssembleOptions{
ArtifactKeys: []string{keys["sow_top"], missing},
OutDir: entriesDir,
})
if err == nil || !strings.Contains(err.Error(), missing) {
t.Fatalf("assemble did not fail closed and name the missing artifact: %v", err)
}
}
func TestRunUsageErrors(t *testing.T) {
cases := [][]string{
nil,
{"nope"},
{"emit"},
{"emit", "artifact-key.hak"},
{"emit", "a", "b", "c"},
{"assemble", "--out", "y"},
}
for _, args := range cases {
var out, errw bytes.Buffer
if code := Run(args, &out, &errw); code != exitUsage {
t.Errorf("Run(%v) exit=%d, want %d", args, code, exitUsage)
}
}
}
-140
View File
@@ -1,140 +0,0 @@
// Package nwsync publishes NWSync repository data: blobs and a per-artifact
// NSYM manifest at each artifact's birth (emit), and one merged manifest at
// module release (assemble).
//
// The split exists because upstream nwn_nwsync_write wants every hak, the TLK
// and the module present in one run on one disk, which our build hosts cannot
// hold. Upstream stays the conformance oracle: manifests compare byte for
// byte, blobs compare after decompression.
package nwsync
import (
"flag"
"fmt"
"io"
)
const (
exitOK = 0
exitUsage = 64
exitInternal = 70
)
// Run executes an nwsync subcommand. args[0] is the subcommand (emit|assemble);
// returns the process exit code.
func Run(args []string, stdout, stderr io.Writer) int {
if len(args) == 0 {
printRunUsage(stderr)
return exitUsage
}
switch args[0] {
case "emit":
return runEmit(args[1:], stdout, stderr)
case "assemble":
return runAssemble(args[1:], stdout, stderr)
case "-h", "--help", "help":
printRunUsage(stdout)
return exitOK
default:
fmt.Fprintf(stderr, "nwsync: unknown subcommand %q\n\n", args[0])
printRunUsage(stderr)
return exitUsage
}
}
func printRunUsage(w io.Writer) {
fmt.Fprint(w, `usage:
nwsync emit [--as NAME] [--out DIR] <artifact-key> <file>
nwsync assemble --group-id N [--tlk-key KEY] [--out DIR] <artifact-key>...
emit explodes one .hak/.erf or one loose file (the TLK) into NWSync blobs plus
a NSYM index covering only that artifact, and uploads both. assemble merges
those indexes into one manifest, reading no bulk data. Artifact keys are depot
keys; an index lives beside its artifact, with the extension replaced.
--out DIR writes to a local repository tree instead of uploading, which is the
conformance path against upstream nwn_nwsync_write. Without it, the zone comes
from NWSYNC_STORAGE_ZONE, NWSYNC_STORAGE_PASSWORD and BUNNY_STORAGE_HOST.
`)
}
// parseArgs parses flags that may appear before, after or between positionals.
// Go's flag package stops at the first non-flag argument, which turns
// `emit <key> <file> --out DIR` into a confusing arity error.
func parseArgs(fs *flag.FlagSet, args []string) ([]string, error) {
var positional []string
for {
if err := fs.Parse(args); err != nil {
return nil, err
}
rest := fs.Args()
if len(rest) == 0 {
return positional, nil
}
positional = append(positional, rest[0])
args = rest[1:]
}
}
func runEmit(args []string, stdout, stderr io.Writer) int {
fs := flag.NewFlagSet("emit", flag.ContinueOnError)
fs.SetOutput(stderr)
as := fs.String("as", "", "published name of the artifact, when it differs from the key")
out := fs.String("out", "", "write to a local repository tree instead of uploading")
positional, err := parseArgs(fs, args)
if err != nil {
return exitUsage
}
if len(positional) != 2 {
fmt.Fprintf(stderr, "nwsync emit: <artifact-key> and <file> are both required\n")
return exitUsage
}
result, err := Emit(EmitOptions{
ArtifactKey: positional[0],
ArtifactPath: positional[1],
As: *as,
OutDir: *out,
})
if err != nil {
fmt.Fprintf(stderr, "nwsync emit: %v\n", err)
return exitInternal
}
fmt.Fprintf(stdout, "emitted %s: %d resources, %d new blobs, index %s\n",
result.Name, result.Entries, result.BlobsWritten, result.ManifestPath)
return exitOK
}
func runAssemble(args []string, stdout, stderr io.Writer) int {
fs := flag.NewFlagSet("assemble", flag.ContinueOnError)
fs.SetOutput(stderr)
tlkKey := fs.String("tlk-key", "", "depot key of the TLK, which shadows nothing and merges last")
out := fs.String("out", "", "write to a local repository tree instead of uploading")
groupID := fs.Int("group-id", 0, "NWSync group id (1 = current, 2 = testing; 0 omits it)")
moduleName := fs.String("module-name", "", "module name recorded in the sidecar")
description := fs.String("description", "", "description recorded in the sidecar")
positional, err := parseArgs(fs, args)
if err != nil {
return exitUsage
}
if len(positional) == 0 {
fmt.Fprintf(stderr, "nwsync assemble: at least one artifact key is required\n")
return exitUsage
}
result, err := Assemble(AssembleOptions{
ArtifactKeys: positional,
TLKKey: *tlkKey,
OutDir: *out,
GroupID: *groupID,
ModuleName: *moduleName,
Description: *description,
})
if err != nil {
fmt.Fprintf(stderr, "nwsync assemble: %v\n", err)
return exitInternal
}
fmt.Fprintf(stdout, "assembled manifest %s: %d resources, %s\n",
result.SHA1, result.Entries, result.ManifestPath)
return exitOK
}
-207
View File
@@ -1,207 +0,0 @@
package nwsync
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"os"
"path"
"path/filepath"
"strings"
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/depot"
)
// sink is where an emit or assemble run puts what it produces. The zone is the
// production sink; a local directory exists only as the conformance path, so
// upstream's output and ours can be diffed on a developer machine.
type sink interface {
// putBlob stores one NWCompressedBuffer blob under its sha1 name and
// returns the bytes stored, or 0 if the blob was already there. Blob names
// are content hashes, so an existing name is existing content — which is
// why body is a thunk: compression is the expensive part of emit and a
// blob that is already stored must not pay for it.
putBlob(sha1Hex string, body func() []byte) (int64, error)
// putIndex stores a NSYM manifest and its sidecar under key, which is
// either an artifact-derived object key or a local path.
putIndex(key string, manifest, sidecar []byte) error
// getIndex reads back a NSYM manifest and its sidecar.
getIndex(key string) (manifest, sidecar []byte, err error)
// describe names the sink for messages.
describe(key string) string
}
// dirSink writes a local NWSync repository tree.
type dirSink struct{ root string }
func (s dirSink) putBlob(sha1Hex string, body func() []byte) (int64, error) {
blob := blobPath(s.root, sha1Hex)
if _, err := os.Stat(blob); err == nil {
return 0, nil
}
if err := os.MkdirAll(filepath.Dir(blob), 0o755); err != nil {
return 0, fmt.Errorf("create blob directory: %w", err)
}
data := body()
if err := os.WriteFile(blob, data, 0o644); err != nil {
return 0, fmt.Errorf("write blob: %w", err)
}
return int64(len(data)), nil
}
func (s dirSink) putIndex(key string, manifest, sidecar []byte) error {
target := filepath.Join(s.root, filepath.FromSlash(key))
if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
return fmt.Errorf("create manifest directory: %w", err)
}
if err := os.WriteFile(target, manifest, 0o644); err != nil {
return fmt.Errorf("write manifest: %w", err)
}
if err := os.WriteFile(target+".json", sidecar, 0o644); err != nil {
return fmt.Errorf("write sidecar: %w", err)
}
return nil
}
func (s dirSink) getIndex(key string) ([]byte, []byte, error) {
target := filepath.Join(s.root, filepath.FromSlash(key))
manifest, err := os.ReadFile(target)
if err != nil {
return nil, nil, fmt.Errorf("read index: %w", err)
}
sidecar, err := os.ReadFile(target + ".json")
if err != nil {
return nil, nil, fmt.Errorf("read sidecar: %w", err)
}
return manifest, sidecar, nil
}
func (s dirSink) describe(key string) string {
return filepath.Join(s.root, filepath.FromSlash(key))
}
// zoneSink uploads straight to the NWSync storage zone. Nothing bulky is ever
// written to the runner's disk: the working set is one resource at a time.
type zoneSink struct {
store depot.KeyStore
ctx context.Context
zone string
}
func (s zoneSink) putBlob(sha1Hex string, body func() []byte) (int64, error) {
key := path.Join("data", "sha1", sha1Hex[0:2], sha1Hex[2:4], sha1Hex)
// A throttled probe must never be read as "missing, re-upload" or as
// "present, skip", so only a confirmed Present skips the upload.
state, _, err := s.store.ProbeKey(s.ctx, key)
if err != nil {
return 0, fmt.Errorf("probe blob %s: %w", sha1Hex, err)
}
if state == depot.Present {
return 0, nil
}
data := body()
if err := s.put(key, data); err != nil {
return 0, fmt.Errorf("upload blob %s: %w", sha1Hex, err)
}
return int64(len(data)), nil
}
func (s zoneSink) putIndex(key string, manifest, sidecar []byte) error {
// The manifest lands last: its presence is the publication marker, so it
// must never appear before the blobs it names.
if err := s.put(key+".json", sidecar); err != nil {
return fmt.Errorf("upload sidecar %s: %w", key, err)
}
if err := s.put(key, manifest); err != nil {
return fmt.Errorf("upload index %s: %w", key, err)
}
return nil
}
func (s zoneSink) getIndex(key string) ([]byte, []byte, error) {
manifest, err := s.store.GetKey(s.ctx, key)
if err != nil {
return nil, nil, fmt.Errorf("read index: %w", err)
}
sidecar, err := s.store.GetKey(s.ctx, key+".json")
if err != nil {
return nil, nil, fmt.Errorf("read sidecar: %w", err)
}
return manifest, sidecar, nil
}
func (s zoneSink) describe(key string) string { return s.zone + "/" + key }
func (s zoneSink) put(key string, body []byte) error {
sum := sha256.Sum256(body)
return s.store.PutReader(s.ctx, key, bytes.NewReader(body), int64(len(body)), hex.EncodeToString(sum[:]))
}
// newZoneSink builds the upload sink from the environment. NWSync data lives
// in its own zone, separate from the asset depot, so it has its own zone and
// credential; only the host is shared, and Crucible has no default host.
func newZoneSink(ctx context.Context, getenv func(string) string) (sink, error) {
cfg := depot.LoadConfig(getenv)
cfg.StorageZone = getenv("NWSYNC_STORAGE_ZONE")
cfg.WriteKey = getenv("NWSYNC_STORAGE_PASSWORD")
cfg.ReadKey = cfg.WriteKey
if cfg.StorageZone == "" {
return nil, fmt.Errorf("NWSYNC_STORAGE_ZONE is unset (or pass --out DIR to write locally)")
}
if cfg.WriteKey == "" {
return nil, fmt.Errorf("NWSYNC_STORAGE_PASSWORD is unset (or pass --out DIR to write locally)")
}
if cfg.StorageHost == "" {
return nil, fmt.Errorf("BUNNY_STORAGE_HOST is unset")
}
store, err := depot.NewKeyStore(cfg)
if err != nil {
return nil, err
}
return zoneSink{store: store, ctx: ctx, zone: cfg.StorageZone}, nil
}
// indexKey is where an artifact's NSYM lives: beside the artifact itself, with
// the final extension replaced. emit and assemble must agree on this one rule,
// so it lives here and nowhere else.
//
// artifacts/haks/sha256/30/46/3046….hak -> artifacts/haks/sha256/30/46/3046….nsym
func indexKey(artifactKey string) (string, error) {
extension := path.Ext(artifactKey)
if extension == "" {
return "", fmt.Errorf("artifact key %q has no extension", artifactKey)
}
return strings.TrimSuffix(artifactKey, extension) + ".nsym", nil
}
// resolveIndexKey is where emit writes an artifact's index and where assemble
// reads it from. On the zone that is beside the artifact; locally the indexes
// sit flat beside the data tree, so upstream's output and ours diff directly.
func resolveIndexKey(artifactKey, outDir string) (string, error) {
key, err := indexKey(artifactKey)
if err != nil {
return "", err
}
if outDir != "" {
return path.Base(key), nil
}
return key, nil
}
// checkArtifactKey fails closed when the key's embedded digest is not the
// digest of the bytes being emitted. Publishing an index under the wrong key
// silently pairs a manifest with the wrong artifact.
func checkArtifactKey(artifactKey string, artifact []byte) error {
base := path.Base(artifactKey)
digest := strings.TrimSuffix(base, path.Ext(base))
if len(digest) != 64 {
return fmt.Errorf("artifact key %q does not name a sha256", artifactKey)
}
sum := sha256.Sum256(artifact)
if got := hex.EncodeToString(sum[:]); got != digest {
return fmt.Errorf("artifact key %q names digest %s but the file hashes to %s", artifactKey, digest, got)
}
return nil
}
-249
View File
@@ -1,249 +0,0 @@
package nwsync
import (
"crypto/sha1"
"crypto/sha256"
"encoding/hex"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"sync"
"testing"
)
// fakeZone is a Bunny-shaped object store: PUT stores, GET reads, and the
// Checksum header is verified the way Bunny verifies it.
type fakeZone struct {
mu sync.Mutex
objects map[string][]byte
puts []string
failOn func(key string) bool // when true, the PUT fails
}
func newFakeZone(t *testing.T) (*fakeZone, func(string) string) {
t.Helper()
zone := &fakeZone{objects: map[string][]byte{}}
server := httptest.NewServer(zone)
t.Cleanup(server.Close)
getenv := func(name string) string {
switch name {
case "NWSYNC_STORAGE_ZONE":
return "sow-nwsync"
case "NWSYNC_STORAGE_PASSWORD":
return "write-key"
case "BUNNY_STORAGE_HOST":
return server.URL
}
return ""
}
return zone, getenv
}
func (z *fakeZone) ServeHTTP(w http.ResponseWriter, r *http.Request) {
key := strings.TrimPrefix(r.URL.Path, "/sow-nwsync/")
switch r.Method {
case http.MethodPut:
if z.failOn != nil && z.failOn(key) {
http.Error(w, "boom", http.StatusInternalServerError)
return
}
body, err := io.ReadAll(r.Body)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
sum := sha256.Sum256(body)
if want := strings.ToUpper(hex.EncodeToString(sum[:])); r.Header.Get("Checksum") != want {
http.Error(w, "checksum mismatch", http.StatusBadRequest)
return
}
z.mu.Lock()
z.objects[key] = body
z.puts = append(z.puts, key)
z.mu.Unlock()
w.WriteHeader(http.StatusCreated)
case http.MethodGet:
z.mu.Lock()
body, ok := z.objects[key]
z.mu.Unlock()
if !ok {
http.Error(w, "not found", http.StatusNotFound)
return
}
_, _ = w.Write(body)
default:
http.Error(w, "unsupported", http.StatusMethodNotAllowed)
}
}
func (z *zoneSinkFixture) emit(t *testing.T, path string) EmitResult {
t.Helper()
result, err := Emit(EmitOptions{
ArtifactKey: artifactKey(t, path),
ArtifactPath: path,
Sink: z.sink,
})
if err != nil {
t.Fatalf("emit %s: %v", path, err)
}
return result
}
type zoneSinkFixture struct {
zone *fakeZone
sink sink
}
func newZoneFixture(t *testing.T) *zoneSinkFixture {
t.Helper()
zone, getenv := newFakeZone(t)
target, err := newZoneSink(t.Context(), getenv)
if err != nil {
t.Fatalf("zone sink: %v", err)
}
return &zoneSinkFixture{zone: zone, sink: target}
}
func sha1Of(body []byte) [20]byte { return sha1.Sum(body) }
func TestEmitUploadsBlobsThenIndex(t *testing.T) {
fixture := newZoneFixture(t)
dir := t.TempDir()
hak := filepath.Join(dir, "sow_test_01.hak")
body := []byte("texture bytes")
writeHak(t, hak, map[string][]byte{"bloodstain1.tga": body, "copy1.txi": body})
key := artifactKey(t, hak)
result := fixture.emit(t, hak)
index, err := indexKey(key)
if err != nil {
t.Fatal(err)
}
if _, ok := fixture.zone.objects[index]; !ok {
t.Fatalf("no index at %s; zone holds %v", index, fixture.zone.puts)
}
if result.BlobsWritten != 1 {
t.Errorf("uploaded %d blobs, want 1 (identical content shares a blob)", result.BlobsWritten)
}
// The index is the publication marker, so it must land after every blob it
// names — including its own sidecar.
last := fixture.zone.puts[len(fixture.zone.puts)-1]
if last != index {
t.Errorf("index landed at position %d of %d; it must be last", len(fixture.zone.puts), len(fixture.zone.puts))
}
for _, key := range fixture.zone.puts[:len(fixture.zone.puts)-1] {
if strings.HasPrefix(key, "data/sha1/") || key == index+".json" {
continue
}
t.Errorf("unexpected object uploaded before the index: %s", key)
}
}
func TestEmitSkipsBlobsAlreadyInTheZone(t *testing.T) {
fixture := newZoneFixture(t)
dir := t.TempDir()
hak := filepath.Join(dir, "sow_test_01.hak")
writeHak(t, hak, map[string][]byte{"bloodstain1.tga": []byte("blood")})
first := fixture.emit(t, hak)
if first.BlobsWritten != 1 {
t.Fatalf("first emit uploaded %d blobs, want 1", first.BlobsWritten)
}
second := fixture.emit(t, hak)
if second.BlobsWritten != 0 {
t.Errorf("re-emit uploaded %d blobs, want 0 (a blob name is its content)", second.BlobsWritten)
}
}
func TestEmitLeavesNoIndexWhenAnUploadFails(t *testing.T) {
fixture := newZoneFixture(t)
fixture.zone.failOn = func(key string) bool { return strings.HasPrefix(key, "data/sha1/") }
dir := t.TempDir()
hak := filepath.Join(dir, "sow_test_01.hak")
writeHak(t, hak, map[string][]byte{"bloodstain1.tga": []byte("blood")})
_, err := Emit(EmitOptions{
ArtifactKey: artifactKey(t, hak),
ArtifactPath: hak,
Sink: fixture.sink,
})
if err == nil {
t.Fatal("emit reported success after an upload failed")
}
for key := range fixture.zone.objects {
if strings.HasSuffix(key, ".nsym") {
t.Errorf("a half-emitted artifact published an index: %s", key)
}
}
}
func TestEmitRejectsAKeyThatDoesNotMatchTheFile(t *testing.T) {
fixture := newZoneFixture(t)
dir := t.TempDir()
hak := filepath.Join(dir, "sow_test_01.hak")
writeHak(t, hak, map[string][]byte{"bloodstain1.tga": []byte("blood")})
wrong := "artifacts/haks/sha256/00/11/" + strings.Repeat("0", 64) + ".hak"
_, err := Emit(EmitOptions{ArtifactKey: wrong, ArtifactPath: hak, Sink: fixture.sink})
if err == nil || !strings.Contains(err.Error(), "hashes to") {
t.Fatalf("emit published under a key that names another artifact: %v", err)
}
}
func TestAssembleReadsIndexesFromTheZone(t *testing.T) {
fixture := newZoneFixture(t)
dir := t.TempDir()
topBody := []byte("2da from sow_top")
assetBody := []byte("2da from the asset hak")
top := filepath.Join(dir, "sow_top.hak")
core := filepath.Join(dir, "sow_core_01.hak")
writeHak(t, top, map[string][]byte{"appearance.2da": topBody})
writeHak(t, core, map[string][]byte{"appearance.2da": assetBody, "bloodstain1.tga": []byte("blood")})
tlkPath := filepath.Join(dir, "sow_tlk.tlk")
if err := os.WriteFile(tlkPath, []byte("TLK V3.0 payload"), 0o644); err != nil {
t.Fatal(err)
}
fixture.emit(t, top)
fixture.emit(t, core)
if _, err := Emit(EmitOptions{
ArtifactKey: artifactKey(t, tlkPath),
ArtifactPath: tlkPath,
As: "sow_tlk.tlk",
Sink: fixture.sink,
}); err != nil {
t.Fatalf("emit tlk: %v", err)
}
result, err := Assemble(AssembleOptions{
ArtifactKeys: []string{artifactKey(t, top), artifactKey(t, core)},
TLKKey: artifactKey(t, tlkPath),
GroupID: 2,
Sink: fixture.sink,
})
if err != nil {
t.Fatalf("assemble: %v", err)
}
if result.Entries != 3 {
t.Fatalf("merged %d entries, want 3 (appearance.2da is shadowed, the TLK adds one)", result.Entries)
}
manifest, ok := fixture.zone.objects["manifests/"+result.SHA1]
if !ok {
t.Fatalf("no merged manifest in the zone; it holds %v", fixture.zone.puts)
}
entries, err := readManifest(manifest)
if err != nil {
t.Fatalf("parse merged manifest: %v", err)
}
for _, entry := range entries {
if entry.ResRef == "appearance" && entry.SHA1 != sha1Of(topBody) {
t.Errorf("appearance.2da resolved to the shadowed hak, not the first one given")
}
}
}
+2 -10
View File
@@ -593,14 +593,9 @@ func envBool(name string) bool {
func collectModuleResources(p *project.Project, moduleHakOrder []string) ([]erf.Resource, error) {
var moduleResources []erf.Resource
palettes, err := collectPaletteDescriptors(p)
if err != nil {
return nil, err
}
for _, rel := range p.Inventory.SourceFiles {
abs := filepath.Join(p.SourceDir(), filepath.FromSlash(rel))
resource, err := resourceFromJSON(abs, moduleHakOrder, palettes)
resource, err := resourceFromJSON(abs, moduleHakOrder)
if err != nil {
return nil, err
}
@@ -1035,7 +1030,7 @@ func compareResourceKeys(a, b erf.Resource) int {
return 0
}
func resourceFromJSON(path string, moduleHakOrder []string, palettes paletteProjection) (erf.Resource, error) {
func resourceFromJSON(path string, moduleHakOrder []string) (erf.Resource, error) {
name, extension, err := splitSourceName(path)
if err != nil {
return erf.Resource{}, err
@@ -1064,9 +1059,6 @@ func resourceFromJSON(path string, moduleHakOrder []string, palettes paletteProj
if extension == ".ifo" && name == "module" && len(moduleHakOrder) > 0 {
setModuleHAKList(&document, moduleHakOrder)
}
if extension == ".itp" && isPaletteProjectionResref(name) {
projectPaletteDocument(&document, palettes[strings.ToLower(name)])
}
var buf bytes.Buffer
if err := gff.Write(&buf, document); err != nil {
-7
View File
@@ -117,10 +117,6 @@ func expectedResources(p *project.Project) (map[string]resourceExpectation, erro
if err != nil {
return nil, err
}
palettes, err := collectPaletteDescriptors(p)
if err != nil {
return nil, err
}
for _, rel := range p.Inventory.SourceFiles {
abs := filepath.Join(p.SourceDir(), filepath.FromSlash(rel))
@@ -140,9 +136,6 @@ func expectedResources(p *project.Project) (map[string]resourceExpectation, erro
if extension == ".ifo" && strings.EqualFold(name, "module") && len(moduleHakOrder) > 0 {
setModuleHAKList(&document, moduleHakOrder)
}
if extension == ".itp" && isPaletteProjectionResref(name) {
projectPaletteDocument(&document, palettes[strings.ToLower(name)])
}
canonical, err := json.Marshal(document)
if err != nil {
-7
View File
@@ -124,13 +124,6 @@ func extractArchiveResources(p *project.Project, archive erf.Archive, desired ma
skippedCount++
continue
}
// *palcus.itp are Toolset-generated palette projections; the module
// build regenerates them from source blueprints, so extraction never
// writes them back into source.
if ext == "itp" && isPaletteProjectionResref(resource.Name) {
skippedCount++
continue
}
target, data, err := extractedFile(p, resource, ext)
if err != nil {
-298
View File
@@ -1,298 +0,0 @@
package pipeline
// Custom palette projections (*palcus.itp) are generated artifacts: the Toolset
// derives them from the category frameworks plus the blueprints present in the
// module. Crucible reproduces that projection deterministically at build time so
// the module source only carries descriptor-free category skeletons and the
// blueprint files themselves. Extract never writes palcus files back.
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"sort"
"strings"
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/gff"
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
)
const noStrref = 0xFFFFFFFF
// hiddenPaletteID suppresses a blueprint from the Custom palette (engine
// convention; see docs in sow-module's palette research notes).
const hiddenPaletteID = 255
var paletteFamilyByExtension = map[string]string{
".utc": "creaturepalcus",
".utd": "doorpalcus",
".ute": "encounterpalcus",
".uti": "itempalcus",
".utm": "storepalcus",
".utp": "placeablepalcus",
".uts": "soundpalcus",
".utt": "triggerpalcus",
".utw": "waypointpalcus",
}
type paletteDescriptor struct {
name string
strref uint32
resref string
creature bool
cr float32
faction string
}
func (d paletteDescriptor) sortKey() string {
if d.strref != noStrref || d.name == "" {
return strings.ToLower(d.resref)
}
return strings.ToLower(d.name)
}
// palette resref (e.g. "itempalcus") -> terminal category ID -> descriptors.
type paletteProjection map[string]map[uint8][]paletteDescriptor
func isPaletteProjectionResref(name string) bool {
return strings.HasSuffix(strings.ToLower(name), "palcus")
}
func collectPaletteDescriptors(p *project.Project) (paletteProjection, error) {
projection := paletteProjection{}
factions, err := loadFactionNames(p)
if err != nil {
return nil, err
}
for _, rel := range p.Inventory.SourceFiles {
abs := filepath.Join(p.SourceDir(), filepath.FromSlash(rel))
name, extension, err := splitSourceName(abs)
if err != nil {
return nil, err
}
family, ok := paletteFamilyByExtension[extension]
if !ok {
continue
}
raw, err := os.ReadFile(abs)
if err != nil {
return nil, fmt.Errorf("read %s: %w", abs, err)
}
var document gff.Document
if err := json.Unmarshal(raw, &document); err != nil {
return nil, fmt.Errorf("parse gff json %s: %w", abs, err)
}
descriptor, paletteID, ok := blueprintDescriptor(document.Root, name, extension, factions)
if !ok {
continue
}
if projection[family] == nil {
projection[family] = map[uint8][]paletteDescriptor{}
}
projection[family][paletteID] = append(projection[family][paletteID], descriptor)
}
for _, byID := range projection {
for _, descriptors := range byID {
sort.SliceStable(descriptors, func(i, j int) bool {
a, b := descriptors[i], descriptors[j]
if a.sortKey() != b.sortKey() {
return a.sortKey() < b.sortKey()
}
return a.resref < b.resref
})
}
}
return projection, nil
}
func blueprintDescriptor(root gff.Struct, fileName, extension string, factions []string) (paletteDescriptor, uint8, bool) {
descriptor := paletteDescriptor{
strref: noStrref,
resref: strings.ToLower(fileName),
creature: extension == ".utc",
}
paletteID := -1
for _, field := range root.Fields {
switch field.Label {
case "PaletteID":
if v, ok := field.Value.(gff.ByteValue); ok {
paletteID = int(v)
}
case "TemplateResRef":
if v, ok := field.Value.(gff.ResRefValue); ok && v != "" {
descriptor.resref = strings.ToLower(string(v))
}
case "LocalizedName", "LocName", "FirstName":
if v, ok := field.Value.(gff.LocString); ok {
name, strref := locStringLabel(v)
if field.Label == "FirstName" {
descriptor.name = strings.TrimSpace(descriptor.name + " " + name)
if descriptor.strref == noStrref {
descriptor.strref = strref
}
} else {
descriptor.name = name
descriptor.strref = strref
}
}
case "LastName":
if v, ok := field.Value.(gff.LocString); ok {
name, _ := locStringLabel(v)
descriptor.name = strings.TrimSpace(descriptor.name + " " + name)
}
case "ChallengeRating":
if v, ok := field.Value.(gff.FloatValue); ok {
descriptor.cr = float32(v)
}
case "FactionID":
if v, ok := field.Value.(gff.WordValue); ok && int(v) < len(factions) {
descriptor.faction = factions[v]
}
}
}
if paletteID < 0 || paletteID == hiddenPaletteID {
return paletteDescriptor{}, 0, false
}
return descriptor, uint8(paletteID), true
}
func locStringLabel(value gff.LocString) (string, uint32) {
if value.StringRef != noStrref {
return "", value.StringRef
}
for _, entry := range value.Entries {
if entry.Value != "" {
return entry.Value, noStrref
}
}
return "", noStrref
}
func loadFactionNames(p *project.Project) ([]string, error) {
for _, rel := range p.Inventory.SourceFiles {
if !strings.HasSuffix(strings.ToLower(rel), "repute.fac.json") {
continue
}
abs := filepath.Join(p.SourceDir(), filepath.FromSlash(rel))
raw, err := os.ReadFile(abs)
if err != nil {
return nil, fmt.Errorf("read %s: %w", abs, err)
}
var document gff.Document
if err := json.Unmarshal(raw, &document); err != nil {
return nil, fmt.Errorf("parse gff json %s: %w", abs, err)
}
var names []string
for _, field := range document.Root.Fields {
if field.Label != "FactionList" {
continue
}
list, ok := field.Value.(gff.ListValue)
if !ok {
continue
}
for _, faction := range list {
name := ""
for _, f := range faction.Fields {
if f.Label == "FactionName" {
if v, ok := f.Value.(gff.StringValue); ok {
name = string(v)
}
}
}
names = append(names, name)
}
}
return names, nil
}
return nil, nil
}
// projectPaletteDocument strips every blueprint descriptor from the palette
// tree and re-inserts the descriptors derived from module source. Category
// structure (branches, terminal IDs, labels) passes through untouched.
func projectPaletteDocument(document *gff.Document, byID map[uint8][]paletteDescriptor) {
for i, field := range document.Root.Fields {
if field.Label != "MAIN" {
continue
}
if list, ok := field.Value.(gff.ListValue); ok {
document.Root.Fields[i] = gff.NewField("MAIN", projectPaletteNodes(list, byID))
}
}
}
func projectPaletteNodes(nodes gff.ListValue, byID map[uint8][]paletteDescriptor) gff.ListValue {
out := make(gff.ListValue, 0, len(nodes))
for _, node := range nodes {
if structHasField(node, "RESREF") {
continue // blueprint descriptor — regenerated below
}
terminalID := -1
fields := make([]gff.Field, 0, len(node.Fields))
for _, field := range node.Fields {
if field.Label == "LIST" {
continue // rebuilt for terminals, recursed for branches
}
if field.Label == "ID" {
if v, ok := field.Value.(gff.ByteValue); ok {
terminalID = int(v)
}
}
fields = append(fields, field)
}
switch {
case terminalID >= 0:
if descriptors := byID[uint8(terminalID)]; len(descriptors) > 0 {
fields = append(fields, gff.NewField("LIST", descriptorList(descriptors)))
}
default:
for _, field := range node.Fields {
if field.Label == "LIST" {
if list, ok := field.Value.(gff.ListValue); ok {
fields = append(fields, gff.NewField("LIST", projectPaletteNodes(list, byID)))
}
}
}
}
out = append(out, gff.Struct{Type: node.Type, Fields: fields})
}
return out
}
func descriptorList(descriptors []paletteDescriptor) gff.ListValue {
list := make(gff.ListValue, 0, len(descriptors))
for _, d := range descriptors {
fields := make([]gff.Field, 0, 4)
if d.strref != noStrref {
fields = append(fields, gff.NewField("STRREF", gff.DWordValue(d.strref)))
} else {
fields = append(fields, gff.NewField("NAME", gff.StringValue(d.name)))
}
fields = append(fields, gff.NewField("RESREF", gff.ResRefValue(d.resref)))
if d.creature {
fields = append(fields, gff.NewField("CR", gff.FloatValue(d.cr)))
fields = append(fields, gff.NewField("FACTION", gff.StringValue(d.faction)))
}
list = append(list, gff.Struct{Fields: fields})
}
return list
}
func structHasField(s gff.Struct, label string) bool {
for _, field := range s.Fields {
if field.Label == label {
return true
}
}
return false
}
-191
View File
@@ -1,191 +0,0 @@
package pipeline
import (
"bytes"
"encoding/json"
"os"
"path/filepath"
"testing"
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf"
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/gff"
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
)
const paletteTestSkeleton = `{
"file_type": "ITP ",
"file_version": "V3.2",
"root": {
"struct_type": 4294967295,
"fields": [
{
"label": "MAIN",
"type": "List",
"value": [
{
"struct_type": 0,
"fields": [
{"label": "STRREF", "type": "DWord", "value": 500},
{
"label": "LIST",
"type": "List",
"value": [
{
"struct_type": 0,
"fields": [
{"label": "STRREF", "type": "DWord", "value": 6699},
{"label": "ID", "type": "Byte", "value": 23},
{
"label": "LIST",
"type": "List",
"value": [
{
"struct_type": 0,
"fields": [
{"label": "NAME", "type": "CExoString", "value": "Stale Junk"},
{"label": "RESREF", "type": "ResRef", "value": "stalejunk"}
]
}
]
}
]
},
{
"struct_type": 0,
"fields": [
{"label": "STRREF", "type": "DWord", "value": 6753},
{"label": "ID", "type": "Byte", "value": 24}
]
}
]
}
]
}
]
}
]
}
}
`
func paletteTestItem(resref, name string, paletteID int) string {
return `{
"file_type": "UTI ",
"file_version": "V3.2",
"root": {
"struct_type": 4294967295,
"fields": [
{"label": "TemplateResRef", "type": "ResRef", "value": "` + resref + `"},
{
"label": "LocalizedName",
"type": "CExoLocString",
"value": {"string_ref": 4294967295, "entries": [{"id": 0, "value": "` + name + `"}]}
},
{"label": "PaletteID", "type": "Byte", "value": ` + itoa(paletteID) + `}
]
}
}
`
}
func itoa(v int) string {
data, _ := json.Marshal(v)
return string(data)
}
func TestBuildProjectsPaletteDescriptorsAndExtractSkipsThem(t *testing.T) {
root := t.TempDir()
mustMkdir(t, filepath.Join(root, "src", "module"))
mustMkdir(t, filepath.Join(root, "src", "palettes"))
mustMkdir(t, filepath.Join(root, "src", "blueprints", "items"))
mustMkdir(t, filepath.Join(root, "build"))
mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{
"module": {"name": "Test Module", "resref": "testmod"},
"paths": {"source": "src", "build": "build"}
}
`)
mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{
"file_type": "IFO ",
"file_version": "V3.2",
"root": {"struct_type": 4294967295, "fields": [{"label": "Mod_Name", "type": "CExoString", "value": "Test Module"}]}
}
`)
mustWriteFile(t, filepath.Join(root, "src", "palettes", "itempalcus.itp.json"), paletteTestSkeleton)
mustWriteFile(t, filepath.Join(root, "src", "blueprints", "items", "i_b.uti.json"), paletteTestItem("i_b", "Bravo Item", 23))
mustWriteFile(t, filepath.Join(root, "src", "blueprints", "items", "i_a.uti.json"), paletteTestItem("i_a", "Alpha Item", 23))
mustWriteFile(t, filepath.Join(root, "src", "blueprints", "items", "i_hidden.uti.json"), paletteTestItem("i_hidden", "Hidden Item", 255))
p, err := project.Load(root)
if err != nil {
t.Fatalf("load project: %v", err)
}
if err := p.Scan(); err != nil {
t.Fatalf("scan: %v", err)
}
if _, err := BuildModule(p); err != nil {
t.Fatalf("build: %v", err)
}
archiveFile, err := os.Open(p.ModuleArchivePath())
if err != nil {
t.Fatalf("open module archive: %v", err)
}
defer archiveFile.Close()
archive, err := erf.Read(archiveFile)
if err != nil {
t.Fatalf("read module archive: %v", err)
}
var palette *gff.Document
for _, resource := range archive.Resources {
if resource.Name == "itempalcus" {
document, err := gff.Read(bytes.NewReader(resource.Data))
if err != nil {
t.Fatalf("decode itempalcus: %v", err)
}
palette = &document
}
}
if palette == nil {
t.Fatal("itempalcus missing from built module")
}
canonical, err := json.Marshal(palette)
if err != nil {
t.Fatalf("marshal palette: %v", err)
}
text := string(canonical)
if bytes.Contains(canonical, []byte("stalejunk")) {
t.Fatalf("stale descriptor survived projection: %s", text)
}
for _, resref := range []string{"i_a", "i_b"} {
if !bytes.Contains(canonical, []byte(resref)) {
t.Fatalf("descriptor %s missing from projection: %s", resref, text)
}
}
if bytes.Contains(canonical, []byte("i_hidden")) {
t.Fatalf("PaletteID 255 blueprint leaked into projection: %s", text)
}
if a, b := bytes.Index(canonical, []byte("i_a")), bytes.Index(canonical, []byte("i_b")); a > b {
t.Fatalf("descriptors not name-sorted: %s", text)
}
if _, err := Compare(p); err != nil {
t.Fatalf("compare after build: %v", err)
}
// Extraction must never write palcus files back into source.
if err := os.Remove(filepath.Join(root, "src", "palettes", "itempalcus.itp.json")); err != nil {
t.Fatalf("remove skeleton: %v", err)
}
if err := p.Scan(); err != nil {
t.Fatalf("rescan: %v", err)
}
if _, err := Extract(p); err != nil {
t.Fatalf("extract: %v", err)
}
if _, err := os.Stat(filepath.Join(root, "src", "palettes", "itempalcus.itp.json")); !os.IsNotExist(err) {
t.Fatalf("extract wrote palcus file back (stat err: %v)", err)
}
}
+1 -1
View File
@@ -31,7 +31,7 @@ var SourceExtensions = []string{
}
var AssetExtensions = []string{
".2da", ".bik", ".bmp", ".bmu", ".dds", ".dwk", ".itp", ".jpg", ".lod", ".lyt", ".mdl", ".mdx", ".mtr", ".plt", ".png", ".pwk", ".set", ".shd", ".tga", ".txi", ".uti", ".vis", ".wav", ".wok",
".2da", ".bik", ".bmp", ".bmu", ".dds", ".dwk", ".gr2", ".itp", ".jpg", ".lod", ".lyt", ".mdb", ".mdl", ".mdx", ".mtr", ".plt", ".png", ".pwk", ".set", ".shd", ".tga", ".txi", ".uti", ".vis", ".wav", ".wlk", ".wok", ".xml",
}
var BuiltinScriptPrefixes = []string{
+2 -16
View File
@@ -109,7 +109,6 @@ type tlkCompiler struct {
active map[string]tlkEntryData
activeKeys map[string]struct{}
reservedByID map[int]string
pinnedByID map[int]string
nextID int
}
@@ -159,7 +158,6 @@ func newTLKCompiler(sourceDir string, legacy *legacyTLKData) (*tlkCompiler, erro
active: map[string]tlkEntryData{},
activeKeys: map[string]struct{}{},
reservedByID: reserved,
pinnedByID: map[int]string{},
nextID: nextID,
}
if legacy != nil {
@@ -382,24 +380,12 @@ func (c *tlkCompiler) registerInlineAtID(key string, id int, entry tlkEntryData)
if id < 0 {
return fmt.Errorf("TLK key %q has negative id %d", key, id)
}
// The pin is authoritative over the per-machine .tlk_state.json cache: a
// stale mapping that dynamically grabbed this id on an older build must
// yield so the pinned key can take it. Only a genuine clash between two
// pins in custom.tlk.yml is an author error.
if owner, ok := c.pinnedByID[id]; ok && owner != key {
return fmt.Errorf("TLK id %d is pinned by both %q and %q", id, owner, key)
}
if mapping, ok := c.state.Entries[key]; ok && mapping.ID != id {
// This key held a different cached id; release it so the pin wins.
if c.reservedByID[mapping.ID] == key {
delete(c.reservedByID, mapping.ID)
}
return fmt.Errorf("TLK key %q changed id from %d to %d", key, mapping.ID, id)
}
if owner, ok := c.reservedByID[id]; ok && owner != key {
// Evict the stale owner; it gets a fresh id when next made active.
delete(c.state.Entries, owner)
return fmt.Errorf("TLK id %d is already reserved by %q", id, owner)
}
c.pinnedByID[id] = key
c.state.Entries[key] = tlkStateMapping{ID: id}
c.reservedByID[id] = key
return c.markActive(key, entry)
-49
View File
@@ -2449,55 +2449,6 @@ strings:
}
}
func TestBuildStandaloneTLKPinEvictsStaleStateOwner(t *testing.T) {
root := testProjectRoot(t)
mkdirAll(t, filepath.Join(root, "topdata", "data", "feat"))
mkdirAll(t, filepath.Join(root, "topdata", "tlk"))
writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n")
writeFile(t, filepath.Join(root, "topdata", "data", "feat", "base.json"), `{
"output": "feat.2da",
"columns": ["LABEL", "FEAT", "DESCRIPTION"],
"rows": [{
"id": 0,
"key": "feat:test",
"LABEL": "TEST_LABEL",
"FEAT": {"tlk": {"key": "feat:test.name", "text": "Test Feat"}},
"DESCRIPTION": "****"
}]
}`+"\n")
writeFile(t, filepath.Join(root, "topdata", "tlk", "custom.tlk.yml"), `schema: sow-topdata/tlk/v1
base_strref: 16777216
strings:
- key: sow.module.name
text: Shadows Over Westgate
id: 50
`)
// Simulate a per-machine state that predates the pinned taxonomy: the feat
// ref dynamically grabbed id 50 on an older build, exactly where the pin
// now lives. The build must self-heal instead of failing.
writeFile(t, filepath.Join(root, "topdata", tlkStateFile),
`{"version":1,"language":"en","entries":{"feat:test.name":{"id":50}}}`+"\n")
if _, err := BuildNative(testProject(root), nil); err != nil {
t.Fatalf("BuildNative failed on stale pin collision: %v", err)
}
stateRaw, err := os.ReadFile(filepath.Join(root, "topdata", tlkStateFile))
if err != nil {
t.Fatalf("read tlk state: %v", err)
}
var state tlkStateDocument
if err := json.Unmarshal(stateRaw, &state); err != nil {
t.Fatalf("parse tlk state: %v", err)
}
if state.Entries["sow.module.name"].ID != 50 {
t.Fatalf("pin must own id 50, got %#v", state.Entries["sow.module.name"])
}
if got := state.Entries["feat:test.name"].ID; got == 50 {
t.Fatalf("stale feat ref should have been reallocated off id 50, got %d", got)
}
}
func TestBuildPreservesTLKStateAcrossTextChanges(t *testing.T) {
root := testProjectRoot(t)
mkdirAll(t, filepath.Join(root, "topdata", "data", "skills"))
+1 -1
View File
@@ -18,7 +18,7 @@ bin=bin
# Keep in sync with internal/dispatch.Registry (Wired flag).
unwired=()
wired=(assets depot hak module nwsync topdata wiki)
wired=(assets depot hak module topdata wiki)
exit_of() { set +e; "$@" >/dev/null 2>&1; local c=$?; set -e; echo "${c}"; }
-67
View File
@@ -1,67 +0,0 @@
#!/usr/bin/env bash
# Delete the binary assets of every release except the newest KEEP ones.
#
# Gitea has no asset retention, so 9 assets x ~57 MB per tag pile up forever on
# the host disk (sow-tools #52). Nothing consumes an old asset: CI takes the
# Crucible binary from the Nix flake, and every deleted file is reproducible
# from its tag. Tags, source archives, release notes and the releases
# themselves are never touched — only attachments.
#
# Nothing is excluded, SHA256SUMS and the wrappers included: the checksums are
# only meaningful next to the binaries they cover, and the drift-check reads
# the wrappers from the latest release, which always stays complete.
#
# Best-effort by design: a stale asset is cheaper than a blocked release, so
# every API failure is a warning, never a non-zero exit.
#
# Env:
# API repo API base, e.g. https://git.example/api/v1/repos/owner/repo [required]
# TOKEN Gitea token with releases:write [required]
# KEEP how many newest releases stay complete (default 2)
set -uo pipefail
: "${API:?set API}"
: "${TOKEN:?set TOKEN}"
keep="${KEEP:-2}"
auth="Authorization: token ${TOKEN}"
per_page=50
warn() { echo "prune-release-assets: $*" >&2; }
# Walk every page so an old backlog is cleared, not only the tag that fell out
# of the window on this run. Each page already embeds its releases' assets, so
# no follow-up request per release is needed.
releases='[]'
page=1
while :; do
body="$(curl -fsS -H "$auth" "${API}/releases?limit=${per_page}&page=${page}&draft=false")" || {
warn "listing releases failed on page ${page}; pruning what was listed so far"
break
}
count="$(jq 'length' <<<"$body")" || { warn "unreadable release page ${page}"; break; }
releases="$(jq -s 'add' <(printf '%s' "$releases") <(printf '%s' "$body"))"
# A short page is the last one; this also stops a server that ignores `page`.
(( count < per_page )) && break
page=$(( page + 1 ))
done
# Sort here rather than trusting the response order, and skip drafts in case
# the server ignored `draft=false` — a draft must not consume a keep slot and
# strip the binaries off the newest real release.
mapfile -t stale < <(jq -r --argjson keep "$keep" '
[.[] | select(.draft != true)]
| sort_by(.created_at) | reverse | .[$keep:]
| .[] | "\(.id) \(.assets // [] | map(.id) | join(","))"
' <<<"$releases")
(( ${#stale[@]} )) || { echo "prune-release-assets: nothing older than the newest ${keep} releases"; exit 0; }
for line in "${stale[@]}"; do
id="${line%% *}"
assets="${line#* }"
for asset in ${assets//,/ }; do
echo "deleting asset ${asset} of release ${id}"
curl -fsS -X DELETE -H "$auth" "${API}/releases/${id}/assets/${asset}" >/dev/null \
|| warn "deleting asset ${asset} of release ${id} failed"
done
done
-78
View File
@@ -1,78 +0,0 @@
#!/usr/bin/env bash
# Contract for scripts/prune-release-assets.sh, driven by a stub curl on PATH:
# only assets of releases outside the keep window go, the window is decided by
# release date rather than response order, drafts never consume a keep slot,
# releases and tags stay, and an API failure never fails the run.
set -euo pipefail
repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
script="$repo_root/scripts/prune-release-assets.sh"
tmp="$(mktemp -d)"
trap 'rm -rf "$tmp"' EXIT
# Stub curl: one short page of releases, deliberately out of date order and
# with a draft that is newer than every published release. Each release embeds
# two assets. Every request is appended to $CALLS for the assertions below.
mkdir -p "$tmp/bin"
cat >"$tmp/bin/curl" <<'STUB'
#!/usr/bin/env bash
url="${!#}"
printf '%s\n' "$*" >>"$CALLS"
rel() { printf '{"id":%s,"created_at":"%s","draft":%s,"assets":[{"id":%s01},{"id":%s02}]}' "$1" "$2" "$3" "$1" "$1"; }
case "$url" in
*releases\?*)
[[ "${FAIL_LIST:-0}" == 1 ]] && exit 22
printf '[%s,%s,%s,%s,%s]\n' \
"$(rel 12 2024-01-02T00:00:00Z false)" \
"$(rel 10 2024-01-04T00:00:00Z false)" \
"$(rel 9 2024-06-01T00:00:00Z true)" \
"$(rel 13 2024-01-01T00:00:00Z false)" \
"$(rel 11 2024-01-03T00:00:00Z false)"
;;
*/assets/*) [[ "${FAIL_DELETE:-0}" == 1 ]] && exit 22 ;;
esac
exit 0
STUB
chmod +x "$tmp/bin/curl"
export PATH="$tmp/bin:$PATH" API=https://git.example/api/v1/repos/o/r TOKEN=t
run() { CALLS="$tmp/calls" bash "$script" >"$tmp/out" 2>"$tmp/err"; }
# Default keep=2: the two newest published releases (10, 11) stay whole, the
# older two (12, 13) lose their assets, and the newer draft is ignored.
: >"$tmp/calls"; run
deletes="$(grep -c -- '-X DELETE' "$tmp/calls" || true)"
[[ "$deletes" == 4 ]] || { echo "expected 4 asset deletes, got $deletes" >&2; exit 1; }
grep -q 'assets/1201' "$tmp/calls" && grep -q 'assets/1302' "$tmp/calls" || {
echo "expected assets of releases 12 and 13 to be deleted" >&2; exit 1; }
if grep -q 'releases/10/assets/\|releases/11/assets/' "$tmp/calls"; then
echo "kept releases lost assets" >&2; exit 1
fi
if grep -q 'releases/9/assets/' "$tmp/calls"; then
echo "a draft release was pruned" >&2; exit 1
fi
# A release or tag must never be deleted, only attachments under /assets/.
if grep -- '-X DELETE' "$tmp/calls" | grep -qv '/assets/'; then
echo "a non-asset DELETE was issued" >&2; exit 1
fi
# A short page ends the sweep: no second page, so a server ignoring `page`
# cannot spin the job until the job timeout.
[[ "$(grep -c 'releases?' "$tmp/calls")" == 1 ]] || {
echo "a short release page did not end the sweep" >&2; exit 1; }
# KEEP=4 covers every published release: nothing to delete.
: >"$tmp/calls"; KEEP=4 run
if grep -q -- '-X DELETE' "$tmp/calls"; then echo "KEEP=4 still deleted" >&2; exit 1; fi
# Failures are warnings, never a non-zero exit.
: >"$tmp/calls"; FAIL_DELETE=1 run || { echo "delete failure failed the run" >&2; exit 1; }
grep -q 'failed' "$tmp/err" || { echo "delete failure was not warned about" >&2; exit 1; }
: >"$tmp/calls"; FAIL_LIST=1 run || { echo "listing failure failed the run" >&2; exit 1; }
grep -q 'listing releases failed' "$tmp/err" || {
echo "listing failure was not warned about" >&2; exit 1; }
if grep -q -- '-X DELETE' "$tmp/calls"; then
echo "deleted assets despite an unreadable release list" >&2; exit 1
fi
echo "prune-release-assets: prunes by release date outside the keep window, ignores drafts, never deletes a release, never fails the build"
-86
View File
@@ -1,86 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
contract_script="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/$(basename "${BASH_SOURCE[0]}")"
default_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
repo_root="${WORKFLOW_ROOT:-$default_root}"
cd "$repo_root"
ci=.gitea/workflows/ci.yml
release=.gitea/workflows/build-binaries.yml
sync=.gitea/workflows/sync-wrappers.yml
checkout='actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683'
assert_exact_events() {
local workflow="$1" expected="$2" actual
actual="$(awk '
/^on:$/ { in_on = 1 }
in_on && /^$/ { next }
in_on && $0 != "on:" && $0 !~ /^[[:space:]]/ { exit }
in_on { print }
' "$workflow")"
[[ "$actual" == "$expected" ]] || {
echo "workflow-contract: unexpected event scope in $workflow" >&2
return 1
}
}
[[ -f "$ci" ]] || { echo "missing consolidated CI workflow" >&2; exit 1; }
[[ ! -e .gitea/workflows/test.yml ]] || { echo "legacy test workflow still present" >&2; exit 1; }
assert_exact_events "$ci" $'on:\n pull_request:'
grep -Fqx 'permissions: read-all' "$ci"
grep -Fqx ' timeout-minutes: 60' "$ci"
[[ "$(grep -Fc "$checkout" "$ci")" -eq 1 ]]
grep -Fqx ' fetch-depth: 0' "$ci"
grep -Fq 'go vet ./...' "$ci"
grep -Fq 'go test ./...' "$ci"
grep -Fq 'shellcheck scripts/*.sh' "$ci"
grep -Fq 'yamllint .gitea' "$ci"
grep -Fq 'make smoke' "$ci"
grep -Fq 'for target in linux/amd64 linux/arm64 darwin/amd64 darwin/arm64 windows/amd64 windows/arm64; do' "$ci"
assert_exact_events "$release" $'on:\n push:\n tags: [\'v*\']'
grep -Fqx ' code: read' "$release"
grep -Fqx ' releases: write' "$release"
grep -Fqx ' timeout-minutes: 30' "$release"
[[ "$(grep -Fc "$checkout" "$release")" -eq 1 ]]
grep -Fq 'secrets.GITEA_TOKEN' "$release"
grep -Fq 'scripts/prune-release-assets.sh' "$release"
grep -Fq 'continue-on-error: true' "$release"
if grep -Fq 'secrets.GITHUB_TOKEN' "$release"; then
echo "workflow-contract: release uses the GitHub token alias" >&2
exit 1
fi
grep -Fqx 'permissions: read-all' "$sync"
grep -Fqx ' timeout-minutes: 30' "$sync"
[[ "$(grep -Fc "$checkout" "$sync")" -eq 1 ]]
assert_exact_events "$sync" $'on:\n push:\n branches: [main]\n paths:\n - \'wrappers/crucible.sh\'\n - \'wrappers/crucible.ps1\''
expect_mutation_rejected() {
local name="$1" workflow="$2" expression="$3" tmp
tmp="$(mktemp -d)"
mkdir -p "$tmp/.gitea"
cp -R "$default_root/.gitea/workflows" "$tmp/.gitea/workflows"
sed -i "$expression" "$tmp/$workflow"
if WORKFLOW_ROOT="$tmp" WORKFLOW_CONTRACT_MUTATION=1 bash "$contract_script" >/dev/null 2>&1; then
echo "workflow-contract: accepted forbidden mutation: $name" >&2
rm -rf "$tmp"
return 1
fi
rm -rf "$tmp"
}
if [[ "${WORKFLOW_CONTRACT_MUTATION:-0}" != 1 ]]; then
mutations_ok=0
expect_mutation_rejected 'CI workflow_dispatch event' "$ci" '/^ pull_request:$/a\ workflow_dispatch:' || mutations_ok=1
expect_mutation_rejected 'release develop branch' "$release" "/^ tags:/a\\ branches: [develop]" || mutations_ok=1
expect_mutation_rejected 'release schedule event' "$release" "/^ tags:/a\\ schedule:\n - cron: '0 0 * * *'" || mutations_ok=1
expect_mutation_rejected 'release extra tag pattern' "$release" "s/tags: \['v\*'\]/tags: ['v*', 'release-*']/" || mutations_ok=1
expect_mutation_rejected 'wrapper sync pull_request event' "$sync" '/^ push:$/i\ pull_request:' || mutations_ok=1
expect_mutation_rejected 'wrapper sync broad path' "$sync" "/^ - 'wrappers\/crucible.ps1'$/a\\ - 'wrappers/**'" || mutations_ok=1
(( mutations_ok == 0 )) || exit 1
fi
echo "workflow-contract: CI is PR-only; release and wrapper sync retain their narrow triggers"