Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f34c25afb0 |
@@ -1,22 +1,25 @@
|
|||||||
# Cross-platform Crucible release binaries: a v* tag builds all targets,
|
# Cross-platform Crucible binaries (D7 trigger standard):
|
||||||
# then uploads them, SHA256SUMS, and the canonical wrappers to its Gitea release.
|
# PR / push main -> cross-build ALL targets to prove they compile (no publish)
|
||||||
|
# tag v* -> build all targets + SHA256SUMS, upload to the Gitea release
|
||||||
# Crucible is pure Go (CGO_ENABLED=0), so cross-compiling is a fast loop.
|
# Crucible is pure Go (CGO_ENABLED=0), so cross-compiling is a fast loop.
|
||||||
name: build-binaries
|
name: build-binaries
|
||||||
|
|
||||||
on:
|
on:
|
||||||
|
pull_request:
|
||||||
push:
|
push:
|
||||||
|
branches: [main]
|
||||||
tags: ['v*']
|
tags: ['v*']
|
||||||
|
paths-ignore:
|
||||||
permissions:
|
- "docs/**"
|
||||||
code: read
|
- "README.md"
|
||||||
releases: write
|
- "AGENTS.md"
|
||||||
|
- "LICENSE"
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
build-binaries:
|
build-binaries:
|
||||||
runs-on: nix-docker
|
runs-on: nix-docker
|
||||||
timeout-minutes: 30
|
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
- uses: actions/checkout@v4
|
||||||
with: { fetch-depth: 0 }
|
with: { fetch-depth: 0 }
|
||||||
|
|
||||||
- name: Cross-build all targets
|
- name: Cross-build all targets
|
||||||
@@ -42,8 +45,9 @@ jobs:
|
|||||||
'
|
'
|
||||||
|
|
||||||
- name: Upload to Gitea release
|
- name: Upload to Gitea release
|
||||||
|
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
|
||||||
env:
|
env:
|
||||||
TOKEN: ${{ secrets.GITEA_TOKEN }}
|
TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||||
TAG: ${{ github.ref_name }}
|
TAG: ${{ github.ref_name }}
|
||||||
REPO: ${{ github.repository }}
|
REPO: ${{ github.repository }}
|
||||||
SERVER: ${{ github.server_url }}
|
SERVER: ${{ github.server_url }}
|
||||||
@@ -65,17 +69,3 @@ jobs:
|
|||||||
"${api}/releases/${id}/assets?name=$(basename "$f")"
|
"${api}/releases/${id}/assets?name=$(basename "$f")"
|
||||||
done
|
done
|
||||||
'
|
'
|
||||||
|
|
||||||
# Gitea has no asset retention (#52): without this every tag keeps its
|
|
||||||
# ~57 MB binary set forever. Best-effort — a stale asset is cheaper than
|
|
||||||
# a blocked publish, so a failure here never fails the release.
|
|
||||||
- name: Prune assets of older releases
|
|
||||||
continue-on-error: true
|
|
||||||
env:
|
|
||||||
TOKEN: ${{ secrets.GITEA_TOKEN }}
|
|
||||||
REPO: ${{ github.repository }}
|
|
||||||
SERVER: ${{ github.server_url }}
|
|
||||||
run: |
|
|
||||||
nix develop --command bash -c '
|
|
||||||
API="${SERVER}/api/v1/repos/${REPO}" scripts/prune-release-assets.sh
|
|
||||||
'
|
|
||||||
|
|||||||
@@ -1,47 +0,0 @@
|
|||||||
# Pull-request validation for Crucible. Releases and wrapper synchronization
|
|
||||||
# have their own narrow workflows because they need tag/main events.
|
|
||||||
name: ci
|
|
||||||
|
|
||||||
on:
|
|
||||||
pull_request:
|
|
||||||
|
|
||||||
permissions: read-all
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
ci:
|
|
||||||
runs-on: nix-docker
|
|
||||||
timeout-minutes: 60
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
||||||
with:
|
|
||||||
fetch-depth: 0
|
|
||||||
|
|
||||||
- name: vet + test + lint
|
|
||||||
run: |
|
|
||||||
nix develop --command bash -c '
|
|
||||||
set -euo pipefail
|
|
||||||
go vet ./...
|
|
||||||
go test ./...
|
|
||||||
shellcheck scripts/*.sh
|
|
||||||
yamllint .gitea
|
|
||||||
'
|
|
||||||
|
|
||||||
- name: binary smoke (fail-closed contract)
|
|
||||||
run: nix develop --command make smoke
|
|
||||||
|
|
||||||
- name: Cross-build all targets
|
|
||||||
run: |
|
|
||||||
nix develop --command bash -c '
|
|
||||||
set -euo pipefail
|
|
||||||
sha="$(git rev-parse --short=12 HEAD)"
|
|
||||||
ldflags="-s -w -X git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/buildinfo.Version=${sha}"
|
|
||||||
rm -rf dist && mkdir -p dist
|
|
||||||
export CGO_ENABLED=0
|
|
||||||
for target in linux/amd64 linux/arm64 darwin/amd64 darwin/arm64 windows/amd64 windows/arm64; do
|
|
||||||
os="${target%/*}"; arch="${target#*/}"
|
|
||||||
ext=""; [ "$os" = windows ] && ext=".exe"
|
|
||||||
echo "building crucible-${os}-${arch}${ext}"
|
|
||||||
GOOS="$os" GOARCH="$arch" go build -trimpath -ldflags "$ldflags" \
|
|
||||||
-o "dist/crucible-${os}-${arch}${ext}" ./cmd/crucible
|
|
||||||
done
|
|
||||||
'
|
|
||||||
@@ -14,14 +14,11 @@ on:
|
|||||||
- 'wrappers/crucible.sh'
|
- 'wrappers/crucible.sh'
|
||||||
- 'wrappers/crucible.ps1'
|
- 'wrappers/crucible.ps1'
|
||||||
|
|
||||||
permissions: read-all
|
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
sync:
|
sync:
|
||||||
runs-on: nix-docker
|
runs-on: nix-docker
|
||||||
timeout-minutes: 30
|
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
- uses: actions/checkout@v4
|
||||||
|
|
||||||
- name: Open sync PRs to consumers
|
- name: Open sync PRs to consumers
|
||||||
env:
|
env:
|
||||||
|
|||||||
@@ -0,0 +1,30 @@
|
|||||||
|
# Lint + unit tests for the Crucible suite. PR-first: PRs + push to main (D7).
|
||||||
|
name: test
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches: [main]
|
||||||
|
paths-ignore:
|
||||||
|
- "docs/**"
|
||||||
|
- "README.md"
|
||||||
|
- "AGENTS.md"
|
||||||
|
- "LICENSE"
|
||||||
|
pull_request:
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
test:
|
||||||
|
runs-on: nix-docker
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
with: { fetch-depth: 0 }
|
||||||
|
- name: vet + test + lint
|
||||||
|
run: |
|
||||||
|
nix develop --command bash -c '
|
||||||
|
set -euo pipefail
|
||||||
|
go vet ./...
|
||||||
|
go test ./...
|
||||||
|
shellcheck scripts/*.sh
|
||||||
|
yamllint .gitea
|
||||||
|
'
|
||||||
|
- name: binary smoke (fail-closed contract)
|
||||||
|
run: nix develop --command make smoke
|
||||||
@@ -87,21 +87,3 @@ make smoke # assert fail-closed contract
|
|||||||
## Tests
|
## Tests
|
||||||
|
|
||||||
Tests must survive harmless changes to constants, defaults, wording, ordering, fixture data, and internal implementation details. A test that fails merely because a basic value changed is usually a bad test. Only assert exact values when the value is part of a documented public contract, external protocol, compatibility requirement, security rule, migration, or business rule.
|
Tests must survive harmless changes to constants, defaults, wording, ordering, fixture data, and internal implementation details. A test that fails merely because a basic value changed is usually a bad test. Only assert exact values when the value is part of a documented public contract, external protocol, compatibility requirement, security rule, migration, or business rule.
|
||||||
|
|
||||||
## Agent skills
|
|
||||||
|
|
||||||
### Issue tracker
|
|
||||||
|
|
||||||
Issues live in Gitea at git.westgate.pw (`ShadowsOverWestgate/sow-tools`), managed with the `tea` CLI. Issues follow ownership — file work in the repo that owns it, not the one you happen to be standing in. See `docs/agents/issue-tracker.md`.
|
|
||||||
|
|
||||||
### Triage labels
|
|
||||||
|
|
||||||
Default label vocabulary (`needs-triage`, `needs-info`, `ready-for-agent`, `ready-for-human`, `wontfix`). See `docs/agents/triage-labels.md`.
|
|
||||||
|
|
||||||
### Domain docs
|
|
||||||
|
|
||||||
Single-context: `CONTEXT.md` at the repo root plus `docs/adr/`. See `docs/agents/domain.md`.
|
|
||||||
|
|
||||||
### Where work lives
|
|
||||||
|
|
||||||
Markdown here is **reference, law, or an ADR — nothing else** (`sow-codebase` ADR-0001). Live work -> wayfinder maps + Gitea issues (closeable, assignable, queryable). Settled decisions -> ADR files in `docs/adr/` (immutable, never closed, only superseded). Standing law -> `DOCTRINE.md` / `AGENTS.md` / `CONTEXT.md`. Current-state reference -> docs describing what the code does now. Everything else — plans, specs, concepts, handoffs, trackers — is process: it belongs in a Gitea issue, not a file. Harvest unfinished intent to an issue before deleting a process doc. `sow-docs` is deprecated and read-only.
|
|
||||||
|
|||||||
@@ -6,8 +6,6 @@ SHELL := bash
|
|||||||
check: vet test
|
check: vet test
|
||||||
shellcheck scripts/*.sh
|
shellcheck scripts/*.sh
|
||||||
yamllint .gitea
|
yamllint .gitea
|
||||||
bash tests/workflow-contract.sh
|
|
||||||
bash tests/prune-release-assets.sh
|
|
||||||
|
|
||||||
vet:
|
vet:
|
||||||
go vet ./...
|
go vet ./...
|
||||||
|
|||||||
@@ -19,7 +19,6 @@ Crucible is how the artifact repos turn source into artifacts.
|
|||||||
| `crucible-depot` | `crucible depot` | content-addressed depot blob verify/move |
|
| `crucible-depot` | `crucible depot` | content-addressed depot blob verify/move |
|
||||||
| `crucible-hak` | `crucible hak` | ERF/HAK pack/unpack + hak manifests |
|
| `crucible-hak` | `crucible hak` | ERF/HAK pack/unpack + hak manifests |
|
||||||
| `crucible-module` | `crucible module` | build/extract/validate/compare the `.mod` |
|
| `crucible-module` | `crucible module` | build/extract/validate/compare the `.mod` |
|
||||||
| `crucible-nwsync` | `crucible nwsync` | NWSync blob emit + manifest assemble |
|
|
||||||
| `crucible-topdata` | `crucible topdata` | compile 2da/tlk topdata + packages |
|
| `crucible-topdata` | `crucible topdata` | compile 2da/tlk topdata + packages |
|
||||||
| `crucible-wiki` | `crucible wiki` | render + deploy mechanical wiki pages |
|
| `crucible-wiki` | `crucible wiki` | render + deploy mechanical wiki pages |
|
||||||
|
|
||||||
@@ -79,16 +78,13 @@ This retires the old habit of checking in `nwn-tool` / `sow-toolkit`.
|
|||||||
|
|
||||||
## CI
|
## CI
|
||||||
|
|
||||||
PR-first (D7): checks run once on pull requests; the only publish event is a
|
PR-first (D7): checks run on pull requests and on push to `main`; the only
|
||||||
`v*` tag (see `runbooks/ci-trigger-standard.md` in sow-docs,
|
publish event is a `v*` tag (see `runbooks/ci-trigger-standard.md` in sow-docs,
|
||||||
https://git.westgate.pw/ShadowsOverWestgate/sow-docs).
|
https://git.westgate.pw/ShadowsOverWestgate/sow-docs).
|
||||||
|
|
||||||
- `ci.yml` — vet, test, shellcheck, yamllint, binary smoke, and cross-build all
|
- `test.yml` — vet, test, shellcheck, yamllint, binary smoke (PR + main).
|
||||||
targets once per pull request.
|
- `build-binaries.yml` — cross-build all targets (PR + main); on a `v*` tag, upload
|
||||||
- `build-binaries.yml` — on a `v*` tag, cross-build and upload the binaries,
|
the binaries, `SHA256SUMS`, and the wrappers to the Gitea release.
|
||||||
`SHA256SUMS`, and the wrappers to the Gitea release, then delete the assets
|
|
||||||
of every release except the newest two — Gitea keeps them forever otherwise,
|
|
||||||
and every binary is reproducible from its tag.
|
|
||||||
- `sync-wrappers.yml` — on a `main` push that touches `wrappers/`, auto-PR the
|
- `sync-wrappers.yml` — on a `main` push that touches `wrappers/`, auto-PR the
|
||||||
canonical wrappers to the consumer repos in `wrappers/consumers.txt`.
|
canonical wrappers to the consumer repos in `wrappers/consumers.txt`.
|
||||||
Consumer drift checks run after those PRs merge to `main`, not on the PRs
|
Consumer drift checks run after those PRs merge to `main`, not on the PRs
|
||||||
|
|||||||
@@ -1,11 +0,0 @@
|
|||||||
// Command crucible-nwsync is the standalone nwsync builder (equivalent to
|
|
||||||
// `crucible nwsync`). A single-token binary keeps consumer wrapper scripts simple.
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"os"
|
|
||||||
|
|
||||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/dispatch"
|
|
||||||
)
|
|
||||||
|
|
||||||
func main() { os.Exit(dispatch.RunBuilder("nwsync", os.Args[1:])) }
|
|
||||||
@@ -1,55 +0,0 @@
|
|||||||
# ADR-0001: Markdown in this repo is reference, law, or an ADR — nothing else
|
|
||||||
|
|
||||||
- **Status:** Accepted
|
|
||||||
- **Date:** 2026-07-24
|
|
||||||
- **Origin:** Adopted workspace-wide from `sow-codebase` ADR-0001, which records
|
|
||||||
the full context (a `docs/` tree that had grown to 434 files, ~27MB, most of
|
|
||||||
it dead process artifacts). This repo adopts the same law so the rule is
|
|
||||||
local, not a cross-repo reference.
|
|
||||||
|
|
||||||
## Context
|
|
||||||
|
|
||||||
Markdown that *claims to describe current state or a plan* rots, because
|
|
||||||
reality diverges and nobody edits the file. Markdown that claims neither — a
|
|
||||||
dated, immutable decision record, or a standing rule — does not rot.
|
|
||||||
|
|
||||||
Two different things get conflated:
|
|
||||||
|
|
||||||
- **Rationale** — why a system is shaped the way it is. Worth keeping.
|
|
||||||
- **Process artifacts** — plans, specs, concepts, handoffs, trackers. A record
|
|
||||||
of how work happened, not what is true now.
|
|
||||||
|
|
||||||
Left unchecked, every completed effort leaves its planning behind and the repo
|
|
||||||
accumulates a "historical" pile that agents and humans must route around to
|
|
||||||
find the few live docs.
|
|
||||||
|
|
||||||
## Decision
|
|
||||||
|
|
||||||
Markdown may exist in this repo only as one of three genres:
|
|
||||||
|
|
||||||
1. **Current-state reference** — describes what the code does now, kept honest
|
|
||||||
by code review touching it.
|
|
||||||
2. **Standing law** — `DOCTRINE.md`, root and folder-scoped `AGENTS.md`,
|
|
||||||
`CONTEXT.md`. States rules and vocabulary, not plans.
|
|
||||||
3. **Immutable decision record** — ADRs under `docs/adr/`. Append-only; never
|
|
||||||
edited, only superseded by a later ADR that points back.
|
|
||||||
|
|
||||||
Anything else — implementation plans, design specs, concepts, handoffs,
|
|
||||||
progress trackers, scratch — is **process** and does not live in the repo. It
|
|
||||||
lives in Gitea issues and wayfinder maps, where it can be assigned, closed, and
|
|
||||||
superseded. Small tasks need no written plan at all.
|
|
||||||
|
|
||||||
Deleting a process document is not destroying history: `git log -- <path>`
|
|
||||||
recovers it. The exception is *unfinished intent* (a design never built, an
|
|
||||||
open question still wanted) — that is live, not history, and must be harvested
|
|
||||||
to a Gitea ticket before its file is deleted. A citation from a living doc or
|
|
||||||
from code must be resolved before the cited file is deleted.
|
|
||||||
|
|
||||||
## Consequences
|
|
||||||
|
|
||||||
- The documentation map lists only current truth; there is no "historical" pile
|
|
||||||
to route around.
|
|
||||||
- No agent, under any plugin or skill, writes process-markdown into the repo.
|
|
||||||
The rule is plugin-agnostic on purpose — it binds the agent, not a named tool.
|
|
||||||
- History of deleted process docs is in git; unfinished intent is in the issue
|
|
||||||
tracker; rationale is in ADRs and law. Each thing has exactly one home.
|
|
||||||
@@ -1,12 +0,0 @@
|
|||||||
# ADR-0002: `sow-tools` becomes the Crucible suite
|
|
||||||
|
|
||||||
- **Status:** Accepted
|
|
||||||
- **Date:** 2026-06-11
|
|
||||||
- **Migrated from:** `sow-docs` `07-decisions-log.md` entry **D11**, on the
|
|
||||||
retirement of `sow-docs`. Content is the original decision, unchanged.
|
|
||||||
|
|
||||||
- **Decision:** Rename the future toolchain surface to Crucible: one Go module,
|
|
||||||
multiple `cmd/` binaries, plus a dispatcher so wrapper commands can stay
|
|
||||||
stable.
|
|
||||||
- **Rationale:** Depot, HAK, module, topdata, and wiki tooling share internals
|
|
||||||
but have different command surfaces. One module avoids premature repo splits.
|
|
||||||
@@ -1,29 +0,0 @@
|
|||||||
# ADR-0003: Crucible binary contract: standalone shims, fail-closed scaffold, no committed binaries
|
|
||||||
|
|
||||||
- **Status:** Accepted
|
|
||||||
- **Date:** 2026-06-11
|
|
||||||
- **Migrated from:** `sow-docs` `07-decisions-log.md` entry **D19**, on the
|
|
||||||
retirement of `sow-docs`. Content is the original decision, unchanged.
|
|
||||||
|
|
||||||
- **Decision:** Phase 5 builds `migration/sow-tools` (Crucible, D11) as a
|
|
||||||
multi-binary Go scaffold:
|
|
||||||
- One `crucible` dispatcher plus standalone `crucible-{depot,hak,module,topdata,wiki}`
|
|
||||||
binaries sharing one registry (`internal/dispatch`). The dispatcher is for
|
|
||||||
humans/CI; the **standalone shims are canonical for consumers** so wrapper
|
|
||||||
scripts resolve a single-token command and `"$builder" args` quoting stays
|
|
||||||
correct.
|
|
||||||
- Consumer resolution order: explicit env override
|
|
||||||
(`$SOW_MODULE_BUILD`/`$SOW_TOPDATA_BUILD`/`$CRUCIBLE`) → `crucible-<name>` →
|
|
||||||
legacy `sow-<name>-build`. CI runs inside the pinned `crucible:<sha>` image.
|
|
||||||
- Every builder is **unwired** in the scaffold and fails closed (exit 70);
|
|
||||||
it never fakes an artifact. The `internal/` logic is migrated from
|
|
||||||
`gitea/sow-tools` by the operator at cutover (hard rule: no source
|
|
||||||
transplant by tooling).
|
|
||||||
- **Binaries are never committed** — they are CI artifacts / image layers.
|
|
||||||
Retires the committed `nwn-tool` / `tools/sow-toolkit`.
|
|
||||||
- Builders take `NWN_ROOT` only via explicit env/flag; no `$HOME` defaulting.
|
|
||||||
- **Rationale:** Locks the command surface, container, and CI shape so migrated
|
|
||||||
logic drops into a stable frame; aligns the three artifact repos onto one
|
|
||||||
Crucible contract while keeping back-compat with the pre-D11 skeletons.
|
|
||||||
- **Image name:** `registry.westgate.pw/deployment/crucible:<git-sha>` (the bare
|
|
||||||
`crucible:<sha>` is the local/dev tag).
|
|
||||||
@@ -1,24 +0,0 @@
|
|||||||
# ADR-0004: Nix-built OCI images start server-side with Crucible; local Anvil images deferred
|
|
||||||
|
|
||||||
- **Status:** Accepted
|
|
||||||
- **Date:** 2026-06-12
|
|
||||||
- **Migrated from:** `sow-docs` `07-decisions-log.md` entry **D29**, on the
|
|
||||||
retirement of `sow-docs`. Content is the original decision, unchanged.
|
|
||||||
|
|
||||||
- **Decision:** Introduce Nix-built OCI images as an artifact-production option,
|
|
||||||
starting with `sow-tools`/Crucible in server-side CI. `sow-tools` will grow a
|
|
||||||
Nix-built `crucible` package plus `crucible-image`; PR CI builds/smokes it and
|
|
||||||
`main` publishes the normal pinned OCI image
|
|
||||||
`registry.westgate.pw/deployment/crucible:<sha>`. `docker/Dockerfile` stays during
|
|
||||||
parity and as the client-compatible fallback.
|
|
||||||
- **Local rule:** Nix users may get optional wrappers that provide tools and
|
|
||||||
call the existing Dockerfiles for local Anvil/NWServer testing. A true
|
|
||||||
`nix build .#nwserver-test-image` is explicitly deferred until real
|
|
||||||
`sow-codebase/src` exists and the Dockerfile image is proven.
|
|
||||||
- **Rationale:** Server-side image build shape is harder to change once deploy
|
|
||||||
promotion and registry gates are active, so prove Nix image builds before
|
|
||||||
cutover. Crucible is the lowest-risk pilot because it is a Go toolchain image;
|
|
||||||
NodeBB and Anvil/NWServer depend more heavily on upstream container filesystem
|
|
||||||
semantics.
|
|
||||||
- **Non-goals:** no source builds in `sow-platform`; no `nix-sidecar` or
|
|
||||||
Kubernetes-style runtime cache layer; no removal of client Dockerfiles.
|
|
||||||
@@ -1,51 +0,0 @@
|
|||||||
# Domain Docs
|
|
||||||
|
|
||||||
How the engineering skills should consume this repo's domain documentation when exploring the codebase.
|
|
||||||
|
|
||||||
## Before exploring, read these
|
|
||||||
|
|
||||||
- **`CONTEXT.md`** at the repo root, or
|
|
||||||
- **`CONTEXT-MAP.md`** at the repo root if it exists — it points at one `CONTEXT.md` per context. Read each one relevant to the topic.
|
|
||||||
- **`docs/adr/`** — read ADRs that touch the area you're about to work in. In multi-context repos, also check `src/<context>/docs/adr/` for context-scoped decisions.
|
|
||||||
|
|
||||||
If any of these files don't exist, **proceed silently**. Don't flag their absence; don't suggest creating them upfront. The `/domain-modeling` skill (reached via `/grill-with-docs` and `/improve-codebase-architecture`) creates them lazily when terms or decisions actually get resolved.
|
|
||||||
|
|
||||||
## File structure
|
|
||||||
|
|
||||||
Single-context repo (most repos):
|
|
||||||
|
|
||||||
```
|
|
||||||
/
|
|
||||||
├── CONTEXT.md
|
|
||||||
├── docs/adr/
|
|
||||||
│ ├── 0001-event-sourced-orders.md
|
|
||||||
│ └── 0002-postgres-for-write-model.md
|
|
||||||
└── src/
|
|
||||||
```
|
|
||||||
|
|
||||||
Multi-context repo (presence of `CONTEXT-MAP.md` at the root):
|
|
||||||
|
|
||||||
```
|
|
||||||
/
|
|
||||||
├── CONTEXT-MAP.md
|
|
||||||
├── docs/adr/ ← system-wide decisions
|
|
||||||
└── src/
|
|
||||||
├── ordering/
|
|
||||||
│ ├── CONTEXT.md
|
|
||||||
│ └── docs/adr/ ← context-specific decisions
|
|
||||||
└── billing/
|
|
||||||
├── CONTEXT.md
|
|
||||||
└── docs/adr/
|
|
||||||
```
|
|
||||||
|
|
||||||
## Use the glossary's vocabulary
|
|
||||||
|
|
||||||
When your output names a domain concept (in an issue title, a refactor proposal, a hypothesis, a test name), use the term as defined in `CONTEXT.md`. Don't drift to synonyms the glossary explicitly avoids.
|
|
||||||
|
|
||||||
If the concept you need isn't in the glossary yet, that's a signal — either you're inventing language the project doesn't use (reconsider) or there's a real gap (note it for `/domain-modeling`).
|
|
||||||
|
|
||||||
## Flag ADR conflicts
|
|
||||||
|
|
||||||
If your output contradicts an existing ADR, surface it explicitly rather than silently overriding:
|
|
||||||
|
|
||||||
> _Contradicts ADR-0007 (event-sourced orders) — but worth reopening because…_
|
|
||||||
@@ -1,88 +0,0 @@
|
|||||||
# Issue tracker: Gitea (via tea)
|
|
||||||
|
|
||||||
Issues for this repo live in Gitea at `git.westgate.pw`, repo
|
|
||||||
`ShadowsOverWestgate/sow-tools`. Use the `tea` CLI for all operations —
|
|
||||||
`gh` does not work here. For anything `tea` lacks a subcommand for, use
|
|
||||||
`tea api <endpoint>` (Gitea's API mirrors GitHub's closely).
|
|
||||||
|
|
||||||
Authenticate with your own `tea` login (`tea login add`); never commit tokens
|
|
||||||
or tea config into this repo. Note Gitea blocks self-review, so approving a PR
|
|
||||||
needs a different account than the one that opened it.
|
|
||||||
|
|
||||||
## Where work lives
|
|
||||||
|
|
||||||
Markdown in this repo is **reference, law, or an ADR — nothing else**
|
|
||||||
(`sow-codebase` ADR-0001). Four homes, no overlap:
|
|
||||||
|
|
||||||
- **Live work** → wayfinder maps + Gitea issues. Closeable, assignable,
|
|
||||||
queryable. Never a markdown file.
|
|
||||||
- **Settled decisions** → ADR files in `docs/adr/`. Immutable, findable, never
|
|
||||||
closed, never edited — only superseded by a later ADR pointing back. When an
|
|
||||||
issue ends in a durable decision, write the ADR, then close the issue
|
|
||||||
pointing at it. Decisions spanning repos go to `sow-platform/docs/adr/`.
|
|
||||||
- **Standing law** → `DOCTRINE.md`, `AGENTS.md`, `CONTEXT.md`. Rules that are
|
|
||||||
always true and vocabulary everyone shares — not the record of one decision.
|
|
||||||
- **Current-state reference** → docs describing what the code does now, kept
|
|
||||||
honest by review touching them.
|
|
||||||
|
|
||||||
Anything else — implementation plans, design specs, concepts, handoffs,
|
|
||||||
progress trackers, scratch — is **process**. It does not live in this repo. It
|
|
||||||
lives in a Gitea issue or a wayfinder map, where it can be assigned, closed,
|
|
||||||
and superseded. Small tasks need no written plan at all.
|
|
||||||
|
|
||||||
Deleting a process doc is not destroying history — `git log -- <path>` recovers
|
|
||||||
it. But *unfinished intent* (a design never built, an open question still
|
|
||||||
wanted) is live, not history: harvest it to a Gitea issue before deleting.
|
|
||||||
|
|
||||||
`sow-docs` is deprecated and read-only. Never add to it, never send work there.
|
|
||||||
|
|
||||||
## Which repo gets the issue
|
|
||||||
|
|
||||||
Issues follow ownership. File the issue in the repo that **owns the work** —
|
|
||||||
see the Repo/Owns/Produces table in the workspace root `AGENTS.md`. Standing
|
|
||||||
in one repo is not a reason to file there.
|
|
||||||
|
|
||||||
If work spans repos, file it in the repo that owns the *outcome* and reference
|
|
||||||
the others from it. A wrong-repo issue is a routing bug, not a filing
|
|
||||||
preference — move it.
|
|
||||||
|
|
||||||
## Conventions
|
|
||||||
|
|
||||||
- **Create an issue**: `tea issues create --title "..." --description "..."`
|
|
||||||
- **Read an issue**: `tea issues <number>` and
|
|
||||||
`tea api repos/ShadowsOverWestgate/sow-tools/issues/<number>/comments` for comments.
|
|
||||||
- **List issues**: `tea issues list --state open` (add `--labels ...` to filter).
|
|
||||||
- **Comment**: `tea comment <number> "..." </dev/null`
|
|
||||||
Always redirect stdin. `tea` reads stdin to EOF and appends it to the body,
|
|
||||||
so any non-interactive shell (every agent) hangs forever without
|
|
||||||
`</dev/null`. Same trap on `tea issues create --description` and
|
|
||||||
`tea pr create`.
|
|
||||||
- **Apply / remove labels**: `tea api --method PATCH` on the issue, or
|
|
||||||
`tea api repos/ShadowsOverWestgate/sow-tools/issues/<number>/labels` endpoints.
|
|
||||||
- **Close**: `tea issues close <number>`
|
|
||||||
|
|
||||||
`tea` infers the repo from the git remote when run inside the clone.
|
|
||||||
Gitea shares one number space across issues and PRs.
|
|
||||||
|
|
||||||
## Pull requests as a triage surface
|
|
||||||
|
|
||||||
**PRs as a request surface: no.**
|
|
||||||
|
|
||||||
## When a skill says "publish to the issue tracker"
|
|
||||||
|
|
||||||
Create a Gitea issue with `tea issues create`.
|
|
||||||
|
|
||||||
## When a skill says "fetch the relevant ticket"
|
|
||||||
|
|
||||||
Run `tea issues <number>` plus the comments API call above.
|
|
||||||
|
|
||||||
## Wayfinding operations
|
|
||||||
|
|
||||||
Used by `/wayfinder`. The **map** is a single issue with **child** issues as tickets.
|
|
||||||
|
|
||||||
- **Map**: a single issue labelled `wayfinder:map`, holding the Notes / Decisions-so-far / Fog body. `tea issues create --title "..." --description "..." --labels wayfinder:map`.
|
|
||||||
- **Child ticket**: this Gitea instance (v1.27) has no native sub-issue hierarchy, so a child issue carries `Part of #<map>` at the top of its description, and is also added to a task list in the map body. Labels: `wayfinder:<type>` (`research`/`prototype`/`grilling`/`task`). Once claimed, the ticket is assigned to the driving dev.
|
|
||||||
- **Blocking**: Gitea's **native dependencies API** — the canonical, UI-visible representation (shows as "Depends on" / "Blocks" on the issue page). Add an edge with `tea api -X POST repos/ShadowsOverWestgate/sow-tools/issues/<child>/dependencies -f owner=ShadowsOverWestgate -f repo=sow-tools -F index=<blocker>`, where `<blocker>` is the blocker's issue **index** (its `#number` — Gitea's dependency API takes the index directly, unlike GitHub's numeric database id). Check status with `tea api repos/ShadowsOverWestgate/sow-tools/issues/<child>/dependencies` (GET) — a ticket is unblocked when every returned issue's `state` is `closed`.
|
|
||||||
- **Frontier query**: list the map's open children (`tea issues list --state open`, keep the ones whose description contains `Part of #<map>`), drop any with an open dependency (per the GET above) or an assignee; first in map order wins.
|
|
||||||
- **Claim**: `tea issues edit <n> --add-assignees <username>` — the session's first write.
|
|
||||||
- **Resolve**: `tea comment <n> "<answer>" </dev/null`, then `tea issues close <n>`, then append a context pointer (gist + link) to the map's Decisions-so-far.
|
|
||||||
@@ -1,15 +0,0 @@
|
|||||||
# Triage Labels
|
|
||||||
|
|
||||||
The skills speak in terms of five canonical triage roles. This file maps those roles to the actual label strings used in this repo's issue tracker (Gitea — see `issue-tracker.md` for how to apply labels with `tea`).
|
|
||||||
|
|
||||||
| Label in mattpocock/skills | Label in our tracker | Meaning |
|
|
||||||
| -------------------------- | -------------------- | ---------------------------------------- |
|
|
||||||
| `needs-triage` | `needs-triage` | Maintainer needs to evaluate this issue |
|
|
||||||
| `needs-info` | `needs-info` | Waiting on reporter for more information |
|
|
||||||
| `ready-for-agent` | `ready-for-agent` | Fully specified, ready for an AFK agent |
|
|
||||||
| `ready-for-human` | `ready-for-human` | Requires human implementation |
|
|
||||||
| `wontfix` | `wontfix` | Will not be actioned |
|
|
||||||
|
|
||||||
When a skill mentions a role (e.g. "apply the AFK-ready triage label"), use the corresponding label string from this table.
|
|
||||||
|
|
||||||
Edit the right-hand column to match whatever vocabulary you actually use.
|
|
||||||
+2
-69
@@ -29,75 +29,9 @@ aliases.
|
|||||||
| `assets` | `fix-mdl` | Lowercase `.mdl` names and rewrite model identity to match. |
|
| `assets` | `fix-mdl` | Lowercase `.mdl` names and rewrite model identity to match. |
|
||||||
| `assets` | `check-dupes` | Report runtime-name (basename) collisions across dirs. |
|
| `assets` | `check-dupes` | Report runtime-name (basename) collisions across dirs. |
|
||||||
| `assets` | `clean-dupes` | Delete clean-tree files whose basename collides with primary. |
|
| `assets` | `clean-dupes` | Delete clean-tree files whose basename collides with primary. |
|
||||||
| `depot` | `status` | Report referenced-vs-present drift against a backend. |
|
|
||||||
| `depot` | `push` | Upload referenced-but-absent blobs from a local depot. |
|
|
||||||
| `depot` | `verify` | Existence sweep plus sampled download re-hash. |
|
|
||||||
| `depot` | `get` | Fetch one blob with sha re-verify. |
|
|
||||||
| `depot` | `pull` | Incremental verified pull of every referenced blob. |
|
|
||||||
| `nwsync` | `emit` | Explode one artifact into NWSync blobs plus its own NSYM manifest. |
|
|
||||||
| `nwsync` | `assemble` | Merge per-artifact NSYM manifests into one merged manifest. |
|
|
||||||
|
|
||||||
`depot status` and `depot get` pick their backend either with `--out DIR`, a
|
`crucible-depot` remains registered but unwired. It fails closed with exit `70`
|
||||||
depot tree on disk, or with `--target bunny|cdn`, a remote backend. The two
|
and never emits placeholder artifacts.
|
||||||
flags are mutually exclusive.
|
|
||||||
|
|
||||||
`nwsync emit` runs where an artifact is born (a `.hak`/`.erf`, or a loose file
|
|
||||||
such as the TLK); `nwsync assemble` runs at module release and reads only the
|
|
||||||
small per-artifact indexes. Both take **depot keys**: an artifact's index lives
|
|
||||||
beside the artifact itself with the extension replaced, so `emit` and
|
|
||||||
`assemble` agree on where it is without being told.
|
|
||||||
|
|
||||||
```
|
|
||||||
nwsync emit [--as NAME] [--out DIR] <artifact-key> <file>
|
|
||||||
nwsync assemble --group-id N [--tlk-key KEY] [--out DIR] <artifact-key>...
|
|
||||||
```
|
|
||||||
|
|
||||||
Both verbs upload by default; nothing bulky is ever written to the runner's
|
|
||||||
disk. `--out DIR` writes a local repository tree instead, which is the
|
|
||||||
conformance path against upstream `nwn_nwsync_write`. The zone comes from
|
|
||||||
`NWSYNC_STORAGE_ZONE` and `NWSYNC_STORAGE_PASSWORD`, with the host from
|
|
||||||
`BUNNY_STORAGE_HOST` — NWSync data is a separate zone from the asset depot.
|
|
||||||
|
|
||||||
`assemble`'s artifact keys are in `Mod_HakList` order, highest priority first: a
|
|
||||||
resref in more than one artifact resolves to the earliest one, the way the game
|
|
||||||
resolves it. `--tlk-key` has its own slot because the TLK shadows nothing.
|
|
||||||
`--group-id` is per channel — 1 is current, 2 is testing, and 0 leaves the field
|
|
||||||
out of the sidecar.
|
|
||||||
|
|
||||||
`emit` uploads blobs first and the index last, so the presence of an index is
|
|
||||||
the publication marker: an artifact whose emit died halfway leaves real blobs in
|
|
||||||
the zone and no index. Blob names are content hashes, so re-running skips
|
|
||||||
whatever already landed, and `assemble` fails closed on an artifact with no
|
|
||||||
index rather than publishing a manifest that is missing a hak.
|
|
||||||
|
|
||||||
### What an emitted tree looks like
|
|
||||||
|
|
||||||
`--out DIR` produces the same tree `emit` would upload, which makes it the way
|
|
||||||
to check a zone by hand without touching one:
|
|
||||||
|
|
||||||
```
|
|
||||||
<artifact-sha>.nsym binary index
|
|
||||||
<artifact-sha>.nsym.json the same index, readable
|
|
||||||
data/sha1/a7/4a/a74aa84a... one blob per resource, two-level fanout
|
|
||||||
```
|
|
||||||
|
|
||||||
A blob's name is the SHA-1 of the resource's **original** bytes, but the file on
|
|
||||||
disk is not those bytes: each blob is wrapped in NWCompressedBuffer framing, a
|
|
||||||
24-byte `NSYC` header followed by a zstd frame. Hashing the file directly will
|
|
||||||
not match its name, which is the obvious
|
|
||||||
first thing to try and the obvious first thing to be confused by. Strip the
|
|
||||||
header first:
|
|
||||||
|
|
||||||
```
|
|
||||||
tail -c +25 <blob> | zstd -dc | sha1sum # == the blob's filename
|
|
||||||
```
|
|
||||||
|
|
||||||
The header carries the uncompressed length as a little-endian `uint32` at offset
|
|
||||||
12, so the decompressed size is checkable without decompressing. Compression is
|
|
||||||
worth roughly a 4:1
|
|
||||||
saving on hak content: a 250 MB hak emitted 2296 blobs totalling 59 MB on disk
|
|
||||||
against 249 MB of resources, as recorded in the sidecar's `on_disk_bytes` and
|
|
||||||
`total_bytes`.
|
|
||||||
|
|
||||||
## Hidden compatibility aliases
|
## Hidden compatibility aliases
|
||||||
|
|
||||||
@@ -115,7 +49,6 @@ but omitted from routine help and the interactive menu:
|
|||||||
| `topdata` | `build-top-package` | `build-top-package` |
|
| `topdata` | `build-top-package` | `build-top-package` |
|
||||||
| `topdata` | `compare-topdata` | `compare-topdata` |
|
| `topdata` | `compare-topdata` | `compare-topdata` |
|
||||||
| `topdata` | `convert-topdata` | `convert-topdata` |
|
| `topdata` | `convert-topdata` | `convert-topdata` |
|
||||||
| `depot` | `--target local` | `--out $DEPOT_DIR` on `status`/`get` |
|
|
||||||
| `wiki` | `build-wiki` | `build-wiki` |
|
| `wiki` | `build-wiki` | `build-wiki` |
|
||||||
| `wiki` | `deploy-wiki` | `deploy-wiki` |
|
| `wiki` | `deploy-wiki` | `deploy-wiki` |
|
||||||
|
|
||||||
|
|||||||
@@ -126,9 +126,8 @@ exactly as the reference does:
|
|||||||
- Beamdog install dirs.
|
- Beamdog install dirs.
|
||||||
- `--nwn <install>` overrides (path to the install root or directly to the
|
- `--nwn <install>` overrides (path to the install root or directly to the
|
||||||
binary).
|
binary).
|
||||||
- **Headless.** The engine is a GUI binary. Require `xvfb-run` and wrap the call
|
- **Headless.** The engine is a GUI binary. If there is no `DISPLAY` and
|
||||||
regardless of the caller's `DISPLAY` so compilation never opens the client
|
`xvfb-run` is present, wrap the call:
|
||||||
UI; fail before invoking the engine when `xvfb-run` is unavailable:
|
|
||||||
`xvfb-run -a --server-args=-screen 0 1024x768x24 nwmain-linux compilemodel <stem>`.
|
`xvfb-run -a --server-args=-screen 0 1024x768x24 nwmain-linux compilemodel <stem>`.
|
||||||
- **Per model (one at a time — the engine's `development/` and `modelcompiler/`
|
- **Per model (one at a time — the engine's `development/` and `modelcompiler/`
|
||||||
folders are flat and single-slot):**
|
folders are flat and single-slot):**
|
||||||
|
|||||||
@@ -22,13 +22,12 @@
|
|||||||
pname = "crucible";
|
pname = "crucible";
|
||||||
inherit version;
|
inherit version;
|
||||||
src = ./.;
|
src = ./.;
|
||||||
vendorHash = "sha256-0I8j7On9YGD2GK9xbj/KkgBrlkMJ6Y6XQv+KCLTgBBU=";
|
vendorHash = "sha256-hm6mrNAtXv0LidzHUfz4eukTFZouizGtxkZ8gKJFUVI=";
|
||||||
subPackages = [
|
subPackages = [
|
||||||
"cmd/crucible"
|
"cmd/crucible"
|
||||||
"cmd/crucible-depot"
|
"cmd/crucible-depot"
|
||||||
"cmd/crucible-hak"
|
"cmd/crucible-hak"
|
||||||
"cmd/crucible-module"
|
"cmd/crucible-module"
|
||||||
"cmd/crucible-nwsync"
|
|
||||||
"cmd/crucible-topdata"
|
"cmd/crucible-topdata"
|
||||||
"cmd/crucible-wiki"
|
"cmd/crucible-wiki"
|
||||||
];
|
];
|
||||||
|
|||||||
@@ -6,5 +6,3 @@ require (
|
|||||||
golang.org/x/text v0.35.0
|
golang.org/x/text v0.35.0
|
||||||
gopkg.in/yaml.v3 v3.0.1
|
gopkg.in/yaml.v3 v3.0.1
|
||||||
)
|
)
|
||||||
|
|
||||||
require github.com/klauspost/compress v1.19.1
|
|
||||||
|
|||||||
@@ -1,5 +1,3 @@
|
|||||||
github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk=
|
|
||||||
github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
|
||||||
golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8=
|
golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8=
|
||||||
golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
|
golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
|
||||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||||
|
|||||||
@@ -40,13 +40,13 @@ func runCompile(args []string, stdout, stderr io.Writer, getenv func(string) str
|
|||||||
}
|
}
|
||||||
binDir := filepath.Dir(nwmain)
|
binDir := filepath.Dir(nwmain)
|
||||||
|
|
||||||
// Always use a virtual X so compilation never opens the client UI.
|
// Headless wrap: no DISPLAY + xvfb-run present -> run under a virtual X.
|
||||||
xvfb := look("xvfb-run")
|
var wrap []string
|
||||||
if xvfb == "" {
|
if getenv("DISPLAY") == "" {
|
||||||
fmt.Fprintln(stderr, "assets compile: xvfb-run not found — install it to run the NWN model compiler headlessly")
|
if xvfb := look("xvfb-run"); xvfb != "" {
|
||||||
return exitTool
|
wrap = []string{xvfb, "-a", "--server-args=-screen 0 1024x768x24"}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
wrap := []string{xvfb, "-a", "--server-args=-screen 0 1024x768x24"}
|
|
||||||
|
|
||||||
files, err := walk(dirs, mdlExt, !*nonRecursive)
|
files, err := walk(dirs, mdlExt, !*nonRecursive)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -4,7 +4,6 @@ import (
|
|||||||
"bytes"
|
"bytes"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
|
||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -24,19 +23,13 @@ func TestCompileDrivesEngineAndReplacesInPlace(t *testing.T) {
|
|||||||
if err := os.WriteFile(nwmain, []byte("#!/bin/sh\n"), 0o755); err != nil {
|
if err := os.WriteFile(nwmain, []byte("#!/bin/sh\n"), 0o755); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
xvfbDir := t.TempDir()
|
|
||||||
xvfb := filepath.Join(xvfbDir, "xvfb-run")
|
|
||||||
if err := os.WriteFile(xvfb, []byte("#!/bin/sh\n"), 0o755); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
t.Setenv("PATH", xvfbDir+string(os.PathListSeparator)+os.Getenv("PATH"))
|
|
||||||
|
|
||||||
getenv := func(k string) string {
|
getenv := func(k string) string {
|
||||||
switch k {
|
switch k {
|
||||||
case "HOME":
|
case "HOME":
|
||||||
return home
|
return home
|
||||||
case "DISPLAY":
|
case "DISPLAY":
|
||||||
return ":0" // a desktop display must not make compilation interactive
|
return ":0" // pretend a display exists so no xvfb wrap is needed
|
||||||
}
|
}
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
@@ -45,11 +38,7 @@ func TestCompileDrivesEngineAndReplacesInPlace(t *testing.T) {
|
|||||||
orig := runner
|
orig := runner
|
||||||
defer func() { runner = orig }()
|
defer func() { runner = orig }()
|
||||||
runner = func(dir string, env []string, name string, args ...string) ([]byte, error) {
|
runner = func(dir string, env []string, name string, args ...string) ([]byte, error) {
|
||||||
if name != xvfb || len(args) != 5 || args[0] != "-a" ||
|
// args: compilemodel <stem>
|
||||||
args[1] != "--server-args=-screen 0 1024x768x24" || args[2] != nwmain ||
|
|
||||||
args[3] != "compilemodel" {
|
|
||||||
t.Fatalf("engine command = %q %q, want xvfb-run wrapping nwmain", name, args)
|
|
||||||
}
|
|
||||||
stem := args[len(args)-1]
|
stem := args[len(args)-1]
|
||||||
compiled := filepath.Join(mc, stem+".mdl")
|
compiled := filepath.Join(mc, stem+".mdl")
|
||||||
return nil, os.WriteFile(compiled, []byte("\x00\x00compiled"), 0o644)
|
return nil, os.WriteFile(compiled, []byte("\x00\x00compiled"), 0o644)
|
||||||
@@ -88,11 +77,6 @@ func TestCompileAbortsOnNameMismatch(t *testing.T) {
|
|||||||
if err := os.WriteFile(nwmain, []byte("#!/bin/sh\n"), 0o755); err != nil {
|
if err := os.WriteFile(nwmain, []byte("#!/bin/sh\n"), 0o755); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
xvfbDir := t.TempDir()
|
|
||||||
if err := os.WriteFile(filepath.Join(xvfbDir, "xvfb-run"), []byte("#!/bin/sh\n"), 0o755); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
t.Setenv("PATH", xvfbDir+string(os.PathListSeparator)+os.Getenv("PATH"))
|
|
||||||
getenv := func(k string) string {
|
getenv := func(k string) string {
|
||||||
if k == "HOME" {
|
if k == "HOME" {
|
||||||
return home
|
return home
|
||||||
@@ -124,48 +108,3 @@ func TestCompileAbortsOnNameMismatch(t *testing.T) {
|
|||||||
t.Fatal("engine should not run for a name-mismatched model")
|
t.Fatal("engine should not run for a name-mismatched model")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCompileFailsClosedWithoutXvfb(t *testing.T) {
|
|
||||||
home := t.TempDir()
|
|
||||||
userData := filepath.Join(home, ".local", "share", "Neverwinter Nights")
|
|
||||||
for _, d := range []string{filepath.Join(userData, "development"), filepath.Join(userData, "modelcompiler")} {
|
|
||||||
if err := os.MkdirAll(d, 0o755); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
nwmain := filepath.Join(t.TempDir(), "nwmain-linux")
|
|
||||||
if err := os.WriteFile(nwmain, []byte("#!/bin/sh\n"), 0o755); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
t.Setenv("PATH", t.TempDir())
|
|
||||||
|
|
||||||
getenv := func(k string) string {
|
|
||||||
if k == "HOME" {
|
|
||||||
return home
|
|
||||||
}
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
orig := runner
|
|
||||||
defer func() { runner = orig }()
|
|
||||||
engineCalled := false
|
|
||||||
runner = func(string, []string, string, ...string) ([]byte, error) {
|
|
||||||
engineCalled = true
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
srcDir := t.TempDir()
|
|
||||||
if err := os.WriteFile(filepath.Join(srcDir, "foo.mdl"),
|
|
||||||
[]byte("newmodel foo\nbeginmodelgeom foo\n node dummy foo\n parent null\n endnode\nendmodelgeom foo\ndonemodel foo\n"), 0o644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
var stdout, stderr bytes.Buffer
|
|
||||||
if code := runCompile([]string{"--nwn", nwmain, srcDir}, &stdout, &stderr, getenv); code != exitTool {
|
|
||||||
t.Fatalf("compile exit = %d, want %d\n%s", code, exitTool, stderr.String())
|
|
||||||
}
|
|
||||||
if engineCalled {
|
|
||||||
t.Fatal("engine must not run without xvfb-run")
|
|
||||||
}
|
|
||||||
if !strings.Contains(stderr.String(), "xvfb-run") {
|
|
||||||
t.Fatalf("missing actionable xvfb-run error: %s", stderr.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,121 +0,0 @@
|
|||||||
package depot
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"os"
|
|
||||||
"strings"
|
|
||||||
)
|
|
||||||
|
|
||||||
// KeyStore is the zone addressed by object key rather than by depot sha. The
|
|
||||||
// depot names every object after the sha256 of its contents; NWSync does not —
|
|
||||||
// a blob is named after the sha1 of its *uncompressed* bytes while the body
|
|
||||||
// uploaded is the compressed form, and a per-artifact index is named after its
|
|
||||||
// artifact. Both addressing modes want the same transport, retry and probe
|
|
||||||
// discipline, so the sha-addressed Backend rides on this rather than the other
|
|
||||||
// way round.
|
|
||||||
type KeyStore interface {
|
|
||||||
// ProbeKey returns the existence state of one key. transient=true means a
|
|
||||||
// retry might change the answer — never read it as "missing, re-upload".
|
|
||||||
ProbeKey(ctx context.Context, key string) (state ProbeState, transient bool, err error)
|
|
||||||
// PutReader uploads size bytes read from r to key. checksum is the
|
|
||||||
// uppercase hex sha256 of those bytes, which Bunny verifies server-side.
|
|
||||||
PutReader(ctx context.Context, key string, r io.Reader, size int64, checksum string) error
|
|
||||||
// GetKey fetches the whole object at key. Small objects only — it holds
|
|
||||||
// the body in memory and does no hash check, because a key is not always
|
|
||||||
// a content hash.
|
|
||||||
GetKey(ctx context.Context, key string) ([]byte, error)
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewKeyStore returns a KeyStore for cfg's storage zone. Fails closed on a
|
|
||||||
// missing host or read key, matching NewBackend.
|
|
||||||
func NewKeyStore(cfg Config) (KeyStore, error) {
|
|
||||||
if cfg.StorageHost == "" {
|
|
||||||
return nil, errors.New("storage backend requires a storage host")
|
|
||||||
}
|
|
||||||
if cfg.StorageZone == "" {
|
|
||||||
return nil, errors.New("storage backend requires a storage zone")
|
|
||||||
}
|
|
||||||
if cfg.ReadKey == "" {
|
|
||||||
return nil, errors.New("storage backend requires a read key")
|
|
||||||
}
|
|
||||||
return &httpBackend{name: "bunny", client: newHTTPClient(cfg), cfg: cfg}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// keyURL is the storage URL of one object key.
|
|
||||||
func (b *httpBackend) keyURL(key string) string {
|
|
||||||
host := b.cfg.StorageHost
|
|
||||||
// StorageHost is normally a bare host ("storage.bunnycdn.com"); allow a
|
|
||||||
// full scheme (used by tests against httptest.NewServer) to pass through
|
|
||||||
// unchanged.
|
|
||||||
if !strings.Contains(host, "://") {
|
|
||||||
host = "https://" + host
|
|
||||||
}
|
|
||||||
return fmt.Sprintf("%s/%s/%s", strings.TrimSuffix(host, "/"), b.cfg.StorageZone, key)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (b *httpBackend) ProbeKey(ctx context.Context, key string) (ProbeState, bool, error) {
|
|
||||||
return b.rangeProbe(ctx, b.keyURL(key), map[string]string{"AccessKey": b.cfg.ReadKey})
|
|
||||||
}
|
|
||||||
|
|
||||||
func (b *httpBackend) PutReader(ctx context.Context, key string, r io.Reader, size int64, checksum string) error {
|
|
||||||
if b.name == "cdn" {
|
|
||||||
return errors.New("cdn backend is read-only")
|
|
||||||
}
|
|
||||||
if b.cfg.WriteKey == "" {
|
|
||||||
return errors.New("storage backend requires a write key to write")
|
|
||||||
}
|
|
||||||
|
|
||||||
req, err := http.NewRequestWithContext(ctx, http.MethodPut, b.keyURL(key), r)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
req.ContentLength = size
|
|
||||||
req.Header.Set("AccessKey", b.cfg.WriteKey)
|
|
||||||
// Bunny defines Checksum as sha256 of the body and rejects a mismatch, so
|
|
||||||
// this is server-side integrity checking, not decoration.
|
|
||||||
req.Header.Set("Checksum", strings.ToUpper(checksum))
|
|
||||||
|
|
||||||
resp, err := b.client.Do(req)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer resp.Body.Close()
|
|
||||||
_, _ = io.Copy(io.Discard, resp.Body)
|
|
||||||
|
|
||||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
|
||||||
return fmt.Errorf("put %s: unexpected status %d", key, resp.StatusCode)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (b *httpBackend) GetKey(ctx context.Context, key string) ([]byte, error) {
|
|
||||||
resp, err := b.get(ctx, b.keyURL(key), map[string]string{"AccessKey": b.cfg.ReadKey})
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
defer resp.Body.Close()
|
|
||||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
|
||||||
_, _ = io.Copy(io.Discard, resp.Body)
|
|
||||||
return nil, fmt.Errorf("get %s: unexpected status %d", key, resp.StatusCode)
|
|
||||||
}
|
|
||||||
return io.ReadAll(resp.Body)
|
|
||||||
}
|
|
||||||
|
|
||||||
// putFile uploads the file at src to key, streaming it. checksum is the
|
|
||||||
// uppercase hex sha256 of the file's bytes.
|
|
||||||
func (b *httpBackend) putFile(ctx context.Context, key, src, checksum string) error {
|
|
||||||
f, err := os.Open(src)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer f.Close()
|
|
||||||
info, err := f.Stat()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return b.PutReader(ctx, key, f, info.Size(), checksum)
|
|
||||||
}
|
|
||||||
@@ -10,6 +10,7 @@ import (
|
|||||||
"net/http"
|
"net/http"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -70,7 +71,16 @@ type httpBackend struct {
|
|||||||
|
|
||||||
func (b *httpBackend) Name() string { return b.name }
|
func (b *httpBackend) Name() string { return b.name }
|
||||||
|
|
||||||
func (b *httpBackend) storageURL(sha string) string { return b.keyURL(BlobKey(sha)) }
|
func (b *httpBackend) storageURL(sha string) string {
|
||||||
|
host := b.cfg.StorageHost
|
||||||
|
// StorageHost is normally a bare host ("storage.bunnycdn.com"); allow a
|
||||||
|
// full scheme (used by tests against httptest.NewServer) to pass through
|
||||||
|
// unchanged.
|
||||||
|
if strings.Contains(host, "://") {
|
||||||
|
return fmt.Sprintf("%s/%s/%s", strings.TrimSuffix(host, "/"), b.cfg.StorageZone, BlobKey(sha))
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("https://%s/%s/%s", host, b.cfg.StorageZone, BlobKey(sha))
|
||||||
|
}
|
||||||
|
|
||||||
func (b *httpBackend) cdnURL(sha string) string {
|
func (b *httpBackend) cdnURL(sha string) string {
|
||||||
return fmt.Sprintf("%s/%s", b.cfg.CDNBase, BlobKey(sha))
|
return fmt.Sprintf("%s/%s", b.cfg.CDNBase, BlobKey(sha))
|
||||||
@@ -130,8 +140,7 @@ func (b *httpBackend) Probe(ctx context.Context, sha string) (ProbeState, bool,
|
|||||||
return b.rangeProbe(ctx, b.storageURL(sha), map[string]string{"AccessKey": b.cfg.ReadKey})
|
return b.rangeProbe(ctx, b.storageURL(sha), map[string]string{"AccessKey": b.cfg.ReadKey})
|
||||||
}
|
}
|
||||||
|
|
||||||
// Put uploads src for sha. cdn is read-only. A depot object is named after the
|
// Put uploads src for sha. cdn is read-only.
|
||||||
// sha256 of its own bytes, so the key's sha doubles as the Checksum header.
|
|
||||||
func (b *httpBackend) Put(ctx context.Context, sha, src string) error {
|
func (b *httpBackend) Put(ctx context.Context, sha, src string) error {
|
||||||
if b.name == "cdn" {
|
if b.name == "cdn" {
|
||||||
return errors.New("cdn backend is read-only")
|
return errors.New("cdn backend is read-only")
|
||||||
@@ -148,7 +157,30 @@ func (b *httpBackend) Put(ctx context.Context, sha, src string) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
return b.putFile(ctx, BlobKey(sha), src, sha)
|
f, err := os.Open(src)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer f.Close()
|
||||||
|
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodPut, b.storageURL(sha), f)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
req.Header.Set("AccessKey", b.cfg.WriteKey)
|
||||||
|
req.Header.Set("Checksum", strings.ToUpper(sha))
|
||||||
|
|
||||||
|
resp, err := b.client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
_, _ = io.Copy(io.Discard, resp.Body)
|
||||||
|
|
||||||
|
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||||
|
return fmt.Errorf("bunny put %s: unexpected status %d", sha, resp.StatusCode)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get fetches sha into dest via temp file + rename, re-hashing and deleting
|
// Get fetches sha into dest via temp file + rename, re-hashing and deleting
|
||||||
|
|||||||
+25
-43
@@ -11,7 +11,6 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
@@ -52,15 +51,13 @@ func Run(args []string, stdout, stderr io.Writer, getenv func(string) string) in
|
|||||||
|
|
||||||
func printRunUsage(w io.Writer) {
|
func printRunUsage(w io.Writer) {
|
||||||
fmt.Fprint(w, `usage:
|
fmt.Fprint(w, `usage:
|
||||||
depot status [--manifests DIR] --target bunny|cdn | --out DIR
|
depot status [--manifests DIR] [--source LOCAL_DEPOT] --target bunny|cdn|local
|
||||||
depot push [--manifests DIR] --source LOCAL_DEPOT --target bunny
|
depot push [--manifests DIR] --source LOCAL_DEPOT --target bunny
|
||||||
depot verify [--manifests DIR] --target bunny|cdn [--sample N]
|
depot verify [--manifests DIR] --target bunny|cdn [--sample N]
|
||||||
depot get <sha> <dest> --target cdn|bunny | --out DIR
|
depot get <sha> <dest> --target cdn|bunny|local
|
||||||
depot pull [--manifests DIR] --dest DIR --target cdn|bunny
|
depot pull [--manifests DIR] --dest DIR --target cdn|bunny
|
||||||
|
|
||||||
--out DIR reads and writes a depot tree on disk at DIR instead of a remote
|
--target local uses the DEPOT_DIR environment variable as the local root.
|
||||||
backend. --target names a remote backend only; the two are mutually exclusive.
|
|
||||||
--target local is a deprecated alias for --out $DEPOT_DIR.
|
|
||||||
`)
|
`)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -68,27 +65,16 @@ backend. --target names a remote backend only; the two are mutually exclusive.
|
|||||||
// than internal/backend failures (exit 70).
|
// than internal/backend failures (exit 70).
|
||||||
var errUsage = errors.New("usage error")
|
var errUsage = errors.New("usage error")
|
||||||
|
|
||||||
// resolveBackend picks the backend for a command that can work either against a
|
// resolveBackend builds the named backend, resolving "local" against DEPOT_DIR.
|
||||||
// depot tree on disk (--out DIR) or a remote backend (--target bunny|cdn).
|
func resolveBackend(target string, getenv func(string) string, cfg Config) (Backend, error) {
|
||||||
// "--target local" stays as a deprecated alias for --out $DEPOT_DIR so older
|
root := ""
|
||||||
// callers keep working.
|
if target == "local" {
|
||||||
func resolveBackend(out, target string, getenv func(string) string, cfg Config) (Backend, error) {
|
root = getenv("DEPOT_DIR")
|
||||||
switch {
|
|
||||||
case out != "" && target != "":
|
|
||||||
return nil, fmt.Errorf("--out and --target are mutually exclusive: %w", errUsage)
|
|
||||||
case out != "":
|
|
||||||
return NewBackend("local", out, cfg)
|
|
||||||
case target == "local":
|
|
||||||
root := getenv("DEPOT_DIR")
|
|
||||||
if root == "" {
|
if root == "" {
|
||||||
return nil, fmt.Errorf("--target local requires DEPOT_DIR to be set (prefer --out DIR): %w", errUsage)
|
return nil, fmt.Errorf("--target local requires DEPOT_DIR to be set: %w", errUsage)
|
||||||
}
|
}
|
||||||
return NewBackend("local", root, cfg)
|
|
||||||
case target == "":
|
|
||||||
return nil, fmt.Errorf("--out DIR or --target bunny|cdn is required: %w", errUsage)
|
|
||||||
default:
|
|
||||||
return NewBackend(target, "", cfg)
|
|
||||||
}
|
}
|
||||||
|
return NewBackend(target, root, cfg)
|
||||||
}
|
}
|
||||||
|
|
||||||
// backendErrExit maps a resolveBackend error onto the exit contract.
|
// backendErrExit maps a resolveBackend error onto the exit contract.
|
||||||
@@ -133,18 +119,18 @@ func runStatus(args []string, stdout, stderr io.Writer, getenv func(string) stri
|
|||||||
fs := flag.NewFlagSet("status", flag.ContinueOnError)
|
fs := flag.NewFlagSet("status", flag.ContinueOnError)
|
||||||
fs.SetOutput(stderr)
|
fs.SetOutput(stderr)
|
||||||
manifests := fs.String("manifests", "assets", "directory of *.yml manifests")
|
manifests := fs.String("manifests", "assets", "directory of *.yml manifests")
|
||||||
source := fs.String("source", "", "deprecated and ignored (use --out DIR for a depot tree on disk)")
|
_ = fs.String("source", "", "local depot root (unused for status target=local; see DEPOT_DIR)")
|
||||||
out := fs.String("out", "", "depot tree on disk to read instead of a remote backend")
|
target := fs.String("target", "", "bunny|cdn|local")
|
||||||
target := fs.String("target", "", "bunny|cdn")
|
|
||||||
if err := fs.Parse(args); err != nil {
|
if err := fs.Parse(args); err != nil {
|
||||||
return exitUsage
|
return exitUsage
|
||||||
}
|
}
|
||||||
if *source != "" {
|
if *target == "" {
|
||||||
fmt.Fprintln(stderr, "depot status: --source is ignored; use --out DIR")
|
fmt.Fprintln(stderr, "depot status: --target is required")
|
||||||
|
return exitUsage
|
||||||
}
|
}
|
||||||
|
|
||||||
cfg := LoadConfig(getenv)
|
cfg := LoadConfig(getenv)
|
||||||
backend, err := resolveBackend(*out, *target, getenv, cfg)
|
backend, err := resolveBackend(*target, getenv, cfg)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Fprintln(stderr, "depot status:", err)
|
fmt.Fprintln(stderr, "depot status:", err)
|
||||||
return backendErrExit(err)
|
return backendErrExit(err)
|
||||||
@@ -323,21 +309,13 @@ func runVerify(args []string, stdout, stderr io.Writer, getenv func(string) stri
|
|||||||
func runGet(args []string, stdout, stderr io.Writer, getenv func(string) string) int {
|
func runGet(args []string, stdout, stderr io.Writer, getenv func(string) string) int {
|
||||||
fs := flag.NewFlagSet("get", flag.ContinueOnError)
|
fs := flag.NewFlagSet("get", flag.ContinueOnError)
|
||||||
fs.SetOutput(stderr)
|
fs.SetOutput(stderr)
|
||||||
out := fs.String("out", "", "depot tree on disk to read instead of a remote backend")
|
target := fs.String("target", "", "cdn|bunny|local")
|
||||||
target := fs.String("target", "", "cdn|bunny")
|
if err := fs.Parse(args); err != nil {
|
||||||
// Positionals come first in the documented usage, and Go's flag package
|
|
||||||
// stops parsing at the first one, so split them off by hand.
|
|
||||||
split := 0
|
|
||||||
for split < len(args) && !strings.HasPrefix(args[split], "-") {
|
|
||||||
split++
|
|
||||||
}
|
|
||||||
positional := args[:split]
|
|
||||||
if err := fs.Parse(args[split:]); err != nil {
|
|
||||||
return exitUsage
|
return exitUsage
|
||||||
}
|
}
|
||||||
positional = append(positional, fs.Args()...)
|
positional := fs.Args()
|
||||||
if len(positional) != 2 {
|
if len(positional) != 2 {
|
||||||
fmt.Fprintln(stderr, "depot get: usage: depot get <sha> <dest> --target cdn|bunny | --out DIR")
|
fmt.Fprintln(stderr, "depot get: usage: depot get <sha> <dest> --target cdn|bunny|local")
|
||||||
return exitUsage
|
return exitUsage
|
||||||
}
|
}
|
||||||
sha, dest := positional[0], positional[1]
|
sha, dest := positional[0], positional[1]
|
||||||
@@ -345,9 +323,13 @@ func runGet(args []string, stdout, stderr io.Writer, getenv func(string) string)
|
|||||||
fmt.Fprintf(stderr, "depot get: invalid sha %q\n", sha)
|
fmt.Fprintf(stderr, "depot get: invalid sha %q\n", sha)
|
||||||
return exitUsage
|
return exitUsage
|
||||||
}
|
}
|
||||||
|
if *target == "" {
|
||||||
|
fmt.Fprintln(stderr, "depot get: --target is required")
|
||||||
|
return exitUsage
|
||||||
|
}
|
||||||
|
|
||||||
cfg := LoadConfig(getenv)
|
cfg := LoadConfig(getenv)
|
||||||
backend, err := resolveBackend(*out, *target, getenv, cfg)
|
backend, err := resolveBackend(*target, getenv, cfg)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Fprintln(stderr, "depot get:", err)
|
fmt.Fprintln(stderr, "depot get:", err)
|
||||||
return backendErrExit(err)
|
return backendErrExit(err)
|
||||||
|
|||||||
@@ -54,91 +54,6 @@ func TestRunUsage(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestOutFlag(t *testing.T) {
|
|
||||||
t.Run("status --out reads the given tree", func(t *testing.T) {
|
|
||||||
sha := shaOf("out-blob")
|
|
||||||
manifests := t.TempDir()
|
|
||||||
writeManifest(t, manifests, sha, 8)
|
|
||||||
depotDir := t.TempDir()
|
|
||||||
writeBlob(t, depotDir, sha, "out-blob")
|
|
||||||
|
|
||||||
var out, errb bytes.Buffer
|
|
||||||
code := Run([]string{"status", "--manifests", manifests, "--out", depotDir}, &out, &errb, testGetenv(nil))
|
|
||||||
if code != 0 {
|
|
||||||
t.Fatalf("expected 0, got %d (stdout=%s stderr=%s)", code, out.String(), errb.String())
|
|
||||||
}
|
|
||||||
if !bytesContains(out.String(), "present=1") {
|
|
||||||
t.Fatalf("expected present=1, got %s", out.String())
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("get --out fetches from the given tree", func(t *testing.T) {
|
|
||||||
sha := shaOf("get-blob")
|
|
||||||
depotDir := t.TempDir()
|
|
||||||
writeBlob(t, depotDir, sha, "get-blob")
|
|
||||||
dest := filepath.Join(t.TempDir(), "fetched")
|
|
||||||
|
|
||||||
var out, errb bytes.Buffer
|
|
||||||
code := Run([]string{"get", sha, dest, "--out", depotDir}, &out, &errb, testGetenv(nil))
|
|
||||||
if code != 0 {
|
|
||||||
t.Fatalf("expected 0, got %d (stderr=%s)", code, errb.String())
|
|
||||||
}
|
|
||||||
got, err := os.ReadFile(dest)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if string(got) != "get-blob" {
|
|
||||||
t.Fatalf("got %q", got)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("--out with any --target is rejected", func(t *testing.T) {
|
|
||||||
for _, target := range []string{"bunny", "cdn", "local"} {
|
|
||||||
var out, errb bytes.Buffer
|
|
||||||
code := Run([]string{"status", "--manifests", t.TempDir(), "--out", t.TempDir(), "--target", target}, &out, &errb, testGetenv(nil))
|
|
||||||
if code != 64 {
|
|
||||||
t.Fatalf("--target %s: expected 64, got %d (stderr=%s)", target, code, errb.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("get accepts flags before the positionals", func(t *testing.T) {
|
|
||||||
sha := shaOf("flags-first-blob")
|
|
||||||
depotDir := t.TempDir()
|
|
||||||
writeBlob(t, depotDir, sha, "flags-first-blob")
|
|
||||||
dest := filepath.Join(t.TempDir(), "fetched")
|
|
||||||
|
|
||||||
var out, errb bytes.Buffer
|
|
||||||
code := Run([]string{"get", "--out", depotDir, sha, dest}, &out, &errb, testGetenv(nil))
|
|
||||||
if code != 0 {
|
|
||||||
t.Fatalf("expected 0, got %d (stderr=%s)", code, errb.String())
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("neither --out nor --target is rejected", func(t *testing.T) {
|
|
||||||
var out, errb bytes.Buffer
|
|
||||||
code := Run([]string{"status", "--manifests", t.TempDir()}, &out, &errb, testGetenv(nil))
|
|
||||||
if code != 64 {
|
|
||||||
t.Fatalf("expected 64, got %d (stderr=%s)", code, errb.String())
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("--target local still works as a hidden alias", func(t *testing.T) {
|
|
||||||
sha := shaOf("alias-blob")
|
|
||||||
manifests := t.TempDir()
|
|
||||||
writeManifest(t, manifests, sha, 11)
|
|
||||||
depotDir := t.TempDir()
|
|
||||||
writeBlob(t, depotDir, sha, "alias-blob")
|
|
||||||
|
|
||||||
var out, errb bytes.Buffer
|
|
||||||
code := Run([]string{"status", "--manifests", manifests, "--target", "local"}, &out, &errb,
|
|
||||||
testGetenv(map[string]string{"DEPOT_DIR": depotDir}))
|
|
||||||
if code != 0 {
|
|
||||||
t.Fatalf("expected 0, got %d (stdout=%s stderr=%s)", code, out.String(), errb.String())
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGetInvalidSHA(t *testing.T) {
|
func TestGetInvalidSHA(t *testing.T) {
|
||||||
var out, errb bytes.Buffer
|
var out, errb bytes.Buffer
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
@@ -257,17 +172,6 @@ func bytesContains(s, substr string) bool {
|
|||||||
return bytes.Contains([]byte(s), []byte(substr))
|
return bytes.Contains([]byte(s), []byte(substr))
|
||||||
}
|
}
|
||||||
|
|
||||||
func writeBlob(t *testing.T, root, sha, content string) {
|
|
||||||
t.Helper()
|
|
||||||
path := filepath.Join(root, BlobKey(sha))
|
|
||||||
if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := os.WriteFile(path, []byte(content), 0644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func writeManifest(t *testing.T, dir, sha string, size int64) {
|
func writeManifest(t *testing.T, dir, sha string, size int64) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
content := fmt.Sprintf("assets:\n - path: foo\n sha256: %s\n size: %d\n", sha, size)
|
content := fmt.Sprintf("assets:\n - path: foo\n sha256: %s\n size: %d\n", sha, size)
|
||||||
|
|||||||
@@ -21,7 +21,6 @@ import (
|
|||||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/buildinfo"
|
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/buildinfo"
|
||||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/depot"
|
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/depot"
|
||||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/menu"
|
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/menu"
|
||||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/nwsync"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// Exit codes follow the sysexits(3) convention so CI can distinguish
|
// Exit codes follow the sysexits(3) convention so CI can distinguish
|
||||||
@@ -95,26 +94,16 @@ var Registry = []Builder{
|
|||||||
{
|
{
|
||||||
Name: "depot",
|
Name: "depot",
|
||||||
Bin: "crucible-depot",
|
Bin: "crucible-depot",
|
||||||
Summary: "content-addressed asset depot (disk/cdn/bunny)",
|
Summary: "content-addressed asset depot (local/cdn/bunny)",
|
||||||
Commands: []Command{
|
Commands: []Command{
|
||||||
{Name: "status", Summary: "report referenced-vs-present drift against a backend", Usage: "crucible depot status [--manifests DIR] --target bunny|cdn | --out DIR"},
|
{Name: "status", Summary: "report referenced-vs-present drift against a backend", Usage: "crucible depot status [--manifests DIR] [--source DIR] --target bunny|cdn|local"},
|
||||||
{Name: "push", Summary: "upload referenced-but-absent blobs from a local depot", Usage: "crucible depot push [--manifests DIR] --source DIR --target bunny"},
|
{Name: "push", Summary: "upload referenced-but-absent blobs from a local depot", Usage: "crucible depot push [--manifests DIR] --source DIR --target bunny"},
|
||||||
{Name: "verify", Summary: "existence sweep plus sampled download re-hash", Usage: "crucible depot verify [--manifests DIR] --target bunny|cdn [--sample N]"},
|
{Name: "verify", Summary: "existence sweep plus sampled download re-hash", Usage: "crucible depot verify [--manifests DIR] --target bunny|cdn [--sample N]"},
|
||||||
{Name: "get", Summary: "fetch one blob with sha re-verify", Usage: "crucible depot get <sha> <dest> --target cdn|bunny | --out DIR"},
|
{Name: "get", Summary: "fetch one blob with sha re-verify", Usage: "crucible depot get <sha> <dest> --target cdn|bunny|local"},
|
||||||
{Name: "pull", Summary: "incremental verified pull of every referenced blob", Usage: "crucible depot pull [--manifests DIR] --dest DIR --target cdn|bunny"},
|
{Name: "pull", Summary: "incremental verified pull of every referenced blob", Usage: "crucible depot pull [--manifests DIR] --dest DIR --target cdn|bunny"},
|
||||||
},
|
},
|
||||||
Wired: true,
|
Wired: true,
|
||||||
},
|
},
|
||||||
{
|
|
||||||
Name: "nwsync",
|
|
||||||
Bin: "crucible-nwsync",
|
|
||||||
Summary: "publish NWSync blobs and manifests (emit/assemble)",
|
|
||||||
Commands: []Command{
|
|
||||||
{Name: "emit", Summary: "explode one artifact into blobs plus its own NSYM manifest", Usage: "crucible nwsync emit <artifact> --out DIR"},
|
|
||||||
{Name: "assemble", Summary: "merge per-artifact NSYM manifests into one", Usage: "crucible nwsync assemble --order NAMES --entries DIR --out DIR [--group-id N]"},
|
|
||||||
},
|
|
||||||
Wired: true,
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
Name: "assets",
|
Name: "assets",
|
||||||
Bin: "crucible-assets",
|
Bin: "crucible-assets",
|
||||||
@@ -403,7 +392,7 @@ func runBuilder(name string, args []string, out, errw io.Writer) int {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if b.Wired {
|
if b.Wired {
|
||||||
// depot, assets and nwsync are self-contained builders: they parse their own
|
// depot and assets are self-contained builders: they parse their own
|
||||||
// subcommands and own their exit contract, bypassing the
|
// subcommands and own their exit contract, bypassing the
|
||||||
// b.Commands/delegateLegacy legacy routing entirely.
|
// b.Commands/delegateLegacy legacy routing entirely.
|
||||||
switch b.Name {
|
switch b.Name {
|
||||||
@@ -411,8 +400,6 @@ func runBuilder(name string, args []string, out, errw io.Writer) int {
|
|||||||
return depot.Run(args, out, errw, os.Getenv)
|
return depot.Run(args, out, errw, os.Getenv)
|
||||||
case "assets":
|
case "assets":
|
||||||
return assets.Run(args, out, errw, os.Getenv)
|
return assets.Run(args, out, errw, os.Getenv)
|
||||||
case "nwsync":
|
|
||||||
return nwsync.Run(args, out, errw)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if !b.Wired {
|
if !b.Wired {
|
||||||
|
|||||||
@@ -148,10 +148,6 @@ func TestBuilderHelpIsOK(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// selfContained builders parse their own subcommands instead of delegating to
|
|
||||||
// the legacy internal/app surface.
|
|
||||||
var selfContained = map[string]bool{"depot": true, "assets": true, "nwsync": true}
|
|
||||||
|
|
||||||
func TestCanonicalCommandSurface(t *testing.T) {
|
func TestCanonicalCommandSurface(t *testing.T) {
|
||||||
want := map[string][]string{
|
want := map[string][]string{
|
||||||
"depot": {"status", "push", "verify", "get", "pull"},
|
"depot": {"status", "push", "verify", "get", "pull"},
|
||||||
@@ -160,7 +156,6 @@ func TestCanonicalCommandSurface(t *testing.T) {
|
|||||||
"module": {"build", "extract", "validate", "compare", "manifest"},
|
"module": {"build", "extract", "validate", "compare", "manifest"},
|
||||||
"topdata": {"validate", "build", "package", "compare", "convert"},
|
"topdata": {"validate", "build", "package", "compare", "convert"},
|
||||||
"wiki": {"build", "deploy"},
|
"wiki": {"build", "deploy"},
|
||||||
"nwsync": {"emit", "assemble"},
|
|
||||||
}
|
}
|
||||||
for _, builder := range Registry {
|
for _, builder := range Registry {
|
||||||
got := builder.subcommands()
|
got := builder.subcommands()
|
||||||
@@ -178,10 +173,10 @@ func TestRegistryCommandNamesAndAliasesAreUnambiguous(t *testing.T) {
|
|||||||
for _, builder := range Registry {
|
for _, builder := range Registry {
|
||||||
seen := map[string]bool{}
|
seen := map[string]bool{}
|
||||||
for _, command := range builder.Commands {
|
for _, command := range builder.Commands {
|
||||||
// depot, assets and nwsync parse their own subcommands and bypass
|
// depot and assets parse their own subcommands and bypass AppCommand
|
||||||
// AppCommand routing entirely (see the self-contained-builder branch
|
// routing entirely (see the self-contained-builder branch in
|
||||||
// in runBuilder), so their Commands carry no AppCommand.
|
// runBuilder), so their Commands carry no AppCommand.
|
||||||
requireAppCommand := !selfContained[builder.Name]
|
requireAppCommand := builder.Name != "depot" && builder.Name != "assets"
|
||||||
if command.Name == "" || command.Summary == "" || command.Usage == "" || (requireAppCommand && command.AppCommand == "") {
|
if command.Name == "" || command.Summary == "" || command.Usage == "" || (requireAppCommand && command.AppCommand == "") {
|
||||||
t.Errorf("%s has incomplete command metadata: %#v", builder.Name, command)
|
t.Errorf("%s has incomplete command metadata: %#v", builder.Name, command)
|
||||||
}
|
}
|
||||||
|
|||||||
+63
-120
@@ -67,19 +67,6 @@ type resourceEntry struct {
|
|||||||
Size uint32
|
Size uint32
|
||||||
}
|
}
|
||||||
|
|
||||||
// extensionTypes and typeExtensions must stay exact inverses of each other;
|
|
||||||
// init() panics if they drift apart.
|
|
||||||
//
|
|
||||||
// Numbers below 0x0BB8 follow upstream neverwinter.nim
|
|
||||||
// (neverwinter/restype.nim). The 0x0BB8 and up entries are lyt/vis/mdx: real
|
|
||||||
// Aurora archive types that NWN1 ships in its own data/*.bif but upstream
|
|
||||||
// happens not to register. xoreos corroborates those three numbers
|
|
||||||
// (src/aurora/types.h).
|
|
||||||
//
|
|
||||||
// This table is NWN:EE only. Do not add NWN2 formats (mdb, gr2, wlk, xml):
|
|
||||||
// NWN:EE either gives that number to something else (2070 is xbc here and mdb
|
|
||||||
// in NWN2) or has no number for it at all, so packing one into a HAK writes a
|
|
||||||
// resource the game misreads. See the reserved list in erf_test.go.
|
|
||||||
var extensionTypes = map[string]uint16{
|
var extensionTypes = map[string]uint16{
|
||||||
"res": 0x0000,
|
"res": 0x0000,
|
||||||
"bmp": 0x0001,
|
"bmp": 0x0001,
|
||||||
@@ -109,13 +96,12 @@ var extensionTypes = map[string]uint16{
|
|||||||
"utt": 0x07F0,
|
"utt": 0x07F0,
|
||||||
"dds": 0x07F1,
|
"dds": 0x07F1,
|
||||||
"uts": 0x07F3,
|
"uts": 0x07F3,
|
||||||
"ltr": 0x07F4,
|
|
||||||
"gff": 0x07F5,
|
|
||||||
"fac": 0x07F6,
|
"fac": 0x07F6,
|
||||||
|
"gff": 0x07F7,
|
||||||
"ute": 0x07F8,
|
"ute": 0x07F8,
|
||||||
"utd": 0x07FA,
|
"utd": 0x07FA,
|
||||||
"utp": 0x07FC,
|
"utp": 0x07FC,
|
||||||
"dft": 0x07FD,
|
"dfa": 0x07FD,
|
||||||
"gic": 0x07FE,
|
"gic": 0x07FE,
|
||||||
"gui": 0x07FF,
|
"gui": 0x07FF,
|
||||||
"utm": 0x0803,
|
"utm": 0x0803,
|
||||||
@@ -131,15 +117,20 @@ var extensionTypes = map[string]uint16{
|
|||||||
"ndb": 0x0810,
|
"ndb": 0x0810,
|
||||||
"ptm": 0x0811,
|
"ptm": 0x0811,
|
||||||
"ptt": 0x0812,
|
"ptt": 0x0812,
|
||||||
|
"ltr": 0x0813,
|
||||||
"shd": 0x0815,
|
"shd": 0x0815,
|
||||||
|
"mdb": 0x0816,
|
||||||
"mtr": 0x0818,
|
"mtr": 0x0818,
|
||||||
|
"jpg": 0x081C,
|
||||||
"lod": 0x081E,
|
"lod": 0x081E,
|
||||||
"gif": 0x081F,
|
"gif": 0x081F,
|
||||||
"png": 0x0820,
|
"png": 0x0820,
|
||||||
"jpg": 0x0821,
|
|
||||||
"lyt": 0x0BB8,
|
"lyt": 0x0BB8,
|
||||||
"vis": 0x0BB9,
|
"vis": 0x0BB9,
|
||||||
"mdx": 0x0BC0,
|
"mdx": 0x0BC0,
|
||||||
|
"wlk": 0x0BCC,
|
||||||
|
"xml": 0x0BCD,
|
||||||
|
"gr2": 0x0FA3,
|
||||||
}
|
}
|
||||||
|
|
||||||
var typeExtensions = map[uint16]string{
|
var typeExtensions = map[uint16]string{
|
||||||
@@ -171,13 +162,12 @@ var typeExtensions = map[uint16]string{
|
|||||||
0x07F0: "utt",
|
0x07F0: "utt",
|
||||||
0x07F1: "dds",
|
0x07F1: "dds",
|
||||||
0x07F3: "uts",
|
0x07F3: "uts",
|
||||||
0x07F4: "ltr",
|
|
||||||
0x07F5: "gff",
|
|
||||||
0x07F6: "fac",
|
0x07F6: "fac",
|
||||||
|
0x07F7: "gff",
|
||||||
0x07F8: "ute",
|
0x07F8: "ute",
|
||||||
0x07FA: "utd",
|
0x07FA: "utd",
|
||||||
0x07FC: "utp",
|
0x07FC: "utp",
|
||||||
0x07FD: "dft",
|
0x07FD: "dfa",
|
||||||
0x07FE: "gic",
|
0x07FE: "gic",
|
||||||
0x07FF: "gui",
|
0x07FF: "gui",
|
||||||
0x0803: "utm",
|
0x0803: "utm",
|
||||||
@@ -193,15 +183,20 @@ var typeExtensions = map[uint16]string{
|
|||||||
0x0810: "ndb",
|
0x0810: "ndb",
|
||||||
0x0811: "ptm",
|
0x0811: "ptm",
|
||||||
0x0812: "ptt",
|
0x0812: "ptt",
|
||||||
|
0x0813: "ltr",
|
||||||
0x0815: "shd",
|
0x0815: "shd",
|
||||||
|
0x0816: "mdb",
|
||||||
0x0818: "mtr",
|
0x0818: "mtr",
|
||||||
|
0x081C: "jpg",
|
||||||
0x081E: "lod",
|
0x081E: "lod",
|
||||||
0x081F: "gif",
|
0x081F: "gif",
|
||||||
0x0820: "png",
|
0x0820: "png",
|
||||||
0x0821: "jpg",
|
|
||||||
0x0BB8: "lyt",
|
0x0BB8: "lyt",
|
||||||
0x0BB9: "vis",
|
0x0BB9: "vis",
|
||||||
0x0BC0: "mdx",
|
0x0BC0: "mdx",
|
||||||
|
0x0BCC: "wlk",
|
||||||
|
0x0BCD: "xml",
|
||||||
|
0x0FA3: "gr2",
|
||||||
}
|
}
|
||||||
|
|
||||||
func init() {
|
func init() {
|
||||||
@@ -210,13 +205,8 @@ func init() {
|
|||||||
if !ok {
|
if !ok {
|
||||||
panic(fmt.Sprintf("missing canonical extension for resource type 0x%04X", resourceType))
|
panic(fmt.Sprintf("missing canonical extension for resource type 0x%04X", resourceType))
|
||||||
}
|
}
|
||||||
if canonicalExt != ext {
|
if canonicalExt == ext {
|
||||||
panic(fmt.Sprintf("resource type 0x%04X is %q in extensionTypes but %q in typeExtensions", resourceType, ext, canonicalExt))
|
continue
|
||||||
}
|
|
||||||
}
|
|
||||||
for resourceType, ext := range typeExtensions {
|
|
||||||
if registered, ok := extensionTypes[ext]; !ok || registered != resourceType {
|
|
||||||
panic(fmt.Sprintf("extension %q is 0x%04X in typeExtensions but 0x%04X in extensionTypes (registered=%v)", ext, resourceType, registered, ok))
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -311,110 +301,63 @@ func Write(w io.Writer, archive Archive) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// IndexEntry locates one resource inside an archive without holding its
|
|
||||||
// payload. Streaming callers read one payload at a time from these, so peak
|
|
||||||
// memory tracks the largest resource instead of the whole archive.
|
|
||||||
type IndexEntry struct {
|
|
||||||
Name string
|
|
||||||
Type uint16
|
|
||||||
Offset int64
|
|
||||||
Size int64
|
|
||||||
}
|
|
||||||
|
|
||||||
// Index is the header plus the resource table of an ERF: everything except the
|
|
||||||
// payloads.
|
|
||||||
type Index struct {
|
|
||||||
FileType string
|
|
||||||
Version string
|
|
||||||
Entries []IndexEntry
|
|
||||||
}
|
|
||||||
|
|
||||||
// ReadIndex parses the tables of an ERF of the given size, reading only the
|
|
||||||
// header, the key list and the resource list.
|
|
||||||
func ReadIndex(r io.ReaderAt, size int64) (Index, error) {
|
|
||||||
if size < headerSize {
|
|
||||||
return Index{}, fmt.Errorf("erf file too small: %d bytes", size)
|
|
||||||
}
|
|
||||||
|
|
||||||
var hdr header
|
|
||||||
if err := binary.Read(io.NewSectionReader(r, 0, headerSize), binary.LittleEndian, &hdr); err != nil {
|
|
||||||
return Index{}, fmt.Errorf("decode erf header: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if int64(hdr.KeyListOffset)+int64(hdr.EntryCount)*24 > size {
|
|
||||||
return Index{}, fmt.Errorf("erf key list exceeds file bounds")
|
|
||||||
}
|
|
||||||
keys := make([]keyEntry, hdr.EntryCount)
|
|
||||||
keyReader := io.NewSectionReader(r, int64(hdr.KeyListOffset), int64(hdr.EntryCount)*24)
|
|
||||||
if err := binary.Read(keyReader, binary.LittleEndian, &keys); err != nil {
|
|
||||||
return Index{}, fmt.Errorf("decode key list: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if int64(hdr.ResourceListOffset)+int64(hdr.EntryCount)*8 > size {
|
|
||||||
return Index{}, fmt.Errorf("erf resource list exceeds file bounds")
|
|
||||||
}
|
|
||||||
entries := make([]resourceEntry, hdr.EntryCount)
|
|
||||||
entryReader := io.NewSectionReader(r, int64(hdr.ResourceListOffset), int64(hdr.EntryCount)*8)
|
|
||||||
if err := binary.Read(entryReader, binary.LittleEndian, &entries); err != nil {
|
|
||||||
return Index{}, fmt.Errorf("decode resource list: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
index := Index{
|
|
||||||
FileType: string(hdr.FileType[:]),
|
|
||||||
Version: string(hdr.Version[:]),
|
|
||||||
Entries: make([]IndexEntry, 0, hdr.EntryCount),
|
|
||||||
}
|
|
||||||
for position, key := range keys {
|
|
||||||
entry := entries[position]
|
|
||||||
if int64(entry.Offset)+int64(entry.Size) > size {
|
|
||||||
return Index{}, fmt.Errorf("resource %d exceeds file bounds", position)
|
|
||||||
}
|
|
||||||
index.Entries = append(index.Entries, IndexEntry{
|
|
||||||
Name: string(bytes.TrimRight(key.ResRef[:], "\x00")),
|
|
||||||
Type: key.ResourceType,
|
|
||||||
Offset: int64(entry.Offset),
|
|
||||||
Size: int64(entry.Size),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return index, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// ReadPayload returns one resource's bytes.
|
|
||||||
func ReadPayload(r io.ReaderAt, entry IndexEntry) ([]byte, error) {
|
|
||||||
payload := make([]byte, entry.Size)
|
|
||||||
if _, err := r.ReadAt(payload, entry.Offset); err != nil {
|
|
||||||
return nil, fmt.Errorf("read resource %q: %w", entry.Name, err)
|
|
||||||
}
|
|
||||||
return payload, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Read materialises a whole archive. Payloads are subslices of the buffer the
|
|
||||||
// archive was read into, so nothing is copied twice: a caller must not mutate
|
|
||||||
// Data. Callers that only need one resource at a time should use ReadIndex
|
|
||||||
// instead, which never holds the archive at all.
|
|
||||||
func Read(r io.Reader) (Archive, error) {
|
func Read(r io.Reader) (Archive, error) {
|
||||||
data, err := io.ReadAll(r)
|
data, err := io.ReadAll(r)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Archive{}, fmt.Errorf("read erf: %w", err)
|
return Archive{}, fmt.Errorf("read erf: %w", err)
|
||||||
}
|
}
|
||||||
index, err := ReadIndex(bytes.NewReader(data), int64(len(data)))
|
if len(data) < headerSize {
|
||||||
if err != nil {
|
return Archive{}, fmt.Errorf("erf file too small: %d bytes", len(data))
|
||||||
return Archive{}, err
|
|
||||||
}
|
}
|
||||||
|
|
||||||
resources := make([]Resource, 0, len(index.Entries))
|
var hdr header
|
||||||
for _, entry := range index.Entries {
|
if err := binary.Read(bytes.NewReader(data[:headerSize]), binary.LittleEndian, &hdr); err != nil {
|
||||||
|
return Archive{}, fmt.Errorf("decode erf header: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
keyStart := int(hdr.KeyListOffset)
|
||||||
|
keyEnd := keyStart + int(hdr.EntryCount)*24
|
||||||
|
if keyEnd > len(data) {
|
||||||
|
return Archive{}, fmt.Errorf("erf key list exceeds file bounds")
|
||||||
|
}
|
||||||
|
keys := make([]keyEntry, hdr.EntryCount)
|
||||||
|
if err := binary.Read(bytes.NewReader(data[keyStart:keyEnd]), binary.LittleEndian, &keys); err != nil {
|
||||||
|
return Archive{}, fmt.Errorf("decode key list: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
resourceStart := int(hdr.ResourceListOffset)
|
||||||
|
resourceEnd := resourceStart + int(hdr.EntryCount)*8
|
||||||
|
if resourceEnd > len(data) {
|
||||||
|
return Archive{}, fmt.Errorf("erf resource list exceeds file bounds")
|
||||||
|
}
|
||||||
|
entries := make([]resourceEntry, hdr.EntryCount)
|
||||||
|
if err := binary.Read(bytes.NewReader(data[resourceStart:resourceEnd]), binary.LittleEndian, &entries); err != nil {
|
||||||
|
return Archive{}, fmt.Errorf("decode resource list: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
resources := make([]Resource, 0, hdr.EntryCount)
|
||||||
|
for index, key := range keys {
|
||||||
|
entry := entries[index]
|
||||||
|
start := int(entry.Offset)
|
||||||
|
end := start + int(entry.Size)
|
||||||
|
if end > len(data) {
|
||||||
|
return Archive{}, fmt.Errorf("resource %d exceeds file bounds", index)
|
||||||
|
}
|
||||||
|
|
||||||
|
resref := string(bytes.TrimRight(key.ResRef[:], "\x00"))
|
||||||
|
payload := make([]byte, entry.Size)
|
||||||
|
copy(payload, data[start:end])
|
||||||
resources = append(resources, Resource{
|
resources = append(resources, Resource{
|
||||||
Name: entry.Name,
|
Name: resref,
|
||||||
Type: entry.Type,
|
Type: key.ResourceType,
|
||||||
Data: data[entry.Offset : entry.Offset+entry.Size],
|
Data: payload,
|
||||||
Size: entry.Size,
|
Size: int64(entry.Size),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
return Archive{
|
return Archive{
|
||||||
FileType: index.FileType,
|
FileType: string(hdr.FileType[:]),
|
||||||
Version: index.Version,
|
Version: string(hdr.Version[:]),
|
||||||
Resources: resources,
|
Resources: resources,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -134,10 +134,14 @@ func TestExtensionMappingsSupportModernAssetTypes(t *testing.T) {
|
|||||||
"mtr": 0x0818,
|
"mtr": 0x0818,
|
||||||
"shd": 0x0815,
|
"shd": 0x0815,
|
||||||
"txi": 0x07E6,
|
"txi": 0x07E6,
|
||||||
"jpg": 0x0821,
|
"jpg": 0x081C,
|
||||||
|
"mdb": 0x0816,
|
||||||
"lyt": 0x0BB8,
|
"lyt": 0x0BB8,
|
||||||
"vis": 0x0BB9,
|
"vis": 0x0BB9,
|
||||||
"mdx": 0x0BC0,
|
"mdx": 0x0BC0,
|
||||||
|
"xml": 0x0BCD,
|
||||||
|
"wlk": 0x0BCC,
|
||||||
|
"gr2": 0x0FA3,
|
||||||
}
|
}
|
||||||
for ext, wantType := range cases {
|
for ext, wantType := range cases {
|
||||||
gotType, ok := ResourceTypeForExtension(ext)
|
gotType, ok := ResourceTypeForExtension(ext)
|
||||||
@@ -187,71 +191,3 @@ func TestExtensionMappingsSupportAllUTBlueprintTypes(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestRestypeTableMatchesUpstream pins the restype numbers Crucible shares with
|
|
||||||
// neverwinter.nim's restype.nim. The values below are upstream's; a mismatch
|
|
||||||
// means a HAK we write is mislabelled for the game.
|
|
||||||
func TestRestypeTableMatchesUpstream(t *testing.T) {
|
|
||||||
upstream := map[string]uint16{
|
|
||||||
"res": 0, "bmp": 1, "mve": 2, "tga": 3, "wav": 4, "plt": 6, "ini": 7,
|
|
||||||
"bmu": 8, "txt": 10, "mdl": 2002, "nss": 2009, "ncs": 2010, "are": 2012,
|
|
||||||
"set": 2013, "ifo": 2014, "bic": 2015, "wok": 2016, "2da": 2017,
|
|
||||||
"tlk": 2018, "txi": 2022, "git": 2023, "uti": 2025, "utc": 2027,
|
|
||||||
"dlg": 2029, "itp": 2030, "utt": 2032, "dds": 2033, "uts": 2035,
|
|
||||||
"ltr": 2036, "gff": 2037, "fac": 2038, "ute": 2040, "utd": 2042,
|
|
||||||
"utp": 2044, "dft": 2045, "gic": 2046, "gui": 2047, "utm": 2051,
|
|
||||||
"dwk": 2052, "pwk": 2053, "utg": 2055, "jrl": 2056, "utw": 2058,
|
|
||||||
"ssf": 2060, "hak": 2061, "nwm": 2062, "bik": 2063, "ndb": 2064,
|
|
||||||
"ptm": 2065, "ptt": 2066, "shd": 2069, "mtr": 2072, "lod": 2078,
|
|
||||||
"gif": 2079, "png": 2080, "jpg": 2081,
|
|
||||||
}
|
|
||||||
for ext, want := range upstream {
|
|
||||||
got, ok := ResourceTypeForExtension(ext)
|
|
||||||
if !ok {
|
|
||||||
t.Errorf("%s: not registered, upstream has %d", ext, want)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if got != want {
|
|
||||||
t.Errorf("%s: registered as %d, upstream has %d", ext, got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Extensions upstream registers that Crucible must not reuse for anything else.
|
|
||||||
reserved := map[uint16]string{2039: "bte", 2067: "bak", 2070: "xbc", 2076: "tml"}
|
|
||||||
for number, upstreamExt := range reserved {
|
|
||||||
if ext, ok := ExtensionForResourceType(number); ok {
|
|
||||||
t.Errorf("%d: registered as %s, upstream reserves it for %s", number, ext, upstreamExt)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestNWN2FormatsAreNotRegistered keeps NWN2 file formats out of an NWN:EE
|
|
||||||
// table. mdb and gr2 have Aurora numbers that mean something else (or nothing)
|
|
||||||
// in NWN:EE; wlk and xml have no archive number in any Aurora game, so the
|
|
||||||
// values Crucible used for them were invented. Packing any of these into a HAK
|
|
||||||
// writes a resource the game misreads.
|
|
||||||
func TestNWN2FormatsAreNotRegistered(t *testing.T) {
|
|
||||||
for _, ext := range []string{"mdb", "gr2", "wlk", "xml"} {
|
|
||||||
if number, ok := ResourceTypeForExtension(ext); ok {
|
|
||||||
t.Errorf("%s is an NWN2 format but is registered as 0x%04X", ext, number)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRestypeTablesAreMutualInverses is the check init() is meant to enforce.
|
|
||||||
func TestRestypeTablesAreMutualInverses(t *testing.T) {
|
|
||||||
for ext, number := range extensionTypes {
|
|
||||||
canonical, ok := typeExtensions[number]
|
|
||||||
if !ok {
|
|
||||||
t.Errorf("%s: number 0x%04X missing from typeExtensions", ext, number)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if canonical != ext {
|
|
||||||
t.Errorf("%s: maps to 0x%04X, which maps back to %s", ext, number, canonical)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for number, ext := range typeExtensions {
|
|
||||||
if got, ok := extensionTypes[ext]; !ok || got != number {
|
|
||||||
t.Errorf("0x%04X: maps to %s, which maps back to 0x%04X (ok=%v)", number, ext, got, ok)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,128 +0,0 @@
|
|||||||
package nwsync
|
|
||||||
|
|
||||||
import (
|
|
||||||
"crypto/sha1"
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"path"
|
|
||||||
)
|
|
||||||
|
|
||||||
// AssembleOptions describes one merged manifest.
|
|
||||||
type AssembleOptions struct {
|
|
||||||
// ArtifactKeys are the depot keys of the artifacts to merge, in
|
|
||||||
// Mod_HakList order — highest priority first. Each one's index is read
|
|
||||||
// from the key beside it.
|
|
||||||
ArtifactKeys []string
|
|
||||||
TLKKey string // the TLK's key, if the manifest carries one
|
|
||||||
OutDir string // write locally instead of uploading — the conformance path
|
|
||||||
GroupID int // 1 = current, 2 = testing; 0 means absent
|
|
||||||
ModuleName string
|
|
||||||
Description string
|
|
||||||
Sink sink // test seam; nil means OutDir or the zone
|
|
||||||
}
|
|
||||||
|
|
||||||
// AssembleResult reports what one assemble run produced.
|
|
||||||
type AssembleResult struct {
|
|
||||||
SHA1 string
|
|
||||||
ManifestPath string
|
|
||||||
Entries int
|
|
||||||
}
|
|
||||||
|
|
||||||
// Assemble merges the per-artifact NSYM manifests named by Order into one
|
|
||||||
// manifest. It reads no bulk data at all — only the small index files.
|
|
||||||
//
|
|
||||||
// Merge rule is resref shadowing, not concatenation: a resref present in more
|
|
||||||
// than one artifact resolves to the earliest artifact in Order, which is how
|
|
||||||
// the game resolves it (upstream's resman adds haks in reverse and lets the
|
|
||||||
// last one win). Get this backwards and the wrong texture ships silently.
|
|
||||||
func Assemble(options AssembleOptions) (AssembleResult, error) {
|
|
||||||
if len(options.ArtifactKeys) == 0 {
|
|
||||||
return AssembleResult{}, fmt.Errorf("assemble: no artifact keys given")
|
|
||||||
}
|
|
||||||
|
|
||||||
target, err := openSink(options.OutDir, options.Sink)
|
|
||||||
if err != nil {
|
|
||||||
return AssembleResult{}, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// The TLK carries no precedence — it is not a hak and shadows nothing —
|
|
||||||
// so it merges last, after every hak has had its say.
|
|
||||||
keys := append([]string{}, options.ArtifactKeys...)
|
|
||||||
if options.TLKKey != "" {
|
|
||||||
keys = append(keys, options.TLKKey)
|
|
||||||
}
|
|
||||||
|
|
||||||
merged := make([]Entry, 0, 1024)
|
|
||||||
winner := make(map[Identity]bool, 1024)
|
|
||||||
var onDiskBytes int64
|
|
||||||
|
|
||||||
for _, artifactKey := range keys {
|
|
||||||
key, err := resolveIndexKey(artifactKey, options.OutDir)
|
|
||||||
if err != nil {
|
|
||||||
return AssembleResult{}, err
|
|
||||||
}
|
|
||||||
data, sidecarBody, err := target.getIndex(key)
|
|
||||||
if err != nil {
|
|
||||||
// An artifact with no index is an artifact whose emit never
|
|
||||||
// finished. Publishing a manifest without it would ship a release
|
|
||||||
// missing a hak, so this fails closed.
|
|
||||||
return AssembleResult{}, fmt.Errorf("assemble: no index for %s: %w", artifactKey, err)
|
|
||||||
}
|
|
||||||
entries, err := readManifest(data)
|
|
||||||
if err != nil {
|
|
||||||
return AssembleResult{}, fmt.Errorf("%s: %w", target.describe(key), err)
|
|
||||||
}
|
|
||||||
sidecar, err := parseSidecar(target.describe(key), sidecarBody)
|
|
||||||
if err != nil {
|
|
||||||
return AssembleResult{}, err
|
|
||||||
}
|
|
||||||
// Two producers of blobs means a skewed emitter can write blobs the
|
|
||||||
// merged manifest quietly disagrees with. Refuse to merge across
|
|
||||||
// mismatched emitter versions.
|
|
||||||
if sidecar.EmitterVersion != emitterVersion {
|
|
||||||
return AssembleResult{}, fmt.Errorf(
|
|
||||||
"assemble: emitter version mismatch: %s was emitted by emitter %q, this is emitter %q",
|
|
||||||
artifactKey, sidecar.EmitterVersion, emitterVersion)
|
|
||||||
}
|
|
||||||
// on_disk_bytes overcounts by the handful of cross-artifact
|
|
||||||
// duplicates. It is a display statistic; no dedupe pass for it.
|
|
||||||
onDiskBytes += sidecar.OnDiskBytes
|
|
||||||
|
|
||||||
for _, entry := range entries {
|
|
||||||
identity := entry.identity()
|
|
||||||
if winner[identity] {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
winner[identity] = true
|
|
||||||
merged = append(merged, entry)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(merged) == 0 {
|
|
||||||
return AssembleResult{}, fmt.Errorf("assemble: merged manifest is empty")
|
|
||||||
}
|
|
||||||
|
|
||||||
data, err := writeManifest(merged)
|
|
||||||
if err != nil {
|
|
||||||
return AssembleResult{}, err
|
|
||||||
}
|
|
||||||
sha1Hex := fmt.Sprintf("%x", sha1.Sum(data))
|
|
||||||
manifestKey := path.Join("manifests", sha1Hex)
|
|
||||||
sidecar := Sidecar{
|
|
||||||
ModuleName: options.ModuleName,
|
|
||||||
Description: options.Description,
|
|
||||||
GroupID: options.GroupID,
|
|
||||||
}
|
|
||||||
if err := putManifestPair(target, manifestKey, data, merged, onDiskBytes, sidecar); err != nil {
|
|
||||||
return AssembleResult{}, err
|
|
||||||
}
|
|
||||||
return AssembleResult{SHA1: sha1Hex, ManifestPath: target.describe(manifestKey), Entries: len(merged)}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func parseSidecar(where string, data []byte) (Sidecar, error) {
|
|
||||||
var sidecar Sidecar
|
|
||||||
if err := json.Unmarshal(data, &sidecar); err != nil {
|
|
||||||
return Sidecar{}, fmt.Errorf("%s: %w", where, err)
|
|
||||||
}
|
|
||||||
return sidecar, nil
|
|
||||||
}
|
|
||||||
@@ -1,80 +0,0 @@
|
|||||||
package nwsync
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"encoding/binary"
|
|
||||||
"fmt"
|
|
||||||
|
|
||||||
"github.com/klauspost/compress/zstd"
|
|
||||||
)
|
|
||||||
|
|
||||||
// NWCompressedBuffer framing, as upstream's neverwinter/compressedbuf.nim
|
|
||||||
// writes it for NWSync blobs. All fields are little-endian uint32:
|
|
||||||
//
|
|
||||||
// magic "NSYC", version 3, algorithm 2 (zstd), uncompressed size,
|
|
||||||
// zstd header version 1, dictionary 0, then the raw zstd frame.
|
|
||||||
const (
|
|
||||||
blobMagic = 0x4359534E // "NSYC" little-endian
|
|
||||||
blobVersion = 3
|
|
||||||
algorithmZstd = 2
|
|
||||||
zstdHeaderVer = 1
|
|
||||||
zstdDictionary = 0
|
|
||||||
blobHeaderBytes = 24
|
|
||||||
)
|
|
||||||
|
|
||||||
// EncodeAll/DecodeAll are single-threaded per call, so the default pool of one
|
|
||||||
// encoder per CPU only buys idle memory: each holds a window-sized history, so
|
|
||||||
// on a 24-core runner that is ~200 MB of live heap doing nothing. Concurrency 1
|
|
||||||
// produces byte-identical output.
|
|
||||||
var (
|
|
||||||
blobEncoder, _ = zstd.NewWriter(nil, zstd.WithEncoderConcurrency(1))
|
|
||||||
blobDecoder, _ = zstd.NewReader(nil, zstd.WithDecoderConcurrency(1))
|
|
||||||
)
|
|
||||||
|
|
||||||
// compressBlob wraps data in NWCompressedBuffer framing.
|
|
||||||
func compressBlob(data []byte) []byte {
|
|
||||||
var out bytes.Buffer
|
|
||||||
header := []uint32{blobMagic, blobVersion, algorithmZstd, uint32(len(data)), zstdHeaderVer, zstdDictionary}
|
|
||||||
for _, field := range header {
|
|
||||||
_ = binary.Write(&out, binary.LittleEndian, field)
|
|
||||||
}
|
|
||||||
out.Write(blobEncoder.EncodeAll(data, nil))
|
|
||||||
return out.Bytes()
|
|
||||||
}
|
|
||||||
|
|
||||||
// decompressBlob unwraps NWCompressedBuffer framing. It exists so a blob this
|
|
||||||
// package wrote — or one upstream wrote — can be compared by its uncompressed
|
|
||||||
// bytes, which is the only comparison that is meaningful across zstd
|
|
||||||
// implementations.
|
|
||||||
func decompressBlob(blob []byte) ([]byte, error) {
|
|
||||||
if len(blob) < blobHeaderBytes {
|
|
||||||
return nil, fmt.Errorf("blob too small: %d bytes", len(blob))
|
|
||||||
}
|
|
||||||
header := make([]uint32, 6)
|
|
||||||
if err := binary.Read(bytes.NewReader(blob[:blobHeaderBytes]), binary.LittleEndian, header); err != nil {
|
|
||||||
return nil, fmt.Errorf("decode blob header: %w", err)
|
|
||||||
}
|
|
||||||
switch {
|
|
||||||
case header[0] != blobMagic:
|
|
||||||
return nil, fmt.Errorf("invalid blob magic: %#x", header[0])
|
|
||||||
case header[1] != blobVersion:
|
|
||||||
return nil, fmt.Errorf("unsupported blob version: %d", header[1])
|
|
||||||
case header[2] != algorithmZstd:
|
|
||||||
return nil, fmt.Errorf("unsupported compression algorithm: %d", header[2])
|
|
||||||
case header[4] != zstdHeaderVer:
|
|
||||||
return nil, fmt.Errorf("unsupported zstd header version: %d", header[4])
|
|
||||||
case header[5] != zstdDictionary:
|
|
||||||
return nil, fmt.Errorf("zstd dictionaries are not supported")
|
|
||||||
}
|
|
||||||
if header[3] == 0 {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
data, err := blobDecoder.DecodeAll(blob[blobHeaderBytes:], nil)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("decompress blob: %w", err)
|
|
||||||
}
|
|
||||||
if uint32(len(data)) != header[3] {
|
|
||||||
return nil, fmt.Errorf("blob size mismatch: header says %d, got %d", header[3], len(data))
|
|
||||||
}
|
|
||||||
return data, nil
|
|
||||||
}
|
|
||||||
@@ -1,278 +0,0 @@
|
|||||||
package nwsync
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"crypto/sha1"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"os"
|
|
||||||
"path"
|
|
||||||
"path/filepath"
|
|
||||||
"slices"
|
|
||||||
"sort"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/buildinfo"
|
|
||||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf"
|
|
||||||
)
|
|
||||||
|
|
||||||
// fileSizeLimit matches upstream's --limit-file-size default of 15 MB. A
|
|
||||||
// resource over it is a hard failure, not a skip: upstream quit(1)s and so do
|
|
||||||
// we. Our largest resource today is 13.66 MiB, so the headroom is thin.
|
|
||||||
const fileSizeLimit = 15 * 1024 * 1024
|
|
||||||
|
|
||||||
// skippedTypes are never published, matching upstream's GobalResTypeSkipList.
|
|
||||||
var skippedTypes = resTypes("nss", "ndb", "gic")
|
|
||||||
|
|
||||||
// emitterVersion identifies the blob/manifest byte format this package
|
|
||||||
// produces. assemble refuses to merge indexes that disagree on it, because two
|
|
||||||
// producers of blobs mean a skewed emitter can otherwise write blobs the merged
|
|
||||||
// manifest quietly disagrees with. Bump it only when emitted bytes change — it
|
|
||||||
// is deliberately not the build revision, which would invalidate every
|
|
||||||
// published index on every unrelated commit.
|
|
||||||
const emitterVersion = "1"
|
|
||||||
|
|
||||||
// serverTypes are loaded only server-side; a manifest holding nothing else
|
|
||||||
// has no client contents. Mirrors upstream's GlobalResTypeServerList, whose
|
|
||||||
// trailing 0 is RESTYPE_INVALID.
|
|
||||||
var serverTypes = append(resTypes(
|
|
||||||
"are", "dlg", "fac", "gic", "git", "ifo", "itp", "jrl", "ncs", "ndb",
|
|
||||||
"nss", "ptm", "utc", "utd", "ute", "uti", "utm", "utp", "uts", "utt", "utw",
|
|
||||||
), 0)
|
|
||||||
|
|
||||||
func resTypes(extensions ...string) []uint16 {
|
|
||||||
types := make([]uint16, 0, len(extensions))
|
|
||||||
for _, extension := range extensions {
|
|
||||||
restype, ok := erf.ResourceTypeForExtension(extension)
|
|
||||||
if !ok {
|
|
||||||
panic("nwsync: unknown restype " + extension)
|
|
||||||
}
|
|
||||||
types = append(types, restype)
|
|
||||||
}
|
|
||||||
return types
|
|
||||||
}
|
|
||||||
|
|
||||||
// EmitResult reports what one emit run produced.
|
|
||||||
type EmitResult struct {
|
|
||||||
Name string // artifact name, without extension
|
|
||||||
ManifestPath string
|
|
||||||
Entries int
|
|
||||||
BlobsWritten int
|
|
||||||
}
|
|
||||||
|
|
||||||
// EmitOptions describes one emit run.
|
|
||||||
type EmitOptions struct {
|
|
||||||
ArtifactKey string // depot key of the artifact; the NSYM key is derived from it
|
|
||||||
ArtifactPath string // the file on disk
|
|
||||||
As string // name override, for a TLK whose filename is not its published name
|
|
||||||
OutDir string // write locally instead of uploading — the conformance path
|
|
||||||
Sink sink // test seam; nil means OutDir or the zone
|
|
||||||
}
|
|
||||||
|
|
||||||
// Emit explodes one artifact — a .hak/.erf or a loose file such as the TLK —
|
|
||||||
// into NWSync blobs plus a NSYM manifest describing only that artifact.
|
|
||||||
//
|
|
||||||
// Blobs go up as they are produced and the index lands last, so the presence of
|
|
||||||
// an index is the publication marker: an artifact whose emit died halfway has
|
|
||||||
// real blobs in the zone and no index, which is unambiguous. Blob names are
|
|
||||||
// content hashes, so re-running skips whatever already landed.
|
|
||||||
func Emit(options EmitOptions) (EmitResult, error) {
|
|
||||||
artifact, err := os.Open(options.ArtifactPath)
|
|
||||||
if err != nil {
|
|
||||||
return EmitResult{}, fmt.Errorf("read artifact: %w", err)
|
|
||||||
}
|
|
||||||
defer artifact.Close()
|
|
||||||
info, err := artifact.Stat()
|
|
||||||
if err != nil {
|
|
||||||
return EmitResult{}, fmt.Errorf("read artifact: %w", err)
|
|
||||||
}
|
|
||||||
// A section reader, not the file itself: hashing must not move the file
|
|
||||||
// offset out from under everything that reads the artifact afterwards.
|
|
||||||
if err := checkArtifactKey(options.ArtifactKey, io.NewSectionReader(artifact, 0, info.Size())); err != nil {
|
|
||||||
return EmitResult{}, err
|
|
||||||
}
|
|
||||||
name := options.As
|
|
||||||
if name == "" {
|
|
||||||
name = path.Base(options.ArtifactKey)
|
|
||||||
}
|
|
||||||
extension := path.Ext(name)
|
|
||||||
name = strings.TrimSuffix(name, extension)
|
|
||||||
|
|
||||||
key, err := resolveIndexKey(options.ArtifactKey, options.OutDir)
|
|
||||||
if err != nil {
|
|
||||||
return EmitResult{}, err
|
|
||||||
}
|
|
||||||
|
|
||||||
index, err := readArtifactIndex(options.ArtifactPath, artifact, info.Size(), name)
|
|
||||||
if err != nil {
|
|
||||||
return EmitResult{}, err
|
|
||||||
}
|
|
||||||
|
|
||||||
target, err := openSink(options.OutDir, options.Sink)
|
|
||||||
if err != nil {
|
|
||||||
return EmitResult{}, err
|
|
||||||
}
|
|
||||||
|
|
||||||
entries, blobs, onDiskBytes, err := emitResources(artifact, index, target)
|
|
||||||
if err != nil {
|
|
||||||
return EmitResult{}, err
|
|
||||||
}
|
|
||||||
if len(entries) == 0 {
|
|
||||||
return EmitResult{}, fmt.Errorf("%s: nothing to index (no publishable resources)", options.ArtifactPath)
|
|
||||||
}
|
|
||||||
|
|
||||||
data, err := writeManifest(entries)
|
|
||||||
if err != nil {
|
|
||||||
return EmitResult{}, err
|
|
||||||
}
|
|
||||||
if err := putManifestPair(target, key, data, entries, onDiskBytes, Sidecar{ModuleName: name}); err != nil {
|
|
||||||
return EmitResult{}, err
|
|
||||||
}
|
|
||||||
return EmitResult{Name: name, ManifestPath: target.describe(key), Entries: len(entries), BlobsWritten: blobs}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// openSink returns the zone sink, or a local directory when outDir is set.
|
|
||||||
func openSink(outDir string, injected sink) (sink, error) {
|
|
||||||
if injected != nil {
|
|
||||||
return injected, nil
|
|
||||||
}
|
|
||||||
if outDir != "" {
|
|
||||||
return dirSink{root: outDir}, nil
|
|
||||||
}
|
|
||||||
return newZoneSink(context.Background(), os.Getenv)
|
|
||||||
}
|
|
||||||
|
|
||||||
// readArtifactIndex locates the resources of an ERF/HAK/MOD, or the single
|
|
||||||
// resource a loose file represents, without reading any payload. Upstream's
|
|
||||||
// resman does the same dispatch on the file's first three bytes. name is the
|
|
||||||
// artifact's published name, which for a loose file is also its resref.
|
|
||||||
func readArtifactIndex(path string, artifact io.ReaderAt, size int64, name string) ([]erf.IndexEntry, error) {
|
|
||||||
magic := make([]byte, 3)
|
|
||||||
if size >= 3 {
|
|
||||||
if _, err := artifact.ReadAt(magic, 0); err != nil {
|
|
||||||
return nil, fmt.Errorf("%s: %w", path, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
switch string(magic) {
|
|
||||||
case "ERF", "HAK":
|
|
||||||
index, err := erf.ReadIndex(artifact, size)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%s: %w", path, err)
|
|
||||||
}
|
|
||||||
return index.Entries, nil
|
|
||||||
case "MOD":
|
|
||||||
// A persistent world never publishes module contents, so the .mod
|
|
||||||
// contributes no bytes to a manifest — it only says which haks and
|
|
||||||
// which TLK the manifest covers.
|
|
||||||
return nil, fmt.Errorf("%s: a module is never emitted; a manifest is haks plus the TLK", path)
|
|
||||||
}
|
|
||||||
extension := filepath.Ext(filepath.Base(path))
|
|
||||||
restype, ok := erf.ResourceTypeForExtension(extension)
|
|
||||||
if !ok {
|
|
||||||
return nil, fmt.Errorf("%s: unknown resource type %q", path, extension)
|
|
||||||
}
|
|
||||||
return []erf.IndexEntry{{Name: name, Type: restype, Offset: 0, Size: size}}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// emitResources hashes, compresses and stores one resource at a time, reading
|
|
||||||
// each payload from the artifact only when its turn comes. Peak memory
|
|
||||||
// therefore tracks the largest single resource, not the archive: a 2 GB hak
|
|
||||||
// must emit inside a runner's few spare GB.
|
|
||||||
func emitResources(artifact io.ReaderAt, index []erf.IndexEntry, target sink) ([]Entry, int, int64, error) {
|
|
||||||
// A resref appearing twice inside one artifact resolves to the last one,
|
|
||||||
// the way resman lets the last container added win.
|
|
||||||
order := make([]Identity, 0, len(index))
|
|
||||||
latest := make(map[Identity]erf.IndexEntry, len(index))
|
|
||||||
var tooBig []string
|
|
||||||
for _, entry := range index {
|
|
||||||
if _, ok := erf.ExtensionForResourceType(entry.Type); !ok {
|
|
||||||
return nil, 0, 0, fmt.Errorf("resref %s is not resolvable (unknown restype %d)", entry.Name, entry.Type)
|
|
||||||
}
|
|
||||||
if slices.Contains(skippedTypes, entry.Type) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if entry.Size > fileSizeLimit {
|
|
||||||
tooBig = append(tooBig, fmt.Sprintf("%s: %d bytes > %d", entry.Name, entry.Size, fileSizeLimit))
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
identity := Identity{ResRef: strings.ToLower(entry.Name), ResType: entry.Type}
|
|
||||||
if _, seen := latest[identity]; !seen {
|
|
||||||
order = append(order, identity)
|
|
||||||
}
|
|
||||||
latest[identity] = entry
|
|
||||||
}
|
|
||||||
if len(tooBig) > 0 {
|
|
||||||
sort.Strings(tooBig)
|
|
||||||
return nil, 0, 0, fmt.Errorf("resources exceed the file size limit:\n %s", strings.Join(tooBig, "\n "))
|
|
||||||
}
|
|
||||||
|
|
||||||
entries := make([]Entry, 0, len(order))
|
|
||||||
var blobs int
|
|
||||||
var onDiskBytes int64
|
|
||||||
for _, identity := range order {
|
|
||||||
payload, err := erf.ReadPayload(artifact, latest[identity])
|
|
||||||
if err != nil {
|
|
||||||
return nil, 0, 0, err
|
|
||||||
}
|
|
||||||
sum := sha1.Sum(payload)
|
|
||||||
entries = append(entries, Entry{
|
|
||||||
SHA1: sum,
|
|
||||||
Size: uint32(len(payload)),
|
|
||||||
ResRef: identity.ResRef,
|
|
||||||
ResType: identity.ResType,
|
|
||||||
})
|
|
||||||
written, err := target.putBlob(fmt.Sprintf("%x", sum), func() []byte { return compressBlob(payload) })
|
|
||||||
if err != nil {
|
|
||||||
return nil, 0, 0, err
|
|
||||||
}
|
|
||||||
if written > 0 {
|
|
||||||
blobs++
|
|
||||||
onDiskBytes += written
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return entries, blobs, onDiskBytes, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// created is the sidecar timestamp. SOURCE_DATE_EPOCH pins it so a build can
|
|
||||||
// be reproduced byte for byte; the manifest itself is deterministic already.
|
|
||||||
func created() int64 {
|
|
||||||
if raw := os.Getenv("SOURCE_DATE_EPOCH"); raw != "" {
|
|
||||||
if seconds, err := strconv.ParseInt(raw, 10, 64); err == nil {
|
|
||||||
return seconds
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return time.Now().Unix()
|
|
||||||
}
|
|
||||||
|
|
||||||
// putManifestPair stores a NSYM manifest and its .json sidecar at key. The
|
|
||||||
// caller supplies the sidecar fields it knows; the rest are derived from the
|
|
||||||
// entries. data must be the serialised form of entries.
|
|
||||||
func putManifestPair(target sink, key string, data []byte, entries []Entry, onDiskBytes int64, sidecar Sidecar) error {
|
|
||||||
var totalBytes int64
|
|
||||||
clientContents := false
|
|
||||||
for _, entry := range entries {
|
|
||||||
totalBytes += int64(entry.Size)
|
|
||||||
if !slices.Contains(serverTypes, entry.ResType) {
|
|
||||||
clientContents = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
sidecar.Version = manifestVersion
|
|
||||||
sidecar.SHA1 = fmt.Sprintf("%x", sha1.Sum(data))
|
|
||||||
sidecar.HashTreeDepth = hashTreeDepth
|
|
||||||
sidecar.IncludesModuleContents = false
|
|
||||||
sidecar.IncludesClientContents = clientContents
|
|
||||||
sidecar.TotalFiles = len(entries)
|
|
||||||
sidecar.TotalBytes = totalBytes
|
|
||||||
sidecar.OnDiskBytes = onDiskBytes
|
|
||||||
sidecar.Created = created()
|
|
||||||
sidecar.CreatedWith = buildinfo.String()
|
|
||||||
sidecar.EmitterVersion = emitterVersion
|
|
||||||
|
|
||||||
body, err := marshalSidecar(sidecar)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return target.putIndex(key, data, body)
|
|
||||||
}
|
|
||||||
@@ -1,200 +0,0 @@
|
|||||||
package nwsync
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"encoding/binary"
|
|
||||||
"encoding/hex"
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"path/filepath"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
)
|
|
||||||
|
|
||||||
// NSYM manifest, version 3, exactly as upstream neverwinter/nwsync.nim writes
|
|
||||||
// it. Everything is little-endian:
|
|
||||||
//
|
|
||||||
// "NSYM", uint32 version, uint32 entry count, uint32 mapping count,
|
|
||||||
// entries: byte[20] raw sha1, uint32 size, char[16] resref, uint16 restype
|
|
||||||
// mappings: uint32 entry index, char[16] resref, uint16 restype
|
|
||||||
//
|
|
||||||
// Entries are sorted by lowercase sha1 hex then resref, and a resource whose
|
|
||||||
// sha1 was already written becomes a mapping instead of a second entry.
|
|
||||||
const (
|
|
||||||
manifestVersion = 3
|
|
||||||
hashTreeDepth = 2
|
|
||||||
resRefBytes = 16
|
|
||||||
)
|
|
||||||
|
|
||||||
// Entry is one resource in a manifest.
|
|
||||||
type Entry struct {
|
|
||||||
SHA1 [20]byte
|
|
||||||
Size uint32
|
|
||||||
ResRef string // lowercase, no extension, at most 16 characters
|
|
||||||
ResType uint16
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e Entry) sha1Hex() string { return hex.EncodeToString(e.SHA1[:]) }
|
|
||||||
|
|
||||||
// Identity is what a resref resolves by: name plus type. It is the merge key
|
|
||||||
// inside one artifact and across artifacts alike.
|
|
||||||
type Identity struct {
|
|
||||||
ResRef string
|
|
||||||
ResType uint16
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e Entry) identity() Identity { return Identity{ResRef: e.ResRef, ResType: e.ResType} }
|
|
||||||
|
|
||||||
// writeManifest serialises entries into NSYM v3 bytes.
|
|
||||||
func writeManifest(entries []Entry) ([]byte, error) {
|
|
||||||
sorted := make([]Entry, len(entries))
|
|
||||||
copy(sorted, entries)
|
|
||||||
sort.SliceStable(sorted, func(i, j int) bool {
|
|
||||||
left, right := sorted[i].sha1Hex(), sorted[j].sha1Hex()
|
|
||||||
if left != right {
|
|
||||||
return left < right
|
|
||||||
}
|
|
||||||
return sorted[i].ResRef < sorted[j].ResRef
|
|
||||||
})
|
|
||||||
|
|
||||||
var body, mappings bytes.Buffer
|
|
||||||
seen := make(map[string]uint32, len(sorted))
|
|
||||||
var entryCount, mappingCount uint32
|
|
||||||
for _, entry := range sorted {
|
|
||||||
padded, err := padResRef(entry.ResRef)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if index, ok := seen[entry.sha1Hex()]; ok {
|
|
||||||
_ = binary.Write(&mappings, binary.LittleEndian, index)
|
|
||||||
mappings.Write(padded)
|
|
||||||
_ = binary.Write(&mappings, binary.LittleEndian, entry.ResType)
|
|
||||||
mappingCount++
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
seen[entry.sha1Hex()] = entryCount
|
|
||||||
entryCount++
|
|
||||||
body.Write(entry.SHA1[:])
|
|
||||||
_ = binary.Write(&body, binary.LittleEndian, entry.Size)
|
|
||||||
body.Write(padded)
|
|
||||||
_ = binary.Write(&body, binary.LittleEndian, entry.ResType)
|
|
||||||
}
|
|
||||||
|
|
||||||
var out bytes.Buffer
|
|
||||||
out.WriteString("NSYM")
|
|
||||||
for _, field := range []uint32{manifestVersion, entryCount, mappingCount} {
|
|
||||||
_ = binary.Write(&out, binary.LittleEndian, field)
|
|
||||||
}
|
|
||||||
out.Write(body.Bytes())
|
|
||||||
out.Write(mappings.Bytes())
|
|
||||||
return out.Bytes(), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// readManifest parses NSYM v3 bytes. Mappings are expanded back into entries,
|
|
||||||
// the way upstream's reader does, so a caller sees one entry per resref.
|
|
||||||
func readManifest(data []byte) ([]Entry, error) {
|
|
||||||
reader := bytes.NewReader(data)
|
|
||||||
magic := make([]byte, 4)
|
|
||||||
if _, err := io.ReadFull(reader, magic); err != nil || string(magic) != "NSYM" {
|
|
||||||
return nil, fmt.Errorf("not a manifest (invalid magic bytes)")
|
|
||||||
}
|
|
||||||
var version, entryCount, mappingCount uint32
|
|
||||||
for _, field := range []*uint32{&version, &entryCount, &mappingCount} {
|
|
||||||
if err := binary.Read(reader, binary.LittleEndian, field); err != nil {
|
|
||||||
return nil, fmt.Errorf("truncated manifest header: %w", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if version != manifestVersion {
|
|
||||||
return nil, fmt.Errorf("unsupported manifest version %d", version)
|
|
||||||
}
|
|
||||||
|
|
||||||
entries := make([]Entry, 0, entryCount+mappingCount)
|
|
||||||
for i := uint32(0); i < entryCount; i++ {
|
|
||||||
var entry Entry
|
|
||||||
if _, err := io.ReadFull(reader, entry.SHA1[:]); err != nil {
|
|
||||||
return nil, fmt.Errorf("truncated entry %d: %w", i, err)
|
|
||||||
}
|
|
||||||
if err := binary.Read(reader, binary.LittleEndian, &entry.Size); err != nil {
|
|
||||||
return nil, fmt.Errorf("truncated entry %d: %w", i, err)
|
|
||||||
}
|
|
||||||
resref, restype, err := readResRef(reader)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("truncated entry %d: %w", i, err)
|
|
||||||
}
|
|
||||||
entry.ResRef, entry.ResType = resref, restype
|
|
||||||
entries = append(entries, entry)
|
|
||||||
}
|
|
||||||
for i := uint32(0); i < mappingCount; i++ {
|
|
||||||
var index uint32
|
|
||||||
if err := binary.Read(reader, binary.LittleEndian, &index); err != nil {
|
|
||||||
return nil, fmt.Errorf("truncated mapping %d: %w", i, err)
|
|
||||||
}
|
|
||||||
if index >= entryCount {
|
|
||||||
return nil, fmt.Errorf("mapping %d references non-existent entry %d", i, index)
|
|
||||||
}
|
|
||||||
resref, restype, err := readResRef(reader)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("truncated mapping %d: %w", i, err)
|
|
||||||
}
|
|
||||||
target := entries[index]
|
|
||||||
entries = append(entries, Entry{SHA1: target.SHA1, Size: target.Size, ResRef: resref, ResType: restype})
|
|
||||||
}
|
|
||||||
return entries, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func readResRef(reader *bytes.Reader) (string, uint16, error) {
|
|
||||||
raw := make([]byte, resRefBytes)
|
|
||||||
if _, err := io.ReadFull(reader, raw); err != nil {
|
|
||||||
return "", 0, err
|
|
||||||
}
|
|
||||||
var restype uint16
|
|
||||||
if err := binary.Read(reader, binary.LittleEndian, &restype); err != nil {
|
|
||||||
return "", 0, err
|
|
||||||
}
|
|
||||||
return strings.ToLower(string(bytes.TrimRight(raw, "\x00"))), restype, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func padResRef(resref string) ([]byte, error) {
|
|
||||||
if len(resref) > resRefBytes {
|
|
||||||
return nil, fmt.Errorf("resref %q exceeds %d characters", resref, resRefBytes)
|
|
||||||
}
|
|
||||||
padded := make([]byte, resRefBytes)
|
|
||||||
copy(padded, strings.ToLower(resref))
|
|
||||||
return padded, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Sidecar is the .json file written next to every manifest. Clients fetch it
|
|
||||||
// (nwn_nwsync_fetch.nim), so the field names and order match upstream.
|
|
||||||
// Upstream omits an integer meta field whose value is 0, hence group_id's
|
|
||||||
// omitempty: 0 means absent, not "group zero".
|
|
||||||
type Sidecar struct {
|
|
||||||
Version int `json:"version"`
|
|
||||||
SHA1 string `json:"sha1"`
|
|
||||||
HashTreeDepth int `json:"hash_tree_depth"`
|
|
||||||
ModuleName string `json:"module_name"`
|
|
||||||
Description string `json:"description"`
|
|
||||||
IncludesModuleContents bool `json:"includes_module_contents"`
|
|
||||||
IncludesClientContents bool `json:"includes_client_contents"`
|
|
||||||
TotalFiles int `json:"total_files"`
|
|
||||||
TotalBytes int64 `json:"total_bytes"`
|
|
||||||
OnDiskBytes int64 `json:"on_disk_bytes"`
|
|
||||||
Created int64 `json:"created"`
|
|
||||||
CreatedWith string `json:"created_with"`
|
|
||||||
EmitterVersion string `json:"emitter_version,omitempty"`
|
|
||||||
GroupID int `json:"group_id,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func marshalSidecar(sidecar Sidecar) ([]byte, error) {
|
|
||||||
body, err := json.MarshalIndent(sidecar, "", " ")
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
// Upstream terminates the file with CRLF; match it.
|
|
||||||
return append(body, '\r', '\n'), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// blobPath is the data store path for a blob, hash tree depth 2.
|
|
||||||
func blobPath(root, sha1Hex string) string {
|
|
||||||
return filepath.Join(root, "data", "sha1", sha1Hex[0:2], sha1Hex[2:4], sha1Hex)
|
|
||||||
}
|
|
||||||
@@ -1,150 +0,0 @@
|
|||||||
package nwsync
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"fmt"
|
|
||||||
"math/rand"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"runtime"
|
|
||||||
"runtime/debug"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/klauspost/compress/zstd"
|
|
||||||
|
|
||||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf"
|
|
||||||
)
|
|
||||||
|
|
||||||
// TestSingleThreadedEncoderMatchesDefault pins the claim the blob encoder's
|
|
||||||
// concurrency setting rests on: it saves memory only, and a published blob is
|
|
||||||
// the same bytes either way.
|
|
||||||
func TestSingleThreadedEncoderMatchesDefault(t *testing.T) {
|
|
||||||
standard, err := zstd.NewWriter(nil)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
defer standard.Close()
|
|
||||||
|
|
||||||
body := make([]byte, 4<<20)
|
|
||||||
random := rand.New(rand.NewSource(1))
|
|
||||||
random.Read(body[:len(body)/2])
|
|
||||||
for _, size := range []int{0, 1, 4 << 10, len(body)} {
|
|
||||||
if !bytes.Equal(blobEncoder.EncodeAll(body[:size], nil), standard.EncodeAll(body[:size], nil)) {
|
|
||||||
t.Fatalf("%d bytes compress differently at concurrency 1", size)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// resourceSize is one payload in the memory fixtures. Real haks hold a few MB
|
|
||||||
// per resource, and peak memory is meant to track that, not the archive.
|
|
||||||
const resourceSize = 1 << 20
|
|
||||||
|
|
||||||
// writeStreamedHak builds a hak of count resources without ever holding the
|
|
||||||
// archive in memory, so the fixture itself does not decide the measurement.
|
|
||||||
// Payloads are distinct, so no blob is deduplicated away.
|
|
||||||
func writeStreamedHak(t *testing.T, path string, count int) {
|
|
||||||
t.Helper()
|
|
||||||
payload := filepath.Join(t.TempDir(), "payload.bin")
|
|
||||||
body := make([]byte, resourceSize)
|
|
||||||
for index := range body {
|
|
||||||
body[index] = byte(index)
|
|
||||||
}
|
|
||||||
|
|
||||||
resources := make([]erf.Resource, 0, count)
|
|
||||||
for index := range count {
|
|
||||||
// A distinct first byte per resource is enough to give every payload
|
|
||||||
// its own sha1 while still streaming from one file per resource.
|
|
||||||
unique := filepath.Join(filepath.Dir(payload), fmt.Sprintf("p%d.bin", index))
|
|
||||||
body[0] = byte(index)
|
|
||||||
body[1] = byte(index >> 8)
|
|
||||||
if err := os.WriteFile(unique, body, 0o644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
resources = append(resources, erf.Resource{
|
|
||||||
Name: fmt.Sprintf("res%05d", index),
|
|
||||||
Type: restype(t, "tga"),
|
|
||||||
SourcePath: unique,
|
|
||||||
Size: resourceSize,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
file, err := os.Create(path)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
defer file.Close()
|
|
||||||
if err := erf.Write(file, erf.New("HAK", resources)); err != nil {
|
|
||||||
t.Fatalf("write hak: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// peakHeapDuring runs work while sampling the heap, and returns the largest
|
|
||||||
// live heap it saw.
|
|
||||||
func peakHeapDuring(work func()) uint64 {
|
|
||||||
runtime.GC()
|
|
||||||
done := make(chan struct{})
|
|
||||||
peak := make(chan uint64, 1)
|
|
||||||
go func() {
|
|
||||||
var highest uint64
|
|
||||||
var stats runtime.MemStats
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case <-done:
|
|
||||||
peak <- highest
|
|
||||||
return
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
runtime.ReadMemStats(&stats)
|
|
||||||
if stats.HeapAlloc > highest {
|
|
||||||
highest = stats.HeapAlloc
|
|
||||||
}
|
|
||||||
time.Sleep(time.Millisecond)
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
work()
|
|
||||||
close(done)
|
|
||||||
return <-peak
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestEmitPeakMemoryDoesNotScaleWithArtifactSize is the regression check for
|
|
||||||
// the OOM kills on large haks: emit used to hold the whole archive (twice), so
|
|
||||||
// a 2 GB hak needed about 10 GB. Emitting an archive 8× bigger must not cost
|
|
||||||
// meaningfully more memory.
|
|
||||||
func TestEmitPeakMemoryDoesNotScaleWithArtifactSize(t *testing.T) {
|
|
||||||
if testing.Short() {
|
|
||||||
t.Skip("writes a 64 MB fixture")
|
|
||||||
}
|
|
||||||
// A lazy GC lets garbage pile up in proportion to the live heap, which
|
|
||||||
// hides the thing under test. Collecting eagerly makes the sampled heap
|
|
||||||
// track what emit actually holds.
|
|
||||||
defer debug.SetGCPercent(debug.SetGCPercent(10))
|
|
||||||
|
|
||||||
measure := func(count int) uint64 {
|
|
||||||
dir := t.TempDir()
|
|
||||||
hak := filepath.Join(dir, "big.hak")
|
|
||||||
writeStreamedHak(t, hak, count)
|
|
||||||
// The key is computed outside the measurement: the test helper reads
|
|
||||||
// the whole file to hash it, which emit itself no longer does.
|
|
||||||
options := EmitOptions{
|
|
||||||
ArtifactKey: artifactKey(t, hak),
|
|
||||||
ArtifactPath: hak,
|
|
||||||
As: filepath.Base(hak),
|
|
||||||
OutDir: filepath.Join(dir, "out"),
|
|
||||||
}
|
|
||||||
return peakHeapDuring(func() {
|
|
||||||
if _, err := Emit(options); err != nil {
|
|
||||||
t.Fatalf("emit %d resources: %v", count, err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
small := measure(8) // 8 MB
|
|
||||||
large := measure(64) // 64 MB
|
|
||||||
const slack = 24 << 20
|
|
||||||
|
|
||||||
t.Logf("peak heap: 8 MB hak %d bytes, 64 MB hak %d bytes", small, large)
|
|
||||||
if large > small+slack {
|
|
||||||
t.Fatalf("peak heap scaled with artifact size: 8 MB hak peaked at %d bytes, 64 MB hak at %d", small, large)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,476 +0,0 @@
|
|||||||
package nwsync
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"crypto/sha1"
|
|
||||||
"crypto/sha256"
|
|
||||||
"encoding/binary"
|
|
||||||
"encoding/hex"
|
|
||||||
"encoding/json"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf"
|
|
||||||
)
|
|
||||||
|
|
||||||
func restype(t *testing.T, extension string) uint16 {
|
|
||||||
t.Helper()
|
|
||||||
value, ok := erf.ResourceTypeForExtension(extension)
|
|
||||||
if !ok {
|
|
||||||
t.Fatalf("unknown restype %q", extension)
|
|
||||||
}
|
|
||||||
return value
|
|
||||||
}
|
|
||||||
|
|
||||||
// writeHak builds a HAK fixture from resref.ext => body pairs.
|
|
||||||
func writeHak(t *testing.T, path string, contents map[string][]byte) {
|
|
||||||
t.Helper()
|
|
||||||
resources := make([]erf.Resource, 0, len(contents))
|
|
||||||
for name, body := range contents {
|
|
||||||
stem, extension, _ := strings.Cut(name, ".")
|
|
||||||
resources = append(resources, erf.Resource{
|
|
||||||
Name: stem,
|
|
||||||
Type: restype(t, extension),
|
|
||||||
Data: body,
|
|
||||||
Size: int64(len(body)),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
var out bytes.Buffer
|
|
||||||
if err := erf.Write(&out, erf.New("HAK", resources)); err != nil {
|
|
||||||
t.Fatalf("write hak: %v", err)
|
|
||||||
}
|
|
||||||
if err := os.WriteFile(path, out.Bytes(), 0o644); err != nil {
|
|
||||||
t.Fatalf("write hak file: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// artifactKey is the depot key a file would be published under: the sha256 of
|
|
||||||
// its bytes, hash-tree depth 2, keeping the extension.
|
|
||||||
func artifactKey(t *testing.T, path string) string {
|
|
||||||
t.Helper()
|
|
||||||
body, err := os.ReadFile(path)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
sum := sha256.Sum256(body)
|
|
||||||
digest := hex.EncodeToString(sum[:])
|
|
||||||
return "artifacts/haks/sha256/" + digest[0:2] + "/" + digest[2:4] + "/" + digest + filepath.Ext(path)
|
|
||||||
}
|
|
||||||
|
|
||||||
// emitLocal emits one artifact into a local tree, the conformance path.
|
|
||||||
func emitLocal(t *testing.T, path, out string) (EmitResult, error) {
|
|
||||||
t.Helper()
|
|
||||||
return Emit(EmitOptions{
|
|
||||||
ArtifactKey: artifactKey(t, path),
|
|
||||||
ArtifactPath: path,
|
|
||||||
As: filepath.Base(path),
|
|
||||||
OutDir: out,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBlobFramingRoundTrips(t *testing.T) {
|
|
||||||
data := []byte("the quick brown fox jumps over the lazy dog, repeatedly and at length")
|
|
||||||
blob := compressBlob(data)
|
|
||||||
|
|
||||||
header := make([]uint32, 6)
|
|
||||||
if err := binary.Read(bytes.NewReader(blob[:blobHeaderBytes]), binary.LittleEndian, header); err != nil {
|
|
||||||
t.Fatalf("read header: %v", err)
|
|
||||||
}
|
|
||||||
want := []uint32{blobMagic, 3, 2, uint32(len(data)), 1, 0}
|
|
||||||
for i := range want {
|
|
||||||
if header[i] != want[i] {
|
|
||||||
t.Errorf("header field %d = %d, want %d", i, header[i], want[i])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if string(blob[:4]) != "NSYC" {
|
|
||||||
t.Errorf("magic bytes = %q, want NSYC", blob[:4])
|
|
||||||
}
|
|
||||||
|
|
||||||
got, err := decompressBlob(blob)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("decompress: %v", err)
|
|
||||||
}
|
|
||||||
if !bytes.Equal(got, data) {
|
|
||||||
t.Errorf("round trip mismatch: %q", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestManifestBytesMatchUpstreamLayout(t *testing.T) {
|
|
||||||
shared := sha1.Sum([]byte("shared"))
|
|
||||||
other := sha1.Sum([]byte("other"))
|
|
||||||
// Deliberately out of order, and with two resrefs sharing one sha1: the
|
|
||||||
// second one must become a mapping, not a second entry.
|
|
||||||
entries := []Entry{
|
|
||||||
{SHA1: other, Size: 5, ResRef: "zzz", ResType: 1},
|
|
||||||
{SHA1: shared, Size: 6, ResRef: "bbb", ResType: 2},
|
|
||||||
{SHA1: shared, Size: 6, ResRef: "aaa", ResType: 3},
|
|
||||||
}
|
|
||||||
data, err := writeManifest(entries)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("write manifest: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if string(data[:4]) != "NSYM" {
|
|
||||||
t.Fatalf("magic = %q", data[:4])
|
|
||||||
}
|
|
||||||
var version, entryCount, mappingCount uint32
|
|
||||||
reader := bytes.NewReader(data[4:16])
|
|
||||||
for _, field := range []*uint32{&version, &entryCount, &mappingCount} {
|
|
||||||
_ = binary.Read(reader, binary.LittleEndian, field)
|
|
||||||
}
|
|
||||||
if version != 3 || entryCount != 2 || mappingCount != 1 {
|
|
||||||
t.Fatalf("header = version %d, %d entries, %d mappings; want 3/2/1", version, entryCount, mappingCount)
|
|
||||||
}
|
|
||||||
wantSize := 16 + int(entryCount)*(20+4+16+2) + int(mappingCount)*(4+16+2)
|
|
||||||
if len(data) != wantSize {
|
|
||||||
t.Fatalf("manifest is %d bytes, want %d", len(data), wantSize)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Sorted by sha1 hex then resref, so the shared hash's "aaa" is the entry
|
|
||||||
// and "bbb" is demoted to a mapping.
|
|
||||||
round, err := readManifest(data)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("read manifest: %v", err)
|
|
||||||
}
|
|
||||||
if len(round) != 3 {
|
|
||||||
t.Fatalf("round trip returned %d entries, want 3", len(round))
|
|
||||||
}
|
|
||||||
byResRef := map[string]Entry{}
|
|
||||||
for _, entry := range round {
|
|
||||||
byResRef[entry.ResRef] = entry
|
|
||||||
}
|
|
||||||
for _, entry := range entries {
|
|
||||||
got, ok := byResRef[entry.ResRef]
|
|
||||||
if !ok {
|
|
||||||
t.Fatalf("resref %q lost in round trip", entry.ResRef)
|
|
||||||
}
|
|
||||||
if got != entry {
|
|
||||||
t.Errorf("resref %q = %+v, want %+v", entry.ResRef, got, entry)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if round[0].ResRef != "aaa" && round[1].ResRef != "aaa" {
|
|
||||||
t.Errorf("entries are not sorted by sha1 then resref: %+v", round)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestEmitWritesBlobsAndManifest(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
hak := filepath.Join(dir, "sow_test_01.hak")
|
|
||||||
body := []byte("texture bytes")
|
|
||||||
writeHak(t, hak, map[string][]byte{
|
|
||||||
"bloodstain1.tga": body,
|
|
||||||
"copy1.txi": body, // same content, different resref: one blob
|
|
||||||
"script1.nss": []byte("void main() {}"),
|
|
||||||
"debug1.ndb": []byte("debug"),
|
|
||||||
"comment1.gic": []byte("comment"),
|
|
||||||
})
|
|
||||||
|
|
||||||
out := filepath.Join(dir, "out")
|
|
||||||
result, err := emitLocal(t, hak, out)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("emit: %v", err)
|
|
||||||
}
|
|
||||||
if result.Entries != 2 {
|
|
||||||
t.Errorf("emitted %d entries, want 2 (nss/ndb/gic are always skipped)", result.Entries)
|
|
||||||
}
|
|
||||||
if result.BlobsWritten != 1 {
|
|
||||||
t.Errorf("wrote %d blobs, want 1 (identical content shares a blob)", result.BlobsWritten)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The blob is named by the sha1 of the uncompressed bytes, under a depth-2
|
|
||||||
// hash tree, and decompresses back to exactly those bytes.
|
|
||||||
sum := sha1.Sum(body)
|
|
||||||
name := hex.EncodeToString(sum[:])
|
|
||||||
path := filepath.Join(out, "data", "sha1", name[0:2], name[2:4], name)
|
|
||||||
blob, err := os.ReadFile(path)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("blob missing at %s: %v", path, err)
|
|
||||||
}
|
|
||||||
got, err := decompressBlob(blob)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("decompress blob: %v", err)
|
|
||||||
}
|
|
||||||
if !bytes.Equal(got, body) {
|
|
||||||
t.Errorf("blob decompressed to %q, want %q", got, body)
|
|
||||||
}
|
|
||||||
|
|
||||||
entries := readEmitted(t, result.ManifestPath)
|
|
||||||
for _, entry := range entries {
|
|
||||||
if entry.ResType == restype(t, "nss") || entry.ResType == restype(t, "ndb") || entry.ResType == restype(t, "gic") {
|
|
||||||
t.Errorf("skipped restype leaked into the manifest: %+v", entry)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
var sidecar Sidecar
|
|
||||||
body2, err := os.ReadFile(result.ManifestPath + ".json")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("sidecar missing: %v", err)
|
|
||||||
}
|
|
||||||
if err := json.Unmarshal(body2, &sidecar); err != nil {
|
|
||||||
t.Fatalf("sidecar json: %v", err)
|
|
||||||
}
|
|
||||||
if sidecar.TotalFiles != 2 || sidecar.HashTreeDepth != 2 || sidecar.Version != 3 {
|
|
||||||
t.Errorf("sidecar = %+v", sidecar)
|
|
||||||
}
|
|
||||||
if sidecar.IncludesModuleContents {
|
|
||||||
t.Error("sidecar claims module contents; a published manifest never has them")
|
|
||||||
}
|
|
||||||
if strings.Contains(string(body2), "group_id") {
|
|
||||||
t.Error("per-artifact sidecar should omit group_id (0 means absent)")
|
|
||||||
}
|
|
||||||
if _, err := os.Stat(filepath.Join(out, "latest")); err == nil {
|
|
||||||
t.Error("a latest file was written; there must never be one")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func readEmitted(t *testing.T, indexPath string) []Entry {
|
|
||||||
t.Helper()
|
|
||||||
data, err := os.ReadFile(indexPath)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("read emitted manifest: %v", err)
|
|
||||||
}
|
|
||||||
entries, err := readManifest(data)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("parse emitted manifest: %v", err)
|
|
||||||
}
|
|
||||||
return entries
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestEmitFailsClosedOnOversizeResource(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
hak := filepath.Join(dir, "big.hak")
|
|
||||||
writeHak(t, hak, map[string][]byte{"huge1.tga": make([]byte, fileSizeLimit+1)})
|
|
||||||
|
|
||||||
if _, err := emitLocal(t, hak, filepath.Join(dir, "out")); err == nil {
|
|
||||||
t.Fatal("emit accepted a resource over the 15 MB limit")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestEmitLooseFile(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
tlk := filepath.Join(dir, "sow_tlk.tlk")
|
|
||||||
if err := os.WriteFile(tlk, []byte("TLK V3.0 payload"), 0o644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
out := filepath.Join(dir, "out")
|
|
||||||
result, err := emitLocal(t, tlk, out)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("emit tlk: %v", err)
|
|
||||||
}
|
|
||||||
entries := readEmitted(t, result.ManifestPath)
|
|
||||||
if len(entries) != 1 || entries[0].ResRef != "sow_tlk" || entries[0].ResType != restype(t, "tlk") {
|
|
||||||
t.Fatalf("tlk emitted as %+v", entries)
|
|
||||||
}
|
|
||||||
if result.BlobsWritten != 1 {
|
|
||||||
t.Errorf("wrote %d blobs, want 1", result.BlobsWritten)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// emitFixture emits two haks that share a resref, so the merge rule is
|
|
||||||
// observable: "top" holds the winning body, "assets" the shadowed one.
|
|
||||||
func emitFixture(t *testing.T) (out string, keys map[string]string, topBody, assetBody []byte) {
|
|
||||||
t.Helper()
|
|
||||||
dir := t.TempDir()
|
|
||||||
topBody = []byte("2da from sow_top")
|
|
||||||
assetBody = []byte("2da from the asset hak")
|
|
||||||
writeHak(t, filepath.Join(dir, "sow_top.hak"), map[string][]byte{"appearance.2da": topBody})
|
|
||||||
writeHak(t, filepath.Join(dir, "sow_core_01.hak"), map[string][]byte{
|
|
||||||
"appearance.2da": assetBody,
|
|
||||||
"bloodstain1.tga": []byte("blood"),
|
|
||||||
})
|
|
||||||
out = filepath.Join(dir, "out")
|
|
||||||
keys = map[string]string{}
|
|
||||||
for _, name := range []string{"sow_top", "sow_core_01"} {
|
|
||||||
path := filepath.Join(dir, name+".hak")
|
|
||||||
keys[name] = artifactKey(t, path)
|
|
||||||
if _, err := emitLocal(t, path, out); err != nil {
|
|
||||||
t.Fatalf("emit %s: %v", name, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out, keys, topBody, assetBody
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAssembleShadowsByOrder(t *testing.T) {
|
|
||||||
entriesDir, keys, topBody, assetBody := emitFixture(t)
|
|
||||||
|
|
||||||
result, err := Assemble(AssembleOptions{
|
|
||||||
ArtifactKeys: []string{keys["sow_top"], keys["sow_core_01"]},
|
|
||||||
OutDir: entriesDir,
|
|
||||||
GroupID: 2,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("assemble: %v", err)
|
|
||||||
}
|
|
||||||
if result.Entries != 2 {
|
|
||||||
t.Fatalf("merged %d entries, want 2 (appearance.2da is shadowed, not duplicated)", result.Entries)
|
|
||||||
}
|
|
||||||
|
|
||||||
data, err := os.ReadFile(result.ManifestPath)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("read merged manifest: %v", err)
|
|
||||||
}
|
|
||||||
merged := sha1.Sum(data)
|
|
||||||
if hex.EncodeToString(merged[:]) != result.SHA1 || filepath.Base(result.ManifestPath) != result.SHA1 {
|
|
||||||
t.Errorf("manifest is not named by its own sha1: %s", result.ManifestPath)
|
|
||||||
}
|
|
||||||
entries, err := readManifest(data)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("parse merged manifest: %v", err)
|
|
||||||
}
|
|
||||||
for _, entry := range entries {
|
|
||||||
if entry.ResRef != "appearance" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if entry.SHA1 != sha1.Sum(topBody) {
|
|
||||||
t.Errorf("appearance.2da resolved to the wrong hak; want the earliest in --order")
|
|
||||||
}
|
|
||||||
if entry.SHA1 == sha1.Sum(assetBody) {
|
|
||||||
t.Error("appearance.2da resolved to the shadowed hak")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
sidecar := Sidecar{}
|
|
||||||
body, err := os.ReadFile(result.ManifestPath + ".json")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("merged sidecar missing: %v", err)
|
|
||||||
}
|
|
||||||
if err := json.Unmarshal(body, &sidecar); err != nil {
|
|
||||||
t.Fatalf("merged sidecar json: %v", err)
|
|
||||||
}
|
|
||||||
if sidecar.GroupID != 2 {
|
|
||||||
t.Errorf("group_id = %d, want 2 (testing)", sidecar.GroupID)
|
|
||||||
}
|
|
||||||
if sidecar.SHA1 != result.SHA1 {
|
|
||||||
t.Errorf("sidecar sha1 = %s, want %s", sidecar.SHA1, result.SHA1)
|
|
||||||
}
|
|
||||||
if sidecar.TotalFiles != 2 {
|
|
||||||
t.Errorf("total_files = %d, want 2", sidecar.TotalFiles)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAssembleReversedOrderPicksTheOtherHak(t *testing.T) {
|
|
||||||
entriesDir, keys, topBody, assetBody := emitFixture(t)
|
|
||||||
|
|
||||||
result, err := Assemble(AssembleOptions{
|
|
||||||
ArtifactKeys: []string{keys["sow_core_01"], keys["sow_top"]},
|
|
||||||
OutDir: entriesDir,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("assemble: %v", err)
|
|
||||||
}
|
|
||||||
data, _ := os.ReadFile(result.ManifestPath)
|
|
||||||
entries, err := readManifest(data)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("parse merged manifest: %v", err)
|
|
||||||
}
|
|
||||||
for _, entry := range entries {
|
|
||||||
if entry.ResRef == "appearance" && entry.SHA1 != sha1.Sum(assetBody) {
|
|
||||||
t.Errorf("appearance.2da did not follow --order; still resolves to %x", entry.SHA1)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
_ = topBody
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAssembleRefusesMismatchedEmitterVersions(t *testing.T) {
|
|
||||||
entriesDir, keys, _, _ := emitFixture(t)
|
|
||||||
|
|
||||||
index, err := resolveIndexKey(keys["sow_core_01"], entriesDir)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
path := filepath.Join(entriesDir, index+".json")
|
|
||||||
var sidecar Sidecar
|
|
||||||
body, err := os.ReadFile(path)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := json.Unmarshal(body, &sidecar); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
sidecar.EmitterVersion = "0"
|
|
||||||
patched, err := marshalSidecar(sidecar)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := os.WriteFile(path, patched, 0o644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = Assemble(AssembleOptions{
|
|
||||||
ArtifactKeys: []string{keys["sow_top"], keys["sow_core_01"]},
|
|
||||||
OutDir: entriesDir,
|
|
||||||
})
|
|
||||||
if err == nil || !strings.Contains(err.Error(), "emitter version mismatch") {
|
|
||||||
t.Fatalf("assemble merged across emitter versions: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestEmitHonoursSourceDateEpoch(t *testing.T) {
|
|
||||||
t.Setenv("SOURCE_DATE_EPOCH", "1700000000")
|
|
||||||
dir := t.TempDir()
|
|
||||||
hak := filepath.Join(dir, "pinned.hak")
|
|
||||||
writeHak(t, hak, map[string][]byte{"one1.tga": []byte("body")})
|
|
||||||
result, err := emitLocal(t, hak, filepath.Join(dir, "out"))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("emit: %v", err)
|
|
||||||
}
|
|
||||||
body, err := os.ReadFile(result.ManifestPath + ".json")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
var sidecar Sidecar
|
|
||||||
if err := json.Unmarshal(body, &sidecar); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if sidecar.Created != 1700000000 {
|
|
||||||
t.Errorf("created = %d, want the pinned SOURCE_DATE_EPOCH", sidecar.Created)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestEmitRejectsAModule(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
path := filepath.Join(dir, "sow.mod")
|
|
||||||
var out bytes.Buffer
|
|
||||||
if err := erf.Write(&out, erf.New("MOD", []erf.Resource{
|
|
||||||
{Name: "module", Type: restype(t, "ifo"), Data: []byte("ifo"), Size: 3},
|
|
||||||
})); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := os.WriteFile(path, out.Bytes(), 0o644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := emitLocal(t, path, filepath.Join(dir, "out")); err == nil {
|
|
||||||
t.Fatal("emit accepted a .mod; a manifest never carries module contents")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAssembleFailsClosedOnMissingIndex(t *testing.T) {
|
|
||||||
entriesDir, keys, _, _ := emitFixture(t)
|
|
||||||
missing := "artifacts/haks/sha256/00/11/" + strings.Repeat("0", 64) + ".hak"
|
|
||||||
_, err := Assemble(AssembleOptions{
|
|
||||||
ArtifactKeys: []string{keys["sow_top"], missing},
|
|
||||||
OutDir: entriesDir,
|
|
||||||
})
|
|
||||||
if err == nil || !strings.Contains(err.Error(), missing) {
|
|
||||||
t.Fatalf("assemble did not fail closed and name the missing artifact: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunUsageErrors(t *testing.T) {
|
|
||||||
cases := [][]string{
|
|
||||||
nil,
|
|
||||||
{"nope"},
|
|
||||||
{"emit"},
|
|
||||||
{"emit", "artifact-key.hak"},
|
|
||||||
{"emit", "a", "b", "c"},
|
|
||||||
{"assemble", "--out", "y"},
|
|
||||||
}
|
|
||||||
for _, args := range cases {
|
|
||||||
var out, errw bytes.Buffer
|
|
||||||
if code := Run(args, &out, &errw); code != exitUsage {
|
|
||||||
t.Errorf("Run(%v) exit=%d, want %d", args, code, exitUsage)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,140 +0,0 @@
|
|||||||
// Package nwsync publishes NWSync repository data: blobs and a per-artifact
|
|
||||||
// NSYM manifest at each artifact's birth (emit), and one merged manifest at
|
|
||||||
// module release (assemble).
|
|
||||||
//
|
|
||||||
// The split exists because upstream nwn_nwsync_write wants every hak, the TLK
|
|
||||||
// and the module present in one run on one disk, which our build hosts cannot
|
|
||||||
// hold. Upstream stays the conformance oracle: manifests compare byte for
|
|
||||||
// byte, blobs compare after decompression.
|
|
||||||
package nwsync
|
|
||||||
|
|
||||||
import (
|
|
||||||
"flag"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
exitOK = 0
|
|
||||||
exitUsage = 64
|
|
||||||
exitInternal = 70
|
|
||||||
)
|
|
||||||
|
|
||||||
// Run executes an nwsync subcommand. args[0] is the subcommand (emit|assemble);
|
|
||||||
// returns the process exit code.
|
|
||||||
func Run(args []string, stdout, stderr io.Writer) int {
|
|
||||||
if len(args) == 0 {
|
|
||||||
printRunUsage(stderr)
|
|
||||||
return exitUsage
|
|
||||||
}
|
|
||||||
switch args[0] {
|
|
||||||
case "emit":
|
|
||||||
return runEmit(args[1:], stdout, stderr)
|
|
||||||
case "assemble":
|
|
||||||
return runAssemble(args[1:], stdout, stderr)
|
|
||||||
case "-h", "--help", "help":
|
|
||||||
printRunUsage(stdout)
|
|
||||||
return exitOK
|
|
||||||
default:
|
|
||||||
fmt.Fprintf(stderr, "nwsync: unknown subcommand %q\n\n", args[0])
|
|
||||||
printRunUsage(stderr)
|
|
||||||
return exitUsage
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func printRunUsage(w io.Writer) {
|
|
||||||
fmt.Fprint(w, `usage:
|
|
||||||
nwsync emit [--as NAME] [--out DIR] <artifact-key> <file>
|
|
||||||
nwsync assemble --group-id N [--tlk-key KEY] [--out DIR] <artifact-key>...
|
|
||||||
|
|
||||||
emit explodes one .hak/.erf or one loose file (the TLK) into NWSync blobs plus
|
|
||||||
a NSYM index covering only that artifact, and uploads both. assemble merges
|
|
||||||
those indexes into one manifest, reading no bulk data. Artifact keys are depot
|
|
||||||
keys; an index lives beside its artifact, with the extension replaced.
|
|
||||||
|
|
||||||
--out DIR writes to a local repository tree instead of uploading, which is the
|
|
||||||
conformance path against upstream nwn_nwsync_write. Without it, the zone comes
|
|
||||||
from NWSYNC_STORAGE_ZONE, NWSYNC_STORAGE_PASSWORD and BUNNY_STORAGE_HOST.
|
|
||||||
`)
|
|
||||||
}
|
|
||||||
|
|
||||||
// parseArgs parses flags that may appear before, after or between positionals.
|
|
||||||
// Go's flag package stops at the first non-flag argument, which turns
|
|
||||||
// `emit <key> <file> --out DIR` into a confusing arity error.
|
|
||||||
func parseArgs(fs *flag.FlagSet, args []string) ([]string, error) {
|
|
||||||
var positional []string
|
|
||||||
for {
|
|
||||||
if err := fs.Parse(args); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
rest := fs.Args()
|
|
||||||
if len(rest) == 0 {
|
|
||||||
return positional, nil
|
|
||||||
}
|
|
||||||
positional = append(positional, rest[0])
|
|
||||||
args = rest[1:]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func runEmit(args []string, stdout, stderr io.Writer) int {
|
|
||||||
fs := flag.NewFlagSet("emit", flag.ContinueOnError)
|
|
||||||
fs.SetOutput(stderr)
|
|
||||||
as := fs.String("as", "", "published name of the artifact, when it differs from the key")
|
|
||||||
out := fs.String("out", "", "write to a local repository tree instead of uploading")
|
|
||||||
positional, err := parseArgs(fs, args)
|
|
||||||
if err != nil {
|
|
||||||
return exitUsage
|
|
||||||
}
|
|
||||||
if len(positional) != 2 {
|
|
||||||
fmt.Fprintf(stderr, "nwsync emit: <artifact-key> and <file> are both required\n")
|
|
||||||
return exitUsage
|
|
||||||
}
|
|
||||||
|
|
||||||
result, err := Emit(EmitOptions{
|
|
||||||
ArtifactKey: positional[0],
|
|
||||||
ArtifactPath: positional[1],
|
|
||||||
As: *as,
|
|
||||||
OutDir: *out,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintf(stderr, "nwsync emit: %v\n", err)
|
|
||||||
return exitInternal
|
|
||||||
}
|
|
||||||
fmt.Fprintf(stdout, "emitted %s: %d resources, %d new blobs, index %s\n",
|
|
||||||
result.Name, result.Entries, result.BlobsWritten, result.ManifestPath)
|
|
||||||
return exitOK
|
|
||||||
}
|
|
||||||
|
|
||||||
func runAssemble(args []string, stdout, stderr io.Writer) int {
|
|
||||||
fs := flag.NewFlagSet("assemble", flag.ContinueOnError)
|
|
||||||
fs.SetOutput(stderr)
|
|
||||||
tlkKey := fs.String("tlk-key", "", "depot key of the TLK, which shadows nothing and merges last")
|
|
||||||
out := fs.String("out", "", "write to a local repository tree instead of uploading")
|
|
||||||
groupID := fs.Int("group-id", 0, "NWSync group id (1 = current, 2 = testing; 0 omits it)")
|
|
||||||
moduleName := fs.String("module-name", "", "module name recorded in the sidecar")
|
|
||||||
description := fs.String("description", "", "description recorded in the sidecar")
|
|
||||||
positional, err := parseArgs(fs, args)
|
|
||||||
if err != nil {
|
|
||||||
return exitUsage
|
|
||||||
}
|
|
||||||
if len(positional) == 0 {
|
|
||||||
fmt.Fprintf(stderr, "nwsync assemble: at least one artifact key is required\n")
|
|
||||||
return exitUsage
|
|
||||||
}
|
|
||||||
|
|
||||||
result, err := Assemble(AssembleOptions{
|
|
||||||
ArtifactKeys: positional,
|
|
||||||
TLKKey: *tlkKey,
|
|
||||||
OutDir: *out,
|
|
||||||
GroupID: *groupID,
|
|
||||||
ModuleName: *moduleName,
|
|
||||||
Description: *description,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintf(stderr, "nwsync assemble: %v\n", err)
|
|
||||||
return exitInternal
|
|
||||||
}
|
|
||||||
fmt.Fprintf(stdout, "assembled manifest %s: %d resources, %s\n",
|
|
||||||
result.SHA1, result.Entries, result.ManifestPath)
|
|
||||||
return exitOK
|
|
||||||
}
|
|
||||||
@@ -1,212 +0,0 @@
|
|||||||
package nwsync
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"crypto/sha256"
|
|
||||||
"encoding/hex"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"os"
|
|
||||||
"path"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/depot"
|
|
||||||
)
|
|
||||||
|
|
||||||
// sink is where an emit or assemble run puts what it produces. The zone is the
|
|
||||||
// production sink; a local directory exists only as the conformance path, so
|
|
||||||
// upstream's output and ours can be diffed on a developer machine.
|
|
||||||
type sink interface {
|
|
||||||
// putBlob stores one NWCompressedBuffer blob under its sha1 name and
|
|
||||||
// returns the bytes stored, or 0 if the blob was already there. Blob names
|
|
||||||
// are content hashes, so an existing name is existing content — which is
|
|
||||||
// why body is a thunk: compression is the expensive part of emit and a
|
|
||||||
// blob that is already stored must not pay for it.
|
|
||||||
putBlob(sha1Hex string, body func() []byte) (int64, error)
|
|
||||||
// putIndex stores a NSYM manifest and its sidecar under key, which is
|
|
||||||
// either an artifact-derived object key or a local path.
|
|
||||||
putIndex(key string, manifest, sidecar []byte) error
|
|
||||||
// getIndex reads back a NSYM manifest and its sidecar.
|
|
||||||
getIndex(key string) (manifest, sidecar []byte, err error)
|
|
||||||
// describe names the sink for messages.
|
|
||||||
describe(key string) string
|
|
||||||
}
|
|
||||||
|
|
||||||
// dirSink writes a local NWSync repository tree.
|
|
||||||
type dirSink struct{ root string }
|
|
||||||
|
|
||||||
func (s dirSink) putBlob(sha1Hex string, body func() []byte) (int64, error) {
|
|
||||||
blob := blobPath(s.root, sha1Hex)
|
|
||||||
if _, err := os.Stat(blob); err == nil {
|
|
||||||
return 0, nil
|
|
||||||
}
|
|
||||||
if err := os.MkdirAll(filepath.Dir(blob), 0o755); err != nil {
|
|
||||||
return 0, fmt.Errorf("create blob directory: %w", err)
|
|
||||||
}
|
|
||||||
data := body()
|
|
||||||
if err := os.WriteFile(blob, data, 0o644); err != nil {
|
|
||||||
return 0, fmt.Errorf("write blob: %w", err)
|
|
||||||
}
|
|
||||||
return int64(len(data)), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s dirSink) putIndex(key string, manifest, sidecar []byte) error {
|
|
||||||
target := filepath.Join(s.root, filepath.FromSlash(key))
|
|
||||||
if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
|
|
||||||
return fmt.Errorf("create manifest directory: %w", err)
|
|
||||||
}
|
|
||||||
if err := os.WriteFile(target, manifest, 0o644); err != nil {
|
|
||||||
return fmt.Errorf("write manifest: %w", err)
|
|
||||||
}
|
|
||||||
if err := os.WriteFile(target+".json", sidecar, 0o644); err != nil {
|
|
||||||
return fmt.Errorf("write sidecar: %w", err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s dirSink) getIndex(key string) ([]byte, []byte, error) {
|
|
||||||
target := filepath.Join(s.root, filepath.FromSlash(key))
|
|
||||||
manifest, err := os.ReadFile(target)
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, fmt.Errorf("read index: %w", err)
|
|
||||||
}
|
|
||||||
sidecar, err := os.ReadFile(target + ".json")
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, fmt.Errorf("read sidecar: %w", err)
|
|
||||||
}
|
|
||||||
return manifest, sidecar, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s dirSink) describe(key string) string {
|
|
||||||
return filepath.Join(s.root, filepath.FromSlash(key))
|
|
||||||
}
|
|
||||||
|
|
||||||
// zoneSink uploads straight to the NWSync storage zone. Nothing bulky is ever
|
|
||||||
// written to the runner's disk: the working set is one resource at a time.
|
|
||||||
type zoneSink struct {
|
|
||||||
store depot.KeyStore
|
|
||||||
ctx context.Context
|
|
||||||
zone string
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s zoneSink) putBlob(sha1Hex string, body func() []byte) (int64, error) {
|
|
||||||
key := path.Join("data", "sha1", sha1Hex[0:2], sha1Hex[2:4], sha1Hex)
|
|
||||||
// A throttled probe must never be read as "missing, re-upload" or as
|
|
||||||
// "present, skip", so only a confirmed Present skips the upload.
|
|
||||||
state, _, err := s.store.ProbeKey(s.ctx, key)
|
|
||||||
if err != nil {
|
|
||||||
return 0, fmt.Errorf("probe blob %s: %w", sha1Hex, err)
|
|
||||||
}
|
|
||||||
if state == depot.Present {
|
|
||||||
return 0, nil
|
|
||||||
}
|
|
||||||
data := body()
|
|
||||||
if err := s.put(key, data); err != nil {
|
|
||||||
return 0, fmt.Errorf("upload blob %s: %w", sha1Hex, err)
|
|
||||||
}
|
|
||||||
return int64(len(data)), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s zoneSink) putIndex(key string, manifest, sidecar []byte) error {
|
|
||||||
// The manifest lands last: its presence is the publication marker, so it
|
|
||||||
// must never appear before the blobs it names.
|
|
||||||
if err := s.put(key+".json", sidecar); err != nil {
|
|
||||||
return fmt.Errorf("upload sidecar %s: %w", key, err)
|
|
||||||
}
|
|
||||||
if err := s.put(key, manifest); err != nil {
|
|
||||||
return fmt.Errorf("upload index %s: %w", key, err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s zoneSink) getIndex(key string) ([]byte, []byte, error) {
|
|
||||||
manifest, err := s.store.GetKey(s.ctx, key)
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, fmt.Errorf("read index: %w", err)
|
|
||||||
}
|
|
||||||
sidecar, err := s.store.GetKey(s.ctx, key+".json")
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, fmt.Errorf("read sidecar: %w", err)
|
|
||||||
}
|
|
||||||
return manifest, sidecar, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s zoneSink) describe(key string) string { return s.zone + "/" + key }
|
|
||||||
|
|
||||||
func (s zoneSink) put(key string, body []byte) error {
|
|
||||||
sum := sha256.Sum256(body)
|
|
||||||
return s.store.PutReader(s.ctx, key, bytes.NewReader(body), int64(len(body)), hex.EncodeToString(sum[:]))
|
|
||||||
}
|
|
||||||
|
|
||||||
// newZoneSink builds the upload sink from the environment. NWSync data lives
|
|
||||||
// in its own zone, separate from the asset depot, so it has its own zone and
|
|
||||||
// credential; only the host is shared, and Crucible has no default host.
|
|
||||||
func newZoneSink(ctx context.Context, getenv func(string) string) (sink, error) {
|
|
||||||
cfg := depot.LoadConfig(getenv)
|
|
||||||
cfg.StorageZone = getenv("NWSYNC_STORAGE_ZONE")
|
|
||||||
cfg.WriteKey = getenv("NWSYNC_STORAGE_PASSWORD")
|
|
||||||
cfg.ReadKey = cfg.WriteKey
|
|
||||||
if cfg.StorageZone == "" {
|
|
||||||
return nil, fmt.Errorf("NWSYNC_STORAGE_ZONE is unset (or pass --out DIR to write locally)")
|
|
||||||
}
|
|
||||||
if cfg.WriteKey == "" {
|
|
||||||
return nil, fmt.Errorf("NWSYNC_STORAGE_PASSWORD is unset (or pass --out DIR to write locally)")
|
|
||||||
}
|
|
||||||
if cfg.StorageHost == "" {
|
|
||||||
return nil, fmt.Errorf("BUNNY_STORAGE_HOST is unset")
|
|
||||||
}
|
|
||||||
store, err := depot.NewKeyStore(cfg)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return zoneSink{store: store, ctx: ctx, zone: cfg.StorageZone}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// indexKey is where an artifact's NSYM lives: beside the artifact itself, with
|
|
||||||
// the final extension replaced. emit and assemble must agree on this one rule,
|
|
||||||
// so it lives here and nowhere else.
|
|
||||||
//
|
|
||||||
// artifacts/haks/sha256/30/46/3046….hak -> artifacts/haks/sha256/30/46/3046….nsym
|
|
||||||
func indexKey(artifactKey string) (string, error) {
|
|
||||||
extension := path.Ext(artifactKey)
|
|
||||||
if extension == "" {
|
|
||||||
return "", fmt.Errorf("artifact key %q has no extension", artifactKey)
|
|
||||||
}
|
|
||||||
return strings.TrimSuffix(artifactKey, extension) + ".nsym", nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// resolveIndexKey is where emit writes an artifact's index and where assemble
|
|
||||||
// reads it from. On the zone that is beside the artifact; locally the indexes
|
|
||||||
// sit flat beside the data tree, so upstream's output and ours diff directly.
|
|
||||||
func resolveIndexKey(artifactKey, outDir string) (string, error) {
|
|
||||||
key, err := indexKey(artifactKey)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
if outDir != "" {
|
|
||||||
return path.Base(key), nil
|
|
||||||
}
|
|
||||||
return key, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkArtifactKey fails closed when the key's embedded digest is not the
|
|
||||||
// digest of the bytes being emitted. Publishing an index under the wrong key
|
|
||||||
// silently pairs a manifest with the wrong artifact.
|
|
||||||
// artifact is hashed by streaming, so a multi-gigabyte hak is never resident.
|
|
||||||
func checkArtifactKey(artifactKey string, artifact io.Reader) error {
|
|
||||||
base := path.Base(artifactKey)
|
|
||||||
digest := strings.TrimSuffix(base, path.Ext(base))
|
|
||||||
if len(digest) != 64 {
|
|
||||||
return fmt.Errorf("artifact key %q does not name a sha256", artifactKey)
|
|
||||||
}
|
|
||||||
hash := sha256.New()
|
|
||||||
if _, err := io.Copy(hash, artifact); err != nil {
|
|
||||||
return fmt.Errorf("hash artifact: %w", err)
|
|
||||||
}
|
|
||||||
if got := hex.EncodeToString(hash.Sum(nil)); got != digest {
|
|
||||||
return fmt.Errorf("artifact key %q names digest %s but the file hashes to %s", artifactKey, digest, got)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -1,249 +0,0 @@
|
|||||||
package nwsync
|
|
||||||
|
|
||||||
import (
|
|
||||||
"crypto/sha1"
|
|
||||||
"crypto/sha256"
|
|
||||||
"encoding/hex"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
// fakeZone is a Bunny-shaped object store: PUT stores, GET reads, and the
|
|
||||||
// Checksum header is verified the way Bunny verifies it.
|
|
||||||
type fakeZone struct {
|
|
||||||
mu sync.Mutex
|
|
||||||
objects map[string][]byte
|
|
||||||
puts []string
|
|
||||||
failOn func(key string) bool // when true, the PUT fails
|
|
||||||
}
|
|
||||||
|
|
||||||
func newFakeZone(t *testing.T) (*fakeZone, func(string) string) {
|
|
||||||
t.Helper()
|
|
||||||
zone := &fakeZone{objects: map[string][]byte{}}
|
|
||||||
server := httptest.NewServer(zone)
|
|
||||||
t.Cleanup(server.Close)
|
|
||||||
getenv := func(name string) string {
|
|
||||||
switch name {
|
|
||||||
case "NWSYNC_STORAGE_ZONE":
|
|
||||||
return "sow-nwsync"
|
|
||||||
case "NWSYNC_STORAGE_PASSWORD":
|
|
||||||
return "write-key"
|
|
||||||
case "BUNNY_STORAGE_HOST":
|
|
||||||
return server.URL
|
|
||||||
}
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
return zone, getenv
|
|
||||||
}
|
|
||||||
|
|
||||||
func (z *fakeZone) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|
||||||
key := strings.TrimPrefix(r.URL.Path, "/sow-nwsync/")
|
|
||||||
switch r.Method {
|
|
||||||
case http.MethodPut:
|
|
||||||
if z.failOn != nil && z.failOn(key) {
|
|
||||||
http.Error(w, "boom", http.StatusInternalServerError)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
body, err := io.ReadAll(r.Body)
|
|
||||||
if err != nil {
|
|
||||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
sum := sha256.Sum256(body)
|
|
||||||
if want := strings.ToUpper(hex.EncodeToString(sum[:])); r.Header.Get("Checksum") != want {
|
|
||||||
http.Error(w, "checksum mismatch", http.StatusBadRequest)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
z.mu.Lock()
|
|
||||||
z.objects[key] = body
|
|
||||||
z.puts = append(z.puts, key)
|
|
||||||
z.mu.Unlock()
|
|
||||||
w.WriteHeader(http.StatusCreated)
|
|
||||||
case http.MethodGet:
|
|
||||||
z.mu.Lock()
|
|
||||||
body, ok := z.objects[key]
|
|
||||||
z.mu.Unlock()
|
|
||||||
if !ok {
|
|
||||||
http.Error(w, "not found", http.StatusNotFound)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
_, _ = w.Write(body)
|
|
||||||
default:
|
|
||||||
http.Error(w, "unsupported", http.StatusMethodNotAllowed)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (z *zoneSinkFixture) emit(t *testing.T, path string) EmitResult {
|
|
||||||
t.Helper()
|
|
||||||
result, err := Emit(EmitOptions{
|
|
||||||
ArtifactKey: artifactKey(t, path),
|
|
||||||
ArtifactPath: path,
|
|
||||||
Sink: z.sink,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("emit %s: %v", path, err)
|
|
||||||
}
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
|
|
||||||
type zoneSinkFixture struct {
|
|
||||||
zone *fakeZone
|
|
||||||
sink sink
|
|
||||||
}
|
|
||||||
|
|
||||||
func newZoneFixture(t *testing.T) *zoneSinkFixture {
|
|
||||||
t.Helper()
|
|
||||||
zone, getenv := newFakeZone(t)
|
|
||||||
target, err := newZoneSink(t.Context(), getenv)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("zone sink: %v", err)
|
|
||||||
}
|
|
||||||
return &zoneSinkFixture{zone: zone, sink: target}
|
|
||||||
}
|
|
||||||
|
|
||||||
func sha1Of(body []byte) [20]byte { return sha1.Sum(body) }
|
|
||||||
|
|
||||||
func TestEmitUploadsBlobsThenIndex(t *testing.T) {
|
|
||||||
fixture := newZoneFixture(t)
|
|
||||||
dir := t.TempDir()
|
|
||||||
hak := filepath.Join(dir, "sow_test_01.hak")
|
|
||||||
body := []byte("texture bytes")
|
|
||||||
writeHak(t, hak, map[string][]byte{"bloodstain1.tga": body, "copy1.txi": body})
|
|
||||||
|
|
||||||
key := artifactKey(t, hak)
|
|
||||||
result := fixture.emit(t, hak)
|
|
||||||
|
|
||||||
index, err := indexKey(key)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, ok := fixture.zone.objects[index]; !ok {
|
|
||||||
t.Fatalf("no index at %s; zone holds %v", index, fixture.zone.puts)
|
|
||||||
}
|
|
||||||
if result.BlobsWritten != 1 {
|
|
||||||
t.Errorf("uploaded %d blobs, want 1 (identical content shares a blob)", result.BlobsWritten)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The index is the publication marker, so it must land after every blob it
|
|
||||||
// names — including its own sidecar.
|
|
||||||
last := fixture.zone.puts[len(fixture.zone.puts)-1]
|
|
||||||
if last != index {
|
|
||||||
t.Errorf("index landed at position %d of %d; it must be last", len(fixture.zone.puts), len(fixture.zone.puts))
|
|
||||||
}
|
|
||||||
for _, key := range fixture.zone.puts[:len(fixture.zone.puts)-1] {
|
|
||||||
if strings.HasPrefix(key, "data/sha1/") || key == index+".json" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
t.Errorf("unexpected object uploaded before the index: %s", key)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestEmitSkipsBlobsAlreadyInTheZone(t *testing.T) {
|
|
||||||
fixture := newZoneFixture(t)
|
|
||||||
dir := t.TempDir()
|
|
||||||
hak := filepath.Join(dir, "sow_test_01.hak")
|
|
||||||
writeHak(t, hak, map[string][]byte{"bloodstain1.tga": []byte("blood")})
|
|
||||||
|
|
||||||
first := fixture.emit(t, hak)
|
|
||||||
if first.BlobsWritten != 1 {
|
|
||||||
t.Fatalf("first emit uploaded %d blobs, want 1", first.BlobsWritten)
|
|
||||||
}
|
|
||||||
second := fixture.emit(t, hak)
|
|
||||||
if second.BlobsWritten != 0 {
|
|
||||||
t.Errorf("re-emit uploaded %d blobs, want 0 (a blob name is its content)", second.BlobsWritten)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestEmitLeavesNoIndexWhenAnUploadFails(t *testing.T) {
|
|
||||||
fixture := newZoneFixture(t)
|
|
||||||
fixture.zone.failOn = func(key string) bool { return strings.HasPrefix(key, "data/sha1/") }
|
|
||||||
dir := t.TempDir()
|
|
||||||
hak := filepath.Join(dir, "sow_test_01.hak")
|
|
||||||
writeHak(t, hak, map[string][]byte{"bloodstain1.tga": []byte("blood")})
|
|
||||||
|
|
||||||
_, err := Emit(EmitOptions{
|
|
||||||
ArtifactKey: artifactKey(t, hak),
|
|
||||||
ArtifactPath: hak,
|
|
||||||
Sink: fixture.sink,
|
|
||||||
})
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("emit reported success after an upload failed")
|
|
||||||
}
|
|
||||||
for key := range fixture.zone.objects {
|
|
||||||
if strings.HasSuffix(key, ".nsym") {
|
|
||||||
t.Errorf("a half-emitted artifact published an index: %s", key)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestEmitRejectsAKeyThatDoesNotMatchTheFile(t *testing.T) {
|
|
||||||
fixture := newZoneFixture(t)
|
|
||||||
dir := t.TempDir()
|
|
||||||
hak := filepath.Join(dir, "sow_test_01.hak")
|
|
||||||
writeHak(t, hak, map[string][]byte{"bloodstain1.tga": []byte("blood")})
|
|
||||||
|
|
||||||
wrong := "artifacts/haks/sha256/00/11/" + strings.Repeat("0", 64) + ".hak"
|
|
||||||
_, err := Emit(EmitOptions{ArtifactKey: wrong, ArtifactPath: hak, Sink: fixture.sink})
|
|
||||||
if err == nil || !strings.Contains(err.Error(), "hashes to") {
|
|
||||||
t.Fatalf("emit published under a key that names another artifact: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAssembleReadsIndexesFromTheZone(t *testing.T) {
|
|
||||||
fixture := newZoneFixture(t)
|
|
||||||
dir := t.TempDir()
|
|
||||||
topBody := []byte("2da from sow_top")
|
|
||||||
assetBody := []byte("2da from the asset hak")
|
|
||||||
top := filepath.Join(dir, "sow_top.hak")
|
|
||||||
core := filepath.Join(dir, "sow_core_01.hak")
|
|
||||||
writeHak(t, top, map[string][]byte{"appearance.2da": topBody})
|
|
||||||
writeHak(t, core, map[string][]byte{"appearance.2da": assetBody, "bloodstain1.tga": []byte("blood")})
|
|
||||||
tlkPath := filepath.Join(dir, "sow_tlk.tlk")
|
|
||||||
if err := os.WriteFile(tlkPath, []byte("TLK V3.0 payload"), 0o644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
fixture.emit(t, top)
|
|
||||||
fixture.emit(t, core)
|
|
||||||
if _, err := Emit(EmitOptions{
|
|
||||||
ArtifactKey: artifactKey(t, tlkPath),
|
|
||||||
ArtifactPath: tlkPath,
|
|
||||||
As: "sow_tlk.tlk",
|
|
||||||
Sink: fixture.sink,
|
|
||||||
}); err != nil {
|
|
||||||
t.Fatalf("emit tlk: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
result, err := Assemble(AssembleOptions{
|
|
||||||
ArtifactKeys: []string{artifactKey(t, top), artifactKey(t, core)},
|
|
||||||
TLKKey: artifactKey(t, tlkPath),
|
|
||||||
GroupID: 2,
|
|
||||||
Sink: fixture.sink,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("assemble: %v", err)
|
|
||||||
}
|
|
||||||
if result.Entries != 3 {
|
|
||||||
t.Fatalf("merged %d entries, want 3 (appearance.2da is shadowed, the TLK adds one)", result.Entries)
|
|
||||||
}
|
|
||||||
|
|
||||||
manifest, ok := fixture.zone.objects["manifests/"+result.SHA1]
|
|
||||||
if !ok {
|
|
||||||
t.Fatalf("no merged manifest in the zone; it holds %v", fixture.zone.puts)
|
|
||||||
}
|
|
||||||
entries, err := readManifest(manifest)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("parse merged manifest: %v", err)
|
|
||||||
}
|
|
||||||
for _, entry := range entries {
|
|
||||||
if entry.ResRef == "appearance" && entry.SHA1 != sha1Of(topBody) {
|
|
||||||
t.Errorf("appearance.2da resolved to the shadowed hak, not the first one given")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -593,14 +593,9 @@ func envBool(name string) bool {
|
|||||||
func collectModuleResources(p *project.Project, moduleHakOrder []string) ([]erf.Resource, error) {
|
func collectModuleResources(p *project.Project, moduleHakOrder []string) ([]erf.Resource, error) {
|
||||||
var moduleResources []erf.Resource
|
var moduleResources []erf.Resource
|
||||||
|
|
||||||
palettes, err := collectPaletteDescriptors(p)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, rel := range p.Inventory.SourceFiles {
|
for _, rel := range p.Inventory.SourceFiles {
|
||||||
abs := filepath.Join(p.SourceDir(), filepath.FromSlash(rel))
|
abs := filepath.Join(p.SourceDir(), filepath.FromSlash(rel))
|
||||||
resource, err := resourceFromJSON(abs, moduleHakOrder, palettes)
|
resource, err := resourceFromJSON(abs, moduleHakOrder)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -1035,7 +1030,7 @@ func compareResourceKeys(a, b erf.Resource) int {
|
|||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
|
|
||||||
func resourceFromJSON(path string, moduleHakOrder []string, palettes paletteProjection) (erf.Resource, error) {
|
func resourceFromJSON(path string, moduleHakOrder []string) (erf.Resource, error) {
|
||||||
name, extension, err := splitSourceName(path)
|
name, extension, err := splitSourceName(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return erf.Resource{}, err
|
return erf.Resource{}, err
|
||||||
@@ -1064,9 +1059,6 @@ func resourceFromJSON(path string, moduleHakOrder []string, palettes paletteProj
|
|||||||
if extension == ".ifo" && name == "module" && len(moduleHakOrder) > 0 {
|
if extension == ".ifo" && name == "module" && len(moduleHakOrder) > 0 {
|
||||||
setModuleHAKList(&document, moduleHakOrder)
|
setModuleHAKList(&document, moduleHakOrder)
|
||||||
}
|
}
|
||||||
if extension == ".itp" && isPaletteProjectionResref(name) {
|
|
||||||
projectPaletteDocument(&document, palettes[strings.ToLower(name)])
|
|
||||||
}
|
|
||||||
|
|
||||||
var buf bytes.Buffer
|
var buf bytes.Buffer
|
||||||
if err := gff.Write(&buf, document); err != nil {
|
if err := gff.Write(&buf, document); err != nil {
|
||||||
|
|||||||
@@ -117,10 +117,6 @@ func expectedResources(p *project.Project) (map[string]resourceExpectation, erro
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
palettes, err := collectPaletteDescriptors(p)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, rel := range p.Inventory.SourceFiles {
|
for _, rel := range p.Inventory.SourceFiles {
|
||||||
abs := filepath.Join(p.SourceDir(), filepath.FromSlash(rel))
|
abs := filepath.Join(p.SourceDir(), filepath.FromSlash(rel))
|
||||||
@@ -140,9 +136,6 @@ func expectedResources(p *project.Project) (map[string]resourceExpectation, erro
|
|||||||
if extension == ".ifo" && strings.EqualFold(name, "module") && len(moduleHakOrder) > 0 {
|
if extension == ".ifo" && strings.EqualFold(name, "module") && len(moduleHakOrder) > 0 {
|
||||||
setModuleHAKList(&document, moduleHakOrder)
|
setModuleHAKList(&document, moduleHakOrder)
|
||||||
}
|
}
|
||||||
if extension == ".itp" && isPaletteProjectionResref(name) {
|
|
||||||
projectPaletteDocument(&document, palettes[strings.ToLower(name)])
|
|
||||||
}
|
|
||||||
|
|
||||||
canonical, err := json.Marshal(document)
|
canonical, err := json.Marshal(document)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -124,13 +124,6 @@ func extractArchiveResources(p *project.Project, archive erf.Archive, desired ma
|
|||||||
skippedCount++
|
skippedCount++
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
// *palcus.itp are Toolset-generated palette projections; the module
|
|
||||||
// build regenerates them from source blueprints, so extraction never
|
|
||||||
// writes them back into source.
|
|
||||||
if ext == "itp" && isPaletteProjectionResref(resource.Name) {
|
|
||||||
skippedCount++
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
target, data, err := extractedFile(p, resource, ext)
|
target, data, err := extractedFile(p, resource, ext)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -1,298 +0,0 @@
|
|||||||
package pipeline
|
|
||||||
|
|
||||||
// Custom palette projections (*palcus.itp) are generated artifacts: the Toolset
|
|
||||||
// derives them from the category frameworks plus the blueprints present in the
|
|
||||||
// module. Crucible reproduces that projection deterministically at build time so
|
|
||||||
// the module source only carries descriptor-free category skeletons and the
|
|
||||||
// blueprint files themselves. Extract never writes palcus files back.
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/gff"
|
|
||||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
|
|
||||||
)
|
|
||||||
|
|
||||||
const noStrref = 0xFFFFFFFF
|
|
||||||
|
|
||||||
// hiddenPaletteID suppresses a blueprint from the Custom palette (engine
|
|
||||||
// convention; see docs in sow-module's palette research notes).
|
|
||||||
const hiddenPaletteID = 255
|
|
||||||
|
|
||||||
var paletteFamilyByExtension = map[string]string{
|
|
||||||
".utc": "creaturepalcus",
|
|
||||||
".utd": "doorpalcus",
|
|
||||||
".ute": "encounterpalcus",
|
|
||||||
".uti": "itempalcus",
|
|
||||||
".utm": "storepalcus",
|
|
||||||
".utp": "placeablepalcus",
|
|
||||||
".uts": "soundpalcus",
|
|
||||||
".utt": "triggerpalcus",
|
|
||||||
".utw": "waypointpalcus",
|
|
||||||
}
|
|
||||||
|
|
||||||
type paletteDescriptor struct {
|
|
||||||
name string
|
|
||||||
strref uint32
|
|
||||||
resref string
|
|
||||||
creature bool
|
|
||||||
cr float32
|
|
||||||
faction string
|
|
||||||
}
|
|
||||||
|
|
||||||
func (d paletteDescriptor) sortKey() string {
|
|
||||||
if d.strref != noStrref || d.name == "" {
|
|
||||||
return strings.ToLower(d.resref)
|
|
||||||
}
|
|
||||||
return strings.ToLower(d.name)
|
|
||||||
}
|
|
||||||
|
|
||||||
// palette resref (e.g. "itempalcus") -> terminal category ID -> descriptors.
|
|
||||||
type paletteProjection map[string]map[uint8][]paletteDescriptor
|
|
||||||
|
|
||||||
func isPaletteProjectionResref(name string) bool {
|
|
||||||
return strings.HasSuffix(strings.ToLower(name), "palcus")
|
|
||||||
}
|
|
||||||
|
|
||||||
func collectPaletteDescriptors(p *project.Project) (paletteProjection, error) {
|
|
||||||
projection := paletteProjection{}
|
|
||||||
factions, err := loadFactionNames(p)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, rel := range p.Inventory.SourceFiles {
|
|
||||||
abs := filepath.Join(p.SourceDir(), filepath.FromSlash(rel))
|
|
||||||
name, extension, err := splitSourceName(abs)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
family, ok := paletteFamilyByExtension[extension]
|
|
||||||
if !ok {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
raw, err := os.ReadFile(abs)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("read %s: %w", abs, err)
|
|
||||||
}
|
|
||||||
var document gff.Document
|
|
||||||
if err := json.Unmarshal(raw, &document); err != nil {
|
|
||||||
return nil, fmt.Errorf("parse gff json %s: %w", abs, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
descriptor, paletteID, ok := blueprintDescriptor(document.Root, name, extension, factions)
|
|
||||||
if !ok {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if projection[family] == nil {
|
|
||||||
projection[family] = map[uint8][]paletteDescriptor{}
|
|
||||||
}
|
|
||||||
projection[family][paletteID] = append(projection[family][paletteID], descriptor)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, byID := range projection {
|
|
||||||
for _, descriptors := range byID {
|
|
||||||
sort.SliceStable(descriptors, func(i, j int) bool {
|
|
||||||
a, b := descriptors[i], descriptors[j]
|
|
||||||
if a.sortKey() != b.sortKey() {
|
|
||||||
return a.sortKey() < b.sortKey()
|
|
||||||
}
|
|
||||||
return a.resref < b.resref
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return projection, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func blueprintDescriptor(root gff.Struct, fileName, extension string, factions []string) (paletteDescriptor, uint8, bool) {
|
|
||||||
descriptor := paletteDescriptor{
|
|
||||||
strref: noStrref,
|
|
||||||
resref: strings.ToLower(fileName),
|
|
||||||
creature: extension == ".utc",
|
|
||||||
}
|
|
||||||
paletteID := -1
|
|
||||||
|
|
||||||
for _, field := range root.Fields {
|
|
||||||
switch field.Label {
|
|
||||||
case "PaletteID":
|
|
||||||
if v, ok := field.Value.(gff.ByteValue); ok {
|
|
||||||
paletteID = int(v)
|
|
||||||
}
|
|
||||||
case "TemplateResRef":
|
|
||||||
if v, ok := field.Value.(gff.ResRefValue); ok && v != "" {
|
|
||||||
descriptor.resref = strings.ToLower(string(v))
|
|
||||||
}
|
|
||||||
case "LocalizedName", "LocName", "FirstName":
|
|
||||||
if v, ok := field.Value.(gff.LocString); ok {
|
|
||||||
name, strref := locStringLabel(v)
|
|
||||||
if field.Label == "FirstName" {
|
|
||||||
descriptor.name = strings.TrimSpace(descriptor.name + " " + name)
|
|
||||||
if descriptor.strref == noStrref {
|
|
||||||
descriptor.strref = strref
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
descriptor.name = name
|
|
||||||
descriptor.strref = strref
|
|
||||||
}
|
|
||||||
}
|
|
||||||
case "LastName":
|
|
||||||
if v, ok := field.Value.(gff.LocString); ok {
|
|
||||||
name, _ := locStringLabel(v)
|
|
||||||
descriptor.name = strings.TrimSpace(descriptor.name + " " + name)
|
|
||||||
}
|
|
||||||
case "ChallengeRating":
|
|
||||||
if v, ok := field.Value.(gff.FloatValue); ok {
|
|
||||||
descriptor.cr = float32(v)
|
|
||||||
}
|
|
||||||
case "FactionID":
|
|
||||||
if v, ok := field.Value.(gff.WordValue); ok && int(v) < len(factions) {
|
|
||||||
descriptor.faction = factions[v]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if paletteID < 0 || paletteID == hiddenPaletteID {
|
|
||||||
return paletteDescriptor{}, 0, false
|
|
||||||
}
|
|
||||||
return descriptor, uint8(paletteID), true
|
|
||||||
}
|
|
||||||
|
|
||||||
func locStringLabel(value gff.LocString) (string, uint32) {
|
|
||||||
if value.StringRef != noStrref {
|
|
||||||
return "", value.StringRef
|
|
||||||
}
|
|
||||||
for _, entry := range value.Entries {
|
|
||||||
if entry.Value != "" {
|
|
||||||
return entry.Value, noStrref
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return "", noStrref
|
|
||||||
}
|
|
||||||
|
|
||||||
func loadFactionNames(p *project.Project) ([]string, error) {
|
|
||||||
for _, rel := range p.Inventory.SourceFiles {
|
|
||||||
if !strings.HasSuffix(strings.ToLower(rel), "repute.fac.json") {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
abs := filepath.Join(p.SourceDir(), filepath.FromSlash(rel))
|
|
||||||
raw, err := os.ReadFile(abs)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("read %s: %w", abs, err)
|
|
||||||
}
|
|
||||||
var document gff.Document
|
|
||||||
if err := json.Unmarshal(raw, &document); err != nil {
|
|
||||||
return nil, fmt.Errorf("parse gff json %s: %w", abs, err)
|
|
||||||
}
|
|
||||||
var names []string
|
|
||||||
for _, field := range document.Root.Fields {
|
|
||||||
if field.Label != "FactionList" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
list, ok := field.Value.(gff.ListValue)
|
|
||||||
if !ok {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
for _, faction := range list {
|
|
||||||
name := ""
|
|
||||||
for _, f := range faction.Fields {
|
|
||||||
if f.Label == "FactionName" {
|
|
||||||
if v, ok := f.Value.(gff.StringValue); ok {
|
|
||||||
name = string(v)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
names = append(names, name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return names, nil
|
|
||||||
}
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// projectPaletteDocument strips every blueprint descriptor from the palette
|
|
||||||
// tree and re-inserts the descriptors derived from module source. Category
|
|
||||||
// structure (branches, terminal IDs, labels) passes through untouched.
|
|
||||||
func projectPaletteDocument(document *gff.Document, byID map[uint8][]paletteDescriptor) {
|
|
||||||
for i, field := range document.Root.Fields {
|
|
||||||
if field.Label != "MAIN" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if list, ok := field.Value.(gff.ListValue); ok {
|
|
||||||
document.Root.Fields[i] = gff.NewField("MAIN", projectPaletteNodes(list, byID))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func projectPaletteNodes(nodes gff.ListValue, byID map[uint8][]paletteDescriptor) gff.ListValue {
|
|
||||||
out := make(gff.ListValue, 0, len(nodes))
|
|
||||||
for _, node := range nodes {
|
|
||||||
if structHasField(node, "RESREF") {
|
|
||||||
continue // blueprint descriptor — regenerated below
|
|
||||||
}
|
|
||||||
|
|
||||||
terminalID := -1
|
|
||||||
fields := make([]gff.Field, 0, len(node.Fields))
|
|
||||||
for _, field := range node.Fields {
|
|
||||||
if field.Label == "LIST" {
|
|
||||||
continue // rebuilt for terminals, recursed for branches
|
|
||||||
}
|
|
||||||
if field.Label == "ID" {
|
|
||||||
if v, ok := field.Value.(gff.ByteValue); ok {
|
|
||||||
terminalID = int(v)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
fields = append(fields, field)
|
|
||||||
}
|
|
||||||
|
|
||||||
switch {
|
|
||||||
case terminalID >= 0:
|
|
||||||
if descriptors := byID[uint8(terminalID)]; len(descriptors) > 0 {
|
|
||||||
fields = append(fields, gff.NewField("LIST", descriptorList(descriptors)))
|
|
||||||
}
|
|
||||||
default:
|
|
||||||
for _, field := range node.Fields {
|
|
||||||
if field.Label == "LIST" {
|
|
||||||
if list, ok := field.Value.(gff.ListValue); ok {
|
|
||||||
fields = append(fields, gff.NewField("LIST", projectPaletteNodes(list, byID)))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
out = append(out, gff.Struct{Type: node.Type, Fields: fields})
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
func descriptorList(descriptors []paletteDescriptor) gff.ListValue {
|
|
||||||
list := make(gff.ListValue, 0, len(descriptors))
|
|
||||||
for _, d := range descriptors {
|
|
||||||
fields := make([]gff.Field, 0, 4)
|
|
||||||
if d.strref != noStrref {
|
|
||||||
fields = append(fields, gff.NewField("STRREF", gff.DWordValue(d.strref)))
|
|
||||||
} else {
|
|
||||||
fields = append(fields, gff.NewField("NAME", gff.StringValue(d.name)))
|
|
||||||
}
|
|
||||||
fields = append(fields, gff.NewField("RESREF", gff.ResRefValue(d.resref)))
|
|
||||||
if d.creature {
|
|
||||||
fields = append(fields, gff.NewField("CR", gff.FloatValue(d.cr)))
|
|
||||||
fields = append(fields, gff.NewField("FACTION", gff.StringValue(d.faction)))
|
|
||||||
}
|
|
||||||
list = append(list, gff.Struct{Fields: fields})
|
|
||||||
}
|
|
||||||
return list
|
|
||||||
}
|
|
||||||
|
|
||||||
func structHasField(s gff.Struct, label string) bool {
|
|
||||||
for _, field := range s.Fields {
|
|
||||||
if field.Label == label {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
@@ -1,191 +0,0 @@
|
|||||||
package pipeline
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"encoding/json"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf"
|
|
||||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/gff"
|
|
||||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
|
|
||||||
)
|
|
||||||
|
|
||||||
const paletteTestSkeleton = `{
|
|
||||||
"file_type": "ITP ",
|
|
||||||
"file_version": "V3.2",
|
|
||||||
"root": {
|
|
||||||
"struct_type": 4294967295,
|
|
||||||
"fields": [
|
|
||||||
{
|
|
||||||
"label": "MAIN",
|
|
||||||
"type": "List",
|
|
||||||
"value": [
|
|
||||||
{
|
|
||||||
"struct_type": 0,
|
|
||||||
"fields": [
|
|
||||||
{"label": "STRREF", "type": "DWord", "value": 500},
|
|
||||||
{
|
|
||||||
"label": "LIST",
|
|
||||||
"type": "List",
|
|
||||||
"value": [
|
|
||||||
{
|
|
||||||
"struct_type": 0,
|
|
||||||
"fields": [
|
|
||||||
{"label": "STRREF", "type": "DWord", "value": 6699},
|
|
||||||
{"label": "ID", "type": "Byte", "value": 23},
|
|
||||||
{
|
|
||||||
"label": "LIST",
|
|
||||||
"type": "List",
|
|
||||||
"value": [
|
|
||||||
{
|
|
||||||
"struct_type": 0,
|
|
||||||
"fields": [
|
|
||||||
{"label": "NAME", "type": "CExoString", "value": "Stale Junk"},
|
|
||||||
{"label": "RESREF", "type": "ResRef", "value": "stalejunk"}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"struct_type": 0,
|
|
||||||
"fields": [
|
|
||||||
{"label": "STRREF", "type": "DWord", "value": 6753},
|
|
||||||
{"label": "ID", "type": "Byte", "value": 24}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
`
|
|
||||||
|
|
||||||
func paletteTestItem(resref, name string, paletteID int) string {
|
|
||||||
return `{
|
|
||||||
"file_type": "UTI ",
|
|
||||||
"file_version": "V3.2",
|
|
||||||
"root": {
|
|
||||||
"struct_type": 4294967295,
|
|
||||||
"fields": [
|
|
||||||
{"label": "TemplateResRef", "type": "ResRef", "value": "` + resref + `"},
|
|
||||||
{
|
|
||||||
"label": "LocalizedName",
|
|
||||||
"type": "CExoLocString",
|
|
||||||
"value": {"string_ref": 4294967295, "entries": [{"id": 0, "value": "` + name + `"}]}
|
|
||||||
},
|
|
||||||
{"label": "PaletteID", "type": "Byte", "value": ` + itoa(paletteID) + `}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
`
|
|
||||||
}
|
|
||||||
|
|
||||||
func itoa(v int) string {
|
|
||||||
data, _ := json.Marshal(v)
|
|
||||||
return string(data)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBuildProjectsPaletteDescriptorsAndExtractSkipsThem(t *testing.T) {
|
|
||||||
root := t.TempDir()
|
|
||||||
mustMkdir(t, filepath.Join(root, "src", "module"))
|
|
||||||
mustMkdir(t, filepath.Join(root, "src", "palettes"))
|
|
||||||
mustMkdir(t, filepath.Join(root, "src", "blueprints", "items"))
|
|
||||||
mustMkdir(t, filepath.Join(root, "build"))
|
|
||||||
|
|
||||||
mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{
|
|
||||||
"module": {"name": "Test Module", "resref": "testmod"},
|
|
||||||
"paths": {"source": "src", "build": "build"}
|
|
||||||
}
|
|
||||||
`)
|
|
||||||
mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{
|
|
||||||
"file_type": "IFO ",
|
|
||||||
"file_version": "V3.2",
|
|
||||||
"root": {"struct_type": 4294967295, "fields": [{"label": "Mod_Name", "type": "CExoString", "value": "Test Module"}]}
|
|
||||||
}
|
|
||||||
`)
|
|
||||||
mustWriteFile(t, filepath.Join(root, "src", "palettes", "itempalcus.itp.json"), paletteTestSkeleton)
|
|
||||||
mustWriteFile(t, filepath.Join(root, "src", "blueprints", "items", "i_b.uti.json"), paletteTestItem("i_b", "Bravo Item", 23))
|
|
||||||
mustWriteFile(t, filepath.Join(root, "src", "blueprints", "items", "i_a.uti.json"), paletteTestItem("i_a", "Alpha Item", 23))
|
|
||||||
mustWriteFile(t, filepath.Join(root, "src", "blueprints", "items", "i_hidden.uti.json"), paletteTestItem("i_hidden", "Hidden Item", 255))
|
|
||||||
|
|
||||||
p, err := project.Load(root)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("load project: %v", err)
|
|
||||||
}
|
|
||||||
if err := p.Scan(); err != nil {
|
|
||||||
t.Fatalf("scan: %v", err)
|
|
||||||
}
|
|
||||||
if _, err := BuildModule(p); err != nil {
|
|
||||||
t.Fatalf("build: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
archiveFile, err := os.Open(p.ModuleArchivePath())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("open module archive: %v", err)
|
|
||||||
}
|
|
||||||
defer archiveFile.Close()
|
|
||||||
archive, err := erf.Read(archiveFile)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("read module archive: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var palette *gff.Document
|
|
||||||
for _, resource := range archive.Resources {
|
|
||||||
if resource.Name == "itempalcus" {
|
|
||||||
document, err := gff.Read(bytes.NewReader(resource.Data))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("decode itempalcus: %v", err)
|
|
||||||
}
|
|
||||||
palette = &document
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if palette == nil {
|
|
||||||
t.Fatal("itempalcus missing from built module")
|
|
||||||
}
|
|
||||||
|
|
||||||
canonical, err := json.Marshal(palette)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("marshal palette: %v", err)
|
|
||||||
}
|
|
||||||
text := string(canonical)
|
|
||||||
if bytes.Contains(canonical, []byte("stalejunk")) {
|
|
||||||
t.Fatalf("stale descriptor survived projection: %s", text)
|
|
||||||
}
|
|
||||||
for _, resref := range []string{"i_a", "i_b"} {
|
|
||||||
if !bytes.Contains(canonical, []byte(resref)) {
|
|
||||||
t.Fatalf("descriptor %s missing from projection: %s", resref, text)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if bytes.Contains(canonical, []byte("i_hidden")) {
|
|
||||||
t.Fatalf("PaletteID 255 blueprint leaked into projection: %s", text)
|
|
||||||
}
|
|
||||||
if a, b := bytes.Index(canonical, []byte("i_a")), bytes.Index(canonical, []byte("i_b")); a > b {
|
|
||||||
t.Fatalf("descriptors not name-sorted: %s", text)
|
|
||||||
}
|
|
||||||
|
|
||||||
if _, err := Compare(p); err != nil {
|
|
||||||
t.Fatalf("compare after build: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Extraction must never write palcus files back into source.
|
|
||||||
if err := os.Remove(filepath.Join(root, "src", "palettes", "itempalcus.itp.json")); err != nil {
|
|
||||||
t.Fatalf("remove skeleton: %v", err)
|
|
||||||
}
|
|
||||||
if err := p.Scan(); err != nil {
|
|
||||||
t.Fatalf("rescan: %v", err)
|
|
||||||
}
|
|
||||||
if _, err := Extract(p); err != nil {
|
|
||||||
t.Fatalf("extract: %v", err)
|
|
||||||
}
|
|
||||||
if _, err := os.Stat(filepath.Join(root, "src", "palettes", "itempalcus.itp.json")); !os.IsNotExist(err) {
|
|
||||||
t.Fatalf("extract wrote palcus file back (stat err: %v)", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -31,7 +31,7 @@ var SourceExtensions = []string{
|
|||||||
}
|
}
|
||||||
|
|
||||||
var AssetExtensions = []string{
|
var AssetExtensions = []string{
|
||||||
".2da", ".bik", ".bmp", ".bmu", ".dds", ".dwk", ".itp", ".jpg", ".lod", ".lyt", ".mdl", ".mdx", ".mtr", ".plt", ".png", ".pwk", ".set", ".shd", ".tga", ".txi", ".uti", ".vis", ".wav", ".wok",
|
".2da", ".bik", ".bmp", ".bmu", ".dds", ".dwk", ".gr2", ".itp", ".jpg", ".lod", ".lyt", ".mdb", ".mdl", ".mdx", ".mtr", ".plt", ".png", ".pwk", ".set", ".shd", ".tga", ".txi", ".uti", ".vis", ".wav", ".wlk", ".wok", ".xml",
|
||||||
}
|
}
|
||||||
|
|
||||||
var BuiltinScriptPrefixes = []string{
|
var BuiltinScriptPrefixes = []string{
|
||||||
|
|||||||
@@ -109,7 +109,6 @@ type tlkCompiler struct {
|
|||||||
active map[string]tlkEntryData
|
active map[string]tlkEntryData
|
||||||
activeKeys map[string]struct{}
|
activeKeys map[string]struct{}
|
||||||
reservedByID map[int]string
|
reservedByID map[int]string
|
||||||
pinnedByID map[int]string
|
|
||||||
nextID int
|
nextID int
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -159,7 +158,6 @@ func newTLKCompiler(sourceDir string, legacy *legacyTLKData) (*tlkCompiler, erro
|
|||||||
active: map[string]tlkEntryData{},
|
active: map[string]tlkEntryData{},
|
||||||
activeKeys: map[string]struct{}{},
|
activeKeys: map[string]struct{}{},
|
||||||
reservedByID: reserved,
|
reservedByID: reserved,
|
||||||
pinnedByID: map[int]string{},
|
|
||||||
nextID: nextID,
|
nextID: nextID,
|
||||||
}
|
}
|
||||||
if legacy != nil {
|
if legacy != nil {
|
||||||
@@ -382,24 +380,12 @@ func (c *tlkCompiler) registerInlineAtID(key string, id int, entry tlkEntryData)
|
|||||||
if id < 0 {
|
if id < 0 {
|
||||||
return fmt.Errorf("TLK key %q has negative id %d", key, id)
|
return fmt.Errorf("TLK key %q has negative id %d", key, id)
|
||||||
}
|
}
|
||||||
// The pin is authoritative over the per-machine .tlk_state.json cache: a
|
|
||||||
// stale mapping that dynamically grabbed this id on an older build must
|
|
||||||
// yield so the pinned key can take it. Only a genuine clash between two
|
|
||||||
// pins in custom.tlk.yml is an author error.
|
|
||||||
if owner, ok := c.pinnedByID[id]; ok && owner != key {
|
|
||||||
return fmt.Errorf("TLK id %d is pinned by both %q and %q", id, owner, key)
|
|
||||||
}
|
|
||||||
if mapping, ok := c.state.Entries[key]; ok && mapping.ID != id {
|
if mapping, ok := c.state.Entries[key]; ok && mapping.ID != id {
|
||||||
// This key held a different cached id; release it so the pin wins.
|
return fmt.Errorf("TLK key %q changed id from %d to %d", key, mapping.ID, id)
|
||||||
if c.reservedByID[mapping.ID] == key {
|
|
||||||
delete(c.reservedByID, mapping.ID)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if owner, ok := c.reservedByID[id]; ok && owner != key {
|
if owner, ok := c.reservedByID[id]; ok && owner != key {
|
||||||
// Evict the stale owner; it gets a fresh id when next made active.
|
return fmt.Errorf("TLK id %d is already reserved by %q", id, owner)
|
||||||
delete(c.state.Entries, owner)
|
|
||||||
}
|
}
|
||||||
c.pinnedByID[id] = key
|
|
||||||
c.state.Entries[key] = tlkStateMapping{ID: id}
|
c.state.Entries[key] = tlkStateMapping{ID: id}
|
||||||
c.reservedByID[id] = key
|
c.reservedByID[id] = key
|
||||||
return c.markActive(key, entry)
|
return c.markActive(key, entry)
|
||||||
|
|||||||
@@ -2449,55 +2449,6 @@ strings:
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestBuildStandaloneTLKPinEvictsStaleStateOwner(t *testing.T) {
|
|
||||||
root := testProjectRoot(t)
|
|
||||||
mkdirAll(t, filepath.Join(root, "topdata", "data", "feat"))
|
|
||||||
mkdirAll(t, filepath.Join(root, "topdata", "tlk"))
|
|
||||||
writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n")
|
|
||||||
writeFile(t, filepath.Join(root, "topdata", "data", "feat", "base.json"), `{
|
|
||||||
"output": "feat.2da",
|
|
||||||
"columns": ["LABEL", "FEAT", "DESCRIPTION"],
|
|
||||||
"rows": [{
|
|
||||||
"id": 0,
|
|
||||||
"key": "feat:test",
|
|
||||||
"LABEL": "TEST_LABEL",
|
|
||||||
"FEAT": {"tlk": {"key": "feat:test.name", "text": "Test Feat"}},
|
|
||||||
"DESCRIPTION": "****"
|
|
||||||
}]
|
|
||||||
}`+"\n")
|
|
||||||
writeFile(t, filepath.Join(root, "topdata", "tlk", "custom.tlk.yml"), `schema: sow-topdata/tlk/v1
|
|
||||||
base_strref: 16777216
|
|
||||||
strings:
|
|
||||||
- key: sow.module.name
|
|
||||||
text: Shadows Over Westgate
|
|
||||||
id: 50
|
|
||||||
`)
|
|
||||||
// Simulate a per-machine state that predates the pinned taxonomy: the feat
|
|
||||||
// ref dynamically grabbed id 50 on an older build, exactly where the pin
|
|
||||||
// now lives. The build must self-heal instead of failing.
|
|
||||||
writeFile(t, filepath.Join(root, "topdata", tlkStateFile),
|
|
||||||
`{"version":1,"language":"en","entries":{"feat:test.name":{"id":50}}}`+"\n")
|
|
||||||
|
|
||||||
if _, err := BuildNative(testProject(root), nil); err != nil {
|
|
||||||
t.Fatalf("BuildNative failed on stale pin collision: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
stateRaw, err := os.ReadFile(filepath.Join(root, "topdata", tlkStateFile))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("read tlk state: %v", err)
|
|
||||||
}
|
|
||||||
var state tlkStateDocument
|
|
||||||
if err := json.Unmarshal(stateRaw, &state); err != nil {
|
|
||||||
t.Fatalf("parse tlk state: %v", err)
|
|
||||||
}
|
|
||||||
if state.Entries["sow.module.name"].ID != 50 {
|
|
||||||
t.Fatalf("pin must own id 50, got %#v", state.Entries["sow.module.name"])
|
|
||||||
}
|
|
||||||
if got := state.Entries["feat:test.name"].ID; got == 50 {
|
|
||||||
t.Fatalf("stale feat ref should have been reallocated off id 50, got %d", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBuildPreservesTLKStateAcrossTextChanges(t *testing.T) {
|
func TestBuildPreservesTLKStateAcrossTextChanges(t *testing.T) {
|
||||||
root := testProjectRoot(t)
|
root := testProjectRoot(t)
|
||||||
mkdirAll(t, filepath.Join(root, "topdata", "data", "skills"))
|
mkdirAll(t, filepath.Join(root, "topdata", "data", "skills"))
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ bin=bin
|
|||||||
|
|
||||||
# Keep in sync with internal/dispatch.Registry (Wired flag).
|
# Keep in sync with internal/dispatch.Registry (Wired flag).
|
||||||
unwired=()
|
unwired=()
|
||||||
wired=(assets depot hak module nwsync topdata wiki)
|
wired=(assets depot hak module topdata wiki)
|
||||||
|
|
||||||
exit_of() { set +e; "$@" >/dev/null 2>&1; local c=$?; set -e; echo "${c}"; }
|
exit_of() { set +e; "$@" >/dev/null 2>&1; local c=$?; set -e; echo "${c}"; }
|
||||||
|
|
||||||
|
|||||||
@@ -1,67 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
# Delete the binary assets of every release except the newest KEEP ones.
|
|
||||||
#
|
|
||||||
# Gitea has no asset retention, so 9 assets x ~57 MB per tag pile up forever on
|
|
||||||
# the host disk (sow-tools #52). Nothing consumes an old asset: CI takes the
|
|
||||||
# Crucible binary from the Nix flake, and every deleted file is reproducible
|
|
||||||
# from its tag. Tags, source archives, release notes and the releases
|
|
||||||
# themselves are never touched — only attachments.
|
|
||||||
#
|
|
||||||
# Nothing is excluded, SHA256SUMS and the wrappers included: the checksums are
|
|
||||||
# only meaningful next to the binaries they cover, and the drift-check reads
|
|
||||||
# the wrappers from the latest release, which always stays complete.
|
|
||||||
#
|
|
||||||
# Best-effort by design: a stale asset is cheaper than a blocked release, so
|
|
||||||
# every API failure is a warning, never a non-zero exit.
|
|
||||||
#
|
|
||||||
# Env:
|
|
||||||
# API repo API base, e.g. https://git.example/api/v1/repos/owner/repo [required]
|
|
||||||
# TOKEN Gitea token with releases:write [required]
|
|
||||||
# KEEP how many newest releases stay complete (default 2)
|
|
||||||
set -uo pipefail
|
|
||||||
|
|
||||||
: "${API:?set API}"
|
|
||||||
: "${TOKEN:?set TOKEN}"
|
|
||||||
keep="${KEEP:-2}"
|
|
||||||
auth="Authorization: token ${TOKEN}"
|
|
||||||
per_page=50
|
|
||||||
|
|
||||||
warn() { echo "prune-release-assets: $*" >&2; }
|
|
||||||
|
|
||||||
# Walk every page so an old backlog is cleared, not only the tag that fell out
|
|
||||||
# of the window on this run. Each page already embeds its releases' assets, so
|
|
||||||
# no follow-up request per release is needed.
|
|
||||||
releases='[]'
|
|
||||||
page=1
|
|
||||||
while :; do
|
|
||||||
body="$(curl -fsS -H "$auth" "${API}/releases?limit=${per_page}&page=${page}&draft=false")" || {
|
|
||||||
warn "listing releases failed on page ${page}; pruning what was listed so far"
|
|
||||||
break
|
|
||||||
}
|
|
||||||
count="$(jq 'length' <<<"$body")" || { warn "unreadable release page ${page}"; break; }
|
|
||||||
releases="$(jq -s 'add' <(printf '%s' "$releases") <(printf '%s' "$body"))"
|
|
||||||
# A short page is the last one; this also stops a server that ignores `page`.
|
|
||||||
(( count < per_page )) && break
|
|
||||||
page=$(( page + 1 ))
|
|
||||||
done
|
|
||||||
|
|
||||||
# Sort here rather than trusting the response order, and skip drafts in case
|
|
||||||
# the server ignored `draft=false` — a draft must not consume a keep slot and
|
|
||||||
# strip the binaries off the newest real release.
|
|
||||||
mapfile -t stale < <(jq -r --argjson keep "$keep" '
|
|
||||||
[.[] | select(.draft != true)]
|
|
||||||
| sort_by(.created_at) | reverse | .[$keep:]
|
|
||||||
| .[] | "\(.id) \(.assets // [] | map(.id) | join(","))"
|
|
||||||
' <<<"$releases")
|
|
||||||
|
|
||||||
(( ${#stale[@]} )) || { echo "prune-release-assets: nothing older than the newest ${keep} releases"; exit 0; }
|
|
||||||
|
|
||||||
for line in "${stale[@]}"; do
|
|
||||||
id="${line%% *}"
|
|
||||||
assets="${line#* }"
|
|
||||||
for asset in ${assets//,/ }; do
|
|
||||||
echo "deleting asset ${asset} of release ${id}"
|
|
||||||
curl -fsS -X DELETE -H "$auth" "${API}/releases/${id}/assets/${asset}" >/dev/null \
|
|
||||||
|| warn "deleting asset ${asset} of release ${id} failed"
|
|
||||||
done
|
|
||||||
done
|
|
||||||
@@ -1,78 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
# Contract for scripts/prune-release-assets.sh, driven by a stub curl on PATH:
|
|
||||||
# only assets of releases outside the keep window go, the window is decided by
|
|
||||||
# release date rather than response order, drafts never consume a keep slot,
|
|
||||||
# releases and tags stay, and an API failure never fails the run.
|
|
||||||
set -euo pipefail
|
|
||||||
|
|
||||||
repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
||||||
script="$repo_root/scripts/prune-release-assets.sh"
|
|
||||||
tmp="$(mktemp -d)"
|
|
||||||
trap 'rm -rf "$tmp"' EXIT
|
|
||||||
|
|
||||||
# Stub curl: one short page of releases, deliberately out of date order and
|
|
||||||
# with a draft that is newer than every published release. Each release embeds
|
|
||||||
# two assets. Every request is appended to $CALLS for the assertions below.
|
|
||||||
mkdir -p "$tmp/bin"
|
|
||||||
cat >"$tmp/bin/curl" <<'STUB'
|
|
||||||
#!/usr/bin/env bash
|
|
||||||
url="${!#}"
|
|
||||||
printf '%s\n' "$*" >>"$CALLS"
|
|
||||||
rel() { printf '{"id":%s,"created_at":"%s","draft":%s,"assets":[{"id":%s01},{"id":%s02}]}' "$1" "$2" "$3" "$1" "$1"; }
|
|
||||||
case "$url" in
|
|
||||||
*releases\?*)
|
|
||||||
[[ "${FAIL_LIST:-0}" == 1 ]] && exit 22
|
|
||||||
printf '[%s,%s,%s,%s,%s]\n' \
|
|
||||||
"$(rel 12 2024-01-02T00:00:00Z false)" \
|
|
||||||
"$(rel 10 2024-01-04T00:00:00Z false)" \
|
|
||||||
"$(rel 9 2024-06-01T00:00:00Z true)" \
|
|
||||||
"$(rel 13 2024-01-01T00:00:00Z false)" \
|
|
||||||
"$(rel 11 2024-01-03T00:00:00Z false)"
|
|
||||||
;;
|
|
||||||
*/assets/*) [[ "${FAIL_DELETE:-0}" == 1 ]] && exit 22 ;;
|
|
||||||
esac
|
|
||||||
exit 0
|
|
||||||
STUB
|
|
||||||
chmod +x "$tmp/bin/curl"
|
|
||||||
export PATH="$tmp/bin:$PATH" API=https://git.example/api/v1/repos/o/r TOKEN=t
|
|
||||||
|
|
||||||
run() { CALLS="$tmp/calls" bash "$script" >"$tmp/out" 2>"$tmp/err"; }
|
|
||||||
|
|
||||||
# Default keep=2: the two newest published releases (10, 11) stay whole, the
|
|
||||||
# older two (12, 13) lose their assets, and the newer draft is ignored.
|
|
||||||
: >"$tmp/calls"; run
|
|
||||||
deletes="$(grep -c -- '-X DELETE' "$tmp/calls" || true)"
|
|
||||||
[[ "$deletes" == 4 ]] || { echo "expected 4 asset deletes, got $deletes" >&2; exit 1; }
|
|
||||||
grep -q 'assets/1201' "$tmp/calls" && grep -q 'assets/1302' "$tmp/calls" || {
|
|
||||||
echo "expected assets of releases 12 and 13 to be deleted" >&2; exit 1; }
|
|
||||||
if grep -q 'releases/10/assets/\|releases/11/assets/' "$tmp/calls"; then
|
|
||||||
echo "kept releases lost assets" >&2; exit 1
|
|
||||||
fi
|
|
||||||
if grep -q 'releases/9/assets/' "$tmp/calls"; then
|
|
||||||
echo "a draft release was pruned" >&2; exit 1
|
|
||||||
fi
|
|
||||||
# A release or tag must never be deleted, only attachments under /assets/.
|
|
||||||
if grep -- '-X DELETE' "$tmp/calls" | grep -qv '/assets/'; then
|
|
||||||
echo "a non-asset DELETE was issued" >&2; exit 1
|
|
||||||
fi
|
|
||||||
# A short page ends the sweep: no second page, so a server ignoring `page`
|
|
||||||
# cannot spin the job until the job timeout.
|
|
||||||
[[ "$(grep -c 'releases?' "$tmp/calls")" == 1 ]] || {
|
|
||||||
echo "a short release page did not end the sweep" >&2; exit 1; }
|
|
||||||
|
|
||||||
# KEEP=4 covers every published release: nothing to delete.
|
|
||||||
: >"$tmp/calls"; KEEP=4 run
|
|
||||||
if grep -q -- '-X DELETE' "$tmp/calls"; then echo "KEEP=4 still deleted" >&2; exit 1; fi
|
|
||||||
|
|
||||||
# Failures are warnings, never a non-zero exit.
|
|
||||||
: >"$tmp/calls"; FAIL_DELETE=1 run || { echo "delete failure failed the run" >&2; exit 1; }
|
|
||||||
grep -q 'failed' "$tmp/err" || { echo "delete failure was not warned about" >&2; exit 1; }
|
|
||||||
|
|
||||||
: >"$tmp/calls"; FAIL_LIST=1 run || { echo "listing failure failed the run" >&2; exit 1; }
|
|
||||||
grep -q 'listing releases failed' "$tmp/err" || {
|
|
||||||
echo "listing failure was not warned about" >&2; exit 1; }
|
|
||||||
if grep -q -- '-X DELETE' "$tmp/calls"; then
|
|
||||||
echo "deleted assets despite an unreadable release list" >&2; exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "prune-release-assets: prunes by release date outside the keep window, ignores drafts, never deletes a release, never fails the build"
|
|
||||||
@@ -1,86 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
set -euo pipefail
|
|
||||||
|
|
||||||
contract_script="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/$(basename "${BASH_SOURCE[0]}")"
|
|
||||||
default_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
||||||
repo_root="${WORKFLOW_ROOT:-$default_root}"
|
|
||||||
cd "$repo_root"
|
|
||||||
|
|
||||||
ci=.gitea/workflows/ci.yml
|
|
||||||
release=.gitea/workflows/build-binaries.yml
|
|
||||||
sync=.gitea/workflows/sync-wrappers.yml
|
|
||||||
checkout='actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683'
|
|
||||||
|
|
||||||
assert_exact_events() {
|
|
||||||
local workflow="$1" expected="$2" actual
|
|
||||||
actual="$(awk '
|
|
||||||
/^on:$/ { in_on = 1 }
|
|
||||||
in_on && /^$/ { next }
|
|
||||||
in_on && $0 != "on:" && $0 !~ /^[[:space:]]/ { exit }
|
|
||||||
in_on { print }
|
|
||||||
' "$workflow")"
|
|
||||||
[[ "$actual" == "$expected" ]] || {
|
|
||||||
echo "workflow-contract: unexpected event scope in $workflow" >&2
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
[[ -f "$ci" ]] || { echo "missing consolidated CI workflow" >&2; exit 1; }
|
|
||||||
[[ ! -e .gitea/workflows/test.yml ]] || { echo "legacy test workflow still present" >&2; exit 1; }
|
|
||||||
|
|
||||||
assert_exact_events "$ci" $'on:\n pull_request:'
|
|
||||||
grep -Fqx 'permissions: read-all' "$ci"
|
|
||||||
grep -Fqx ' timeout-minutes: 60' "$ci"
|
|
||||||
[[ "$(grep -Fc "$checkout" "$ci")" -eq 1 ]]
|
|
||||||
grep -Fqx ' fetch-depth: 0' "$ci"
|
|
||||||
grep -Fq 'go vet ./...' "$ci"
|
|
||||||
grep -Fq 'go test ./...' "$ci"
|
|
||||||
grep -Fq 'shellcheck scripts/*.sh' "$ci"
|
|
||||||
grep -Fq 'yamllint .gitea' "$ci"
|
|
||||||
grep -Fq 'make smoke' "$ci"
|
|
||||||
grep -Fq 'for target in linux/amd64 linux/arm64 darwin/amd64 darwin/arm64 windows/amd64 windows/arm64; do' "$ci"
|
|
||||||
|
|
||||||
assert_exact_events "$release" $'on:\n push:\n tags: [\'v*\']'
|
|
||||||
grep -Fqx ' code: read' "$release"
|
|
||||||
grep -Fqx ' releases: write' "$release"
|
|
||||||
grep -Fqx ' timeout-minutes: 30' "$release"
|
|
||||||
[[ "$(grep -Fc "$checkout" "$release")" -eq 1 ]]
|
|
||||||
grep -Fq 'secrets.GITEA_TOKEN' "$release"
|
|
||||||
grep -Fq 'scripts/prune-release-assets.sh' "$release"
|
|
||||||
grep -Fq 'continue-on-error: true' "$release"
|
|
||||||
if grep -Fq 'secrets.GITHUB_TOKEN' "$release"; then
|
|
||||||
echo "workflow-contract: release uses the GitHub token alias" >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
grep -Fqx 'permissions: read-all' "$sync"
|
|
||||||
grep -Fqx ' timeout-minutes: 30' "$sync"
|
|
||||||
[[ "$(grep -Fc "$checkout" "$sync")" -eq 1 ]]
|
|
||||||
assert_exact_events "$sync" $'on:\n push:\n branches: [main]\n paths:\n - \'wrappers/crucible.sh\'\n - \'wrappers/crucible.ps1\''
|
|
||||||
|
|
||||||
expect_mutation_rejected() {
|
|
||||||
local name="$1" workflow="$2" expression="$3" tmp
|
|
||||||
tmp="$(mktemp -d)"
|
|
||||||
mkdir -p "$tmp/.gitea"
|
|
||||||
cp -R "$default_root/.gitea/workflows" "$tmp/.gitea/workflows"
|
|
||||||
sed -i "$expression" "$tmp/$workflow"
|
|
||||||
if WORKFLOW_ROOT="$tmp" WORKFLOW_CONTRACT_MUTATION=1 bash "$contract_script" >/dev/null 2>&1; then
|
|
||||||
echo "workflow-contract: accepted forbidden mutation: $name" >&2
|
|
||||||
rm -rf "$tmp"
|
|
||||||
return 1
|
|
||||||
fi
|
|
||||||
rm -rf "$tmp"
|
|
||||||
}
|
|
||||||
|
|
||||||
if [[ "${WORKFLOW_CONTRACT_MUTATION:-0}" != 1 ]]; then
|
|
||||||
mutations_ok=0
|
|
||||||
expect_mutation_rejected 'CI workflow_dispatch event' "$ci" '/^ pull_request:$/a\ workflow_dispatch:' || mutations_ok=1
|
|
||||||
expect_mutation_rejected 'release develop branch' "$release" "/^ tags:/a\\ branches: [develop]" || mutations_ok=1
|
|
||||||
expect_mutation_rejected 'release schedule event' "$release" "/^ tags:/a\\ schedule:\n - cron: '0 0 * * *'" || mutations_ok=1
|
|
||||||
expect_mutation_rejected 'release extra tag pattern' "$release" "s/tags: \['v\*'\]/tags: ['v*', 'release-*']/" || mutations_ok=1
|
|
||||||
expect_mutation_rejected 'wrapper sync pull_request event' "$sync" '/^ push:$/i\ pull_request:' || mutations_ok=1
|
|
||||||
expect_mutation_rejected 'wrapper sync broad path' "$sync" "/^ - 'wrappers\/crucible.ps1'$/a\\ - 'wrappers/**'" || mutations_ok=1
|
|
||||||
(( mutations_ok == 0 )) || exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "workflow-contract: CI is PR-only; release and wrapper sync retain their narrow triggers"
|
|
||||||
Reference in New Issue
Block a user