depot: report sweep progress to stderr instead of discarding it (#35)

Sweep prints `probed N/M` every 1000 blobs, but every caller passed `io.Discard`, so a full ~71k-blob verify ran ~56 minutes with zero output and looked hung in CI (sow-assets-manifest run 245).

`go test ./internal/depot` passes; verified live against the cdn (progress lines now appear on stderr).

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-authored-by: vickydotbat <vickydotbat@tutamail.com>
Reviewed-on: #35
Co-authored-by: gitea-bot <gitea-bot@noreply.git.westgate.pw>
Co-committed-by: gitea-bot <gitea-bot@noreply.git.westgate.pw>

AGENTS.md

@@ -9,6 +9,47 @@ This repo owns the **builder logic:** one Go module
 (`git.westgate.pw/ShadowsOverWestgate/sow-tools`) producing the `crucible`
 dispatcher and the `crucible-<name>` binaries.
 
+## What it is / part it serves
+
+Shadows Over Westgate (SoW) is a Neverwinter Nights: Enhanced Edition
+persistent world, split into single-purpose repos. This repo is Crucible, the
+Go build toolkit. The content repos hold game source (module areas, rules
+data, binary assets) and call Crucible to turn that source into artifacts:
+the `.mod` file, 2DA/TLK tables, HAKs, wiki pages. Crucible is the only place
+builder logic lives; content repos only run it through thin wrapper scripts.
+
+## Guidance map
+
+| Where | What |
+|-------|------|
+| `cmd/crucible/`, `cmd/crucible-<name>/` | dispatcher + per-builder shims (thin `main.go` files) |
+| `internal/dispatch/` | the command registry — single source of truth for the command surface |
+| `internal/` (app, pipeline, project, erf, gff, topdata, changelog, validator, depot, menu, buildinfo) | the actual builder logic |
+| `wrappers/` | canonical bootstrap wrappers (`crucible`, `crucible.ps1`) synced to consumer repos; `wrappers/consumers.txt` lists targets |
+| `docs/command-surface.md` | every command, old `nwn-tool` name → new home |
+| `docs/consumer-contract.md` | how consumer repos resolve/pin a Crucible binary |
+| `docs/migration-from-nwn-tool.md` | migration status, what remains |
+| `tests/`, `Makefile`, `flake.nix` | checks, targets, dev shell |
+
+Task routing: adding/changing a command → read `docs/command-surface.md`
+first, then `internal/dispatch`. Changing how consumers get binaries →
+`docs/consumer-contract.md` + `wrappers/`. Release/CI questions → README "CI"
+section and `.gitea/workflows/`.
+
+## How it is used
+
+- Dev loop: `nix develop`, then `make check` / `make build` / `make smoke`
+  (see Commands below).
+- Release: push a `v*` tag. CI uploads cross-built binaries + wrappers to the
+  Gitea release and publishes the `crucible` container image.
+- Consumers (they download released binaries via the wrapper; they never
+  vendor a toolkit):
+  - sow-module — https://git.westgate.pw/ShadowsOverWestgate/sow-module
+  - sow-topdata — https://git.westgate.pw/ShadowsOverWestgate/sow-topdata
+  - sow-assets-manifest — https://git.westgate.pw/ShadowsOverWestgate/sow-assets-manifest
+  - sow-platform (deploys the released image/pins) —
+    https://git.westgate.pw/ShadowsOverWestgate/sow-platform
+
 ## What this repo owns / does not own
 
 Owns: build/extract/validate/compare pipeline, ERF/HAK packing, topdata 2da/tlk

README.md

@@ -80,7 +80,8 @@ This retires the old habit of checking in `nwn-tool` / `sow-toolkit`.
 ## CI
 
 PR-first (D7): checks run on pull requests and on push to `main`; the only
-publish event is a `v*` tag (see `sow-docs/runbooks/ci-trigger-standard.md`).
+publish event is a `v*` tag (see `runbooks/ci-trigger-standard.md` in sow-docs,
+https://git.westgate.pw/ShadowsOverWestgate/sow-docs).
 
 - `test.yml` — vet, test, shellcheck, yamllint, binary smoke (PR + main).
 - `test-image.yml` — build the OCI image to prove it compiles (PR + main, no push).

internal/depot/run.go

@@ -143,7 +143,7 @@ func runStatus(args []string, stdout, stderr io.Writer, getenv func(string) stri
 	}
 	shas := shaKeys(shaSizes)
 
-	res, err := Sweep(context.Background(), backend, shas, cfg, io.Discard)
+	res, err := Sweep(context.Background(), backend, shas, cfg, stderr)
 	if err != nil {
 		fmt.Fprintln(stderr, "depot status:", err)
 		return exitInternal
@@ -185,7 +185,7 @@ func runPush(args []string, stdout, stderr io.Writer, getenv func(string) string
 	}
 	shas := shaKeys(shaSizes)
 
-	res, err := Sweep(context.Background(), backend, shas, cfg, io.Discard)
+	res, err := Sweep(context.Background(), backend, shas, cfg, stderr)
 	if err != nil {
 		fmt.Fprintln(stderr, "depot push:", err)
 		return exitInternal
@@ -266,7 +266,7 @@ func runVerify(args []string, stdout, stderr io.Writer, getenv func(string) stri
 	}
 	shas := shaKeys(shaSizes)
 
-	res, err := Sweep(context.Background(), backend, shas, cfg, io.Discard)
+	res, err := Sweep(context.Background(), backend, shas, cfg, stderr)
 	if err != nil {
 		fmt.Fprintln(stderr, "depot verify:", err)
 		return exitInternal
@@ -389,7 +389,7 @@ func runPull(args []string, stdout, stderr io.Writer, getenv func(string) string
 
 	// Sweep the source to distinguish "not there" (exit 1, listed) from
 	// "there, download it".
-	res, err := Sweep(context.Background(), backend, toCheck, cfg, io.Discard)
+	res, err := Sweep(context.Background(), backend, toCheck, cfg, stderr)
 	if err != nil {
 		fmt.Fprintln(stderr, "depot pull:", err)
 		return exitInternal

internal/erf/erf.go

@@ -123,6 +123,7 @@ var extensionTypes = map[string]uint16{
 	"mtr": 0x0818,
 	"jpg": 0x081C,
 	"lod": 0x081E,
+	"gif": 0x081F,
 	"png": 0x0820,
 	"lyt": 0x0BB8,
 	"vis": 0x0BB9,
@@ -188,6 +189,7 @@ var typeExtensions = map[uint16]string{
 	0x0818: "mtr",
 	0x081C: "jpg",
 	0x081E: "lod",
+	0x081F: "gif",
 	0x0820: "png",
 	0x0BB8: "lyt",
 	0x0BB9: "vis",

internal/topdata/class_feat_accessibility.go

@@ -0,0 +1,98 @@
+package topdata
+
+import (
+	"fmt"
+	"path/filepath"
+	"slices"
+	"strings"
+)
+
+// validateClassFeatMasterfeatAccessibility warns when a class feat table
+// references a masterfeat whose member feats are silently dropped from
+// expansion by classFeatExpansionCanUseFeat. Only the ambiguous case is
+// flagged: ALLCLASSESCANUSE=0 with no MinLevelClass, where the feat cannot
+// be obtained by any class at all. Exclusions with an explicit MinLevelClass
+// are deliberate cross-class restrictions and stay silent.
+func validateClassFeatMasterfeatAccessibility(dataDir string, report *ValidationReport) {
+	datasets, err := discoverNativeDatasets(dataDir)
+	if err != nil {
+		return
+	}
+	keyToID := map[string]int{}
+	var featRowByKey map[string]map[string]any
+	type classFeatTable struct {
+		path     string
+		classKey string
+		rows     []map[string]any
+	}
+	classTables := []classFeatTable{}
+	for _, dataset := range datasets {
+		name := filepath.ToSlash(dataset.Name)
+		isClassFeats := strings.HasPrefix(name, "classes/feats/")
+		if name != "feat" && name != "masterfeats" && name != "classes/core" && !isClassFeats {
+			continue
+		}
+		collected, err := collectNativeDataset(dataset)
+		if err != nil {
+			return
+		}
+		for key, id := range collected.LockData {
+			keyToID[key] = id
+		}
+		switch {
+		case name == "feat":
+			featRowByKey = rowsByKey(collected.Rows)
+		case isClassFeats:
+			classTables = append(classTables, classFeatTable{
+				path:     dataset.BasePath,
+				classKey: "classes:" + name[strings.LastIndex(name, "/")+1:],
+				rows:     collected.Rows,
+			})
+		}
+	}
+	if featRowByKey == nil {
+		return
+	}
+	for _, table := range classTables {
+		for _, row := range table.rows {
+			featRef, ok := row["FeatIndex"].(map[string]any)
+			if !ok {
+				continue
+			}
+			refKey, _ := featRef["id"].(string)
+			if !strings.HasPrefix(refKey, "masterfeats:") {
+				continue
+			}
+			masterfeatKey := resolveCanonicalDatasetKey(refKey, keyToID)
+			if _, ok := keyToID[masterfeatKey]; !ok {
+				continue
+			}
+			featKeys := make([]string, 0, len(featRowByKey))
+			for featKey := range featRowByKey {
+				if strings.HasPrefix(featKey, "feat:") {
+					featKeys = append(featKeys, featKey)
+				}
+			}
+			slices.Sort(featKeys)
+			for _, featKey := range featKeys {
+				featRow := featRowByKey[featKey]
+				if !rowReferencesMasterfeat(featRow, masterfeatKey, keyToID) {
+					continue
+				}
+				if classFeatExpansionCanUseFeat(featRow, table.classKey, keyToID) {
+					continue
+				}
+				if minLevelClass, ok := lookupField(featRow, "MinLevelClass"); ok && !isNullishValue(minLevelClass) {
+					continue
+				}
+				report.Diagnostics = append(report.Diagnostics, Diagnostic{
+					Severity: SeverityWarning,
+					Path:     table.path,
+					Message: fmt.Sprintf(
+						"%s references %s but %s is excluded from expansion (ALLCLASSESCANUSE=0 with no MinLevelClass makes it unobtainable); override ALLCLASSESCANUSE or set MinLevelClass",
+						table.classKey, masterfeatKey, featKey),
+				})
+			}
+		}
+	}
+}

internal/topdata/class_feat_accessibility_test.go

@@ -0,0 +1,82 @@
+package topdata
+
+import (
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+func TestValidateClassFeatMasterfeatAccessibilityWarnsOnUnobtainableExpansion(t *testing.T) {
+	root := testProjectRoot(t)
+	mkdirAll(t, filepath.Join(root, "topdata", "data", "classes", "feats"))
+	mkdirAll(t, filepath.Join(root, "topdata", "data", "feat"))
+	mkdirAll(t, filepath.Join(root, "topdata", "data", "masterfeats"))
+	writeFile(t, filepath.Join(root, "topdata", "data", "classes", "base.json"), `{
+  "output": "classes.2da",
+  "columns": ["Label", "Name", "FeatsTable"],
+  "rows": [
+    {"id": 4, "key": "classes:fighter", "Label": "Fighter", "Name": "111", "FeatsTable": "cls_feat_fighter"},
+    {"id": 10, "key": "classes:wizard", "Label": "Wizard", "Name": "112", "FeatsTable": "cls_feat_wizard"}
+  ]
+}`+"\n")
+	writeFile(t, filepath.Join(root, "topdata", "data", "classes", "lock.json"), `{"classes:fighter":4,"classes:wizard":10}`+"\n")
+	writeFile(t, filepath.Join(root, "topdata", "data", "classes", "feats", "wizard.json"), `{
+  "key": "classes/feats:wizard",
+  "output": "cls_feat_wizard.2da",
+  "columns": ["FeatLabel", "FeatIndex", "List", "GrantedOnLevel", "OnMenu"],
+  "rows": [
+    {"FeatIndex": {"id": "masterfeats:spellfocus"}, "List": "1", "GrantedOnLevel": "-1", "OnMenu": "0"},
+    {"FeatIndex": {"id": "masterfeats:weaponspecialization"}, "List": "1", "GrantedOnLevel": "-1", "OnMenu": "0"}
+  ]
+}`+"\n")
+	writeFile(t, filepath.Join(root, "topdata", "data", "feat", "base.json"), `{
+  "output": "feat.2da",
+  "columns": ["LABEL", "FEAT", "DESCRIPTION", "MASTERFEAT", "ALLCLASSESCANUSE", "MinLevelClass"],
+  "rows": [
+    {"id": 100, "key": "feat:spellfocus_abjuration", "LABEL": "SpellFocusAbj", "FEAT": "1000", "DESCRIPTION": "1001", "MASTERFEAT": "1", "ALLCLASSESCANUSE": "0", "MinLevelClass": "****"},
+    {"id": 101, "key": "feat:spellfocus_conjuration", "LABEL": "SpellFocusCon", "FEAT": "1002", "DESCRIPTION": "1003", "MASTERFEAT": "1", "ALLCLASSESCANUSE": "1", "MinLevelClass": "****"},
+    {"id": 200, "key": "feat:weaponspecialization_longsword", "LABEL": "WeaponSpecLongsword", "FEAT": "2000", "DESCRIPTION": "2001", "MASTERFEAT": "2", "ALLCLASSESCANUSE": "0", "MinLevelClass": {"id": "classes:fighter"}}
+  ]
+}`+"\n")
+	writeFile(t, filepath.Join(root, "topdata", "data", "feat", "lock.json"), `{
+  "feat:spellfocus_abjuration": 100,
+  "feat:spellfocus_conjuration": 101,
+  "feat:weaponspecialization_longsword": 200
+}`+"\n")
+	writeFile(t, filepath.Join(root, "topdata", "data", "masterfeats", "base.json"), `{
+  "output": "masterfeats.2da",
+  "columns": ["LABEL", "STRREF", "DESCRIPTION", "ICON"],
+  "rows": [
+    {"id": 1, "key": "masterfeats:spellfocus", "LABEL": "SpellFocus", "STRREF": "6492", "DESCRIPTION": "426", "ICON": "ife_magic"},
+    {"id": 2, "key": "masterfeats:weaponspecialization", "LABEL": "WeaponSpecialization", "STRREF": "6491", "DESCRIPTION": "437", "ICON": "ife_wepspec"}
+  ]
+}`+"\n")
+	writeFile(t, filepath.Join(root, "topdata", "data", "masterfeats", "lock.json"), `{
+  "masterfeats:spellfocus": 1,
+  "masterfeats:weaponspecialization": 2
+}`+"\n")
+
+	report := ValidationReport{}
+	validateClassFeatMasterfeatAccessibility(filepath.Join(root, "topdata", "data"), &report)
+
+	warned := map[string]bool{}
+	for _, diagnostic := range report.Diagnostics {
+		if diagnostic.Severity != SeverityWarning {
+			continue
+		}
+		for _, featKey := range []string{"feat:spellfocus_abjuration", "feat:spellfocus_conjuration", "feat:weaponspecialization_longsword"} {
+			if strings.Contains(diagnostic.Message, featKey) {
+				warned[featKey] = true
+			}
+		}
+	}
+	if !warned["feat:spellfocus_abjuration"] {
+		t.Fatalf("expected warning for unobtainable ALLCLASSESCANUSE=0 feat, got: %+v", report.Diagnostics)
+	}
+	if warned["feat:spellfocus_conjuration"] {
+		t.Fatalf("did not expect warning for ALLCLASSESCANUSE=1 feat, got: %+v", report.Diagnostics)
+	}
+	if warned["feat:weaponspecialization_longsword"] {
+		t.Fatalf("did not expect warning for feat with explicit MinLevelClass, got: %+v", report.Diagnostics)
+	}
+}

internal/topdata/top_package.go

@@ -214,6 +214,9 @@ func collectTopPackageResources(p *project.Project, compiled2DADir string) ([]er
 			if strings.HasPrefix(filepath.Base(path), ".") {
 				return nil
 			}
+			if strings.EqualFold(filepath.Ext(path), ".md") {
+				return nil // docs (AGENTS.md, README.md, ...) are never HAK resources
+			}
 			resource, err := topPackageResourceFromPath(path)
 			if err != nil {
 				return err

internal/topdata/topdata.go

@@ -268,6 +268,7 @@ func ValidateProject(p *project.Project) ValidationReport {
 	validateNativeLockAllocation(dataDir, p.EffectiveConfig().TopData.RowGeneration, &report)
 	validateGeneratedFeatFamilies(dataDir, &report)
 	validateClassSpellbooks(dataDir, &report)
+	validateClassFeatMasterfeatAccessibility(dataDir, &report)
 	validateItempropsRegistryGraph(dataDir, &report)
 	if _, err := os.Stat(statePath); err == nil {
 		report.Files++
@@ -2232,6 +2233,9 @@ func validateTopPackageAssets(sourceDir, dataDir string, report *ValidationRepor
 		if strings.HasPrefix(filepath.Base(path), ".") {
 			return nil
 		}
+		if strings.EqualFold(filepath.Ext(path), ".md") {
+			return nil // docs (AGENTS.md, README.md, ...) are never HAK resources
+		}
 		base := strings.ToLower(strings.TrimSuffix(filepath.Base(path), filepath.Ext(path)))
 		ext := strings.TrimPrefix(strings.ToLower(filepath.Ext(path)), ".")
 		if _, ok := erf.HAKResourceTypeForExtension(ext); !ok {

internal/topdata/topdata_test.go

@@ -15115,3 +15115,23 @@ func writeBytes(t *testing.T, path string, content []byte) {
 		t.Fatalf("write %s: %v", path, err)
 	}
 }
+
+func TestValidateTopPackageAssetsSkipsMarkdown(t *testing.T) {
+	dir := t.TempDir()
+	assets := filepath.Join(dir, "assets", "gui")
+	if err := os.MkdirAll(assets, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(assets, "AGENTS.md"), []byte("docs"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	dataDir := filepath.Join(dir, "data")
+	if err := os.MkdirAll(dataDir, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	var report ValidationReport
+	validateTopPackageAssets(dir, dataDir, &report)
+	for _, d := range report.Diagnostics {
+		t.Errorf("unexpected diagnostic: %s: %s", d.Path, d.Message)
+	}
+}