Files
archvillainette 7cc53aeb68
build-binaries / build-binaries (push) Successful in 2m40s
fix(nwsync): declare Frame_Content_Size on every blob, and verify what is published (#87)
Closes #86. Closes #85.

These land together on purpose. Fixing the encoder alone changes nothing for the blobs already in the zone, because `emit` skips whatever is already present.

## #86 — the framing fix

`klauspost/compress` omits the zstd `Frame_Content_Size` field for inputs under 256 bytes, which the format permits. Reference libzstd never does, so the NWN client — which sizes its output buffer from `ZSTD_getFrameContentSize` and has therefore never met a frame without one — rejected roughly 6% of our blobs outright. Any single one stops a sync dead, so no client could complete a sync of the live manifest.

No encoder option changes this, so `compressBlob` re-headers the affected frames into the shape libzstd itself emits: `Single_Segment_flag` set, `Window_Descriptor` dropped, and the freed byte spent on a one-byte `Frame_Content_Size`. Same length in, same length out, and the same descriptor byte (`0x24`) the issue recorded from libzstd.

`compressBlob` then asserts its own output. An encoder upgrade that finds another way to omit the field would otherwise reproduce #86 in silence, and a blob is skipped by every later emit once written.

`emitter_version` goes to `2`, so `assemble` refuses to merge an index written by the encoder that omitted the field.

**Proved against the reference decoder, not just a round trip.** A real emitted 175-byte blob:

```
Frames  Skips  Compressed  Uncompressed  Ratio  Check  Filename
     1      0      48   B       175   B  3.646  XXH64  frame.zst
c59d6620d4ffd4bf3fe73df43b19b7afcfe8fea4  -            <- zstd -dc | sha1sum
c59d6620d4ffd4bf3fe73df43b19b7afcfe8fea4               <- the blob's own name
```

Before the fix that `Uncompressed` column was blank.

## #85 — `nwsync verify`

`crucible nwsync verify <manifest-sha1>` reads a manifest and its blobs back through the **public pull zone**, with no credential, because what matters is the bytes a client is served, edge behaviour included. Every distinct blob is decompressed and hashed; failures are reported per blob as missing / malformed framing / size mismatch / hash mismatch, and the exit code is 1.

- `--sample N` makes a routine check cheap against a manifest that is ~69,000 blobs and 15 GB; the default is a full sweep.
- `--base URL` / `NWSYNC_PULL_BASE` overrides the public host.
- The manifest is checked against its own sha1 before a single blob is fetched.
- `emit --verify` applies the same check where `emit` would otherwise trust presence, and replaces a stored blob that is not what its name claims. This is what makes the #86 blobs repairable.

## Why the existing checks missed this

Both new checks assert the **frame property**, not just a round trip. The conformance suite (#59) compares decompressed bytes, so a frame that decodes correctly passes regardless of its header; and the earlier zone audit decompressed 68 blobs with the `zstd` CLI, a *more* capable decoder than the client's, which certified exactly the blobs the client rejects.

## Checks

`make check` and `make smoke` green. Second commit is the fixes from a two-axis review of the first.

## Not in this PR

Three follow-ups, filed separately: the backfill has not been run, replacing a blob does not purge the pull-zone edge cache, and #85's runbook line belongs to `sow-platform`.

🤖 Generated with [Claude Code](https://claude.com/claude-code)Reviewed-on: #87

Co-authored-by: vickydotbat <vickydotbat@tutamail.com>
2026-07-31 22:33:12 +00:00

334 lines
11 KiB
Go

package dispatch
import (
"bytes"
"strings"
"testing"
)
func TestVersion(t *testing.T) {
var out, errw bytes.Buffer
for _, arg := range []string{"version", "-V", "--version"} {
out.Reset()
errw.Reset()
if code := run([]string{arg}, &out, &errw); code != exitOK {
t.Fatalf("%s: exit=%d want %d", arg, code, exitOK)
}
if !strings.HasPrefix(out.String(), "crucible ") {
t.Fatalf("%s: version line %q missing prefix", arg, out.String())
}
}
}
func TestHelpAndNoArgs(t *testing.T) {
var out, errw bytes.Buffer
if code := run([]string{"help"}, &out, &errw); code != exitOK {
t.Fatalf("help exit=%d want %d", code, exitOK)
}
for _, builder := range Registry {
if !strings.Contains(out.String(), builder.Name) {
t.Errorf("help output missing builder %q:\n%s", builder.Name, out.String())
}
}
// No args is a usage error (exit 64) but still prints help.
out.Reset()
if code := run(nil, &out, &errw); code != exitUsage {
t.Fatalf("no-args exit=%d want %d", code, exitUsage)
}
if out.Len() == 0 {
t.Fatal("no-args usage error should include help")
}
}
func TestListCoversRegistry(t *testing.T) {
var out bytes.Buffer
list(&out)
lines := strings.Split(strings.TrimSpace(out.String()), "\n")
if len(lines) != len(Registry) {
t.Fatalf("list returned %d rows for %d builders:\n%s", len(lines), len(Registry), out.String())
}
builders := make(map[string]Builder, len(Registry))
for _, builder := range Registry {
builders[builder.Name] = builder
}
seen := make(map[string]bool, len(lines))
for _, line := range lines {
fields := strings.Split(strings.TrimSpace(line), "\t")
if len(fields) != 3 {
t.Fatalf("list row must be name<TAB>bin<TAB>summary, got %q", line)
}
builder, ok := builders[fields[0]]
if !ok {
t.Errorf("list returned unregistered builder %q", fields[0])
continue
}
if fields[1] != builder.Bin {
t.Errorf("builder %q listed binary %q, want %q", builder.Name, fields[1], builder.Bin)
}
if strings.TrimSpace(fields[2]) == "" {
t.Errorf("builder %q listed an empty summary", builder.Name)
}
seen[fields[0]] = true
}
for _, builder := range Registry {
if !seen[builder.Name] {
t.Errorf("list missing builder %q", builder.Name)
}
}
}
func TestUnknownBuilderFailsUsage(t *testing.T) {
var out, errw bytes.Buffer
if code := run([]string{"frobnicate"}, &out, &errw); code != exitUsage {
t.Fatalf("unknown builder exit=%d want %d", code, exitUsage)
}
if errw.Len() == 0 {
t.Fatal("unknown builder should explain the usage error")
}
}
func TestUnwiredBuilderFailsClosed(t *testing.T) {
for _, b := range Registry {
if b.Wired {
continue
}
var out, errw bytes.Buffer
// Via dispatcher.
if code := run([]string{b.Name}, &out, &errw); code != exitUnwired {
t.Errorf("crucible %s: exit=%d want %d (must fail closed)", b.Name, code, exitUnwired)
}
if errw.Len() == 0 {
t.Errorf("crucible %s: missing fail-closed explanation", b.Name)
}
// Via standalone shim path.
out.Reset()
errw.Reset()
if code := runBuilder(b.Name, nil, &out, &errw); code != exitUnwired {
t.Errorf("%s: exit=%d want %d (must fail closed)", b.Bin, code, exitUnwired)
}
}
}
func TestWiredBuilderRejectsBadInvocation(t *testing.T) {
for _, b := range Registry {
if !b.Wired {
continue
}
// No subcommand is a usage error (it must never silently delegate).
var out, errw bytes.Buffer
if code := runBuilder(b.Name, nil, &out, &errw); code != exitUsage {
t.Errorf("crucible %s (no subcommand): exit=%d want %d", b.Name, code, exitUsage)
}
// Unknown subcommand is a usage error, not a delegate.
out.Reset()
errw.Reset()
if code := runBuilder(b.Name, []string{"frobnicate"}, &out, &errw); code != exitUsage {
t.Errorf("crucible %s frobnicate: exit=%d want %d", b.Name, code, exitUsage)
}
if errw.Len() == 0 {
t.Errorf("crucible %s frobnicate: missing usage explanation", b.Name)
}
}
}
func TestBuilderHelpIsOK(t *testing.T) {
for _, b := range Registry {
var out, errw bytes.Buffer
if code := runBuilder(b.Name, []string{"--help"}, &out, &errw); code != exitOK {
t.Errorf("%s --help: exit=%d want %d", b.Name, code, exitOK)
}
if !strings.Contains(out.String(), b.Name) || !strings.Contains(out.String(), b.Bin) {
t.Errorf("%s --help: missing builder identity", b.Name)
}
for _, subcommand := range b.subcommands() {
if !strings.Contains(out.String(), subcommand) {
t.Errorf("%s --help: missing subcommand %q", b.Name, subcommand)
}
}
}
}
// selfContained builders parse their own subcommands instead of delegating to
// the legacy internal/app surface.
var selfContained = map[string]bool{"depot": true, "assets": true, "nwsync": true}
func TestCanonicalCommandSurface(t *testing.T) {
want := map[string][]string{
"depot": {"status", "push", "verify", "get", "pull"},
"assets": {"compile", "convert", "upscale", "check-mdl", "fix-mdl", "check-dupes", "clean-dupes"},
"hak": {"build", "manifest"},
"module": {"build", "extract", "validate", "compare", "manifest"},
"topdata": {"validate", "build", "package", "compare", "convert"},
"wiki": {"build", "deploy"},
"nwsync": {"emit", "assemble", "verify"},
}
for _, builder := range Registry {
got := builder.subcommands()
expected, ok := want[builder.Name]
if !ok {
t.Fatalf("unexpected builder %q", builder.Name)
}
if strings.Join(got, ",") != strings.Join(expected, ",") {
t.Errorf("%s commands = %v, want %v", builder.Name, got, expected)
}
}
}
func TestRegistryCommandNamesAndAliasesAreUnambiguous(t *testing.T) {
for _, builder := range Registry {
seen := map[string]bool{}
for _, command := range builder.Commands {
// depot, assets and nwsync parse their own subcommands and bypass
// AppCommand routing entirely (see the self-contained-builder branch
// in runBuilder), so their Commands carry no AppCommand.
requireAppCommand := !selfContained[builder.Name]
if command.Name == "" || command.Summary == "" || command.Usage == "" || (requireAppCommand && command.AppCommand == "") {
t.Errorf("%s has incomplete command metadata: %#v", builder.Name, command)
}
if seen[command.Name] {
t.Errorf("%s repeats command name %q", builder.Name, command.Name)
}
seen[command.Name] = true
for _, alias := range command.Aliases {
if alias.Name == "" {
t.Errorf("%s %s has an empty alias", builder.Name, command.Name)
continue
}
if seen[alias.Name] {
t.Errorf("%s repeats command or alias %q", builder.Name, alias.Name)
}
seen[alias.Name] = true
}
}
}
}
func TestMusicCommandsAreNotAccepted(t *testing.T) {
for _, builderName := range []string{"hak", "module"} {
builder, ok := find(builderName)
if !ok {
t.Fatalf("missing builder %q", builderName)
}
if builder.accepts("music") {
t.Errorf("%s must not accept the removed music command", builderName)
}
}
}
func TestHiddenAliasesPreserveImplementationTargets(t *testing.T) {
tests := []struct {
builder string
alias string
target string
}{
{"hak", "build-haks", "build-haks"},
{"hak", "apply-hak-manifest", "apply-hak-manifest"},
{"module", "build-module", "build-module"},
{"module", "apply-hak-manifest", "apply-hak-manifest"},
{"topdata", "validate-topdata", "validate-topdata"},
{"topdata", "build-topdata", "build-topdata"},
{"topdata", "build-top-package", "build-top-package"},
{"topdata", "compare-topdata", "compare-topdata"},
{"topdata", "convert-topdata", "convert-topdata"},
{"wiki", "build-wiki", "build-wiki"},
{"wiki", "deploy-wiki", "deploy-wiki"},
}
for _, tt := range tests {
t.Run(tt.builder+"/"+tt.alias, func(t *testing.T) {
builder, ok := find(tt.builder)
if !ok {
t.Fatalf("missing builder %q", tt.builder)
}
command, target, ok := builder.command(tt.alias)
if !ok {
t.Fatalf("hidden alias %q is not accepted", tt.alias)
}
if target != tt.target {
t.Fatalf("alias %q target = %q, want %q", tt.alias, target, tt.target)
}
if command.Name == tt.alias {
t.Fatalf("alias %q must not be a visible command", tt.alias)
}
})
}
}
func TestBuilderHelpHidesCompatibilityAliases(t *testing.T) {
aliases := map[string][]string{
"hak": {"build-haks", "apply-hak-manifest"},
"module": {"build-module", "apply-hak-manifest"},
"topdata": {"validate-topdata", "build-topdata", "build-top-package", "compare-topdata", "convert-topdata"},
"wiki": {"build-wiki", "deploy-wiki"},
}
for builderName, hidden := range aliases {
var out, errw bytes.Buffer
if code := runBuilder(builderName, []string{"--help"}, &out, &errw); code != exitOK {
t.Fatalf("%s help exit = %d", builderName, code)
}
for _, alias := range hidden {
if strings.Contains(out.String(), alias) {
t.Errorf("%s help exposed hidden alias %q:\n%s", builderName, alias, out.String())
}
}
}
}
func TestCommandHelpUsesCanonicalGuidanceWithoutLoadingProject(t *testing.T) {
var out, errw bytes.Buffer
if code := runBuilder("topdata", []string{"build", "--help"}, &out, &errw); code != exitOK {
t.Fatalf("command help exit = %d, stderr:\n%s", code, errw.String())
}
for _, want := range []string{
"compile topdata",
"usage: crucible topdata build",
"--force",
"--skip-lfs",
} {
if !strings.Contains(out.String(), want) {
t.Errorf("command help missing %q:\n%s", want, out.String())
}
}
if errw.Len() != 0 {
t.Fatalf("command help must not load a project or emit errors:\n%s", errw.String())
}
}
func TestMenuItemsSkipsUnwiredIncludesWired(t *testing.T) {
items := menuItems()
if len(items) == 0 {
t.Fatal("expected menu items")
}
sawModuleBuild := false
sawDepotStatus := false
for _, it := range items {
if len(it.Args) == 2 && it.Args[0] == "module" && it.Args[1] == "build" {
sawModuleBuild = true
}
if len(it.Args) == 2 && it.Args[0] == "depot" && it.Args[1] == "status" {
sawDepotStatus = true
}
}
if !sawModuleBuild {
t.Error("expected 'module build' in the menu")
}
if !sawDepotStatus {
t.Error("expected wired builder 'depot status' in the menu")
}
}
// TestDepotRoutesToDepotRun asserts the depot special-case in runBuilder
// reaches depot.Run rather than the generic Commands/delegateLegacy path or
// the unwired fail-closed path. With no manifests dir present, depot.Run's
// status command fails resolving the backend/manifest (exit 70), but the
// stderr text must come from depot, never the dispatcher's unwiredMsg.
func TestDepotRoutesToDepotRun(t *testing.T) {
var out, errw bytes.Buffer
code := runBuilder("depot", []string{"status", "--target", "local"}, &out, &errw)
if errw.Len() == 0 {
t.Fatalf("depot status with no manifests dir: expected an error from depot.Run, got no stderr (exit=%d)", code)
}
if strings.Contains(errw.String(), "not wired yet") {
t.Fatalf("depot status: stderr shows the dispatcher's unwired message, want depot.Run's own error:\n%s", errw.String())
}
}