diff --git a/docs/command-surface.md b/docs/command-surface.md index 1de8876..15128c9 100644 --- a/docs/command-surface.md +++ b/docs/command-surface.md @@ -43,10 +43,32 @@ flags are mutually exclusive. `nwsync emit` runs where an artifact is born (a `.hak`/`.erf`, or a loose file such as the TLK); `nwsync assemble` runs at module release and reads only the -small per-artifact manifests. `--order` lists artifact names highest priority -first: a resref in more than one artifact resolves to the earliest one, the way -the game resolves it. `--group-id` is per channel — 1 is current, 2 is testing, -and 0 leaves the field out of the sidecar. +small per-artifact indexes. Both take **depot keys**: an artifact's index lives +beside the artifact itself with the extension replaced, so `emit` and +`assemble` agree on where it is without being told. + +``` +nwsync emit [--as NAME] [--out DIR] +nwsync assemble --group-id N [--tlk-key KEY] [--out DIR] ... +``` + +Both verbs upload by default; nothing bulky is ever written to the runner's +disk. `--out DIR` writes a local repository tree instead, which is the +conformance path against upstream `nwn_nwsync_write`. The zone comes from +`NWSYNC_STORAGE_ZONE` and `NWSYNC_STORAGE_PASSWORD`, with the host from +`BUNNY_STORAGE_HOST` — NWSync data is a separate zone from the asset depot. + +`assemble`'s artifact keys are in `Mod_HakList` order, highest priority first: a +resref in more than one artifact resolves to the earliest one, the way the game +resolves it. `--tlk-key` has its own slot because the TLK shadows nothing. +`--group-id` is per channel — 1 is current, 2 is testing, and 0 leaves the field +out of the sidecar. + +`emit` uploads blobs first and the index last, so the presence of an index is +the publication marker: an artifact whose emit died halfway leaves real blobs in +the zone and no index. Blob names are content hashes, so re-running skips +whatever already landed, and `assemble` fails closed on an artifact with no +index rather than publishing a manifest that is missing a hak. ## Hidden compatibility aliases diff --git a/internal/depot/keystore.go b/internal/depot/keystore.go new file mode 100644 index 0000000..6112e80 --- /dev/null +++ b/internal/depot/keystore.go @@ -0,0 +1,121 @@ +package depot + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "os" + "strings" +) + +// KeyStore is the zone addressed by object key rather than by depot sha. The +// depot names every object after the sha256 of its contents; NWSync does not — +// a blob is named after the sha1 of its *uncompressed* bytes while the body +// uploaded is the compressed form, and a per-artifact index is named after its +// artifact. Both addressing modes want the same transport, retry and probe +// discipline, so the sha-addressed Backend rides on this rather than the other +// way round. +type KeyStore interface { + // ProbeKey returns the existence state of one key. transient=true means a + // retry might change the answer — never read it as "missing, re-upload". + ProbeKey(ctx context.Context, key string) (state ProbeState, transient bool, err error) + // PutReader uploads size bytes read from r to key. checksum is the + // uppercase hex sha256 of those bytes, which Bunny verifies server-side. + PutReader(ctx context.Context, key string, r io.Reader, size int64, checksum string) error + // GetKey fetches the whole object at key. Small objects only — it holds + // the body in memory and does no hash check, because a key is not always + // a content hash. + GetKey(ctx context.Context, key string) ([]byte, error) +} + +// NewKeyStore returns a KeyStore for cfg's storage zone. Fails closed on a +// missing host or read key, matching NewBackend. +func NewKeyStore(cfg Config) (KeyStore, error) { + if cfg.StorageHost == "" { + return nil, errors.New("storage backend requires a storage host") + } + if cfg.StorageZone == "" { + return nil, errors.New("storage backend requires a storage zone") + } + if cfg.ReadKey == "" { + return nil, errors.New("storage backend requires a read key") + } + return &httpBackend{name: "bunny", client: newHTTPClient(cfg), cfg: cfg}, nil +} + +// keyURL is the storage URL of one object key. +func (b *httpBackend) keyURL(key string) string { + host := b.cfg.StorageHost + // StorageHost is normally a bare host ("storage.bunnycdn.com"); allow a + // full scheme (used by tests against httptest.NewServer) to pass through + // unchanged. + if !strings.Contains(host, "://") { + host = "https://" + host + } + return fmt.Sprintf("%s/%s/%s", strings.TrimSuffix(host, "/"), b.cfg.StorageZone, key) +} + +func (b *httpBackend) ProbeKey(ctx context.Context, key string) (ProbeState, bool, error) { + return b.rangeProbe(ctx, b.keyURL(key), map[string]string{"AccessKey": b.cfg.ReadKey}) +} + +func (b *httpBackend) PutReader(ctx context.Context, key string, r io.Reader, size int64, checksum string) error { + if b.name == "cdn" { + return errors.New("cdn backend is read-only") + } + if b.cfg.WriteKey == "" { + return errors.New("storage backend requires a write key to write") + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPut, b.keyURL(key), r) + if err != nil { + return err + } + req.ContentLength = size + req.Header.Set("AccessKey", b.cfg.WriteKey) + // Bunny defines Checksum as sha256 of the body and rejects a mismatch, so + // this is server-side integrity checking, not decoration. + req.Header.Set("Checksum", strings.ToUpper(checksum)) + + resp, err := b.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + _, _ = io.Copy(io.Discard, resp.Body) + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return fmt.Errorf("put %s: unexpected status %d", key, resp.StatusCode) + } + return nil +} + +func (b *httpBackend) GetKey(ctx context.Context, key string) ([]byte, error) { + resp, err := b.get(ctx, b.keyURL(key), map[string]string{"AccessKey": b.cfg.ReadKey}) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + _, _ = io.Copy(io.Discard, resp.Body) + return nil, fmt.Errorf("get %s: unexpected status %d", key, resp.StatusCode) + } + return io.ReadAll(resp.Body) +} + +// putFile uploads the file at src to key, streaming it. checksum is the +// uppercase hex sha256 of the file's bytes. +func (b *httpBackend) putFile(ctx context.Context, key, src, checksum string) error { + f, err := os.Open(src) + if err != nil { + return err + } + defer f.Close() + info, err := f.Stat() + if err != nil { + return err + } + return b.PutReader(ctx, key, f, info.Size(), checksum) +} diff --git a/internal/depot/remote.go b/internal/depot/remote.go index cdd38dc..4fe6db9 100644 --- a/internal/depot/remote.go +++ b/internal/depot/remote.go @@ -10,7 +10,6 @@ import ( "net/http" "os" "path/filepath" - "strings" "time" ) @@ -71,16 +70,7 @@ type httpBackend struct { func (b *httpBackend) Name() string { return b.name } -func (b *httpBackend) storageURL(sha string) string { - host := b.cfg.StorageHost - // StorageHost is normally a bare host ("storage.bunnycdn.com"); allow a - // full scheme (used by tests against httptest.NewServer) to pass through - // unchanged. - if strings.Contains(host, "://") { - return fmt.Sprintf("%s/%s/%s", strings.TrimSuffix(host, "/"), b.cfg.StorageZone, BlobKey(sha)) - } - return fmt.Sprintf("https://%s/%s/%s", host, b.cfg.StorageZone, BlobKey(sha)) -} +func (b *httpBackend) storageURL(sha string) string { return b.keyURL(BlobKey(sha)) } func (b *httpBackend) cdnURL(sha string) string { return fmt.Sprintf("%s/%s", b.cfg.CDNBase, BlobKey(sha)) @@ -140,7 +130,8 @@ func (b *httpBackend) Probe(ctx context.Context, sha string) (ProbeState, bool, return b.rangeProbe(ctx, b.storageURL(sha), map[string]string{"AccessKey": b.cfg.ReadKey}) } -// Put uploads src for sha. cdn is read-only. +// Put uploads src for sha. cdn is read-only. A depot object is named after the +// sha256 of its own bytes, so the key's sha doubles as the Checksum header. func (b *httpBackend) Put(ctx context.Context, sha, src string) error { if b.name == "cdn" { return errors.New("cdn backend is read-only") @@ -157,30 +148,7 @@ func (b *httpBackend) Put(ctx context.Context, sha, src string) error { return nil } - f, err := os.Open(src) - if err != nil { - return err - } - defer f.Close() - - req, err := http.NewRequestWithContext(ctx, http.MethodPut, b.storageURL(sha), f) - if err != nil { - return err - } - req.Header.Set("AccessKey", b.cfg.WriteKey) - req.Header.Set("Checksum", strings.ToUpper(sha)) - - resp, err := b.client.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - _, _ = io.Copy(io.Discard, resp.Body) - - if resp.StatusCode < 200 || resp.StatusCode >= 300 { - return fmt.Errorf("bunny put %s: unexpected status %d", sha, resp.StatusCode) - } - return nil + return b.putFile(ctx, BlobKey(sha), src, sha) } // Get fetches sha into dest via temp file + rename, re-hashing and deleting diff --git a/internal/nwsync/assemble.go b/internal/nwsync/assemble.go index 666cb41..cd18472 100644 --- a/internal/nwsync/assemble.go +++ b/internal/nwsync/assemble.go @@ -4,18 +4,21 @@ import ( "crypto/sha1" "encoding/json" "fmt" - "os" - "path/filepath" + "path" ) // AssembleOptions describes one merged manifest. type AssembleOptions struct { - Order []string // artifact names, highest priority first - EntriesDir string // directory holding .nsym and .nsym.json - OutDir string // repository root; the manifest lands in /manifests - GroupID int // 1 = current, 2 = testing; 0 means absent - ModuleName string - Description string + // ArtifactKeys are the depot keys of the artifacts to merge, in + // Mod_HakList order — highest priority first. Each one's index is read + // from the key beside it. + ArtifactKeys []string + TLKKey string // the TLK's key, if the manifest carries one + OutDir string // write locally instead of uploading — the conformance path + GroupID int // 1 = current, 2 = testing; 0 means absent + ModuleName string + Description string + Sink sink // test seam; nil means OutDir or the zone } // AssembleResult reports what one assemble run produced. @@ -33,25 +36,43 @@ type AssembleResult struct { // the game resolves it (upstream's resman adds haks in reverse and lets the // last one win). Get this backwards and the wrong texture ships silently. func Assemble(options AssembleOptions) (AssembleResult, error) { - if len(options.Order) == 0 { - return AssembleResult{}, fmt.Errorf("assemble: --order names no artifacts") + if len(options.ArtifactKeys) == 0 { + return AssembleResult{}, fmt.Errorf("assemble: no artifact keys given") + } + + target, err := openSink(options.OutDir, options.Sink) + if err != nil { + return AssembleResult{}, err + } + + // The TLK carries no precedence — it is not a hak and shadows nothing — + // so it merges last, after every hak has had its say. + keys := append([]string{}, options.ArtifactKeys...) + if options.TLKKey != "" { + keys = append(keys, options.TLKKey) } merged := make([]Entry, 0, 1024) winner := make(map[Identity]bool, 1024) var onDiskBytes int64 - for _, name := range options.Order { - manifestPath := filepath.Join(options.EntriesDir, name+".nsym") - data, err := os.ReadFile(manifestPath) + for _, artifactKey := range keys { + key, err := resolveIndexKey(artifactKey, options.OutDir) if err != nil { - return AssembleResult{}, fmt.Errorf("assemble: no index for %q: %w", name, err) + return AssembleResult{}, err + } + data, sidecarBody, err := target.getIndex(key) + if err != nil { + // An artifact with no index is an artifact whose emit never + // finished. Publishing a manifest without it would ship a release + // missing a hak, so this fails closed. + return AssembleResult{}, fmt.Errorf("assemble: no index for %s: %w", artifactKey, err) } entries, err := readManifest(data) if err != nil { - return AssembleResult{}, fmt.Errorf("%s: %w", manifestPath, err) + return AssembleResult{}, fmt.Errorf("%s: %w", target.describe(key), err) } - sidecar, err := readSidecar(manifestPath + ".json") + sidecar, err := parseSidecar(target.describe(key), sidecarBody) if err != nil { return AssembleResult{}, err } @@ -61,7 +82,7 @@ func Assemble(options AssembleOptions) (AssembleResult, error) { if sidecar.EmitterVersion != emitterVersion { return AssembleResult{}, fmt.Errorf( "assemble: emitter version mismatch: %s was emitted by emitter %q, this is emitter %q", - name, sidecar.EmitterVersion, emitterVersion) + artifactKey, sidecar.EmitterVersion, emitterVersion) } // on_disk_bytes overcounts by the handful of cross-artifact // duplicates. It is a display statistic; no dedupe pass for it. @@ -86,26 +107,22 @@ func Assemble(options AssembleOptions) (AssembleResult, error) { return AssembleResult{}, err } sha1Hex := fmt.Sprintf("%x", sha1.Sum(data)) - manifestPath := filepath.Join(options.OutDir, "manifests", sha1Hex) + manifestKey := path.Join("manifests", sha1Hex) sidecar := Sidecar{ ModuleName: options.ModuleName, Description: options.Description, GroupID: options.GroupID, } - if err := writeManifestPair(manifestPath, data, merged, onDiskBytes, sidecar); err != nil { + if err := putManifestPair(target, manifestKey, data, merged, onDiskBytes, sidecar); err != nil { return AssembleResult{}, err } - return AssembleResult{SHA1: sha1Hex, ManifestPath: manifestPath, Entries: len(merged)}, nil + return AssembleResult{SHA1: sha1Hex, ManifestPath: target.describe(manifestKey), Entries: len(merged)}, nil } -func readSidecar(path string) (Sidecar, error) { - data, err := os.ReadFile(path) - if err != nil { - return Sidecar{}, fmt.Errorf("assemble: missing sidecar: %w", err) - } +func parseSidecar(where string, data []byte) (Sidecar, error) { var sidecar Sidecar if err := json.Unmarshal(data, &sidecar); err != nil { - return Sidecar{}, fmt.Errorf("%s: %w", path, err) + return Sidecar{}, fmt.Errorf("%s: %w", where, err) } return sidecar, nil } diff --git a/internal/nwsync/emit.go b/internal/nwsync/emit.go index 8317ae7..df00477 100644 --- a/internal/nwsync/emit.go +++ b/internal/nwsync/emit.go @@ -2,9 +2,11 @@ package nwsync import ( "bytes" + "context" "crypto/sha1" "fmt" "os" + "path" "path/filepath" "slices" "sort" @@ -60,42 +62,86 @@ type EmitResult struct { BlobsWritten int } -// Emit explodes one artifact — a .hak/.erf/.mod or a loose file such as the -// TLK — into NWSync blobs plus a NSYM manifest describing only that artifact. -func Emit(artifactPath, outDir string) (EmitResult, error) { - resources, err := readArtifact(artifactPath) +// EmitOptions describes one emit run. +type EmitOptions struct { + ArtifactKey string // depot key of the artifact; the NSYM key is derived from it + ArtifactPath string // the file on disk + As string // name override, for a TLK whose filename is not its published name + OutDir string // write locally instead of uploading — the conformance path + Sink sink // test seam; nil means OutDir or the zone +} + +// Emit explodes one artifact — a .hak/.erf or a loose file such as the TLK — +// into NWSync blobs plus a NSYM manifest describing only that artifact. +// +// Blobs go up as they are produced and the index lands last, so the presence of +// an index is the publication marker: an artifact whose emit died halfway has +// real blobs in the zone and no index, which is unambiguous. Blob names are +// content hashes, so re-running skips whatever already landed. +func Emit(options EmitOptions) (EmitResult, error) { + artifact, err := os.ReadFile(options.ArtifactPath) + if err != nil { + return EmitResult{}, fmt.Errorf("read artifact: %w", err) + } + if err := checkArtifactKey(options.ArtifactKey, artifact); err != nil { + return EmitResult{}, err + } + name := options.As + if name == "" { + name = path.Base(options.ArtifactKey) + } + extension := path.Ext(name) + name = strings.TrimSuffix(name, extension) + + key, err := resolveIndexKey(options.ArtifactKey, options.OutDir) if err != nil { return EmitResult{}, err } - name := strings.TrimSuffix(filepath.Base(artifactPath), filepath.Ext(artifactPath)) - entries, blobs, onDiskBytes, err := emitResources(resources, outDir) + resources, err := readArtifact(options.ArtifactPath, artifact, name) + if err != nil { + return EmitResult{}, err + } + + target, err := openSink(options.OutDir, options.Sink) + if err != nil { + return EmitResult{}, err + } + + entries, blobs, onDiskBytes, err := emitResources(resources, target) if err != nil { return EmitResult{}, err } if len(entries) == 0 { - return EmitResult{}, fmt.Errorf("%s: nothing to index (no publishable resources)", artifactPath) + return EmitResult{}, fmt.Errorf("%s: nothing to index (no publishable resources)", options.ArtifactPath) } data, err := writeManifest(entries) if err != nil { return EmitResult{}, err } - manifestPath := filepath.Join(outDir, name+".nsym") - if err := writeManifestPair(manifestPath, data, entries, onDiskBytes, Sidecar{ModuleName: name}); err != nil { + if err := putManifestPair(target, key, data, entries, onDiskBytes, Sidecar{ModuleName: name}); err != nil { return EmitResult{}, err } - return EmitResult{Name: name, ManifestPath: manifestPath, Entries: len(entries), BlobsWritten: blobs}, nil + return EmitResult{Name: name, ManifestPath: target.describe(key), Entries: len(entries), BlobsWritten: blobs}, nil +} + +// openSink returns the zone sink, or a local directory when outDir is set. +func openSink(outDir string, injected sink) (sink, error) { + if injected != nil { + return injected, nil + } + if outDir != "" { + return dirSink{root: outDir}, nil + } + return newZoneSink(context.Background(), os.Getenv) } // readArtifact returns the resources of an ERF/HAK/MOD, or the single resource // a loose file represents. Upstream's resman does the same dispatch on the -// file's first three bytes. -func readArtifact(path string) ([]erf.Resource, error) { - data, err := os.ReadFile(path) - if err != nil { - return nil, fmt.Errorf("read artifact: %w", err) - } +// file's first three bytes. name is the artifact's published name, which for a +// loose file is also its resref. +func readArtifact(path string, data []byte, name string) ([]erf.Resource, error) { if len(data) >= 3 { switch string(data[:3]) { case "ERF", "HAK": @@ -111,20 +157,19 @@ func readArtifact(path string) ([]erf.Resource, error) { return nil, fmt.Errorf("%s: a module is never emitted; a manifest is haks plus the TLK", path) } } - base := filepath.Base(path) - extension := filepath.Ext(base) + extension := filepath.Ext(filepath.Base(path)) restype, ok := erf.ResourceTypeForExtension(extension) if !ok { return nil, fmt.Errorf("%s: unknown resource type %q", path, extension) } return []erf.Resource{{ - Name: strings.TrimSuffix(base, extension), + Name: name, Type: restype, Data: data, }}, nil } -func emitResources(resources []erf.Resource, outDir string) ([]Entry, int, int64, error) { +func emitResources(resources []erf.Resource, target sink) ([]Entry, int, int64, error) { // A resref appearing twice inside one artifact resolves to the last one, // the way resman lets the last container added win. order := make([]Identity, 0, len(resources)) @@ -164,7 +209,7 @@ func emitResources(resources []erf.Resource, outDir string) ([]Entry, int, int64 ResRef: identity.ResRef, ResType: identity.ResType, }) - written, err := writeBlob(outDir, sum, resource.Data) + written, err := target.putBlob(fmt.Sprintf("%x", sum), func() []byte { return compressBlob(resource.Data) }) if err != nil { return nil, 0, 0, err } @@ -187,35 +232,10 @@ func created() int64 { return time.Now().Unix() } -// writeBlob writes one NWCompressedBuffer blob and returns its size on disk, -// or 0 if the blob already existed. Blob names are content hashes, so an -// existing name is existing content. -func writeBlob(outDir string, sum [20]byte, data []byte) (int64, error) { - path := blobPath(outDir, fmt.Sprintf("%x", sum)) - if _, err := os.Stat(path); err == nil { - return 0, nil - } - if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { - return 0, fmt.Errorf("create blob directory: %w", err) - } - blob := compressBlob(data) - if err := os.WriteFile(path, blob, 0o644); err != nil { - return 0, fmt.Errorf("write blob: %w", err) - } - return int64(len(blob)), nil -} - -// writeManifestPair writes a NSYM manifest and its .json sidecar. The caller -// supplies the sidecar fields it knows; the rest are derived from the entries. -// data must be the serialised form of entries. -func writeManifestPair(manifestPath string, data []byte, entries []Entry, onDiskBytes int64, sidecar Sidecar) error { - if err := os.MkdirAll(filepath.Dir(manifestPath), 0o755); err != nil { - return fmt.Errorf("create manifest directory: %w", err) - } - if err := os.WriteFile(manifestPath, data, 0o644); err != nil { - return fmt.Errorf("write manifest: %w", err) - } - +// putManifestPair stores a NSYM manifest and its .json sidecar at key. The +// caller supplies the sidecar fields it knows; the rest are derived from the +// entries. data must be the serialised form of entries. +func putManifestPair(target sink, key string, data []byte, entries []Entry, onDiskBytes int64, sidecar Sidecar) error { var totalBytes int64 clientContents := false for _, entry := range entries { @@ -240,8 +260,5 @@ func writeManifestPair(manifestPath string, data []byte, entries []Entry, onDisk if err != nil { return err } - if err := os.WriteFile(manifestPath+".json", body, 0o644); err != nil { - return fmt.Errorf("write sidecar: %w", err) - } - return nil + return target.putIndex(key, data, body) } diff --git a/internal/nwsync/nwsync_test.go b/internal/nwsync/nwsync_test.go index 5af837f..0eb5c5c 100644 --- a/internal/nwsync/nwsync_test.go +++ b/internal/nwsync/nwsync_test.go @@ -3,6 +3,7 @@ package nwsync import ( "bytes" "crypto/sha1" + "crypto/sha256" "encoding/binary" "encoding/hex" "encoding/json" @@ -45,6 +46,30 @@ func writeHak(t *testing.T, path string, contents map[string][]byte) { } } +// artifactKey is the depot key a file would be published under: the sha256 of +// its bytes, hash-tree depth 2, keeping the extension. +func artifactKey(t *testing.T, path string) string { + t.Helper() + body, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + sum := sha256.Sum256(body) + digest := hex.EncodeToString(sum[:]) + return "artifacts/haks/sha256/" + digest[0:2] + "/" + digest[2:4] + "/" + digest + filepath.Ext(path) +} + +// emitLocal emits one artifact into a local tree, the conformance path. +func emitLocal(t *testing.T, path, out string) (EmitResult, error) { + t.Helper() + return Emit(EmitOptions{ + ArtifactKey: artifactKey(t, path), + ArtifactPath: path, + As: filepath.Base(path), + OutDir: out, + }) +} + func TestBlobFramingRoundTrips(t *testing.T) { data := []byte("the quick brown fox jumps over the lazy dog, repeatedly and at length") blob := compressBlob(data) @@ -143,7 +168,7 @@ func TestEmitWritesBlobsAndManifest(t *testing.T) { }) out := filepath.Join(dir, "out") - result, err := Emit(hak, out) + result, err := emitLocal(t, hak, out) if err != nil { t.Fatalf("emit: %v", err) } @@ -171,7 +196,7 @@ func TestEmitWritesBlobsAndManifest(t *testing.T) { t.Errorf("blob decompressed to %q, want %q", got, body) } - entries := readEmitted(t, out, "sow_test_01") + entries := readEmitted(t, result.ManifestPath) for _, entry := range entries { if entry.ResType == restype(t, "nss") || entry.ResType == restype(t, "ndb") || entry.ResType == restype(t, "gic") { t.Errorf("skipped restype leaked into the manifest: %+v", entry) @@ -200,9 +225,9 @@ func TestEmitWritesBlobsAndManifest(t *testing.T) { } } -func readEmitted(t *testing.T, dir, name string) []Entry { +func readEmitted(t *testing.T, indexPath string) []Entry { t.Helper() - data, err := os.ReadFile(filepath.Join(dir, name+".nsym")) + data, err := os.ReadFile(indexPath) if err != nil { t.Fatalf("read emitted manifest: %v", err) } @@ -218,7 +243,7 @@ func TestEmitFailsClosedOnOversizeResource(t *testing.T) { hak := filepath.Join(dir, "big.hak") writeHak(t, hak, map[string][]byte{"huge1.tga": make([]byte, fileSizeLimit+1)}) - if _, err := Emit(hak, filepath.Join(dir, "out")); err == nil { + if _, err := emitLocal(t, hak, filepath.Join(dir, "out")); err == nil { t.Fatal("emit accepted a resource over the 15 MB limit") } } @@ -230,11 +255,11 @@ func TestEmitLooseFile(t *testing.T) { t.Fatal(err) } out := filepath.Join(dir, "out") - result, err := Emit(tlk, out) + result, err := emitLocal(t, tlk, out) if err != nil { t.Fatalf("emit tlk: %v", err) } - entries := readEmitted(t, out, "sow_tlk") + entries := readEmitted(t, result.ManifestPath) if len(entries) != 1 || entries[0].ResRef != "sow_tlk" || entries[0].ResType != restype(t, "tlk") { t.Fatalf("tlk emitted as %+v", entries) } @@ -245,34 +270,35 @@ func TestEmitLooseFile(t *testing.T) { // emitFixture emits two haks that share a resref, so the merge rule is // observable: "top" holds the winning body, "assets" the shadowed one. -func emitFixture(t *testing.T) (string, []byte, []byte) { +func emitFixture(t *testing.T) (out string, keys map[string]string, topBody, assetBody []byte) { t.Helper() dir := t.TempDir() - topBody := []byte("2da from sow_top") - assetBody := []byte("2da from the asset hak") + topBody = []byte("2da from sow_top") + assetBody = []byte("2da from the asset hak") writeHak(t, filepath.Join(dir, "sow_top.hak"), map[string][]byte{"appearance.2da": topBody}) writeHak(t, filepath.Join(dir, "sow_core_01.hak"), map[string][]byte{ "appearance.2da": assetBody, "bloodstain1.tga": []byte("blood"), }) - out := filepath.Join(dir, "out") + out = filepath.Join(dir, "out") + keys = map[string]string{} for _, name := range []string{"sow_top", "sow_core_01"} { - if _, err := Emit(filepath.Join(dir, name+".hak"), out); err != nil { + path := filepath.Join(dir, name+".hak") + keys[name] = artifactKey(t, path) + if _, err := emitLocal(t, path, out); err != nil { t.Fatalf("emit %s: %v", name, err) } } - return out, topBody, assetBody + return out, keys, topBody, assetBody } func TestAssembleShadowsByOrder(t *testing.T) { - entriesDir, topBody, assetBody := emitFixture(t) - out := t.TempDir() + entriesDir, keys, topBody, assetBody := emitFixture(t) result, err := Assemble(AssembleOptions{ - Order: []string{"sow_top", "sow_core_01"}, - EntriesDir: entriesDir, - OutDir: out, - GroupID: 2, + ArtifactKeys: []string{keys["sow_top"], keys["sow_core_01"]}, + OutDir: entriesDir, + GroupID: 2, }) if err != nil { t.Fatalf("assemble: %v", err) @@ -325,13 +351,11 @@ func TestAssembleShadowsByOrder(t *testing.T) { } func TestAssembleReversedOrderPicksTheOtherHak(t *testing.T) { - entriesDir, topBody, assetBody := emitFixture(t) - out := t.TempDir() + entriesDir, keys, topBody, assetBody := emitFixture(t) result, err := Assemble(AssembleOptions{ - Order: []string{"sow_core_01", "sow_top"}, - EntriesDir: entriesDir, - OutDir: out, + ArtifactKeys: []string{keys["sow_core_01"], keys["sow_top"]}, + OutDir: entriesDir, }) if err != nil { t.Fatalf("assemble: %v", err) @@ -350,9 +374,13 @@ func TestAssembleReversedOrderPicksTheOtherHak(t *testing.T) { } func TestAssembleRefusesMismatchedEmitterVersions(t *testing.T) { - entriesDir, _, _ := emitFixture(t) + entriesDir, keys, _, _ := emitFixture(t) - path := filepath.Join(entriesDir, "sow_core_01.nsym.json") + index, err := resolveIndexKey(keys["sow_core_01"], entriesDir) + if err != nil { + t.Fatal(err) + } + path := filepath.Join(entriesDir, index+".json") var sidecar Sidecar body, err := os.ReadFile(path) if err != nil { @@ -371,9 +399,8 @@ func TestAssembleRefusesMismatchedEmitterVersions(t *testing.T) { } _, err = Assemble(AssembleOptions{ - Order: []string{"sow_top", "sow_core_01"}, - EntriesDir: entriesDir, - OutDir: t.TempDir(), + ArtifactKeys: []string{keys["sow_top"], keys["sow_core_01"]}, + OutDir: entriesDir, }) if err == nil || !strings.Contains(err.Error(), "emitter version mismatch") { t.Fatalf("assemble merged across emitter versions: %v", err) @@ -385,7 +412,7 @@ func TestEmitHonoursSourceDateEpoch(t *testing.T) { dir := t.TempDir() hak := filepath.Join(dir, "pinned.hak") writeHak(t, hak, map[string][]byte{"one1.tga": []byte("body")}) - result, err := Emit(hak, filepath.Join(dir, "out")) + result, err := emitLocal(t, hak, filepath.Join(dir, "out")) if err != nil { t.Fatalf("emit: %v", err) } @@ -414,19 +441,19 @@ func TestEmitRejectsAModule(t *testing.T) { if err := os.WriteFile(path, out.Bytes(), 0o644); err != nil { t.Fatal(err) } - if _, err := Emit(path, filepath.Join(dir, "out")); err == nil { + if _, err := emitLocal(t, path, filepath.Join(dir, "out")); err == nil { t.Fatal("emit accepted a .mod; a manifest never carries module contents") } } func TestAssembleFailsClosedOnMissingIndex(t *testing.T) { - entriesDir, _, _ := emitFixture(t) + entriesDir, keys, _, _ := emitFixture(t) + missing := "artifacts/haks/sha256/00/11/" + strings.Repeat("0", 64) + ".hak" _, err := Assemble(AssembleOptions{ - Order: []string{"sow_top", "sow_never_published"}, - EntriesDir: entriesDir, - OutDir: t.TempDir(), + ArtifactKeys: []string{keys["sow_top"], missing}, + OutDir: entriesDir, }) - if err == nil || !strings.Contains(err.Error(), "sow_never_published") { + if err == nil || !strings.Contains(err.Error(), missing) { t.Fatalf("assemble did not fail closed and name the missing artifact: %v", err) } } @@ -436,10 +463,9 @@ func TestRunUsageErrors(t *testing.T) { nil, {"nope"}, {"emit"}, - {"emit", "artifact.hak"}, - {"assemble", "--entries", "x", "--out", "y"}, - {"assemble", "--order", "a", "--out", "y"}, - {"assemble", "--order", "a", "--entries", "x"}, + {"emit", "artifact-key.hak"}, + {"emit", "a", "b", "c"}, + {"assemble", "--out", "y"}, } for _, args := range cases { var out, errw bytes.Buffer diff --git a/internal/nwsync/run.go b/internal/nwsync/run.go index 53a7365..40dffd2 100644 --- a/internal/nwsync/run.go +++ b/internal/nwsync/run.go @@ -12,7 +12,6 @@ import ( "flag" "fmt" "io" - "strings" ) const ( @@ -45,37 +44,63 @@ func Run(args []string, stdout, stderr io.Writer) int { func printRunUsage(w io.Writer) { fmt.Fprint(w, `usage: - nwsync emit --out DIR - nwsync assemble --order NAMES --entries DIR --out DIR [--group-id N] + nwsync emit [--as NAME] [--out DIR] + nwsync assemble --group-id N [--tlk-key KEY] [--out DIR] ... emit explodes one .hak/.erf or one loose file (the TLK) into NWSync blobs plus -a NSYM manifest covering only that artifact. assemble merges those per-artifact -manifests into one, reading no bulk data. +a NSYM index covering only that artifact, and uploads both. assemble merges +those indexes into one manifest, reading no bulk data. Artifact keys are depot +keys; an index lives beside its artifact, with the extension replaced. + +--out DIR writes to a local repository tree instead of uploading, which is the +conformance path against upstream nwn_nwsync_write. Without it, the zone comes +from NWSYNC_STORAGE_ZONE, NWSYNC_STORAGE_PASSWORD and BUNNY_STORAGE_HOST. `) } +// parseArgs parses flags that may appear before, after or between positionals. +// Go's flag package stops at the first non-flag argument, which turns +// `emit --out DIR` into a confusing arity error. +func parseArgs(fs *flag.FlagSet, args []string) ([]string, error) { + var positional []string + for { + if err := fs.Parse(args); err != nil { + return nil, err + } + rest := fs.Args() + if len(rest) == 0 { + return positional, nil + } + positional = append(positional, rest[0]) + args = rest[1:] + } +} + func runEmit(args []string, stdout, stderr io.Writer) int { fs := flag.NewFlagSet("emit", flag.ContinueOnError) fs.SetOutput(stderr) - out := fs.String("out", "", "output directory (blobs plus the per-artifact manifest)") - if err := fs.Parse(args); err != nil { + as := fs.String("as", "", "published name of the artifact, when it differs from the key") + out := fs.String("out", "", "write to a local repository tree instead of uploading") + positional, err := parseArgs(fs, args) + if err != nil { return exitUsage } - if fs.NArg() != 1 { - fmt.Fprintf(stderr, "nwsync emit: exactly one artifact is required\n") - return exitUsage - } - if *out == "" { - fmt.Fprintf(stderr, "nwsync emit: --out is required\n") + if len(positional) != 2 { + fmt.Fprintf(stderr, "nwsync emit: and are both required\n") return exitUsage } - result, err := Emit(fs.Arg(0), *out) + result, err := Emit(EmitOptions{ + ArtifactKey: positional[0], + ArtifactPath: positional[1], + As: *as, + OutDir: *out, + }) if err != nil { fmt.Fprintf(stderr, "nwsync emit: %v\n", err) return exitInternal } - fmt.Fprintf(stdout, "emitted %s: %d resources, %d new blobs, manifest %s\n", + fmt.Fprintf(stdout, "emitted %s: %d resources, %d new blobs, index %s\n", result.Name, result.Entries, result.BlobsWritten, result.ManifestPath) return exitOK } @@ -83,47 +108,33 @@ func runEmit(args []string, stdout, stderr io.Writer) int { func runAssemble(args []string, stdout, stderr io.Writer) int { fs := flag.NewFlagSet("assemble", flag.ContinueOnError) fs.SetOutput(stderr) - order := fs.String("order", "", "comma-separated artifact names, highest priority first") - entries := fs.String("entries", "", "directory holding the per-artifact .nsym files") - out := fs.String("out", "", "repository root; the manifest lands in /manifests") + tlkKey := fs.String("tlk-key", "", "depot key of the TLK, which shadows nothing and merges last") + out := fs.String("out", "", "write to a local repository tree instead of uploading") groupID := fs.Int("group-id", 0, "NWSync group id (1 = current, 2 = testing; 0 omits it)") moduleName := fs.String("module-name", "", "module name recorded in the sidecar") description := fs.String("description", "", "description recorded in the sidecar") - if err := fs.Parse(args); err != nil { + positional, err := parseArgs(fs, args) + if err != nil { return exitUsage } - switch { - case *order == "": - fmt.Fprintf(stderr, "nwsync assemble: --order is required\n") + if len(positional) == 0 { + fmt.Fprintf(stderr, "nwsync assemble: at least one artifact key is required\n") return exitUsage - case *entries == "": - fmt.Fprintf(stderr, "nwsync assemble: --entries is required\n") - return exitUsage - case *out == "": - fmt.Fprintf(stderr, "nwsync assemble: --out is required\n") - return exitUsage - } - - names := make([]string, 0, 8) - for _, name := range strings.Split(*order, ",") { - if name = strings.TrimSpace(name); name != "" { - names = append(names, name) - } } result, err := Assemble(AssembleOptions{ - Order: names, - EntriesDir: *entries, - OutDir: *out, - GroupID: *groupID, - ModuleName: *moduleName, - Description: *description, + ArtifactKeys: positional, + TLKKey: *tlkKey, + OutDir: *out, + GroupID: *groupID, + ModuleName: *moduleName, + Description: *description, }) if err != nil { fmt.Fprintf(stderr, "nwsync assemble: %v\n", err) return exitInternal } - fmt.Fprintf(stdout, "assembled %s: %d resources from %d artifacts\n", - result.SHA1, result.Entries, len(names)) + fmt.Fprintf(stdout, "assembled manifest %s: %d resources, %s\n", + result.SHA1, result.Entries, result.ManifestPath) return exitOK } diff --git a/internal/nwsync/sink.go b/internal/nwsync/sink.go new file mode 100644 index 0000000..f6ce812 --- /dev/null +++ b/internal/nwsync/sink.go @@ -0,0 +1,207 @@ +package nwsync + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "os" + "path" + "path/filepath" + "strings" + + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/depot" +) + +// sink is where an emit or assemble run puts what it produces. The zone is the +// production sink; a local directory exists only as the conformance path, so +// upstream's output and ours can be diffed on a developer machine. +type sink interface { + // putBlob stores one NWCompressedBuffer blob under its sha1 name and + // returns the bytes stored, or 0 if the blob was already there. Blob names + // are content hashes, so an existing name is existing content — which is + // why body is a thunk: compression is the expensive part of emit and a + // blob that is already stored must not pay for it. + putBlob(sha1Hex string, body func() []byte) (int64, error) + // putIndex stores a NSYM manifest and its sidecar under key, which is + // either an artifact-derived object key or a local path. + putIndex(key string, manifest, sidecar []byte) error + // getIndex reads back a NSYM manifest and its sidecar. + getIndex(key string) (manifest, sidecar []byte, err error) + // describe names the sink for messages. + describe(key string) string +} + +// dirSink writes a local NWSync repository tree. +type dirSink struct{ root string } + +func (s dirSink) putBlob(sha1Hex string, body func() []byte) (int64, error) { + blob := blobPath(s.root, sha1Hex) + if _, err := os.Stat(blob); err == nil { + return 0, nil + } + if err := os.MkdirAll(filepath.Dir(blob), 0o755); err != nil { + return 0, fmt.Errorf("create blob directory: %w", err) + } + data := body() + if err := os.WriteFile(blob, data, 0o644); err != nil { + return 0, fmt.Errorf("write blob: %w", err) + } + return int64(len(data)), nil +} + +func (s dirSink) putIndex(key string, manifest, sidecar []byte) error { + target := filepath.Join(s.root, filepath.FromSlash(key)) + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + return fmt.Errorf("create manifest directory: %w", err) + } + if err := os.WriteFile(target, manifest, 0o644); err != nil { + return fmt.Errorf("write manifest: %w", err) + } + if err := os.WriteFile(target+".json", sidecar, 0o644); err != nil { + return fmt.Errorf("write sidecar: %w", err) + } + return nil +} + +func (s dirSink) getIndex(key string) ([]byte, []byte, error) { + target := filepath.Join(s.root, filepath.FromSlash(key)) + manifest, err := os.ReadFile(target) + if err != nil { + return nil, nil, fmt.Errorf("read index: %w", err) + } + sidecar, err := os.ReadFile(target + ".json") + if err != nil { + return nil, nil, fmt.Errorf("read sidecar: %w", err) + } + return manifest, sidecar, nil +} + +func (s dirSink) describe(key string) string { + return filepath.Join(s.root, filepath.FromSlash(key)) +} + +// zoneSink uploads straight to the NWSync storage zone. Nothing bulky is ever +// written to the runner's disk: the working set is one resource at a time. +type zoneSink struct { + store depot.KeyStore + ctx context.Context + zone string +} + +func (s zoneSink) putBlob(sha1Hex string, body func() []byte) (int64, error) { + key := path.Join("data", "sha1", sha1Hex[0:2], sha1Hex[2:4], sha1Hex) + // A throttled probe must never be read as "missing, re-upload" or as + // "present, skip", so only a confirmed Present skips the upload. + state, _, err := s.store.ProbeKey(s.ctx, key) + if err != nil { + return 0, fmt.Errorf("probe blob %s: %w", sha1Hex, err) + } + if state == depot.Present { + return 0, nil + } + data := body() + if err := s.put(key, data); err != nil { + return 0, fmt.Errorf("upload blob %s: %w", sha1Hex, err) + } + return int64(len(data)), nil +} + +func (s zoneSink) putIndex(key string, manifest, sidecar []byte) error { + // The manifest lands last: its presence is the publication marker, so it + // must never appear before the blobs it names. + if err := s.put(key+".json", sidecar); err != nil { + return fmt.Errorf("upload sidecar %s: %w", key, err) + } + if err := s.put(key, manifest); err != nil { + return fmt.Errorf("upload index %s: %w", key, err) + } + return nil +} + +func (s zoneSink) getIndex(key string) ([]byte, []byte, error) { + manifest, err := s.store.GetKey(s.ctx, key) + if err != nil { + return nil, nil, fmt.Errorf("read index: %w", err) + } + sidecar, err := s.store.GetKey(s.ctx, key+".json") + if err != nil { + return nil, nil, fmt.Errorf("read sidecar: %w", err) + } + return manifest, sidecar, nil +} + +func (s zoneSink) describe(key string) string { return s.zone + "/" + key } + +func (s zoneSink) put(key string, body []byte) error { + sum := sha256.Sum256(body) + return s.store.PutReader(s.ctx, key, bytes.NewReader(body), int64(len(body)), hex.EncodeToString(sum[:])) +} + +// newZoneSink builds the upload sink from the environment. NWSync data lives +// in its own zone, separate from the asset depot, so it has its own zone and +// credential; only the host is shared, and Crucible has no default host. +func newZoneSink(ctx context.Context, getenv func(string) string) (sink, error) { + cfg := depot.LoadConfig(getenv) + cfg.StorageZone = getenv("NWSYNC_STORAGE_ZONE") + cfg.WriteKey = getenv("NWSYNC_STORAGE_PASSWORD") + cfg.ReadKey = cfg.WriteKey + if cfg.StorageZone == "" { + return nil, fmt.Errorf("NWSYNC_STORAGE_ZONE is unset (or pass --out DIR to write locally)") + } + if cfg.WriteKey == "" { + return nil, fmt.Errorf("NWSYNC_STORAGE_PASSWORD is unset (or pass --out DIR to write locally)") + } + if cfg.StorageHost == "" { + return nil, fmt.Errorf("BUNNY_STORAGE_HOST is unset") + } + store, err := depot.NewKeyStore(cfg) + if err != nil { + return nil, err + } + return zoneSink{store: store, ctx: ctx, zone: cfg.StorageZone}, nil +} + +// indexKey is where an artifact's NSYM lives: beside the artifact itself, with +// the final extension replaced. emit and assemble must agree on this one rule, +// so it lives here and nowhere else. +// +// artifacts/haks/sha256/30/46/3046….hak -> artifacts/haks/sha256/30/46/3046….nsym +func indexKey(artifactKey string) (string, error) { + extension := path.Ext(artifactKey) + if extension == "" { + return "", fmt.Errorf("artifact key %q has no extension", artifactKey) + } + return strings.TrimSuffix(artifactKey, extension) + ".nsym", nil +} + +// resolveIndexKey is where emit writes an artifact's index and where assemble +// reads it from. On the zone that is beside the artifact; locally the indexes +// sit flat beside the data tree, so upstream's output and ours diff directly. +func resolveIndexKey(artifactKey, outDir string) (string, error) { + key, err := indexKey(artifactKey) + if err != nil { + return "", err + } + if outDir != "" { + return path.Base(key), nil + } + return key, nil +} + +// checkArtifactKey fails closed when the key's embedded digest is not the +// digest of the bytes being emitted. Publishing an index under the wrong key +// silently pairs a manifest with the wrong artifact. +func checkArtifactKey(artifactKey string, artifact []byte) error { + base := path.Base(artifactKey) + digest := strings.TrimSuffix(base, path.Ext(base)) + if len(digest) != 64 { + return fmt.Errorf("artifact key %q does not name a sha256", artifactKey) + } + sum := sha256.Sum256(artifact) + if got := hex.EncodeToString(sum[:]); got != digest { + return fmt.Errorf("artifact key %q names digest %s but the file hashes to %s", artifactKey, digest, got) + } + return nil +} diff --git a/internal/nwsync/zone_test.go b/internal/nwsync/zone_test.go new file mode 100644 index 0000000..eeb4df3 --- /dev/null +++ b/internal/nwsync/zone_test.go @@ -0,0 +1,249 @@ +package nwsync + +import ( + "crypto/sha1" + "crypto/sha256" + "encoding/hex" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" +) + +// fakeZone is a Bunny-shaped object store: PUT stores, GET reads, and the +// Checksum header is verified the way Bunny verifies it. +type fakeZone struct { + mu sync.Mutex + objects map[string][]byte + puts []string + failOn func(key string) bool // when true, the PUT fails +} + +func newFakeZone(t *testing.T) (*fakeZone, func(string) string) { + t.Helper() + zone := &fakeZone{objects: map[string][]byte{}} + server := httptest.NewServer(zone) + t.Cleanup(server.Close) + getenv := func(name string) string { + switch name { + case "NWSYNC_STORAGE_ZONE": + return "sow-nwsync" + case "NWSYNC_STORAGE_PASSWORD": + return "write-key" + case "BUNNY_STORAGE_HOST": + return server.URL + } + return "" + } + return zone, getenv +} + +func (z *fakeZone) ServeHTTP(w http.ResponseWriter, r *http.Request) { + key := strings.TrimPrefix(r.URL.Path, "/sow-nwsync/") + switch r.Method { + case http.MethodPut: + if z.failOn != nil && z.failOn(key) { + http.Error(w, "boom", http.StatusInternalServerError) + return + } + body, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + sum := sha256.Sum256(body) + if want := strings.ToUpper(hex.EncodeToString(sum[:])); r.Header.Get("Checksum") != want { + http.Error(w, "checksum mismatch", http.StatusBadRequest) + return + } + z.mu.Lock() + z.objects[key] = body + z.puts = append(z.puts, key) + z.mu.Unlock() + w.WriteHeader(http.StatusCreated) + case http.MethodGet: + z.mu.Lock() + body, ok := z.objects[key] + z.mu.Unlock() + if !ok { + http.Error(w, "not found", http.StatusNotFound) + return + } + _, _ = w.Write(body) + default: + http.Error(w, "unsupported", http.StatusMethodNotAllowed) + } +} + +func (z *zoneSinkFixture) emit(t *testing.T, path string) EmitResult { + t.Helper() + result, err := Emit(EmitOptions{ + ArtifactKey: artifactKey(t, path), + ArtifactPath: path, + Sink: z.sink, + }) + if err != nil { + t.Fatalf("emit %s: %v", path, err) + } + return result +} + +type zoneSinkFixture struct { + zone *fakeZone + sink sink +} + +func newZoneFixture(t *testing.T) *zoneSinkFixture { + t.Helper() + zone, getenv := newFakeZone(t) + target, err := newZoneSink(t.Context(), getenv) + if err != nil { + t.Fatalf("zone sink: %v", err) + } + return &zoneSinkFixture{zone: zone, sink: target} +} + +func sha1Of(body []byte) [20]byte { return sha1.Sum(body) } + +func TestEmitUploadsBlobsThenIndex(t *testing.T) { + fixture := newZoneFixture(t) + dir := t.TempDir() + hak := filepath.Join(dir, "sow_test_01.hak") + body := []byte("texture bytes") + writeHak(t, hak, map[string][]byte{"bloodstain1.tga": body, "copy1.txi": body}) + + key := artifactKey(t, hak) + result := fixture.emit(t, hak) + + index, err := indexKey(key) + if err != nil { + t.Fatal(err) + } + if _, ok := fixture.zone.objects[index]; !ok { + t.Fatalf("no index at %s; zone holds %v", index, fixture.zone.puts) + } + if result.BlobsWritten != 1 { + t.Errorf("uploaded %d blobs, want 1 (identical content shares a blob)", result.BlobsWritten) + } + + // The index is the publication marker, so it must land after every blob it + // names — including its own sidecar. + last := fixture.zone.puts[len(fixture.zone.puts)-1] + if last != index { + t.Errorf("index landed at position %d of %d; it must be last", len(fixture.zone.puts), len(fixture.zone.puts)) + } + for _, key := range fixture.zone.puts[:len(fixture.zone.puts)-1] { + if strings.HasPrefix(key, "data/sha1/") || key == index+".json" { + continue + } + t.Errorf("unexpected object uploaded before the index: %s", key) + } +} + +func TestEmitSkipsBlobsAlreadyInTheZone(t *testing.T) { + fixture := newZoneFixture(t) + dir := t.TempDir() + hak := filepath.Join(dir, "sow_test_01.hak") + writeHak(t, hak, map[string][]byte{"bloodstain1.tga": []byte("blood")}) + + first := fixture.emit(t, hak) + if first.BlobsWritten != 1 { + t.Fatalf("first emit uploaded %d blobs, want 1", first.BlobsWritten) + } + second := fixture.emit(t, hak) + if second.BlobsWritten != 0 { + t.Errorf("re-emit uploaded %d blobs, want 0 (a blob name is its content)", second.BlobsWritten) + } +} + +func TestEmitLeavesNoIndexWhenAnUploadFails(t *testing.T) { + fixture := newZoneFixture(t) + fixture.zone.failOn = func(key string) bool { return strings.HasPrefix(key, "data/sha1/") } + dir := t.TempDir() + hak := filepath.Join(dir, "sow_test_01.hak") + writeHak(t, hak, map[string][]byte{"bloodstain1.tga": []byte("blood")}) + + _, err := Emit(EmitOptions{ + ArtifactKey: artifactKey(t, hak), + ArtifactPath: hak, + Sink: fixture.sink, + }) + if err == nil { + t.Fatal("emit reported success after an upload failed") + } + for key := range fixture.zone.objects { + if strings.HasSuffix(key, ".nsym") { + t.Errorf("a half-emitted artifact published an index: %s", key) + } + } +} + +func TestEmitRejectsAKeyThatDoesNotMatchTheFile(t *testing.T) { + fixture := newZoneFixture(t) + dir := t.TempDir() + hak := filepath.Join(dir, "sow_test_01.hak") + writeHak(t, hak, map[string][]byte{"bloodstain1.tga": []byte("blood")}) + + wrong := "artifacts/haks/sha256/00/11/" + strings.Repeat("0", 64) + ".hak" + _, err := Emit(EmitOptions{ArtifactKey: wrong, ArtifactPath: hak, Sink: fixture.sink}) + if err == nil || !strings.Contains(err.Error(), "hashes to") { + t.Fatalf("emit published under a key that names another artifact: %v", err) + } +} + +func TestAssembleReadsIndexesFromTheZone(t *testing.T) { + fixture := newZoneFixture(t) + dir := t.TempDir() + topBody := []byte("2da from sow_top") + assetBody := []byte("2da from the asset hak") + top := filepath.Join(dir, "sow_top.hak") + core := filepath.Join(dir, "sow_core_01.hak") + writeHak(t, top, map[string][]byte{"appearance.2da": topBody}) + writeHak(t, core, map[string][]byte{"appearance.2da": assetBody, "bloodstain1.tga": []byte("blood")}) + tlkPath := filepath.Join(dir, "sow_tlk.tlk") + if err := os.WriteFile(tlkPath, []byte("TLK V3.0 payload"), 0o644); err != nil { + t.Fatal(err) + } + + fixture.emit(t, top) + fixture.emit(t, core) + if _, err := Emit(EmitOptions{ + ArtifactKey: artifactKey(t, tlkPath), + ArtifactPath: tlkPath, + As: "sow_tlk.tlk", + Sink: fixture.sink, + }); err != nil { + t.Fatalf("emit tlk: %v", err) + } + + result, err := Assemble(AssembleOptions{ + ArtifactKeys: []string{artifactKey(t, top), artifactKey(t, core)}, + TLKKey: artifactKey(t, tlkPath), + GroupID: 2, + Sink: fixture.sink, + }) + if err != nil { + t.Fatalf("assemble: %v", err) + } + if result.Entries != 3 { + t.Fatalf("merged %d entries, want 3 (appearance.2da is shadowed, the TLK adds one)", result.Entries) + } + + manifest, ok := fixture.zone.objects["manifests/"+result.SHA1] + if !ok { + t.Fatalf("no merged manifest in the zone; it holds %v", fixture.zone.puts) + } + entries, err := readManifest(manifest) + if err != nil { + t.Fatalf("parse merged manifest: %v", err) + } + for _, entry := range entries { + if entry.ResRef == "appearance" && entry.SHA1 != sha1Of(topBody) { + t.Errorf("appearance.2da resolved to the shadowed hak, not the first one given") + } + } +}