nwsync: upload sink, key-addressed CLI, fail-closed publication marker (#73)
build-binaries / build-binaries (push) Successful in 2m18s
build-binaries / build-binaries (push) Successful in 2m18s
Builds the code half of #60, and closes the CLI gap the #53 sweep found: PR #71 shipped #53's original surface rather than the one #56, #62 and #65 settled. ## What lands **`depot.KeyStore`** — `ProbeKey` / `PutReader` / `GetKey`, addressing the zone by object key rather than by depot sha. NWSync cannot use the sha-addressed path: a blob is named after the sha1 of its *uncompressed* bytes while the body uploaded is the compressed form, and Bunny's `Checksum` header is sha256 of the body. Per #55 this reuses `internal/depot`'s `httpBackend` — same IPv4-pinned transport, same retry, same tri-state probe — and the sha-addressed `Backend` is now rewritten on top of it. No second HTTP client, no per-instance hash-function fields: the caller passes the key and the checksum, which turned out simpler than #55 expected. **A sink in `internal/nwsync`** — the zone by default, a local tree under `--out DIR` as the conformance path. Blobs upload as they are produced and the index lands last, so the presence of an index is the publication marker. A blob already in the zone is skipped via #55's probe *without* paying for compression (the body is a thunk) — which matters for the backfill, where compression is the expensive part. **The settled CLI** ``` nwsync emit [--as NAME] [--out DIR] <artifact-key> <file> nwsync assemble --group-id N [--tlk-key KEY] [--out DIR] <artifact-key>... ``` Artifact keys are depot keys; an index lives beside its artifact with the extension replaced (#62), derived in exactly one place so `emit` and `assemble` cannot disagree. Flags may now follow positionals — Go's `flag` stops at the first non-flag argument, which cost a run during #59. **Fail-closed in two places** — an artifact key whose embedded digest does not match the file is refused (publishing an index under the wrong key silently pairs a manifest with the wrong artifact), and `assemble` refuses an artifact with no index rather than publishing a manifest missing a hak. ## Checks `make check` green. Six new tests run against a Bunny-shaped `httptest` zone that verifies the `Checksum` header the way Bunny does: blobs-then-index ordering, skip-if-present, no index after a failed upload, key/file mismatch, and assemble reading indexes back out of the zone. Conformance re-run through the new CLI against upstream `nwn_nwsync_write` 2.1.2 over `sow_vfxs_01.hak` (#59's oracle): the manifest is still **byte-identical**. ## Not in this PR - **Live upload against the real zone.** The nwsync zone and its credential are #61, still open. Everything here is proven against a fake zone only. - **Consumer wiring** — #65, in the three producer repos. - **The mid-hak failure *policy*.** The mechanism is here (fail closed, orphan blobs left, re-run resumes); whether a module release may proceed when an emit failed is a human call, still open on #60. 🤖 Generated with [Claude Code](https://claude.com/claude-code)Reviewed-on: #73 Co-authored-by: vickydotbat <vickydotbat@tutamail.com>
This commit was merged in pull request #73.
This commit is contained in:
@@ -0,0 +1,121 @@
|
||||
package depot
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// KeyStore is the zone addressed by object key rather than by depot sha. The
|
||||
// depot names every object after the sha256 of its contents; NWSync does not —
|
||||
// a blob is named after the sha1 of its *uncompressed* bytes while the body
|
||||
// uploaded is the compressed form, and a per-artifact index is named after its
|
||||
// artifact. Both addressing modes want the same transport, retry and probe
|
||||
// discipline, so the sha-addressed Backend rides on this rather than the other
|
||||
// way round.
|
||||
type KeyStore interface {
|
||||
// ProbeKey returns the existence state of one key. transient=true means a
|
||||
// retry might change the answer — never read it as "missing, re-upload".
|
||||
ProbeKey(ctx context.Context, key string) (state ProbeState, transient bool, err error)
|
||||
// PutReader uploads size bytes read from r to key. checksum is the
|
||||
// uppercase hex sha256 of those bytes, which Bunny verifies server-side.
|
||||
PutReader(ctx context.Context, key string, r io.Reader, size int64, checksum string) error
|
||||
// GetKey fetches the whole object at key. Small objects only — it holds
|
||||
// the body in memory and does no hash check, because a key is not always
|
||||
// a content hash.
|
||||
GetKey(ctx context.Context, key string) ([]byte, error)
|
||||
}
|
||||
|
||||
// NewKeyStore returns a KeyStore for cfg's storage zone. Fails closed on a
|
||||
// missing host or read key, matching NewBackend.
|
||||
func NewKeyStore(cfg Config) (KeyStore, error) {
|
||||
if cfg.StorageHost == "" {
|
||||
return nil, errors.New("storage backend requires a storage host")
|
||||
}
|
||||
if cfg.StorageZone == "" {
|
||||
return nil, errors.New("storage backend requires a storage zone")
|
||||
}
|
||||
if cfg.ReadKey == "" {
|
||||
return nil, errors.New("storage backend requires a read key")
|
||||
}
|
||||
return &httpBackend{name: "bunny", client: newHTTPClient(cfg), cfg: cfg}, nil
|
||||
}
|
||||
|
||||
// keyURL is the storage URL of one object key.
|
||||
func (b *httpBackend) keyURL(key string) string {
|
||||
host := b.cfg.StorageHost
|
||||
// StorageHost is normally a bare host ("storage.bunnycdn.com"); allow a
|
||||
// full scheme (used by tests against httptest.NewServer) to pass through
|
||||
// unchanged.
|
||||
if !strings.Contains(host, "://") {
|
||||
host = "https://" + host
|
||||
}
|
||||
return fmt.Sprintf("%s/%s/%s", strings.TrimSuffix(host, "/"), b.cfg.StorageZone, key)
|
||||
}
|
||||
|
||||
func (b *httpBackend) ProbeKey(ctx context.Context, key string) (ProbeState, bool, error) {
|
||||
return b.rangeProbe(ctx, b.keyURL(key), map[string]string{"AccessKey": b.cfg.ReadKey})
|
||||
}
|
||||
|
||||
func (b *httpBackend) PutReader(ctx context.Context, key string, r io.Reader, size int64, checksum string) error {
|
||||
if b.name == "cdn" {
|
||||
return errors.New("cdn backend is read-only")
|
||||
}
|
||||
if b.cfg.WriteKey == "" {
|
||||
return errors.New("storage backend requires a write key to write")
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPut, b.keyURL(key), r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.ContentLength = size
|
||||
req.Header.Set("AccessKey", b.cfg.WriteKey)
|
||||
// Bunny defines Checksum as sha256 of the body and rejects a mismatch, so
|
||||
// this is server-side integrity checking, not decoration.
|
||||
req.Header.Set("Checksum", strings.ToUpper(checksum))
|
||||
|
||||
resp, err := b.client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
return fmt.Errorf("put %s: unexpected status %d", key, resp.StatusCode)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *httpBackend) GetKey(ctx context.Context, key string) ([]byte, error) {
|
||||
resp, err := b.get(ctx, b.keyURL(key), map[string]string{"AccessKey": b.cfg.ReadKey})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
return nil, fmt.Errorf("get %s: unexpected status %d", key, resp.StatusCode)
|
||||
}
|
||||
return io.ReadAll(resp.Body)
|
||||
}
|
||||
|
||||
// putFile uploads the file at src to key, streaming it. checksum is the
|
||||
// uppercase hex sha256 of the file's bytes.
|
||||
func (b *httpBackend) putFile(ctx context.Context, key, src, checksum string) error {
|
||||
f, err := os.Open(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
info, err := f.Stat()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return b.PutReader(ctx, key, f, info.Size(), checksum)
|
||||
}
|
||||
Reference in New Issue
Block a user