feat(nwsync): upload sink, key-addressed CLI, fail-closed publication marker
ci / ci (pull_request) Successful in 3m21s
ci / ci (pull_request) Successful in 3m21s
Resolves the build half of sow-tools#60 and closes the CLI gap sow-tools#53's sweep found: PR #71 shipped #53's original surface, not the one #56, #62 and #65 settled. - `depot.KeyStore` (`ProbeKey`/`PutReader`/`GetKey`) addresses the zone by object key instead of by depot sha, reusing the existing IPv4-pinned transport, retry and tri-state probe. The sha-addressed `Backend` now rides on it; no new HTTP client. - `nwsync` gains a sink: the zone by default, a local tree with `--out DIR` as the conformance path. Blobs upload as they are produced, the index lands last, and a blob already in the zone is skipped without paying for compression. - CLI is now `emit [--as NAME] [--out DIR] <artifact-key> <file>` and `assemble --group-id N [--tlk-key KEY] [--out DIR] <artifact-key>...`. Indexes live beside their artifact with the extension replaced, derived in one place. Flags may follow positionals, which cost a run during sow-tools#59. - Fail-closed: an artifact key whose digest does not match the file is refused, and `assemble` refuses an artifact with no index rather than publishing a manifest missing a hak. Conformance re-checked through the new CLI against upstream nwn_nwsync_write 2.1.2 on sow_vfxs_01.hak: the manifest is still byte-identical. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -10,7 +10,6 @@ import (
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -71,16 +70,7 @@ type httpBackend struct {
|
||||
|
||||
func (b *httpBackend) Name() string { return b.name }
|
||||
|
||||
func (b *httpBackend) storageURL(sha string) string {
|
||||
host := b.cfg.StorageHost
|
||||
// StorageHost is normally a bare host ("storage.bunnycdn.com"); allow a
|
||||
// full scheme (used by tests against httptest.NewServer) to pass through
|
||||
// unchanged.
|
||||
if strings.Contains(host, "://") {
|
||||
return fmt.Sprintf("%s/%s/%s", strings.TrimSuffix(host, "/"), b.cfg.StorageZone, BlobKey(sha))
|
||||
}
|
||||
return fmt.Sprintf("https://%s/%s/%s", host, b.cfg.StorageZone, BlobKey(sha))
|
||||
}
|
||||
func (b *httpBackend) storageURL(sha string) string { return b.keyURL(BlobKey(sha)) }
|
||||
|
||||
func (b *httpBackend) cdnURL(sha string) string {
|
||||
return fmt.Sprintf("%s/%s", b.cfg.CDNBase, BlobKey(sha))
|
||||
@@ -140,7 +130,8 @@ func (b *httpBackend) Probe(ctx context.Context, sha string) (ProbeState, bool,
|
||||
return b.rangeProbe(ctx, b.storageURL(sha), map[string]string{"AccessKey": b.cfg.ReadKey})
|
||||
}
|
||||
|
||||
// Put uploads src for sha. cdn is read-only.
|
||||
// Put uploads src for sha. cdn is read-only. A depot object is named after the
|
||||
// sha256 of its own bytes, so the key's sha doubles as the Checksum header.
|
||||
func (b *httpBackend) Put(ctx context.Context, sha, src string) error {
|
||||
if b.name == "cdn" {
|
||||
return errors.New("cdn backend is read-only")
|
||||
@@ -157,30 +148,7 @@ func (b *httpBackend) Put(ctx context.Context, sha, src string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
f, err := os.Open(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPut, b.storageURL(sha), f)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("AccessKey", b.cfg.WriteKey)
|
||||
req.Header.Set("Checksum", strings.ToUpper(sha))
|
||||
|
||||
resp, err := b.client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
return fmt.Errorf("bunny put %s: unexpected status %d", sha, resp.StatusCode)
|
||||
}
|
||||
return nil
|
||||
return b.putFile(ctx, BlobKey(sha), src, sha)
|
||||
}
|
||||
|
||||
// Get fetches sha into dest via temp file + rename, re-hashing and deleting
|
||||
|
||||
Reference in New Issue
Block a user