sow-tools/internal/nwsync/run.go

// Package nwsync publishes NWSync repository data: blobs and a per-artifact
// NSYM manifest at each artifact's birth (emit), and one merged manifest at
// module release (assemble).
//
// The split exists because upstream nwn_nwsync_write wants every hak, the TLK
// and the module present in one run on one disk, which our build hosts cannot
// hold. Upstream stays the conformance oracle: manifests compare byte for
// byte, blobs compare after decompression.
package nwsync

import (
	"flag"
	"fmt"
	"io"
	"os"
)

const (
	exitOK       = 0
	exitUsage    = 64
	exitInternal = 70
	// exitDrift says the command worked and the zone is wrong, which is a
	// different thing for CI to act on than the command failing. It matches
	// depot's code for the same meaning.
	exitDrift = 1
)

// Run executes an nwsync subcommand. args[0] is the subcommand (emit|assemble);
// returns the process exit code.
func Run(args []string, stdout, stderr io.Writer) int {
	if len(args) == 0 {
		printRunUsage(stderr)
		return exitUsage
	}
	switch args[0] {
	case "emit":
		return runEmit(args[1:], stderr)
	case "assemble":
		return runAssemble(args[1:], stderr)
	case "verify":
		return runVerify(args[1:], stdout, stderr, os.Getenv)
	case "-h", "--help", "help":
		printRunUsage(stdout)
		return exitOK
	default:
		fmt.Fprintf(stderr, "nwsync: unknown subcommand %q\n\n", args[0])
		printRunUsage(stderr)
		return exitUsage
	}
}

func printRunUsage(w io.Writer) {
	fmt.Fprint(w, `usage:
  nwsync emit     [--as NAME] [--out DIR] [--verify] <artifact-key> <file>
  nwsync assemble --group-id N [--tlk-key KEY] [--out DIR] <artifact-key>...
  nwsync verify   [--sample N] [--base URL] <manifest-sha1>

emit explodes one .hak/.erf or one loose file (the TLK) into NWSync blobs plus
a NSYM index covering only that artifact, and uploads both. assemble merges
those indexes into one manifest, reading no bulk data. Artifact keys are depot
keys; an index lives beside its artifact, with the extension replaced.

verify reads a published manifest and its blobs back through the public pull
zone, with no credential, and decompresses and hashes every one. It is the only
check on a published blob upstream of a player's client.

--verify makes emit hash what it would otherwise skip. emit normally treats a
blob's presence as proof of its contents, so without this an object written
truncated, or written by an emitter since found broken, is skipped forever.
--verify repairs the storage zone, while verify reads the edge in front of it.
So a verify run right after a repair is a survey, not a verdict: it names the
keys the edge still serves stale. Purge those, then run it again.

--out DIR writes to a local repository tree instead of uploading, which is the
conformance path against upstream nwn_nwsync_write. Without it, the zone comes
from NWSYNC_STORAGE_ZONE, NWSYNC_STORAGE_PASSWORD and BUNNY_STORAGE_HOST.
verify needs none of those; its base comes from --base or NWSYNC_PULL_BASE.
`)
}

// parseArgs parses flags that may appear before, after or between positionals.
// Go's flag package stops at the first non-flag argument, which turns
// `emit <key> <file> --out DIR` into a confusing arity error.
func parseArgs(fs *flag.FlagSet, args []string) ([]string, error) {
	var positional []string
	for {
		if err := fs.Parse(args); err != nil {
			return nil, err
		}
		rest := fs.Args()
		if len(rest) == 0 {
			return positional, nil
		}
		positional = append(positional, rest[0])
		args = rest[1:]
	}
}

func runEmit(args []string, stderr io.Writer) int {
	fs := flag.NewFlagSet("emit", flag.ContinueOnError)
	fs.SetOutput(stderr)
	as := fs.String("as", "", "published name of the artifact, when it differs from the key")
	out := fs.String("out", "", "write to a local repository tree instead of uploading")
	jobs := fs.Int("jobs", defaultEmitJobs, "resources to hash, compress and store at once")
	verify := fs.Bool("verify", false, "read back and hash blobs that already exist instead of trusting their presence")
	positional, err := parseArgs(fs, args)
	if err != nil {
		return exitUsage
	}
	if len(positional) != 2 {
		fmt.Fprintf(stderr, "nwsync emit: <artifact-key> and <file> are both required\n")
		return exitUsage
	}
	if *jobs < 1 {
		fmt.Fprintf(stderr, "nwsync emit: -jobs must be at least 1, got %d\n", *jobs)
		return exitUsage
	}

	result, err := Emit(EmitOptions{
		ArtifactKey:  positional[0],
		ArtifactPath: positional[1],
		As:           *as,
		OutDir:       *out,
		Jobs:         *jobs,
		Verify:       *verify,
	})
	if err != nil {
		fmt.Fprintf(stderr, "nwsync emit: %v\n", err)
		return exitInternal
	}
	fmt.Fprintf(stderr, "emitted %s: %d resources, %d new blobs, index %s\n",
		result.Name, result.Entries, result.BlobsWritten, result.ManifestPath)
	return exitOK
}

func runVerify(args []string, stdout, stderr io.Writer, getenv func(string) string) int {
	fs := flag.NewFlagSet("verify", flag.ContinueOnError)
	fs.SetOutput(stderr)
	base := fs.String("base", getenv("NWSYNC_PULL_BASE"), "pull zone base URL to read through")
	sample := fs.Int("sample", 0, "check this many random blobs instead of all of them")
	jobs := fs.Int("jobs", defaultEmitJobs, "blobs to fetch and hash at once")
	positional, err := parseArgs(fs, args)
	if err != nil {
		return exitUsage
	}
	if len(positional) != 1 {
		fmt.Fprintf(stderr, "nwsync verify: exactly one <manifest-sha1> is required\n")
		return exitUsage
	}

	result, err := Verify(VerifyOptions{
		ManifestSHA1: positional[0],
		Base:         *base,
		Sample:       *sample,
		Jobs:         *jobs,
		Log:          stderr,
	})
	if err != nil {
		fmt.Fprintf(stderr, "nwsync verify: %v\n", err)
		return exitInternal
	}
	fmt.Fprintf(stdout, "verified %d of %d blobs behind %d resources: %d failures, %d bytes checked\n",
		result.Checked, result.Blobs, result.Entries, result.Failures, result.Bytes)
	if result.Failures > 0 {
		return exitDrift
	}
	return exitOK
}

func runAssemble(args []string, stderr io.Writer) int {
	fs := flag.NewFlagSet("assemble", flag.ContinueOnError)
	fs.SetOutput(stderr)
	tlkKey := fs.String("tlk-key", "", "depot key of the TLK, which shadows nothing and merges last")
	out := fs.String("out", "", "write to a local repository tree instead of uploading")
	groupID := fs.Int("group-id", 0, "NWSync group id (1 = current, 2 = testing; 0 omits it)")
	moduleName := fs.String("module-name", "", "module name recorded in the sidecar")
	description := fs.String("description", "", "description recorded in the sidecar")
	positional, err := parseArgs(fs, args)
	if err != nil {
		return exitUsage
	}
	if len(positional) == 0 {
		fmt.Fprintf(stderr, "nwsync assemble: at least one artifact key is required\n")
		return exitUsage
	}

	result, err := Assemble(AssembleOptions{
		ArtifactKeys: positional,
		TLKKey:       *tlkKey,
		OutDir:       *out,
		GroupID:      *groupID,
		ModuleName:   *moduleName,
		Description:  *description,
	})
	if err != nil {
		fmt.Fprintf(stderr, "nwsync assemble: %v\n", err)
		return exitInternal
	}
	fmt.Fprintf(stderr, "assembled manifest %s: %d resources, %s\n",
		result.SHA1, result.Entries, result.ManifestPath)
	return exitOK
}