// Package nwsync publishes NWSync repository data: blobs and a per-artifact // NSYM manifest at each artifact's birth (emit), and one merged manifest at // module release (assemble). // // The split exists because upstream nwn_nwsync_write wants every hak, the TLK // and the module present in one run on one disk, which our build hosts cannot // hold. Upstream stays the conformance oracle: manifests compare byte for // byte, blobs compare after decompression. package nwsync import ( "flag" "fmt" "io" "os" ) const ( exitOK = 0 exitUsage = 64 exitInternal = 70 // exitDrift says the command worked and the zone is wrong, which is a // different thing for CI to act on than the command failing. It matches // depot's code for the same meaning. exitDrift = 1 ) // Run executes an nwsync subcommand. args[0] is the subcommand (emit|assemble); // returns the process exit code. func Run(args []string, stdout, stderr io.Writer) int { if len(args) == 0 { printRunUsage(stderr) return exitUsage } switch args[0] { case "emit": return runEmit(args[1:], stderr) case "assemble": return runAssemble(args[1:], stderr) case "verify": return runVerify(args[1:], stdout, stderr, os.Getenv) case "-h", "--help", "help": printRunUsage(stdout) return exitOK default: fmt.Fprintf(stderr, "nwsync: unknown subcommand %q\n\n", args[0]) printRunUsage(stderr) return exitUsage } } func printRunUsage(w io.Writer) { fmt.Fprint(w, `usage: nwsync emit [--as NAME] [--out DIR] [--verify] nwsync assemble --group-id N [--tlk-key KEY] [--out DIR] ... nwsync verify [--sample N] [--base URL] emit explodes one .hak/.erf or one loose file (the TLK) into NWSync blobs plus a NSYM index covering only that artifact, and uploads both. assemble merges those indexes into one manifest, reading no bulk data. Artifact keys are depot keys; an index lives beside its artifact, with the extension replaced. verify reads a published manifest and its blobs back through the public pull zone, with no credential, and decompresses and hashes every one. It is the only check on a published blob upstream of a player's client. --verify makes emit hash what it would otherwise skip. emit normally treats a blob's presence as proof of its contents, so without this an object written truncated, or written by an emitter since found broken, is skipped forever. --out DIR writes to a local repository tree instead of uploading, which is the conformance path against upstream nwn_nwsync_write. Without it, the zone comes from NWSYNC_STORAGE_ZONE, NWSYNC_STORAGE_PASSWORD and BUNNY_STORAGE_HOST. verify needs none of those; its base comes from --base or NWSYNC_PULL_BASE. `) } // parseArgs parses flags that may appear before, after or between positionals. // Go's flag package stops at the first non-flag argument, which turns // `emit --out DIR` into a confusing arity error. func parseArgs(fs *flag.FlagSet, args []string) ([]string, error) { var positional []string for { if err := fs.Parse(args); err != nil { return nil, err } rest := fs.Args() if len(rest) == 0 { return positional, nil } positional = append(positional, rest[0]) args = rest[1:] } } func runEmit(args []string, stderr io.Writer) int { fs := flag.NewFlagSet("emit", flag.ContinueOnError) fs.SetOutput(stderr) as := fs.String("as", "", "published name of the artifact, when it differs from the key") out := fs.String("out", "", "write to a local repository tree instead of uploading") jobs := fs.Int("jobs", defaultEmitJobs, "resources to hash, compress and store at once") verify := fs.Bool("verify", false, "read back and hash blobs that already exist instead of trusting their presence") positional, err := parseArgs(fs, args) if err != nil { return exitUsage } if len(positional) != 2 { fmt.Fprintf(stderr, "nwsync emit: and are both required\n") return exitUsage } if *jobs < 1 { fmt.Fprintf(stderr, "nwsync emit: -jobs must be at least 1, got %d\n", *jobs) return exitUsage } result, err := Emit(EmitOptions{ ArtifactKey: positional[0], ArtifactPath: positional[1], As: *as, OutDir: *out, Jobs: *jobs, Verify: *verify, }) if err != nil { fmt.Fprintf(stderr, "nwsync emit: %v\n", err) return exitInternal } fmt.Fprintf(stderr, "emitted %s: %d resources, %d new blobs, index %s\n", result.Name, result.Entries, result.BlobsWritten, result.ManifestPath) return exitOK } func runVerify(args []string, stdout, stderr io.Writer, getenv func(string) string) int { fs := flag.NewFlagSet("verify", flag.ContinueOnError) fs.SetOutput(stderr) base := fs.String("base", getenv("NWSYNC_PULL_BASE"), "pull zone base URL to read through") sample := fs.Int("sample", 0, "check this many random blobs instead of all of them") jobs := fs.Int("jobs", defaultEmitJobs, "blobs to fetch and hash at once") positional, err := parseArgs(fs, args) if err != nil { return exitUsage } if len(positional) != 1 { fmt.Fprintf(stderr, "nwsync verify: exactly one is required\n") return exitUsage } result, err := Verify(VerifyOptions{ ManifestSHA1: positional[0], Base: *base, Sample: *sample, Jobs: *jobs, Log: stderr, }) if err != nil { fmt.Fprintf(stderr, "nwsync verify: %v\n", err) return exitInternal } fmt.Fprintf(stdout, "verified %d of %d blobs behind %d resources: %d failures, %d bytes checked\n", result.Checked, result.Blobs, result.Entries, result.Failures, result.Bytes) if result.Failures > 0 { return exitDrift } return exitOK } func runAssemble(args []string, stderr io.Writer) int { fs := flag.NewFlagSet("assemble", flag.ContinueOnError) fs.SetOutput(stderr) tlkKey := fs.String("tlk-key", "", "depot key of the TLK, which shadows nothing and merges last") out := fs.String("out", "", "write to a local repository tree instead of uploading") groupID := fs.Int("group-id", 0, "NWSync group id (1 = current, 2 = testing; 0 omits it)") moduleName := fs.String("module-name", "", "module name recorded in the sidecar") description := fs.String("description", "", "description recorded in the sidecar") positional, err := parseArgs(fs, args) if err != nil { return exitUsage } if len(positional) == 0 { fmt.Fprintf(stderr, "nwsync assemble: at least one artifact key is required\n") return exitUsage } result, err := Assemble(AssembleOptions{ ArtifactKeys: positional, TLKKey: *tlkKey, OutDir: *out, GroupID: *groupID, ModuleName: *moduleName, Description: *description, }) if err != nil { fmt.Fprintf(stderr, "nwsync assemble: %v\n", err) return exitInternal } fmt.Fprintf(stderr, "assembled manifest %s: %d resources, %s\n", result.SHA1, result.Entries, result.ManifestPath) return exitOK }