package nwsync import ( "bytes" "crypto/sha1" "fmt" "os" "path/filepath" "slices" "sort" "strconv" "strings" "time" "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/buildinfo" "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf" ) // fileSizeLimit matches upstream's --limit-file-size default of 15 MB. A // resource over it is a hard failure, not a skip: upstream quit(1)s and so do // we. Our largest resource today is 13.66 MiB, so the headroom is thin. const fileSizeLimit = 15 * 1024 * 1024 // skippedTypes are never published, matching upstream's GobalResTypeSkipList. var skippedTypes = resTypes("nss", "ndb", "gic") // emitterVersion identifies the blob/manifest byte format this package // produces. assemble refuses to merge indexes that disagree on it, because two // producers of blobs mean a skewed emitter can otherwise write blobs the merged // manifest quietly disagrees with. Bump it only when emitted bytes change — it // is deliberately not the build revision, which would invalidate every // published index on every unrelated commit. const emitterVersion = "1" // serverTypes are loaded only server-side; a manifest holding nothing else // has no client contents. Mirrors upstream's GlobalResTypeServerList, whose // trailing 0 is RESTYPE_INVALID. var serverTypes = append(resTypes( "are", "dlg", "fac", "gic", "git", "ifo", "itp", "jrl", "ncs", "ndb", "nss", "ptm", "utc", "utd", "ute", "uti", "utm", "utp", "uts", "utt", "utw", ), 0) func resTypes(extensions ...string) []uint16 { types := make([]uint16, 0, len(extensions)) for _, extension := range extensions { restype, ok := erf.ResourceTypeForExtension(extension) if !ok { panic("nwsync: unknown restype " + extension) } types = append(types, restype) } return types } // EmitResult reports what one emit run produced. type EmitResult struct { Name string // artifact name, without extension ManifestPath string Entries int BlobsWritten int } // Emit explodes one artifact — a .hak/.erf/.mod or a loose file such as the // TLK — into NWSync blobs plus a NSYM manifest describing only that artifact. func Emit(artifactPath, outDir string) (EmitResult, error) { resources, err := readArtifact(artifactPath) if err != nil { return EmitResult{}, err } name := strings.TrimSuffix(filepath.Base(artifactPath), filepath.Ext(artifactPath)) entries, blobs, onDiskBytes, err := emitResources(resources, outDir) if err != nil { return EmitResult{}, err } if len(entries) == 0 { return EmitResult{}, fmt.Errorf("%s: nothing to index (no publishable resources)", artifactPath) } data, err := writeManifest(entries) if err != nil { return EmitResult{}, err } manifestPath := filepath.Join(outDir, name+".nsym") if err := writeManifestPair(manifestPath, data, entries, onDiskBytes, Sidecar{ModuleName: name}); err != nil { return EmitResult{}, err } return EmitResult{Name: name, ManifestPath: manifestPath, Entries: len(entries), BlobsWritten: blobs}, nil } // readArtifact returns the resources of an ERF/HAK/MOD, or the single resource // a loose file represents. Upstream's resman does the same dispatch on the // file's first three bytes. func readArtifact(path string) ([]erf.Resource, error) { data, err := os.ReadFile(path) if err != nil { return nil, fmt.Errorf("read artifact: %w", err) } if len(data) >= 3 { switch string(data[:3]) { case "ERF", "HAK": archive, err := erf.Read(bytes.NewReader(data)) if err != nil { return nil, fmt.Errorf("%s: %w", path, err) } return archive.Resources, nil case "MOD": // A persistent world never publishes module contents, so the .mod // contributes no bytes to a manifest — it only says which haks and // which TLK the manifest covers. return nil, fmt.Errorf("%s: a module is never emitted; a manifest is haks plus the TLK", path) } } base := filepath.Base(path) extension := filepath.Ext(base) restype, ok := erf.ResourceTypeForExtension(extension) if !ok { return nil, fmt.Errorf("%s: unknown resource type %q", path, extension) } return []erf.Resource{{ Name: strings.TrimSuffix(base, extension), Type: restype, Data: data, }}, nil } func emitResources(resources []erf.Resource, outDir string) ([]Entry, int, int64, error) { // A resref appearing twice inside one artifact resolves to the last one, // the way resman lets the last container added win. order := make([]Identity, 0, len(resources)) latest := make(map[Identity]erf.Resource, len(resources)) var tooBig []string for _, resource := range resources { if _, ok := erf.ExtensionForResourceType(resource.Type); !ok { return nil, 0, 0, fmt.Errorf("resref %s is not resolvable (unknown restype %d)", resource.Name, resource.Type) } if slices.Contains(skippedTypes, resource.Type) { continue } if len(resource.Data) > fileSizeLimit { tooBig = append(tooBig, fmt.Sprintf("%s: %d bytes > %d", resource.Name, len(resource.Data), fileSizeLimit)) continue } identity := Identity{ResRef: strings.ToLower(resource.Name), ResType: resource.Type} if _, seen := latest[identity]; !seen { order = append(order, identity) } latest[identity] = resource } if len(tooBig) > 0 { sort.Strings(tooBig) return nil, 0, 0, fmt.Errorf("resources exceed the file size limit:\n %s", strings.Join(tooBig, "\n ")) } entries := make([]Entry, 0, len(order)) var blobs int var onDiskBytes int64 for _, identity := range order { resource := latest[identity] sum := sha1.Sum(resource.Data) entries = append(entries, Entry{ SHA1: sum, Size: uint32(len(resource.Data)), ResRef: identity.ResRef, ResType: identity.ResType, }) written, err := writeBlob(outDir, sum, resource.Data) if err != nil { return nil, 0, 0, err } if written > 0 { blobs++ onDiskBytes += written } } return entries, blobs, onDiskBytes, nil } // created is the sidecar timestamp. SOURCE_DATE_EPOCH pins it so a build can // be reproduced byte for byte; the manifest itself is deterministic already. func created() int64 { if raw := os.Getenv("SOURCE_DATE_EPOCH"); raw != "" { if seconds, err := strconv.ParseInt(raw, 10, 64); err == nil { return seconds } } return time.Now().Unix() } // writeBlob writes one NWCompressedBuffer blob and returns its size on disk, // or 0 if the blob already existed. Blob names are content hashes, so an // existing name is existing content. func writeBlob(outDir string, sum [20]byte, data []byte) (int64, error) { path := blobPath(outDir, fmt.Sprintf("%x", sum)) if _, err := os.Stat(path); err == nil { return 0, nil } if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { return 0, fmt.Errorf("create blob directory: %w", err) } blob := compressBlob(data) if err := os.WriteFile(path, blob, 0o644); err != nil { return 0, fmt.Errorf("write blob: %w", err) } return int64(len(blob)), nil } // writeManifestPair writes a NSYM manifest and its .json sidecar. The caller // supplies the sidecar fields it knows; the rest are derived from the entries. // data must be the serialised form of entries. func writeManifestPair(manifestPath string, data []byte, entries []Entry, onDiskBytes int64, sidecar Sidecar) error { if err := os.MkdirAll(filepath.Dir(manifestPath), 0o755); err != nil { return fmt.Errorf("create manifest directory: %w", err) } if err := os.WriteFile(manifestPath, data, 0o644); err != nil { return fmt.Errorf("write manifest: %w", err) } var totalBytes int64 clientContents := false for _, entry := range entries { totalBytes += int64(entry.Size) if !slices.Contains(serverTypes, entry.ResType) { clientContents = true } } sidecar.Version = manifestVersion sidecar.SHA1 = fmt.Sprintf("%x", sha1.Sum(data)) sidecar.HashTreeDepth = hashTreeDepth sidecar.IncludesModuleContents = false sidecar.IncludesClientContents = clientContents sidecar.TotalFiles = len(entries) sidecar.TotalBytes = totalBytes sidecar.OnDiskBytes = onDiskBytes sidecar.Created = created() sidecar.CreatedWith = buildinfo.String() sidecar.EmitterVersion = emitterVersion body, err := marshalSidecar(sidecar) if err != nil { return err } if err := os.WriteFile(manifestPath+".json", body, 0o644); err != nil { return fmt.Errorf("write sidecar: %w", err) } return nil }