sow-tools/internal/nwsync/emit.go

package nwsync

import (
	"bytes"
	"crypto/sha1"
	"fmt"
	"os"
	"path/filepath"
	"slices"
	"sort"
	"strconv"
	"strings"
	"time"

	"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/buildinfo"
	"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf"
)

// fileSizeLimit matches upstream's --limit-file-size default of 15 MB. A
// resource over it is a hard failure, not a skip: upstream quit(1)s and so do
// we. Our largest resource today is 13.66 MiB, so the headroom is thin.
const fileSizeLimit = 15 * 1024 * 1024

// skippedTypes are never published, matching upstream's GobalResTypeSkipList.
var skippedTypes = resTypes("nss", "ndb", "gic")

// emitterVersion identifies the blob/manifest byte format this package
// produces. assemble refuses to merge indexes that disagree on it, because two
// producers of blobs mean a skewed emitter can otherwise write blobs the merged
// manifest quietly disagrees with. Bump it only when emitted bytes change — it
// is deliberately not the build revision, which would invalidate every
// published index on every unrelated commit.
const emitterVersion = "1"

// serverTypes are loaded only server-side; a manifest holding nothing else
// has no client contents. Mirrors upstream's GlobalResTypeServerList, whose
// trailing 0 is RESTYPE_INVALID.
var serverTypes = append(resTypes(
	"are", "dlg", "fac", "gic", "git", "ifo", "itp", "jrl", "ncs", "ndb",
	"nss", "ptm", "utc", "utd", "ute", "uti", "utm", "utp", "uts", "utt", "utw",
), 0)

func resTypes(extensions ...string) []uint16 {
	types := make([]uint16, 0, len(extensions))
	for _, extension := range extensions {
		restype, ok := erf.ResourceTypeForExtension(extension)
		if !ok {
			panic("nwsync: unknown restype " + extension)
		}
		types = append(types, restype)
	}
	return types
}

// EmitResult reports what one emit run produced.
type EmitResult struct {
	Name         string // artifact name, without extension
	ManifestPath string
	Entries      int
	BlobsWritten int
}

// Emit explodes one artifact — a .hak/.erf/.mod or a loose file such as the
// TLK — into NWSync blobs plus a NSYM manifest describing only that artifact.
func Emit(artifactPath, outDir string) (EmitResult, error) {
	resources, err := readArtifact(artifactPath)
	if err != nil {
		return EmitResult{}, err
	}
	name := strings.TrimSuffix(filepath.Base(artifactPath), filepath.Ext(artifactPath))

	entries, blobs, onDiskBytes, err := emitResources(resources, outDir)
	if err != nil {
		return EmitResult{}, err
	}
	if len(entries) == 0 {
		return EmitResult{}, fmt.Errorf("%s: nothing to index (no publishable resources)", artifactPath)
	}

	data, err := writeManifest(entries)
	if err != nil {
		return EmitResult{}, err
	}
	manifestPath := filepath.Join(outDir, name+".nsym")
	if err := writeManifestPair(manifestPath, data, entries, onDiskBytes, Sidecar{ModuleName: name}); err != nil {
		return EmitResult{}, err
	}
	return EmitResult{Name: name, ManifestPath: manifestPath, Entries: len(entries), BlobsWritten: blobs}, nil
}

// readArtifact returns the resources of an ERF/HAK/MOD, or the single resource
// a loose file represents. Upstream's resman does the same dispatch on the
// file's first three bytes.
func readArtifact(path string) ([]erf.Resource, error) {
	data, err := os.ReadFile(path)
	if err != nil {
		return nil, fmt.Errorf("read artifact: %w", err)
	}
	if len(data) >= 3 {
		switch string(data[:3]) {
		case "ERF", "HAK":
			archive, err := erf.Read(bytes.NewReader(data))
			if err != nil {
				return nil, fmt.Errorf("%s: %w", path, err)
			}
			return archive.Resources, nil
		case "MOD":
			// A persistent world never publishes module contents, so the .mod
			// contributes no bytes to a manifest — it only says which haks and
			// which TLK the manifest covers.
			return nil, fmt.Errorf("%s: a module is never emitted; a manifest is haks plus the TLK", path)
		}
	}
	base := filepath.Base(path)
	extension := filepath.Ext(base)
	restype, ok := erf.ResourceTypeForExtension(extension)
	if !ok {
		return nil, fmt.Errorf("%s: unknown resource type %q", path, extension)
	}
	return []erf.Resource{{
		Name: strings.TrimSuffix(base, extension),
		Type: restype,
		Data: data,
	}}, nil
}

func emitResources(resources []erf.Resource, outDir string) ([]Entry, int, int64, error) {
	// A resref appearing twice inside one artifact resolves to the last one,
	// the way resman lets the last container added win.
	order := make([]Identity, 0, len(resources))
	latest := make(map[Identity]erf.Resource, len(resources))
	var tooBig []string
	for _, resource := range resources {
		if _, ok := erf.ExtensionForResourceType(resource.Type); !ok {
			return nil, 0, 0, fmt.Errorf("resref %s is not resolvable (unknown restype %d)", resource.Name, resource.Type)
		}
		if slices.Contains(skippedTypes, resource.Type) {
			continue
		}
		if len(resource.Data) > fileSizeLimit {
			tooBig = append(tooBig, fmt.Sprintf("%s: %d bytes > %d", resource.Name, len(resource.Data), fileSizeLimit))
			continue
		}
		identity := Identity{ResRef: strings.ToLower(resource.Name), ResType: resource.Type}
		if _, seen := latest[identity]; !seen {
			order = append(order, identity)
		}
		latest[identity] = resource
	}
	if len(tooBig) > 0 {
		sort.Strings(tooBig)
		return nil, 0, 0, fmt.Errorf("resources exceed the file size limit:\n  %s", strings.Join(tooBig, "\n  "))
	}

	entries := make([]Entry, 0, len(order))
	var blobs int
	var onDiskBytes int64
	for _, identity := range order {
		resource := latest[identity]
		sum := sha1.Sum(resource.Data)
		entries = append(entries, Entry{
			SHA1:    sum,
			Size:    uint32(len(resource.Data)),
			ResRef:  identity.ResRef,
			ResType: identity.ResType,
		})
		written, err := writeBlob(outDir, sum, resource.Data)
		if err != nil {
			return nil, 0, 0, err
		}
		if written > 0 {
			blobs++
			onDiskBytes += written
		}
	}
	return entries, blobs, onDiskBytes, nil
}

// created is the sidecar timestamp. SOURCE_DATE_EPOCH pins it so a build can
// be reproduced byte for byte; the manifest itself is deterministic already.
func created() int64 {
	if raw := os.Getenv("SOURCE_DATE_EPOCH"); raw != "" {
		if seconds, err := strconv.ParseInt(raw, 10, 64); err == nil {
			return seconds
		}
	}
	return time.Now().Unix()
}

// writeBlob writes one NWCompressedBuffer blob and returns its size on disk,
// or 0 if the blob already existed. Blob names are content hashes, so an
// existing name is existing content.
func writeBlob(outDir string, sum [20]byte, data []byte) (int64, error) {
	path := blobPath(outDir, fmt.Sprintf("%x", sum))
	if _, err := os.Stat(path); err == nil {
		return 0, nil
	}
	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
		return 0, fmt.Errorf("create blob directory: %w", err)
	}
	blob := compressBlob(data)
	if err := os.WriteFile(path, blob, 0o644); err != nil {
		return 0, fmt.Errorf("write blob: %w", err)
	}
	return int64(len(blob)), nil
}

// writeManifestPair writes a NSYM manifest and its .json sidecar. The caller
// supplies the sidecar fields it knows; the rest are derived from the entries.
// data must be the serialised form of entries.
func writeManifestPair(manifestPath string, data []byte, entries []Entry, onDiskBytes int64, sidecar Sidecar) error {
	if err := os.MkdirAll(filepath.Dir(manifestPath), 0o755); err != nil {
		return fmt.Errorf("create manifest directory: %w", err)
	}
	if err := os.WriteFile(manifestPath, data, 0o644); err != nil {
		return fmt.Errorf("write manifest: %w", err)
	}

	var totalBytes int64
	clientContents := false
	for _, entry := range entries {
		totalBytes += int64(entry.Size)
		if !slices.Contains(serverTypes, entry.ResType) {
			clientContents = true
		}
	}
	sidecar.Version = manifestVersion
	sidecar.SHA1 = fmt.Sprintf("%x", sha1.Sum(data))
	sidecar.HashTreeDepth = hashTreeDepth
	sidecar.IncludesModuleContents = false
	sidecar.IncludesClientContents = clientContents
	sidecar.TotalFiles = len(entries)
	sidecar.TotalBytes = totalBytes
	sidecar.OnDiskBytes = onDiskBytes
	sidecar.Created = created()
	sidecar.CreatedWith = buildinfo.String()
	sidecar.EmitterVersion = emitterVersion

	body, err := marshalSidecar(sidecar)
	if err != nil {
		return err
	}
	if err := os.WriteFile(manifestPath+".json", body, 0o644); err != nil {
		return fmt.Errorf("write sidecar: %w", err)
	}
	return nil
}