package pipeline import ( "bytes" "crypto/sha256" "encoding/json" "errors" "fmt" "os" "os/exec" "path/filepath" "runtime" "slices" "strconv" "strings" "time" "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf" "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/gff" "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/topdata" "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/validator" ) type BuildResult struct { ModulePath string HAKPaths []string Manifest string Resources int HAKAssets int TopPackageHAK string TopPackageTLK string TopPackageFiles int } type BuildManifest struct { ModuleHAKs []string `json:"module_haks"` HAKs []BuildManifestHAK `json:"haks"` } type BuildManifestHAK struct { Name string `json:"name"` Group string `json:"group"` Priority int `json:"priority"` MaxBytes int64 `json:"max_bytes"` SizeBytes int64 `json:"size_bytes"` Optional bool `json:"optional,omitempty"` Assets []string `json:"assets"` ContentHash string `json:"content_hash,omitempty"` } type assetResource struct { Rel string Resource erf.Resource Size int64 ChangedAt time.Time ContentID string } type lfsAssetInfo struct { OID string Size int64 ObjectPath string } type hakChunk struct { Config project.HAKConfig Index int Name string Assets []assetResource Size int64 } type ProgressFunc func(string) type BuildHAKOptions struct { Progress ProgressFunc ArchiveNames []string } func Build(p *project.Project) (BuildResult, error) { var topPackage topdata.PackageResult var err error if p.HasTopData() { topPackage, err = topdata.BuildAndPackage(p, nil) if err != nil { return BuildResult{}, err } } moduleResult, err := BuildModule(p) if err != nil { return BuildResult{}, err } hakResult, err := BuildHAKs(p) if err != nil { return BuildResult{}, err } moduleResult.HAKPaths = hakResult.HAKPaths moduleResult.Manifest = hakResult.Manifest moduleResult.HAKAssets = hakResult.HAKAssets moduleResult.TopPackageHAK = topPackage.OutputHAKPath moduleResult.TopPackageTLK = topPackage.OutputTLKPath moduleResult.TopPackageFiles = topPackage.HAKResources return moduleResult, nil } func BuildModule(p *project.Project) (BuildResult, error) { return buildModule(p, nil) } func BuildModuleWithProgress(p *project.Project, progress ProgressFunc) (BuildResult, error) { return buildModule(p, progress) } func buildModule(p *project.Project, progress ProgressFunc) (BuildResult, error) { progressf(progress, "Validating project...") if err := validateForBuild(p); err != nil { return BuildResult{}, err } progressf(progress, "Resolving module HAK order...") moduleHakOrder, err := plannedModuleHAKOrder(p) if err != nil { return BuildResult{}, err } progressf(progress, "Collecting module resources...") moduleResources, err := collectModuleResources(p, moduleHakOrder) if err != nil { return BuildResult{}, err } progressf(progress, "Writing module archive...") outputPath := filepath.Join(p.BuildDir(), p.Config.Module.ResRef+".mod") output, err := os.Create(outputPath) if err != nil { return BuildResult{}, fmt.Errorf("create module archive: %w", err) } defer output.Close() if err := erf.Write(output, erf.New("MOD ", moduleResources)); err != nil { return BuildResult{}, fmt.Errorf("write module archive: %w", err) } return BuildResult{ ModulePath: outputPath, Resources: len(moduleResources), }, nil } func BuildHAKs(p *project.Project) (BuildResult, error) { return buildHAKs(p, nil) } func BuildHAKsWithProgress(p *project.Project, progress ProgressFunc) (BuildResult, error) { return buildHAKs(p, progress) } func BuildHAKsWithOptions(p *project.Project, opts BuildHAKOptions) (BuildResult, error) { return planOrBuildHAKs(p, opts.Progress, true, opts.ArchiveNames) } func PlanHAKs(p *project.Project) (BuildResult, error) { return planHAKs(p, nil) } func PlanHAKsWithProgress(p *project.Project, progress ProgressFunc) (BuildResult, error) { return planHAKs(p, progress) } func buildHAKs(p *project.Project, progress ProgressFunc) (BuildResult, error) { return planOrBuildHAKs(p, progress, true, nil) } func planHAKs(p *project.Project, progress ProgressFunc) (BuildResult, error) { return planOrBuildHAKs(p, progress, false, nil) } func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bool, archiveNames []string) (BuildResult, error) { preserveExistingHAKs := envBool("SOW_BUILD_HAKS_KEEP_EXISTING") progressf(progress, "Validating project...") if err := validateForBuild(p); err != nil { return BuildResult{}, err } progressf(progress, "Collecting asset resources...") assetResources, err := collectAssetResources(p, false) if err != nil { return BuildResult{}, err } result := BuildResult{HAKAssets: len(assetResources)} if len(assetResources) == 0 { progressf(progress, "Cleaning previous generated HAKs...") if err := cleanupGeneratedHAKs(p.BuildDir(), p.Config.Module.ResRef); err != nil { return BuildResult{}, err } return result, nil } progressf(progress, "Planning HAK chunks...") chunks, err := planHAKChunks(p, assetResources) if err != nil { return BuildResult{}, err } chunks, err = filterHAKChunksByName(chunks, archiveNames) if err != nil { return BuildResult{}, err } if writeArchives { chunks, err = hydrateHAKChunks(p, chunks) if err != nil { return BuildResult{}, err } } progressf(progress, "Resolving module HAK order...") moduleHakOrder, err := resolveModuleHAKOrder(p, chunks) if err != nil { return BuildResult{}, err } manifest := BuildManifest{ ModuleHAKs: moduleHakOrder, HAKs: make([]BuildManifestHAK, 0, len(chunks)), } currentNames := make(map[string]struct{}, len(chunks)) for _, chunk := range chunks { manifestEntry := buildManifestEntry(chunk) currentNames[chunk.Name] = struct{}{} manifest.HAKs = append(manifest.HAKs, manifestEntry) } manifestPath := filepath.Join(p.BuildDir(), "haks.json") manifestBytes, err := json.MarshalIndent(manifest, "", " ") if err != nil { return BuildResult{}, fmt.Errorf("marshal hak manifest: %w", err) } manifestBytes = append(manifestBytes, '\n') if !writeArchives { progressf(progress, "Cleaning previous generated HAKs...") if err := cleanupGeneratedHAKs(p.BuildDir(), p.Config.Module.ResRef); err != nil { return BuildResult{}, err } progressf(progress, "Writing HAK manifest...") if err := os.WriteFile(manifestPath, manifestBytes, 0o644); err != nil { return BuildResult{}, fmt.Errorf("write hak manifest: %w", err) } result.Manifest = manifestPath return result, nil } previousManifest, err := loadPreviousBuildManifest(p.BuildDir()) if err != nil { return BuildResult{}, err } progressf(progress, "Reusing unchanged generated HAKs where possible...") result.HAKPaths = make([]string, 0, len(chunks)) for index, chunk := range chunks { hakPath := filepath.Join(p.BuildDir(), chunk.Name+".hak") manifestEntry := manifest.HAKs[index] reused, err := reuseExistingHAK(previousManifest, manifestEntry, hakPath) if err != nil { return BuildResult{}, err } if reused { progressf(progress, fmt.Sprintf("Reusing HAK %d/%d: %s (%d assets)", index+1, len(chunks), chunk.Name, len(chunk.Assets))) } else { progressf(progress, fmt.Sprintf("Writing HAK %d/%d: %s (%d assets)", index+1, len(chunks), chunk.Name, len(chunk.Assets))) hakOutput, err := os.Create(hakPath) if err != nil { return BuildResult{}, fmt.Errorf("create hak archive: %w", err) } if err := erf.Write(hakOutput, erf.New("HAK ", resourceSlice(chunk.Assets))); err != nil { hakOutput.Close() return BuildResult{}, fmt.Errorf("write hak archive: %w", err) } if err := hakOutput.Close(); err != nil { return BuildResult{}, fmt.Errorf("close hak archive: %w", err) } } result.HAKPaths = append(result.HAKPaths, hakPath) } progressf(progress, "Cleaning stale generated HAKs...") if !preserveExistingHAKs { if err := cleanupStaleGeneratedHAKs(p.BuildDir(), p.Config.Module.ResRef, currentNames); err != nil { return BuildResult{}, err } } progressf(progress, "Writing HAK manifest...") if err := os.WriteFile(manifestPath, manifestBytes, 0o644); err != nil { return BuildResult{}, fmt.Errorf("write hak manifest: %w", err) } result.Manifest = manifestPath return result, nil } func filterHAKChunksByName(chunks []hakChunk, archiveNames []string) ([]hakChunk, error) { if len(archiveNames) == 0 { return chunks, nil } wanted := make(map[string]struct{}, len(archiveNames)) for _, name := range archiveNames { trimmed := strings.TrimSpace(strings.ToLower(name)) if trimmed == "" { continue } wanted[trimmed] = struct{}{} } if len(wanted) == 0 { return chunks, nil } filtered := make([]hakChunk, 0, len(wanted)) found := make(map[string]struct{}, len(wanted)) for _, chunk := range chunks { key := strings.ToLower(strings.TrimSpace(chunk.Name)) if _, ok := wanted[key]; !ok { continue } filtered = append(filtered, chunk) found[key] = struct{}{} } var missing []string for _, name := range archiveNames { key := strings.TrimSpace(strings.ToLower(name)) if key == "" { continue } if _, ok := found[key]; !ok { missing = append(missing, name) } } if len(missing) > 0 { return nil, fmt.Errorf("unknown hak archive(s): %s", strings.Join(missing, ", ")) } return filtered, nil } func hydrateHAKChunks(p *project.Project, chunks []hakChunk) ([]hakChunk, error) { if len(chunks) == 0 { return chunks, nil } lfsAssets, err := collectLFSAssetInfo(p) if err != nil { return nil, err } hydrated := make([]hakChunk, 0, len(chunks)) for _, chunk := range chunks { nextChunk := chunk nextChunk.Assets = make([]assetResource, 0, len(chunk.Assets)) for _, asset := range chunk.Assets { if !strings.HasPrefix(asset.ContentID, "lfs:") { nextChunk.Assets = append(nextChunk.Assets, asset) continue } info, ok := lfsAssets[filepath.ToSlash(asset.Rel)] if !ok { return nil, fmt.Errorf("missing LFS metadata for %s", asset.Rel) } resource, err := lfsPathResource(filepath.Join(p.AssetsDir(), filepath.FromSlash(asset.Rel)), info, true) if err != nil { return nil, err } asset.Resource = resource nextChunk.Assets = append(nextChunk.Assets, asset) } hydrated = append(hydrated, nextChunk) } return hydrated, nil } func progressf(progress ProgressFunc, message string) { if progress != nil { progress(message) } } func envBool(name string) bool { value := strings.TrimSpace(strings.ToLower(os.Getenv(name))) switch value { case "1", "true", "yes", "on": return true default: return false } } func collectModuleResources(p *project.Project, moduleHakOrder []string) ([]erf.Resource, error) { var moduleResources []erf.Resource for _, rel := range p.Inventory.SourceFiles { abs := filepath.Join(p.SourceDir(), filepath.FromSlash(rel)) resource, err := resourceFromJSON(abs, moduleHakOrder) if err != nil { return nil, err } moduleResources = append(moduleResources, resource) } for _, rel := range p.Inventory.ScriptFiles { abs := filepath.Join(p.SourceDir(), filepath.FromSlash(rel)) resource, err := rawResource(abs) if err != nil { return nil, err } moduleResources = append(moduleResources, resource) } compiledScripts, err := compileReferencedScripts(p) if err != nil { return nil, err } moduleResources = append(moduleResources, compiledScripts...) sortResources(moduleResources) return moduleResources, nil } func compileReferencedScripts(p *project.Project) ([]erf.Resource, error) { referenced, err := referencedScriptResrefs(p) if err != nil { return nil, err } if len(referenced) == 0 { return nil, nil } scriptSources := make(map[string]string, len(p.Inventory.ScriptFiles)) for _, rel := range p.Inventory.ScriptFiles { resref := strings.ToLower(strings.TrimSuffix(filepath.Base(rel), filepath.Ext(rel))) scriptSources[resref] = filepath.Join(p.SourceDir(), filepath.FromSlash(rel)) } compiler, err := resolveScriptCompiler(p) if err != nil { return nil, err } compilerEnv, err := resolveScriptCompilerEnvironment() if err != nil { return nil, err } tmpDir, err := os.MkdirTemp("", "sow-script-compile-*") if err != nil { return nil, fmt.Errorf("create script compiler temp dir: %w", err) } defer os.RemoveAll(tmpDir) resources := make([]erf.Resource, 0, len(referenced)) scriptsDir := filepath.Join(p.SourceDir(), "scripts") for _, resref := range referenced { sourcePath, ok := scriptSources[resref] if !ok { continue } outputPath := filepath.Join(tmpDir, resref+".ncs") cmd := exec.Command(compiler, "--dirs", scriptsDir, "-o", outputPath, sourcePath) cmd.Env = compilerEnv output, err := cmd.CombinedOutput() if err != nil { message := strings.TrimSpace(string(output)) if message == "" { message = err.Error() } return nil, fmt.Errorf("compile script %s: %s", filepath.Base(sourcePath), message) } resource, err := rawResource(outputPath) if err != nil { return nil, fmt.Errorf("load compiled script %s: %w", filepath.Base(outputPath), err) } resources = append(resources, resource) } return resources, nil } func referencedScriptResrefs(p *project.Project) ([]string, error) { referenced := map[string]struct{}{} for _, rel := range p.Inventory.SourceFiles { abs := filepath.Join(p.SourceDir(), filepath.FromSlash(rel)) document, _, _, err := validatorLoadDocument(abs) if err != nil { return nil, err } validatorWalkFields(document.Root, func(fieldLabel, value string) { if !validatorIsScriptField(fieldLabel) { return } value = strings.TrimSpace(value) if value == "" { return } referenced[strings.ToLower(value)] = struct{}{} }) } resrefs := make([]string, 0, len(referenced)) for resref := range referenced { resrefs = append(resrefs, resref) } slices.Sort(resrefs) return resrefs, nil } func resolveScriptCompiler(p *project.Project) (string, error) { if configured := strings.TrimSpace(os.Getenv("SOW_NWN_SCRIPT_COMPILER")); configured != "" { return configured, nil } candidates := []string{ filepath.Join(p.Root, "tools", "script-compiler", "nwn_script_comp"), filepath.Join(p.Root, "tools", "script-compiler", "nwn_script_comp.exe"), filepath.Join(p.Root, "tools", "nwn_script_comp"), filepath.Join(p.Root, "tools", "nwn_script_comp.exe"), } if runtime.GOOS == "windows" { candidates = []string{ filepath.Join(p.Root, "tools", "script-compiler", "nwn_script_comp.exe"), filepath.Join(p.Root, "tools", "script-compiler", "nwn_script_comp"), filepath.Join(p.Root, "tools", "nwn_script_comp.exe"), filepath.Join(p.Root, "tools", "nwn_script_comp"), } } for _, candidate := range candidates { if info, err := os.Stat(candidate); err == nil && !info.IsDir() { return candidate, nil } } if compiler, err := exec.LookPath("nwn_script_comp"); err == nil { return compiler, nil } if runtime.GOOS == "windows" { if compiler, err := exec.LookPath("nwn_script_comp.exe"); err == nil { return compiler, nil } } return "", errors.New("script compiler not found; install nwn_script_comp into ./tools or set SOW_NWN_SCRIPT_COMPILER") } func validatorLoadDocument(path string) (gff.Document, string, string, error) { raw, err := os.ReadFile(path) if err != nil { return gff.Document{}, "", "", fmt.Errorf("read %s: %w", path, err) } stem := strings.TrimSuffix(filepath.Base(path), ".json") extension := strings.TrimPrefix(strings.ToLower(filepath.Ext(stem)), ".") if extension == "" { return gff.Document{}, "", "", fmt.Errorf("missing resource extension before .json: %s", path) } resref := strings.ToLower(strings.TrimSuffix(stem, "."+extension)) var document gff.Document if err := json.Unmarshal(raw, &document); err != nil { return gff.Document{}, "", "", fmt.Errorf("parse %s: %w", path, err) } return document, resref, extension, nil } func validatorWalkFields(root gff.Struct, visit func(string, string)) { walkScriptFields(root, visit) } func validatorIsScriptField(label string) bool { return strings.HasPrefix(label, "On") || strings.HasSuffix(label, "Script") || strings.Contains(label, "Script") } func walkScriptFields(s gff.Struct, visit func(string, string)) { for _, field := range s.Fields { if value, ok := fieldStringValueForScripts(field.Value); ok { visit(field.Label, value) } switch typed := field.Value.(type) { case gff.Struct: walkScriptFields(typed, visit) case gff.ListValue: for _, item := range typed { walkScriptFields(item, visit) } } } } func fieldStringValueForScripts(value gff.Value) (string, bool) { switch typed := value.(type) { case gff.StringValue: return string(typed), true case gff.ResRefValue: return string(typed), true default: return "", false } } func collectAssetResources(p *project.Project, requireContent bool) ([]assetResource, error) { var hakResources []assetResource assetChangedAt, err := collectGitAssetChangeTimes(p) if err != nil { return nil, err } lfsAssets, err := collectLFSAssetInfo(p) if err != nil { return nil, err } assetsRelPath, err := filepath.Rel(p.Root, p.AssetsDir()) if err != nil { return nil, fmt.Errorf("resolve assets dir relative path: %w", err) } assetsRelPath = filepath.ToSlash(assetsRelPath) for _, rel := range p.Inventory.AssetFiles { abs := filepath.Join(p.AssetsDir(), filepath.FromSlash(rel)) resourceInfo, err := assetResourceFromPath(abs, filepath.ToSlash(rel), lfsAssets[filepath.ToSlash(rel)], requireContent) if err != nil { return nil, err } info, err := os.Stat(abs) if err != nil { return nil, fmt.Errorf("stat %s: %w", abs, err) } repoRel := filepath.ToSlash(filepath.Join(assetsRelPath, filepath.FromSlash(rel))) changedAt := assetChangedAt[repoRel] if changedAt.IsZero() { changedAt = info.ModTime() } hakResources = append(hakResources, assetResource{ Rel: rel, Resource: resourceInfo.Resource, Size: erf.ArchiveSize([]erf.Resource{resourceInfo.Resource}), ChangedAt: changedAt, ContentID: resourceInfo.ContentID, }) } slices.SortFunc(hakResources, func(a, b assetResource) int { return compareResourceKeys(a.Resource, b.Resource) }) return hakResources, nil } func collectLFSAssetInfo(p *project.Project) (map[string]lfsAssetInfo, error) { result := make(map[string]lfsAssetInfo) for _, rel := range p.Inventory.AssetFiles { rel = filepath.ToSlash(rel) abs := filepath.Join(p.AssetsDir(), filepath.FromSlash(rel)) info, ok, err := parseLFSPointerFile(abs, p.Root) if err != nil { return nil, err } if ok { result[rel] = info } } return result, nil } func parseLFSPointerFile(path, repoRoot string) (lfsAssetInfo, bool, error) { entry, err := os.Lstat(path) if err != nil { return lfsAssetInfo{}, false, fmt.Errorf("stat %s: %w", path, err) } if entry.Mode()&os.ModeSymlink != 0 || entry.Size() > 1024 { return lfsAssetInfo{}, false, nil } raw, err := os.ReadFile(path) if err != nil { return lfsAssetInfo{}, false, fmt.Errorf("read %s: %w", path, err) } text := strings.ReplaceAll(string(raw), "\r\n", "\n") if !strings.HasPrefix(text, "version https://git-lfs.github.com/spec/v1\n") { return lfsAssetInfo{}, false, nil } info := lfsAssetInfo{} for _, line := range strings.Split(text, "\n") { line = strings.TrimSpace(line) switch { case strings.HasPrefix(line, "oid sha256:"): info.OID = strings.TrimSpace(strings.TrimPrefix(line, "oid sha256:")) case strings.HasPrefix(line, "size "): size, parseErr := strconv.ParseInt(strings.TrimSpace(strings.TrimPrefix(line, "size ")), 10, 64) if parseErr != nil { return lfsAssetInfo{}, false, fmt.Errorf("parse LFS pointer size in %s: %w", path, parseErr) } info.Size = size } } if info.OID == "" || info.Size <= 0 { return lfsAssetInfo{}, false, nil } if len(info.OID) >= 4 { info.ObjectPath = filepath.Join(repoRoot, ".git", "lfs", "objects", info.OID[:2], info.OID[2:4], info.OID) } return info, true, nil } func assetResourceFromPath(path, rel string, lfsInfo lfsAssetInfo, requireContent bool) (assetResource, error) { if lfsInfo.OID != "" { resource, err := lfsPathResource(path, lfsInfo, requireContent) if err != nil { return assetResource{}, err } return assetResource{ Rel: rel, Resource: resource, ContentID: "lfs:" + lfsInfo.OID, }, nil } resource, err := pathResource(path) if err != nil { return assetResource{}, err } fileHash, err := fileSHA256(path) if err != nil { return assetResource{}, err } return assetResource{ Rel: rel, Resource: resource, ContentID: "file:" + fileHash, }, nil } func lfsPathResource(path string, info lfsAssetInfo, requireContent bool) (erf.Resource, error) { extension := strings.TrimPrefix(strings.ToLower(filepath.Ext(path)), ".") resourceType, ok := erf.HAKResourceTypeForExtension(extension) if !ok { return erf.Resource{}, fmt.Errorf("unsupported HAK resource extension %q", filepath.Ext(path)) } name := strings.ToLower(strings.TrimSuffix(filepath.Base(path), filepath.Ext(path))) sourcePath := "" if requireContent { if info.ObjectPath == "" { return erf.Resource{}, fmt.Errorf("missing downloaded LFS object for %s", path) } if _, err := os.Stat(info.ObjectPath); err != nil { return erf.Resource{}, fmt.Errorf("missing downloaded LFS object for %s: %w", path, err) } sourcePath = info.ObjectPath } return erf.Resource{ Name: name, Type: resourceType, SourcePath: sourcePath, Size: info.Size, }, nil } func fileSHA256(path string) (string, error) { raw, err := os.ReadFile(path) if err != nil { return "", fmt.Errorf("read %s: %w", path, err) } sum := sha256.Sum256(raw) return fmt.Sprintf("%x", sum[:]), nil } func collectGitAssetChangeTimes(p *project.Project) (map[string]time.Time, error) { result := make(map[string]time.Time, len(p.Inventory.AssetFiles)) if len(p.Inventory.AssetFiles) == 0 { return result, nil } assetsRelPath, err := filepath.Rel(p.Root, p.AssetsDir()) if err != nil { return result, fmt.Errorf("resolve assets dir relative path: %w", err) } assetsRelPath = filepath.ToSlash(assetsRelPath) output, err := gitOutput(p.Root, "log", "--format=commit:%ct", "--name-only", "--", assetsRelPath) if err != nil { return result, nil } wanted := make(map[string]struct{}, len(p.Inventory.AssetFiles)) for _, rel := range p.Inventory.AssetFiles { wanted[filepath.ToSlash(filepath.Join(assetsRelPath, filepath.FromSlash(rel)))] = struct{}{} } var currentCommitTime time.Time for _, line := range strings.Split(output, "\n") { line = strings.TrimSpace(line) if line == "" { continue } if strings.HasPrefix(line, "commit:") { sec, err := strconv.ParseInt(strings.TrimPrefix(line, "commit:"), 10, 64) if err != nil { currentCommitTime = time.Time{} continue } currentCommitTime = time.Unix(sec, 0).UTC() continue } if currentCommitTime.IsZero() { continue } if _, ok := wanted[line]; !ok { continue } if _, exists := result[line]; exists { continue } result[line] = currentCommitTime if len(result) == len(wanted) { break } } return result, nil } func gitOutput(dir string, args ...string) (string, error) { cmd := exec.Command("git", args...) if dir != "" { cmd.Dir = dir } output, err := cmd.CombinedOutput() if err != nil { return "", fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(string(output))) } return string(output), nil } func sortResources(resources []erf.Resource) { slices.SortFunc(resources, func(a, b erf.Resource) int { return compareResourceKeys(a, b) }) } func compareResourceKeys(a, b erf.Resource) int { if cmp := strings.Compare(a.Name, b.Name); cmp != 0 { return cmp } if a.Type < b.Type { return -1 } if a.Type > b.Type { return 1 } return 0 } func resourceFromJSON(path string, moduleHakOrder []string) (erf.Resource, error) { name, extension, err := splitSourceName(path) if err != nil { return erf.Resource{}, err } resourceType, ok := erf.ResourceTypeForExtension(extension) if !ok { return erf.Resource{}, fmt.Errorf("unsupported source extension %q", extension) } raw, err := os.ReadFile(path) if err != nil { return erf.Resource{}, fmt.Errorf("read %s: %w", path, err) } var document gff.Document if err := json.Unmarshal(raw, &document); err != nil { return erf.Resource{}, fmt.Errorf("parse gff json %s: %w", path, err) } if document.FileType == "" { document.FileType = strings.ToUpper(strings.TrimPrefix(extension, ".")) } if document.FileVersion == "" { document.FileVersion = "V3.2" } if extension == ".ifo" && name == "module" && len(moduleHakOrder) > 0 { setModuleHAKList(&document, moduleHakOrder) } var buf bytes.Buffer if err := gff.Write(&buf, document); err != nil { return erf.Resource{}, fmt.Errorf("encode gff %s: %w", path, err) } return erf.Resource{ Name: strings.ToLower(name), Type: resourceType, Data: buf.Bytes(), }, nil } func pathResource(path string) (erf.Resource, error) { extension := strings.TrimPrefix(strings.ToLower(filepath.Ext(path)), ".") resourceType, ok := erf.HAKResourceTypeForExtension(extension) if !ok { return erf.Resource{}, fmt.Errorf("unsupported HAK resource extension %q", filepath.Ext(path)) } name := strings.ToLower(strings.TrimSuffix(filepath.Base(path), filepath.Ext(path))) info, err := os.Stat(path) if err != nil { return erf.Resource{}, fmt.Errorf("stat %s: %w", path, err) } return erf.Resource{ Name: name, Type: resourceType, SourcePath: path, Size: info.Size(), }, nil } func rawResource(path string) (erf.Resource, error) { extension := strings.TrimPrefix(strings.ToLower(filepath.Ext(path)), ".") resourceType, ok := erf.HAKResourceTypeForExtension(extension) if !ok { return erf.Resource{}, fmt.Errorf("unsupported HAK resource extension %q", filepath.Ext(path)) } name := strings.ToLower(strings.TrimSuffix(filepath.Base(path), filepath.Ext(path))) data, err := os.ReadFile(path) if err != nil { return erf.Resource{}, fmt.Errorf("read %s: %w", path, err) } return erf.Resource{ Name: strings.ToLower(name), Type: resourceType, Data: data, }, nil } func splitSourceName(path string) (string, string, error) { base := filepath.Base(path) if !strings.HasSuffix(base, ".json") { return "", "", fmt.Errorf("expected .json source file: %s", path) } stem := strings.TrimSuffix(base, ".json") extension := filepath.Ext(stem) if extension == "" { return "", "", fmt.Errorf("source file must include target resource extension before .json: %s", path) } return strings.ToLower(strings.TrimSuffix(stem, extension)), strings.ToLower(extension), nil } func planHAKChunks(p *project.Project, assets []assetResource) ([]hakChunk, error) { if len(assets) == 0 { return nil, nil } if len(p.Config.HAKs) == 0 { chunk := hakChunk{ Config: project.HAKConfig{ Name: p.Config.Module.ResRef, Priority: 1, }, Index: 1, Name: p.Config.Module.ResRef, Size: erf.ArchiveSize(resourceSlice(assets)), Assets: assets, } return []hakChunk{chunk}, nil } configs := make([]project.HAKConfig, len(p.Config.HAKs)) copy(configs, p.Config.HAKs) slices.SortFunc(configs, func(a, b project.HAKConfig) int { if a.Priority != b.Priority { if a.Priority < b.Priority { return -1 } return 1 } return strings.Compare(a.Name, b.Name) }) grouped := make([][]assetResource, len(configs)) for _, asset := range assets { matched := false for index, cfg := range configs { if matchesAnyPattern(asset.Rel, cfg.Include) { grouped[index] = append(grouped[index], asset) matched = true break } } if !matched { return nil, fmt.Errorf("asset %s does not match any hak include pattern", asset.Rel) } } var chunks []hakChunk for index, cfg := range configs { groupAssets := grouped[index] if len(groupAssets) == 0 { continue } slices.SortFunc(groupAssets, compareAssetBuildOrder) groupChunks, err := splitHAKGroup(cfg, groupAssets) if err != nil { return nil, err } chunks = append(chunks, groupChunks...) } return chunks, nil } func splitHAKGroup(cfg project.HAKConfig, assets []assetResource) ([]hakChunk, error) { maxBytes := cfg.MaxBytes if maxBytes == 0 { chunk := hakChunk{ Config: cfg, Index: 1, Name: cfg.Name, Assets: assets, Size: erf.ArchiveSize(resourceSlice(assets)), } if err := ensureUniqueChunkResources(chunk); err != nil { return nil, err } return []hakChunk{chunk}, nil } var chunks []hakChunk current := hakChunk{Config: cfg, Index: 1} for _, asset := range assets { candidateAssets := append(append([]assetResource{}, current.Assets...), asset) candidateResources := resourceSlice(candidateAssets) candidateSize := erf.ArchiveSize(candidateResources) if len(current.Assets) == 0 { if candidateSize > maxBytes { return nil, fmt.Errorf("asset %s alone exceeds max_bytes for hak %s", asset.Rel, cfg.Name) } current.Assets = candidateAssets current.Size = candidateSize continue } if candidateSize <= maxBytes { current.Assets = candidateAssets current.Size = candidateSize continue } if !cfg.Split { return nil, fmt.Errorf("hak %s exceeds max_bytes and split is disabled", cfg.Name) } current.Name = chunkName(cfg.Name, current.Index, true) if err := ensureUniqueChunkResources(current); err != nil { return nil, err } chunks = append(chunks, current) current = hakChunk{ Config: cfg, Index: current.Index + 1, Assets: []assetResource{asset}, Size: erf.ArchiveSize([]erf.Resource{asset.Resource}), } if current.Size > maxBytes { return nil, fmt.Errorf("asset %s alone exceeds max_bytes for hak %s", asset.Rel, cfg.Name) } } if len(current.Assets) > 0 { current.Name = chunkName(cfg.Name, current.Index, cfg.Split || len(chunks) > 0) if err := ensureUniqueChunkResources(current); err != nil { return nil, err } chunks = append(chunks, current) } return chunks, nil } func compareAssetBuildOrder(a, b assetResource) int { if cmp := a.ChangedAt.Compare(b.ChangedAt); cmp != 0 { return cmp } if cmp := compareResourceKeys(a.Resource, b.Resource); cmp != 0 { return cmp } return strings.Compare(a.Rel, b.Rel) } func chunkName(base string, index int, split bool) string { if !split { return base } return fmt.Sprintf("%s_%02d", base, index) } func resourceSlice(assets []assetResource) []erf.Resource { out := make([]erf.Resource, 0, len(assets)) for _, asset := range assets { out = append(out, asset.Resource) } return out } func buildManifestEntry(chunk hakChunk) BuildManifestHAK { assets := make([]string, 0, len(chunk.Assets)) for _, asset := range chunk.Assets { assets = append(assets, asset.Rel) } return BuildManifestHAK{ Name: chunk.Name, Group: chunk.Config.Name, Priority: chunk.Config.Priority, MaxBytes: chunk.Config.MaxBytes, SizeBytes: chunk.Size, Optional: chunk.Config.Optional, Assets: assets, ContentHash: chunkContentHash(chunk), } } func chunkContentHash(chunk hakChunk) string { sum := sha256.New() fmt.Fprintf(sum, "name:%s\n", chunk.Name) fmt.Fprintf(sum, "group:%s\n", chunk.Config.Name) fmt.Fprintf(sum, "priority:%d\n", chunk.Config.Priority) fmt.Fprintf(sum, "max_bytes:%d\n", chunk.Config.MaxBytes) fmt.Fprintf(sum, "optional:%t\n", chunk.Config.Optional) fmt.Fprintf(sum, "size_bytes:%d\n", chunk.Size) for _, asset := range chunk.Assets { fmt.Fprintf(sum, "asset:%s\n", asset.Rel) fmt.Fprintf(sum, "resref:%s\n", asset.Resource.Name) fmt.Fprintf(sum, "restype:%d\n", asset.Resource.Type) fmt.Fprintf(sum, "datasize:%d\n", asset.Resource.Size) fmt.Fprintf(sum, "content:%s\n", asset.ContentID) } return fmt.Sprintf("%x", sum.Sum(nil)) } func loadPreviousBuildManifest(buildDir string) (*BuildManifest, error) { manifestPath := filepath.Join(buildDir, "haks.json") raw, err := os.ReadFile(manifestPath) if err != nil { if errors.Is(err, os.ErrNotExist) { return nil, nil } return nil, fmt.Errorf("read previous hak manifest: %w", err) } var manifest BuildManifest if err := json.Unmarshal(raw, &manifest); err != nil { return nil, fmt.Errorf("parse previous hak manifest: %w", err) } return &manifest, nil } func reuseExistingHAK(previousManifest *BuildManifest, current BuildManifestHAK, hakPath string) (bool, error) { if previousManifest == nil { return false, nil } var previous *BuildManifestHAK for index := range previousManifest.HAKs { entry := &previousManifest.HAKs[index] if entry.Name == current.Name { previous = entry break } } if previous == nil { return false, nil } if previous.Group != current.Group || previous.Priority != current.Priority || previous.MaxBytes != current.MaxBytes || previous.SizeBytes != current.SizeBytes || previous.ContentHash != current.ContentHash || !slices.Equal(previous.Assets, current.Assets) { return false, nil } if _, err := os.Stat(hakPath); err != nil { if errors.Is(err, os.ErrNotExist) { return false, nil } return false, fmt.Errorf("stat existing hak archive %s: %w", hakPath, err) } return true, nil } func cleanupStaleGeneratedHAKs(buildDir, moduleResRef string, keepNames map[string]struct{}) error { paths, err := manifestHAKPaths(buildDir) if err != nil { return err } for _, path := range paths { name := strings.TrimSuffix(filepath.Base(path), filepath.Ext(path)) if _, keep := keepNames[name]; keep { continue } if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { return fmt.Errorf("remove stale generated hak %s: %w", path, err) } } legacy := filepath.Join(buildDir, moduleResRef+".hak") if _, keep := keepNames[moduleResRef]; !keep { if err := os.Remove(legacy); err != nil && !errors.Is(err, os.ErrNotExist) { return fmt.Errorf("remove legacy generated hak %s: %w", legacy, err) } } return nil } func matchesAnyPattern(path string, patterns []string) bool { for _, pattern := range patterns { if matchPathPattern(filepath.ToSlash(path), filepath.ToSlash(pattern)) { return true } } return false } func matchPathPattern(path, pattern string) bool { pathSegs := strings.Split(strings.Trim(path, "/"), "/") patternSegs := strings.Split(strings.Trim(pattern, "/"), "/") return matchSegments(pathSegs, patternSegs) } func cleanupGeneratedHAKs(buildDir, moduleResRef string) error { paths, err := manifestHAKPaths(buildDir) if err != nil { return err } for _, path := range paths { if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { return fmt.Errorf("remove previous generated hak %s: %w", path, err) } } legacy := filepath.Join(buildDir, moduleResRef+".hak") if err := os.Remove(legacy); err != nil && !errors.Is(err, os.ErrNotExist) { return fmt.Errorf("remove legacy generated hak %s: %w", legacy, err) } manifest := filepath.Join(buildDir, "haks.json") if err := os.Remove(manifest); err != nil && !errors.Is(err, os.ErrNotExist) { return fmt.Errorf("remove previous hak manifest: %w", err) } return nil } func plannedModuleHAKOrder(p *project.Project) ([]string, error) { manifest, err := loadPreviousBuildManifest(p.BuildDir()) if err != nil { return nil, err } if manifest != nil && len(manifest.ModuleHAKs) > 0 { return append([]string(nil), manifest.ModuleHAKs...), nil } assetResources, err := collectAssetResources(p, false) if err != nil { return nil, err } chunks, err := planHAKChunks(p, assetResources) if err != nil { return nil, err } return resolveModuleHAKOrder(p, chunks) } func resolveModuleHAKOrder(p *project.Project, chunks []hakChunk) ([]string, error) { if len(p.Config.Module.HAKOrder) == 0 { return nil, nil } byGroup := map[string][]string{} optionalNames := map[string]struct{}{} for _, chunk := range chunks { if chunk.Config.Optional { optionalNames[chunk.Name] = struct{}{} continue } byGroup[chunk.Config.Name] = append(byGroup[chunk.Config.Name], chunk.Name) } seen := map[string]struct{}{} var ordered []string for _, entry := range p.Config.Module.HAKOrder { if strings.HasPrefix(entry, "group:") { groupName := strings.TrimPrefix(entry, "group:") names, ok := byGroup[groupName] if !ok { continue } for _, name := range names { if _, exists := seen[name]; exists { continue } ordered = append(ordered, name) seen[name] = struct{}{} } continue } if _, optional := optionalNames[entry]; optional { continue } if _, exists := seen[entry]; exists { continue } ordered = append(ordered, entry) seen[entry] = struct{}{} } return ordered, nil } func setModuleHAKList(document *gff.Document, hakNames []string) { list := make(gff.ListValue, 0, len(hakNames)) for _, name := range hakNames { list = append(list, gff.Struct{ Type: 8, Fields: []gff.Field{ gff.NewField("Mod_Hak", gff.StringValue(name)), }, }) } for index, field := range document.Root.Fields { if field.Label == "Mod_HakList" { document.Root.Fields[index] = gff.NewField("Mod_HakList", list) return } } document.Root.Fields = append(document.Root.Fields, gff.NewField("Mod_HakList", list)) } func validateForBuild(p *project.Project) error { report := validator.ValidateProject(p) if report.HasErrors() { summary := report.SummaryLines(3) if len(summary) > 0 { return fmt.Errorf("validation failed with %d error(s): %s", report.ErrorCount(), summary[0]) } return fmt.Errorf("validation failed with %d error(s)", report.ErrorCount()) } return nil } func matchSegments(pathSegs, patternSegs []string) bool { if len(patternSegs) == 0 { return len(pathSegs) == 0 } if patternSegs[0] == "**" { if matchSegments(pathSegs, patternSegs[1:]) { return true } if len(pathSegs) > 0 { return matchSegments(pathSegs[1:], patternSegs) } return false } if len(pathSegs) == 0 { return false } ok, err := filepath.Match(patternSegs[0], pathSegs[0]) if err != nil || !ok { return false } return matchSegments(pathSegs[1:], patternSegs[1:]) }