From 4f8bfc133a63956cadcffc0f9b0fc6d455857de2 Mon Sep 17 00:00:00 2001 From: vickydotbat Date: Thu, 9 Apr 2026 11:01:39 +0200 Subject: [PATCH] Native Workflow Support --- internal/app/app.go | 39 ++++ internal/pipeline/build.go | 53 ++++- internal/pipeline/compare.go | 75 +++++++- internal/pipeline/pipeline_test.go | 257 ++++++++++++++++++++++++- internal/pipeline/zz_debug_tmp_test.go | 41 ++++ internal/topdata/tlk_native.go | 60 +++++- internal/topdata/top_package.go | 209 ++++++++++++++++++++ internal/topdata/topdata.go | 115 ++++++++++- internal/topdata/topdata_test.go | 80 ++++++++ 9 files changed, 912 insertions(+), 17 deletions(-) create mode 100644 internal/pipeline/zz_debug_tmp_test.go create mode 100644 internal/topdata/top_package.go diff --git a/internal/app/app.go b/internal/app/app.go index 64a344e..a001f96 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -74,6 +74,11 @@ var commands = []command{ description: "Build canonical topdata outputs from topdata/data into build/topdata.", run: runBuildTopData, }, + { + name: "build-top-package", + description: "Build sow_top.hak and sow_tlk.tlk from native topdata output and topdata/assets.", + run: runBuildTopPackage, + }, { name: "compare-topdata", description: "Rebuild topdata natively and compare the current build output against that fresh native result.", @@ -143,6 +148,10 @@ func runBuild(ctx context) error { fmt.Fprintf(ctx.stdout, "project: %s\n", p.Config.Module.Name) fmt.Fprintf(ctx.stdout, "module: %s\n", result.ModulePath) fmt.Fprintf(ctx.stdout, "resources: %d\n", result.Resources) + if result.TopPackageHAK != "" { + fmt.Fprintf(ctx.stdout, "top package hak: %s\n", result.TopPackageHAK) + fmt.Fprintf(ctx.stdout, "top package tlk: %s\n", result.TopPackageTLK) + } if len(result.HAKPaths) > 0 { fmt.Fprintf(ctx.stdout, "hak archives: %d\n", len(result.HAKPaths)) fmt.Fprintf(ctx.stdout, "hak assets: %d\n", result.HAKAssets) @@ -167,6 +176,10 @@ func runBuildModule(ctx context) error { fmt.Fprintf(ctx.stdout, "project: %s\n", p.Config.Module.Name) fmt.Fprintf(ctx.stdout, "module: %s\n", result.ModulePath) fmt.Fprintf(ctx.stdout, "resources: %d\n", result.Resources) + if result.TopPackageHAK != "" { + fmt.Fprintf(ctx.stdout, "top package hak: %s\n", result.TopPackageHAK) + fmt.Fprintf(ctx.stdout, "top package tlk: %s\n", result.TopPackageTLK) + } return nil } @@ -378,6 +391,32 @@ func runBuildTopData(ctx context) error { return nil } +func runBuildTopPackage(ctx context) error { + p, err := loadProject(ctx.cwd) + if err != nil { + return err + } + + result, err := topdata.BuildPackage(p, func(message string) { + fmt.Fprintf(ctx.stdout, "[build-top-package] %s\n", message) + }) + if err != nil { + return err + } + + fmt.Fprintf(ctx.stdout, "project: %s\n", p.Config.Module.Name) + fmt.Fprintf(ctx.stdout, "mode: %s\n", result.Mode) + fmt.Fprintf(ctx.stdout, "topdata 2da output: %s\n", result.Output2DADir) + fmt.Fprintf(ctx.stdout, "topdata tlk output: %s\n", result.OutputTLKDir) + fmt.Fprintf(ctx.stdout, "top package hak: %s\n", result.OutputHAKPath) + fmt.Fprintf(ctx.stdout, "top package tlk: %s\n", result.OutputTLKPath) + fmt.Fprintf(ctx.stdout, "2da files: %d\n", result.Files2DA) + fmt.Fprintf(ctx.stdout, "tlk files: %d\n", result.FilesTLK) + fmt.Fprintf(ctx.stdout, "top package resources: %d\n", result.HAKResources) + fmt.Fprintf(ctx.stdout, "top package assets: %d\n", result.AssetFiles) + return nil +} + func runCompareTopData(ctx context) error { p, err := loadProject(ctx.cwd) if err != nil { diff --git a/internal/pipeline/build.go b/internal/pipeline/build.go index 9410e96..26a8af4 100644 --- a/internal/pipeline/build.go +++ b/internal/pipeline/build.go @@ -16,15 +16,19 @@ import ( "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf" "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/gff" "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" + "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/topdata" "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/validator" ) type BuildResult struct { - ModulePath string - HAKPaths []string - Manifest string - Resources int - HAKAssets int + ModulePath string + HAKPaths []string + Manifest string + Resources int + HAKAssets int + TopPackageHAK string + TopPackageTLK string + TopPackageFiles int } type BuildManifest struct { @@ -87,8 +91,26 @@ func buildModule(p *project.Project, progress ProgressFunc) (BuildResult, error) return BuildResult{}, err } + var topPackage topdata.PackageResult + var err error + if p.HasTopData() { + progressf(progress, "Building native top package...") + topPackage, err = topdata.BuildPackage(p, func(message string) { + progressf(progress, "Top package: "+message) + }) + if err != nil { + return BuildResult{}, err + } + } + + progressf(progress, "Resolving module HAK order...") + moduleHakOrder, err := plannedModuleHAKOrder(p) + if err != nil { + return BuildResult{}, err + } + progressf(progress, "Collecting module resources...") - moduleResources, err := collectModuleResources(p, nil) + moduleResources, err := collectModuleResources(p, moduleHakOrder) if err != nil { return BuildResult{}, err } @@ -106,8 +128,11 @@ func buildModule(p *project.Project, progress ProgressFunc) (BuildResult, error) } return BuildResult{ - ModulePath: outputPath, - Resources: len(moduleResources), + ModulePath: outputPath, + Resources: len(moduleResources), + TopPackageHAK: topPackage.OutputHAKPath, + TopPackageTLK: topPackage.OutputTLKPath, + TopPackageFiles: topPackage.HAKResources, }, nil } @@ -841,6 +866,18 @@ func cleanupGeneratedHAKs(buildDir, moduleResRef string) error { return nil } +func plannedModuleHAKOrder(p *project.Project) ([]string, error) { + assetResources, err := collectAssetResources(p) + if err != nil { + return nil, err + } + chunks, err := planHAKChunks(p, assetResources) + if err != nil { + return nil, err + } + return resolveModuleHAKOrder(p, chunks) +} + func resolveModuleHAKOrder(p *project.Project, chunks []hakChunk) ([]string, error) { if len(p.Config.Module.HAKOrder) == 0 { return nil, nil diff --git a/internal/pipeline/compare.go b/internal/pipeline/compare.go index 94ab425..2915c15 100644 --- a/internal/pipeline/compare.go +++ b/internal/pipeline/compare.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "io/fs" "os" "path/filepath" "slices" @@ -14,6 +15,7 @@ import ( "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf" "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/gff" "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" + "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/topdata" ) type CompareResult struct { @@ -112,6 +114,10 @@ func Compare(p *project.Project) (CompareResult, error) { func expectedResources(p *project.Project) (map[string]resourceExpectation, error) { out := map[string]resourceExpectation{} + moduleHakOrder, err := plannedModuleHAKOrder(p) + if err != nil { + return nil, err + } for _, rel := range p.Inventory.SourceFiles { abs := filepath.Join(p.SourceDir(), filepath.FromSlash(rel)) @@ -128,6 +134,9 @@ func expectedResources(p *project.Project) (map[string]resourceExpectation, erro if err := json.Unmarshal(raw, &document); err != nil { return nil, fmt.Errorf("parse gff json %s: %w", abs, err) } + if extension == ".ifo" && strings.EqualFold(name, "module") && len(moduleHakOrder) > 0 { + setModuleHAKList(&document, moduleHakOrder) + } canonical, err := json.Marshal(document) if err != nil { @@ -195,6 +204,12 @@ func ensureBuildIsCurrent(p *project.Project, modulePath string, hakPaths []stri archivePaths := make([]string, 0, 1+len(hakPaths)) archivePaths = append(archivePaths, modulePath) archivePaths = append(archivePaths, hakPaths...) + if p.HasTopData() { + archivePaths = append(archivePaths, + filepath.Join(p.BuildDir(), topdata.PackageHAKFileName), + filepath.Join(p.BuildDir(), topdata.PackageTLKFileName), + ) + } oldestArchive := time.Time{} for _, path := range archivePaths { @@ -217,6 +232,13 @@ func ensureBuildIsCurrent(p *project.Project, modulePath string, hakPaths []stri for _, rel := range p.Inventory.AssetFiles { sourcePaths = append(sourcePaths, filepath.Join(p.AssetsDir(), filepath.FromSlash(rel))) } + if p.HasTopData() { + topSources, err := topPackageSourcePaths(p) + if err != nil { + return err + } + sourcePaths = append(sourcePaths, topSources...) + } newestSource := time.Time{} newestPath := "" @@ -308,6 +330,57 @@ func manifestHAKPaths(buildDir string) ([]string, error) { if err != nil { return nil, fmt.Errorf("scan hak archives: %w", err) } - slices.Sort(paths) + filtered := make([]string, 0, len(paths)) + for _, path := range paths { + if filepath.Base(path) == topdata.PackageHAKFileName { + continue + } + filtered = append(filtered, path) + } + slices.Sort(filtered) + return filtered, nil +} + +func topPackageSourcePaths(p *project.Project) ([]string, error) { + sourceDir := p.TopDataSourceDir() + paths := make([]string, 0) + for _, candidate := range []string{ + filepath.Join(sourceDir, ".tlk_state.json"), + filepath.Join(sourceDir, "base_dialog.json"), + } { + if _, err := os.Stat(candidate); err == nil { + paths = append(paths, candidate) + } else if err != nil && !errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("stat topdata source %s: %w", candidate, err) + } + } + for _, dir := range []string{ + filepath.Join(sourceDir, "data"), + filepath.Join(sourceDir, "assets"), + } { + info, err := os.Stat(dir) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + continue + } + return nil, fmt.Errorf("stat topdata path %s: %w", dir, err) + } + if !info.IsDir() { + continue + } + err = filepath.WalkDir(dir, func(path string, d fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if d.IsDir() { + return nil + } + paths = append(paths, path) + return nil + }) + if err != nil { + return nil, fmt.Errorf("scan topdata path %s: %w", dir, err) + } + } return paths, nil } diff --git a/internal/pipeline/pipeline_test.go b/internal/pipeline/pipeline_test.go index ca5c7e5..8d16e01 100644 --- a/internal/pipeline/pipeline_test.go +++ b/internal/pipeline/pipeline_test.go @@ -1,6 +1,7 @@ package pipeline import ( + "bytes" "encoding/json" "os" "path/filepath" @@ -9,6 +10,7 @@ import ( "time" "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf" + "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/gff" "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" ) @@ -32,7 +34,7 @@ func TestBuildThenExtract(t *testing.T) { `) mustMkdir(t, filepath.Join(root, "src", "module")) -mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{ + mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{ "file_type": "IFO ", "file_version": "V3.2", "root": { @@ -427,6 +429,259 @@ printf 'compiled:%s\n' "$(basename "$src")" > "$out" } } +func TestBuildModuleBuildsTopPackageAndInjectsHAKList(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src", "module")) + mustMkdir(t, filepath.Join(root, "assets")) + mustMkdir(t, filepath.Join(root, "build")) + mustMkdir(t, filepath.Join(root, "topdata", "data", "repadjust")) + mustMkdir(t, filepath.Join(root, "topdata", "assets", "gui")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{ + "module": { + "name": "Test Module", + "resref": "testmod", + "hak_order": ["sow_top"] + }, + "paths": { + "source": "src", + "assets": "assets", + "build": "build" + }, + "topdata": { + "source": "topdata", + "build": "build/topdata" + } +} +`) + mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{ + "file_type": "IFO ", + "file_version": "V3.2", + "root": { + "struct_type": 0, + "fields": [ + {"label": "Mod_Name", "type": "CExoString", "value": "Test Module"}, + {"label": "Mod_CustomTlk", "type": "CExoString", "value": "sow_tlk"} + ] + } +} +`) + mustWriteFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + mustWriteFile(t, filepath.Join(root, "topdata", "data", "repadjust", "base.json"), `{ + "output": "repadjust.2da", + "columns": ["Label"], + "rows": [{"id": 0, "Label": "TEST_LABEL"}] +} +`) + mustWriteFile(t, filepath.Join(root, "topdata", "assets", "gui", "testicon.tga"), "icon-data") + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + + result, err := BuildModule(p) + if err != nil { + t.Fatalf("build module: %v", err) + } + if _, err := os.Stat(filepath.Join(root, "build", "sow_top.hak")); err != nil { + t.Fatalf("expected sow_top.hak: %v", err) + } + if _, err := os.Stat(filepath.Join(root, "build", "sow_tlk.tlk")); err != nil { + t.Fatalf("expected sow_tlk.tlk: %v", err) + } + if result.TopPackageHAK == "" || result.TopPackageTLK == "" { + t.Fatalf("expected top package paths in build result: %#v", result) + } + + input, err := os.Open(result.ModulePath) + if err != nil { + t.Fatalf("open module: %v", err) + } + defer input.Close() + archive, err := erf.Read(input) + if err != nil { + t.Fatalf("read module archive: %v", err) + } + var ifoData []byte + for _, resource := range archive.Resources { + ext, _ := erf.ExtensionForResourceType(resource.Type) + if strings.ToLower(resource.Name)+"."+ext == "module.ifo" { + ifoData = resource.Data + break + } + } + if ifoData == nil { + t.Fatalf("expected module.ifo in built archive") + } + document, err := gff.Read(bytes.NewReader(ifoData)) + if err != nil { + t.Fatalf("decode module.ifo: %v", err) + } + foundHak := false + for _, field := range document.Root.Fields { + if field.Label != "Mod_HakList" { + continue + } + list, ok := field.Value.(gff.ListValue) + if !ok { + t.Fatalf("expected Mod_HakList list, got %T", field.Value) + } + for _, item := range list { + for _, nested := range item.Fields { + if nested.Label == "Mod_Hak" && nested.Value == gff.StringValue("sow_top") { + foundHak = true + } + } + } + } + if !foundHak { + t.Fatalf("expected built module to reference sow_top") + } +} + +func TestCompareFailsWhenTopPackageIsStale(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src", "module")) + mustMkdir(t, filepath.Join(root, "assets")) + mustMkdir(t, filepath.Join(root, "build")) + mustMkdir(t, filepath.Join(root, "topdata", "data", "repadjust")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{ + "module": { + "name": "Test Module", + "resref": "testmod", + "hak_order": ["sow_top"] + }, + "paths": { + "source": "src", + "assets": "assets", + "build": "build" + }, + "topdata": { + "source": "topdata", + "build": "build/topdata" + } +} +`) + mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{ + "file_type": "IFO ", + "file_version": "V3.2", + "root": { + "struct_type": 0, + "fields": [ + {"label": "Mod_Name", "type": "CExoString", "value": "Test Module"}, + {"label": "Mod_CustomTlk", "type": "CExoString", "value": "sow_tlk"} + ] + } +} +`) + mustWriteFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + topdataBase := filepath.Join(root, "topdata", "data", "repadjust", "base.json") + mustWriteFile(t, topdataBase, `{ + "output": "repadjust.2da", + "columns": ["Label"], + "rows": [{"id": 0, "Label": "TEST_LABEL"}] +} +`) + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + + if _, err := BuildModule(p); err != nil { + t.Fatalf("build module: %v", err) + } + newTime := time.Now().Add(2 * time.Second) + if err := os.Chtimes(topdataBase, newTime, newTime); err != nil { + t.Fatalf("touch topdata source: %v", err) + } + if _, err := Compare(p); err == nil { + t.Fatalf("expected compare to fail when top package is stale") + } +} + +func TestCompareNormalizesModuleHakListFromConfig(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src", "module")) + mustMkdir(t, filepath.Join(root, "assets")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{ + "module": { + "name": "Test Module", + "resref": "testmod", + "hak_order": ["manual_top"] + }, + "paths": { + "source": "src", + "assets": "assets", + "build": "build" + } +}`) + mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{ + "file_type": "IFO ", + "file_version": "V3.2", + "root": { + "struct_type": 0, + "fields": [ + { + "label": "Mod_Name", + "type": "CExoString", + "value": "Test Module" + }, + { + "label": "Mod_HakList", + "type": "List", + "value": [ + { + "struct_type": 8, + "fields": [ + { + "label": "Mod_Hak", + "type": "CExoString", + "value": "stale_old_hak" + } + ] + } + ] + } + ] + } +}`) + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + + if _, err := BuildModule(p); err != nil { + t.Fatalf("build module: %v", err) + } + if _, err := Compare(p); err != nil { + t.Fatalf("compare project: %v", err) + } +} + func TestBuildOrdersMultipleHAKGroups(t *testing.T) { root := t.TempDir() mustMkdir(t, filepath.Join(root, "src")) diff --git a/internal/pipeline/zz_debug_tmp_test.go b/internal/pipeline/zz_debug_tmp_test.go new file mode 100644 index 0000000..08bab7b --- /dev/null +++ b/internal/pipeline/zz_debug_tmp_test.go @@ -0,0 +1,41 @@ +package pipeline + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "path/filepath" + "testing" + + "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf" + "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/gff" + "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" +) + +func TestDebugModuleIFO(t *testing.T) { + root := "/home/vicky/Projects/sow-ShadowsOverWestgate/repositories/sow-module" + p, err := project.Load(root) + if err != nil { t.Fatal(err) } + if err := p.Scan(); err != nil { t.Fatal(err) } + exp, err := expectedResources(p) + if err != nil { t.Fatal(err) } + gotArchive, err := readArchive(filepath.Join(p.BuildDir(), p.Config.Module.ResRef+".mod")) + if err != nil { t.Fatal(err) } + got := archiveIndex(gotArchive) + e := exp["module.ifo"].Bytes + a := got["module.ifo"].Data + fmt.Printf("EQUAL=%v\n", bytes.Equal(e,a)) + fmt.Printf("EXPECTED=%s\n", string(e)) + fmt.Printf("ACTUAL=%s\n", string(a)) + + var edoc gff.Document + sourceRaw, _ := os.ReadFile(filepath.Join(root, "src/module/module.ifo.json")) + _ = json.Unmarshal(sourceRaw, &edoc) + fmt.Printf("SOURCE=%s\n", string(sourceRaw)) + + mod, _ := erf.Read(bytes.NewReader(mustRead(filepath.Join(p.BuildDir(), p.Config.Module.ResRef+".mod")))) + _ = mod +} + +func mustRead(path string) []byte { b, _ := os.ReadFile(path); return b } diff --git a/internal/topdata/tlk_native.go b/internal/topdata/tlk_native.go index 90efa34..a4f56be 100644 --- a/internal/topdata/tlk_native.go +++ b/internal/topdata/tlk_native.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" "slices" + "strconv" "strings" "golang.org/x/text/encoding/charmap" @@ -178,9 +179,9 @@ func loadBaseDialogData(path string) (*legacyTLKData, error) { if !ok { return result, nil } - entries, ok := rawEntries.(map[string]any) - if !ok { - return nil, fmt.Errorf("base_dialog.json entries must be an object") + entries, err := normalizeBaseDialogEntries(rawEntries) + if err != nil { + return nil, err } normalized, err := normalizeLegacyTLKEntries(entries) if err != nil { @@ -192,6 +193,59 @@ func loadBaseDialogData(path string) (*legacyTLKData, error) { return result, nil } +func normalizeBaseDialogEntries(raw any) (map[string]any, error) { + switch typed := raw.(type) { + case map[string]any: + return typed, nil + case []any: + entries := make(map[string]any, len(typed)) + for index, item := range typed { + obj, ok := item.(map[string]any) + if !ok { + return nil, fmt.Errorf("base_dialog.json entry %d must be an object", index) + } + rawID, ok := obj["id"] + if !ok { + return nil, fmt.Errorf("base_dialog.json entry %d is missing id", index) + } + key, err := legacyDialogIDKey(rawID) + if err != nil { + return nil, fmt.Errorf("base_dialog.json entry %d: %w", index, err) + } + entry := map[string]any{} + if text, ok := obj["text"]; ok { + entry["text"] = text + } + if sound, ok := obj["sound_resref"]; ok { + entry["sound_resref"] = sound + } + if length, ok := obj["sound_length"]; ok { + entry["sound_length"] = length + } + entries[key] = entry + } + return entries, nil + default: + return nil, fmt.Errorf("base_dialog.json entries must be an object or array") + } +} + +func legacyDialogIDKey(raw any) (string, error) { + switch typed := raw.(type) { + case string: + if strings.TrimSpace(typed) == "" { + return "", fmt.Errorf("id must not be empty") + } + return typed, nil + case float64: + return strconv.Itoa(int(typed)), nil + case int: + return strconv.Itoa(typed), nil + default: + return "", fmt.Errorf("id must be a string or number") + } +} + func loadTLKState(path string) (tlkStateDocument, error) { doc := tlkStateDocument{ Version: 1, diff --git a/internal/topdata/top_package.go b/internal/topdata/top_package.go new file mode 100644 index 0000000..02300bb --- /dev/null +++ b/internal/topdata/top_package.go @@ -0,0 +1,209 @@ +package topdata + +import ( + "fmt" + "io" + "os" + "path/filepath" + "slices" + "strings" + + "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf" + "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" +) + +const ( + PackageHAKResRef = "sow_top" + PackageHAKFileName = PackageHAKResRef + ".hak" + PackageTLKFileName = defaultTLKName +) + +func BuildPackage(p *project.Project, progress func(string)) (PackageResult, error) { + if progress == nil { + progress = func(string) {} + } + nativeResult, err := BuildNative(p, progress) + if err != nil { + return PackageResult{}, err + } + return packageBuiltTopData(p, nativeResult, progress) +} + +func packageBuiltTopData(p *project.Project, nativeResult BuildResult, progress func(string)) (PackageResult, error) { + if progress == nil { + progress = func(string) {} + } + buildDir := p.BuildDir() + outputHAK := filepath.Join(buildDir, PackageHAKFileName) + outputTLK := filepath.Join(buildDir, PackageTLKFileName) + + progress("Packaging compiled topdata resources into sow_top.hak...") + resources, assetFiles, err := collectTopPackageResources(p, nativeResult.Output2DADir) + if err != nil { + return PackageResult{}, err + } + if err := writeHAK(outputHAK, resources); err != nil { + return PackageResult{}, err + } + + progress("Preparing compiled TLK release output...") + sourceTLK, err := resolveCompiledTLKPath(nativeResult.OutputTLKDir) + if err != nil { + return PackageResult{}, err + } + if err := copyFile(outputTLK, sourceTLK); err != nil { + return PackageResult{}, err + } + + return PackageResult{ + Mode: nativeResult.Mode, + Output2DADir: nativeResult.Output2DADir, + OutputTLKDir: nativeResult.OutputTLKDir, + OutputHAKPath: outputHAK, + OutputTLKPath: outputTLK, + Files2DA: nativeResult.Files2DA, + FilesTLK: nativeResult.FilesTLK, + HAKResources: len(resources), + AssetFiles: assetFiles, + }, nil +} + +func collectTopPackageResources(p *project.Project, compiled2DADir string) ([]erf.Resource, int, error) { + resourceByKey := map[string]erf.Resource{} + + entries, err := os.ReadDir(compiled2DADir) + if err != nil { + return nil, 0, fmt.Errorf("read compiled 2da output dir: %w", err) + } + for _, entry := range entries { + if entry.IsDir() || !strings.EqualFold(filepath.Ext(entry.Name()), ".2da") { + continue + } + path := filepath.Join(compiled2DADir, entry.Name()) + resource, err := topPackageResourceFromPath(path) + if err != nil { + return nil, 0, err + } + resourceByKey[topPackageResourceKey(resource)] = resource + } + + assetFiles := 0 + assetsDir := filepath.Join(p.TopDataSourceDir(), "assets") + info, err := os.Stat(assetsDir) + if err == nil && info.IsDir() { + err = filepath.WalkDir(assetsDir, func(path string, d os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if d.IsDir() { + return nil + } + if strings.HasPrefix(filepath.Base(path), ".") { + return nil + } + resource, err := topPackageResourceFromPath(path) + if err != nil { + return err + } + key := topPackageResourceKey(resource) + if _, ok := resourceByKey[key]; ok { + return fmt.Errorf("topdata asset %s collides with generated top package resource %s", path, key) + } + resourceByKey[key] = resource + assetFiles++ + return nil + }) + if err != nil { + return nil, 0, err + } + } else if err != nil && !os.IsNotExist(err) { + return nil, 0, fmt.Errorf("stat topdata assets dir: %w", err) + } + + keys := make([]string, 0, len(resourceByKey)) + for key := range resourceByKey { + keys = append(keys, key) + } + slices.Sort(keys) + + resources := make([]erf.Resource, 0, len(keys)) + for _, key := range keys { + resources = append(resources, resourceByKey[key]) + } + return resources, assetFiles, nil +} + +func topPackageResourceFromPath(path string) (erf.Resource, error) { + extension := strings.TrimPrefix(strings.ToLower(filepath.Ext(path)), ".") + resourceType, ok := erf.ResourceTypeForExtension(extension) + if !ok { + return erf.Resource{}, fmt.Errorf("unsupported top package resource extension %q", filepath.Ext(path)) + } + info, err := os.Stat(path) + if err != nil { + return erf.Resource{}, fmt.Errorf("stat %s: %w", path, err) + } + return erf.Resource{ + Name: strings.ToLower(strings.TrimSuffix(filepath.Base(path), filepath.Ext(path))), + Type: resourceType, + SourcePath: path, + Size: info.Size(), + }, nil +} + +func topPackageResourceKey(resource erf.Resource) string { + extension, ok := erf.ExtensionForResourceType(resource.Type) + if !ok { + return strings.ToLower(resource.Name) + } + return strings.ToLower(resource.Name) + "." + strings.TrimPrefix(strings.ToLower(extension), ".") +} + +func writeHAK(path string, resources []erf.Resource) error { + output, err := os.Create(path) + if err != nil { + return fmt.Errorf("create top package hak: %w", err) + } + defer output.Close() + if err := erf.Write(output, erf.New("HAK ", resources)); err != nil { + return fmt.Errorf("write top package hak: %w", err) + } + return nil +} + +func resolveCompiledTLKPath(outputDir string) (string, error) { + entries, err := os.ReadDir(outputDir) + if err != nil { + return "", fmt.Errorf("read compiled tlk output dir: %w", err) + } + var tlkFiles []string + for _, entry := range entries { + if entry.IsDir() || !strings.EqualFold(filepath.Ext(entry.Name()), ".tlk") { + continue + } + tlkFiles = append(tlkFiles, filepath.Join(outputDir, entry.Name())) + } + if len(tlkFiles) != 1 { + return "", fmt.Errorf("expected exactly 1 compiled tlk output, found %d", len(tlkFiles)) + } + return tlkFiles[0], nil +} + +func copyFile(dst, src string) error { + input, err := os.Open(src) + if err != nil { + return fmt.Errorf("open %s: %w", src, err) + } + defer input.Close() + + output, err := os.Create(dst) + if err != nil { + return fmt.Errorf("create %s: %w", dst, err) + } + defer output.Close() + + if _, err := io.Copy(output, input); err != nil { + return fmt.Errorf("copy %s to %s: %w", src, dst, err) + } + return nil +} diff --git a/internal/topdata/topdata.go b/internal/topdata/topdata.go index 9239596..46ebe41 100644 --- a/internal/topdata/topdata.go +++ b/internal/topdata/topdata.go @@ -12,6 +12,7 @@ import ( "strconv" "strings" + "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf" "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" ) @@ -72,6 +73,18 @@ type BuildResult struct { FilesTLK int } +type PackageResult struct { + Mode string + Output2DADir string + OutputTLKDir string + OutputHAKPath string + OutputTLKPath string + Files2DA int + FilesTLK int + HAKResources int + AssetFiles int +} + type CompareResult struct { Mode string Compared2DA int @@ -192,6 +205,7 @@ func ValidateProject(p *project.Project) ValidationReport { report.DataFiles++ _ = validateJSONFile(path, "data", &report) }) + validateTopPackageAssets(sourceDir, dataDir, &report) validateNativeOutputCatalog(dataDir, &report) validateItempropsRegistryGraph(dataDir, &report) if _, err := os.Stat(statePath); err == nil { @@ -635,20 +649,26 @@ func validateBaseDialogObject(path string, obj map[string]any, report *Validatio func validateLegacyDialogEntries(path string, obj map[string]any, report *ValidationReport, label string) { entries, ok := obj["entries"] if ok { - if _, ok := entries.(map[string]any); !ok { + switch entries.(type) { + case map[string]any, []any: + // accepted compatibility shapes + default: report.Diagnostics = append(report.Diagnostics, Diagnostic{ Severity: SeverityError, Path: path, - Message: fmt.Sprintf("%s entries must be a JSON object", label), + Message: fmt.Sprintf("%s entries must be a JSON object or array", label), }) } } if language, ok := obj["language"]; ok { - if _, ok := language.(string); !ok { + switch language.(type) { + case string, float64: + // accepted compatibility shapes + default: report.Diagnostics = append(report.Diagnostics, Diagnostic{ Severity: SeverityError, Path: path, - Message: fmt.Sprintf("%s language must be a string when present", label), + Message: fmt.Sprintf("%s language must be a string or numeric language id when present", label), }) } } @@ -866,6 +886,93 @@ func validateNativeOutputCatalog(dataDir string, report *ValidationReport) { } } +func validateTopPackageAssets(sourceDir, dataDir string, report *ValidationReport) { + assetsDir := filepath.Join(sourceDir, "assets") + info, err := os.Stat(assetsDir) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return + } + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: assetsDir, + Message: err.Error(), + }) + return + } + if !info.IsDir() { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: assetsDir, + Message: "must be a directory", + }) + return + } + + projectOutputs, err := discoverNativeOutputCatalog(dataDir) + if err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: assetsDir, + Message: err.Error(), + }) + return + } + + seen := map[string]string{} + err = filepath.WalkDir(assetsDir, func(path string, d fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if d.IsDir() { + return nil + } + report.Files++ + rel, err := filepath.Rel(assetsDir, path) + if err != nil { + return err + } + if strings.HasPrefix(filepath.Base(path), ".") { + return nil + } + base := strings.ToLower(strings.TrimSuffix(filepath.Base(path), filepath.Ext(path))) + ext := strings.TrimPrefix(strings.ToLower(filepath.Ext(path)), ".") + if _, ok := erf.ResourceTypeForExtension(ext); !ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("unsupported topdata asset resource extension %q", filepath.Ext(path)), + }) + return nil + } + key := base + "." + ext + if previous, ok := seen[key]; ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("duplicate topdata asset resource %q also defined by %s", key, previous), + }) + return nil + } + seen[key] = rel + if owner, ok := projectOutputs[key]; ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("topdata asset resource %q collides with compiled topdata output from %s", key, owner), + }) + } + return nil + }) + if err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: assetsDir, + Message: err.Error(), + }) + } +} + func recordNativeOutputOwner(outputName, owner string, outputOwners map[string]string, report *ValidationReport) { if existing, ok := outputOwners[outputName]; ok { report.Diagnostics = append(report.Diagnostics, Diagnostic{ diff --git a/internal/topdata/topdata_test.go b/internal/topdata/topdata_test.go index 16b7810..5e3b249 100644 --- a/internal/topdata/topdata_test.go +++ b/internal/topdata/topdata_test.go @@ -7,6 +7,7 @@ import ( "strings" "testing" + "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf" "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" ) @@ -8061,6 +8062,85 @@ func TestBuildNativeWithSiblingAssetsRepo(t *testing.T) { } } +func TestBuildPackageIncludesCompiled2DAAndTopAssets(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "repadjust")) + mkdirAll(t, filepath.Join(root, "topdata", "assets", "gui")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "repadjust", "base.json"), `{ + "output": "repadjust.2da", + "columns": ["Label"], + "rows": [{"id": 0, "Label": "TEST_LABEL"}] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "assets", "gui", "testicon.tga"), "icon-data") + + proj := testProject(root) + proj.Config.TopData.ReferenceBuilder = "" + + result, err := BuildPackage(proj, nil) + if err != nil { + t.Fatalf("BuildPackage failed: %v", err) + } + if result.OutputHAKPath != filepath.Join(root, "build", PackageHAKFileName) { + t.Fatalf("unexpected hak path: %s", result.OutputHAKPath) + } + if result.OutputTLKPath != filepath.Join(root, "build", PackageTLKFileName) { + t.Fatalf("unexpected tlk path: %s", result.OutputTLKPath) + } + + input, err := os.Open(result.OutputHAKPath) + if err != nil { + t.Fatalf("open hak: %v", err) + } + defer input.Close() + archive, err := erf.Read(input) + if err != nil { + t.Fatalf("read hak: %v", err) + } + found2DA := false + foundAsset := false + for _, resource := range archive.Resources { + ext, _ := erf.ExtensionForResourceType(resource.Type) + key := strings.ToLower(resource.Name) + "." + ext + if key == "repadjust.2da" { + found2DA = true + } + if key == "testicon.tga" { + foundAsset = true + } + } + if !found2DA { + t.Fatalf("expected repadjust.2da in top package") + } + if !foundAsset { + t.Fatalf("expected testicon.tga in top package") + } + if _, err := os.Stat(result.OutputTLKPath); err != nil { + t.Fatalf("expected packaged tlk output: %v", err) + } +} + +func TestValidateProjectErrorsOnTopPackageAssetCollision(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "repadjust")) + mkdirAll(t, filepath.Join(root, "topdata", "assets", "gui")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "repadjust", "base.json"), `{ + "output": "repadjust.2da", + "columns": ["Label"], + "rows": [{"id": 0, "Label": "TEST_LABEL"}] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "assets", "gui", "repadjust.2da"), "collision") + + report := ValidateProject(testProject(root)) + if !report.HasErrors() { + t.Fatalf("expected topdata asset collision to be an error") + } + if !strings.Contains(diagnosticsText(report.Diagnostics), "collides with compiled topdata output") { + t.Fatalf("expected collision diagnostic, got %#v", report.Diagnostics) + } +} + func diagnosticsText(diags []Diagnostic) string { lines := make([]string, 0, len(diags)) for _, diag := range diags {