diff --git a/README.md b/README.md index b138474..8ba03fe 100644 --- a/README.md +++ b/README.md @@ -166,6 +166,22 @@ outputs: build: keep_existing_haks: false +generated_assets: + topdata_2da: + - id: parts + source: topdata + output: "{paths.cache}/generated-assets/parts-2da" + include_datasets: [parts/**] + package_root: part + autogen: + id: parts + mode: parts_rows + root: part + include: ["**/*.mdl"] + derive: + kind: trailing_numeric_suffix + group_from: first_path_segment + inventory: source_extensions: [] # empty means built-in NWN source extensions asset_extensions: [] # empty means built-in NWN asset extensions @@ -234,6 +250,11 @@ of source, but they must not resolve to the repository root (`.`). Output and cache paths must stay relative to the repository root unless a specific setting documents otherwise. +`generated_assets.topdata_2da` is for HAK repositories that need a small +native-topdata subset packaged as HAK resources. It builds only `.2da` output, +injects those generated files into `build-haks`, and rejects TLK-backed text +because this path does not produce TLK packages. + Extraction is deliberately guarded because it writes source files from binary archives. `extract.archives` is a list of glob patterns relative to `paths.build`; the default is only the configured module `.mod`. HAK extraction diff --git a/internal/pipeline/build.go b/internal/pipeline/build.go index e812c88..54e98a7 100644 --- a/internal/pipeline/build.go +++ b/internal/pipeline/build.go @@ -249,7 +249,11 @@ func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bo err = cleanupErr } }() - assetResources, err := collectAssetResources(p, false, allowedAssets, musicAssets) + generated2DAAssets, err := topdata.BuildGenerated2DAAssets(p, progress) + if err != nil { + return BuildResult{}, err + } + assetResources, err := collectAssetResources(p, false, allowedAssets, musicAssets, generated2DAAssets) if err != nil { return BuildResult{}, err } @@ -762,7 +766,7 @@ func fieldStringValueForScripts(value gff.Value) (string, bool) { } } -func collectAssetResources(p *project.Project, requireContent bool, allowed map[string]struct{}, musicAssets *preparedMusicAssets) ([]assetResource, error) { +func collectAssetResources(p *project.Project, requireContent bool, allowed map[string]struct{}, musicAssets *preparedMusicAssets, generated2DAAssets []topdata.Generated2DAAsset) ([]assetResource, error) { var hakResources []assetResource assetCreatedAt, err := collectGitAssetCreationTimes(p) if err != nil { @@ -821,6 +825,29 @@ func collectAssetResources(p *project.Project, requireContent bool, allowed map[ hakResources = append(hakResources, asset) } } + for _, asset := range generated2DAAssets { + rel := filepath.ToSlash(asset.Rel) + if len(allowed) > 0 { + if _, ok := allowed[rel]; !ok { + continue + } + } + resourceInfo, err := assetResourceFromPath(asset.SourcePath, rel, lfsAssetInfo{}, requireContent) + if err != nil { + return nil, err + } + info, err := os.Stat(asset.SourcePath) + if err != nil { + return nil, fmt.Errorf("stat %s: %w", asset.SourcePath, err) + } + hakResources = append(hakResources, assetResource{ + Rel: rel, + Resource: resourceInfo.Resource, + Size: erf.ArchiveSize([]erf.Resource{resourceInfo.Resource}), + CreatedAt: info.ModTime(), + ContentID: resourceInfo.ContentID, + }) + } slices.SortFunc(hakResources, func(a, b assetResource) int { return compareResourceKeys(a.Resource, b.Resource) @@ -1701,7 +1728,11 @@ func plannedModuleHAKOrder(p *project.Project) (order []string, err error) { err = cleanupErr } }() - assetResources, err := collectAssetResources(p, false, nil, musicAssets) + generated2DAAssets, err := topdata.BuildGenerated2DAAssets(p, nil) + if err != nil { + return nil, err + } + assetResources, err := collectAssetResources(p, false, nil, musicAssets, generated2DAAssets) if err != nil { return nil, err } diff --git a/internal/pipeline/pipeline_test.go b/internal/pipeline/pipeline_test.go index 6495180..b710328 100644 --- a/internal/pipeline/pipeline_test.go +++ b/internal/pipeline/pipeline_test.go @@ -2120,10 +2120,9 @@ func TestPlanHAKsWritesManifestWithoutArchives(t *testing.T) { } } -func TestBuildHAKsWritesConfiguredAutogenManifests(t *testing.T) { +func TestBuildHAKsWritesConfiguredHeadVisualeffectsAutogenManifest(t *testing.T) { root := t.TempDir() mustMkdir(t, filepath.Join(root, "src")) - mustMkdir(t, filepath.Join(root, "assets", "part", "belt")) mustMkdir(t, filepath.Join(root, "assets", "vfxs", "head_accessories")) mustMkdir(t, filepath.Join(root, "assets", "vfxs", "head_features", "hair")) mustMkdir(t, filepath.Join(root, "build")) @@ -2141,20 +2140,6 @@ func TestBuildHAKsWritesConfiguredAutogenManifests(t *testing.T) { }, "autogen": { "producers": [ - { - "id": "parts", - "root": "part", - "include": ["**/*.mdl"], - "derive": { - "kind": "trailing_numeric_suffix", - "group_from": "first_path_segment" - }, - "manifest": { - "release_tag": "parts-manifest-current", - "asset_name": "sow-parts-manifest.json", - "cache_name": "sow-parts-manifest.json" - } - }, { "id": "head_visualeffects", "root": "vfxs", @@ -2177,12 +2162,12 @@ func TestBuildHAKsWritesConfiguredAutogenManifests(t *testing.T) { "priority": 1, "max_bytes": 0, "split": false, - "include": ["part/**", "vfxs/**"] + "include": ["vfxs/**"] } ] } `) - mustWriteFile(t, filepath.Join(root, "assets", "part", "belt", "pfa0_belt018.mdl"), "belt") + mustWriteFile(t, filepath.Join(root, "assets", "vfxs", "head_accessories", "hfx_bandana.mdl"), "bandana") mustWriteFile(t, filepath.Join(root, "assets", "vfxs", "head_features", "hair", "hfx_hair_bangs.mdl"), "hair") @@ -2201,16 +2186,11 @@ func TestBuildHAKsWritesConfiguredAutogenManifests(t *testing.T) { if err != nil { t.Fatalf("build haks: %v", err) } - if len(result.AutogenManifestPaths) != 2 { - t.Fatalf("expected 2 autogen manifests, got %d", len(result.AutogenManifestPaths)) + if len(result.AutogenManifestPaths) != 1 { + t.Fatalf("expected 1 autogen manifest, got %d", len(result.AutogenManifestPaths)) } - - partsRaw, err := os.ReadFile(filepath.Join(root, "build", "sow-parts-manifest.json")) - if err != nil { - t.Fatalf("read parts manifest: %v", err) - } - if !strings.Contains(string(partsRaw), `"id": "parts"`) || !strings.Contains(string(partsRaw), `"row_id": 18`) { - t.Fatalf("unexpected parts manifest contents:\n%s", string(partsRaw)) + if _, err := os.Stat(filepath.Join(root, "build", "sow-parts-manifest.json")); !os.IsNotExist(err) { + t.Fatalf("expected no parts manifest, got stat error %v", err) } headRaw, err := os.ReadFile(filepath.Join(root, "build", "sow-head-vfx-manifest.json")) @@ -2231,6 +2211,167 @@ func TestBuildHAKsWritesConfiguredAutogenManifests(t *testing.T) { } } +func TestBuildHAKsGeneratesParts2DAAssetsFromLocalModels(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src")) + mustMkdir(t, filepath.Join(root, "assets", "part", "belt")) + mustMkdir(t, filepath.Join(root, "topdata", "data", "parts")) + mustMkdir(t, filepath.Join(root, "topdata", "data", "parts", "modules")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{ + "module": { + "name": "Test Module", + "resref": "testmod", + "hak_order": ["group:sow_part"] + }, + "paths": { + "source": "src", + "assets": "assets", + "build": "build" + }, + "generated_assets": { + "topdata_2da": [ + { + "id": "parts", + "source": "topdata", + "output": "{paths.cache}/generated-assets/parts-2da", + "include_datasets": ["parts/**"], + "package_root": "part", + "autogen": { + "id": "parts", + "mode": "parts_rows", + "root": "part", + "include": ["**/*.mdl"], + "derive": { + "kind": "trailing_numeric_suffix", + "group_from": "first_path_segment" + } + } + } + ] + }, + "haks": [ + { + "name": "sow_part", + "priority": 1, + "max_bytes": 0, + "split": false, + "include": ["part/**"] + } + ] +} +`) + mustWriteFile(t, filepath.Join(root, "assets", "part", "belt", "pfa0_belt018.mdl"), "belt") + mustWriteFile(t, filepath.Join(root, "topdata", "data", "parts", "belt.json"), `{ + "output": "parts_belt.2da", + "columns": ["COSTMODIFIER", "ACBONUS"], + "rows": [ + {"id": 0, "COSTMODIFIER": 1, "ACBONUS": "0.25"} + ] +} +`) + mustWriteFile(t, filepath.Join(root, "topdata", "data", "parts", "modules", "ovr_belt_acbonus_zero.json"), `{ + "overrides": [ + {"id": 18, "ACBONUS": "0.75"} + ] +} +`) + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + + result, err := BuildHAKs(p) + if err != nil { + t.Fatalf("build haks: %v", err) + } + if len(result.HAKPaths) != 1 { + t.Fatalf("expected one HAK, got %#v", result.HAKPaths) + } + raw, err := os.ReadFile(filepath.Join(root, "build", "haks.json")) + if err != nil { + t.Fatalf("read haks manifest: %v", err) + } + manifestText := string(raw) + if !strings.Contains(manifestText, `"part/parts_belt.2da"`) { + t.Fatalf("expected generated parts 2da in manifest, got:\n%s", manifestText) + } + partsRaw, err := os.ReadFile(filepath.Join(root, ".cache", "generated-assets", "parts-2da", "parts_belt.2da")) + if err != nil { + t.Fatalf("read generated parts 2da: %v", err) + } + partsText := string(partsRaw) + if !strings.Contains(partsText, "0\t1\t0.25") || !strings.Contains(partsText, "18\t0\t0.75") { + t.Fatalf("expected authored and generated rows in parts 2da, got:\n%s", partsText) + } +} + +func TestBuildHAKsRejectsTLKBackedGeneratedParts2DAAssets(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src")) + mustMkdir(t, filepath.Join(root, "assets", "part", "belt")) + mustMkdir(t, filepath.Join(root, "topdata", "data", "parts")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{ + "module": {"name": "Test Module", "resref": "testmod"}, + "paths": {"source": "src", "assets": "assets", "build": "build"}, + "generated_assets": { + "topdata_2da": [ + { + "id": "parts", + "source": "topdata", + "output": "{paths.cache}/generated-assets/parts-2da", + "include_datasets": ["parts/**"], + "package_root": "part", + "autogen": { + "id": "parts", + "mode": "parts_rows", + "root": "part", + "include": ["**/*.mdl"], + "derive": {"kind": "trailing_numeric_suffix", "group_from": "first_path_segment"} + } + } + ] + }, + "haks": [{"name": "sow_part", "priority": 1, "max_bytes": 0, "include": ["part/**"]}] +} +`) + mustWriteFile(t, filepath.Join(root, "assets", "part", "belt", "pfa0_belt018.mdl"), "belt") + mustWriteFile(t, filepath.Join(root, "topdata", "data", "parts", "belt.json"), `{ + "output": "parts_belt.2da", + "columns": ["COSTMODIFIER", "ACBONUS", "NAME"], + "rows": [ + {"id": 0, "COSTMODIFIER": 1, "ACBONUS": "0.25", "NAME": {"tlk": {"text": "Belt"}}} + ] +} +`) + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + + _, err = BuildHAKs(p) + if err == nil || !strings.Contains(err.Error(), "2DA-only generated asset") { + t.Fatalf("expected TLK-backed generated asset error, got %v", err) + } +} + func TestBuildSkipsEmptyHAKGroupsInModuleOrder(t *testing.T) { root := t.TempDir() mustMkdir(t, filepath.Join(root, "src")) @@ -2752,7 +2893,7 @@ func TestPlanHAKChunksPutsNewestAssetsInLastChunk(t *testing.T) { t.Fatalf("scan: %v", err) } - assets, err := collectAssetResources(p, false, nil, nil) + assets, err := collectAssetResources(p, false, nil, nil, nil) if err != nil { t.Fatalf("collect asset resources: %v", err) } diff --git a/internal/project/effective.go b/internal/project/effective.go index 745cd1c..ce7cced 100644 --- a/internal/project/effective.go +++ b/internal/project/effective.go @@ -75,6 +75,7 @@ type EffectiveConfig struct { Paths EffectivePathConfig `json:"paths" yaml:"paths"` Outputs EffectiveOutputConfig `json:"outputs" yaml:"outputs"` Build BuildConfig `json:"build" yaml:"build"` + Generated GeneratedConfig `json:"generated_assets" yaml:"generated_assets"` Inventory InventoryConfig `json:"inventory" yaml:"inventory"` Validation ValidationConfig `json:"validation" yaml:"validation"` Scripts EffectiveScriptsConfig `json:"scripts" yaml:"scripts"` @@ -256,6 +257,11 @@ func (p *Project) EffectiveConfig() EffectiveConfig { }, }, } + generated := p.Config.Generated + for index := range generated.TopData2DA { + generated.TopData2DA[index].Output = expandPathTemplate(defaultString(generated.TopData2DA[index].Output, "{paths.cache}/generated-assets/"+generated.TopData2DA[index].ID), paths) + } + extract := p.Config.Extract extract.Layout = defaultString(extract.Layout, DefaultExtractLayout) extract.HAKDiscovery = defaultString(extract.HAKDiscovery, DefaultExtractHAKDiscovery) @@ -352,6 +358,7 @@ func (p *Project) EffectiveConfig() EffectiveConfig { Paths: paths, Outputs: outputs, Build: p.Config.Build, + Generated: generated, Inventory: inventory, Validation: validation, Scripts: scripts, @@ -439,6 +446,7 @@ func markMissingDefaults(provenance ConfigProvenance) { "outputs.module_archive": DefaultModuleArchiveTemplate, "outputs.hak_manifest": DefaultHAKManifest, "outputs.hak_archive": DefaultHAKArchiveTemplate, + "generated_assets.topdata_2da": "no generated topdata 2DA assets", "inventory.source_extensions": "NWN source resource extensions", "inventory.asset_extensions": "NWN asset resource extensions", "inventory.source_json_pattern": DefaultSourceJSONPattern, diff --git a/internal/project/project.go b/internal/project/project.go index 028186d..ab0c7a9 100644 --- a/internal/project/project.go +++ b/internal/project/project.go @@ -83,6 +83,7 @@ type Config struct { Paths PathConfig `json:"paths" yaml:"paths"` Outputs OutputConfig `json:"outputs" yaml:"outputs"` Build BuildConfig `json:"build" yaml:"build"` + Generated GeneratedConfig `json:"generated_assets" yaml:"generated_assets"` HAKs []HAKConfig `json:"haks" yaml:"haks"` Inventory InventoryConfig `json:"inventory" yaml:"inventory"` Validation ValidationConfig `json:"validation" yaml:"validation"` @@ -118,6 +119,19 @@ type BuildConfig struct { KeepExistingHAKs bool `json:"keep_existing_haks,omitempty" yaml:"keep_existing_haks,omitempty"` } +type GeneratedConfig struct { + TopData2DA []GeneratedTopData2DAConfig `json:"topdata_2da" yaml:"topdata_2da"` +} + +type GeneratedTopData2DAConfig struct { + ID string `json:"id" yaml:"id"` + Source string `json:"source" yaml:"source"` + Output string `json:"output" yaml:"output"` + IncludeDatasets []string `json:"include_datasets" yaml:"include_datasets"` + PackageRoot string `json:"package_root" yaml:"package_root"` + Autogen AutogenConsumerConfig `json:"autogen" yaml:"autogen"` +} + type InventoryConfig struct { SourceExtensions []string `json:"source_extensions" yaml:"source_extensions"` AssetExtensions []string `json:"asset_extensions" yaml:"asset_extensions"` @@ -459,6 +473,7 @@ func (p *Project) ValidateLayout() error { "outputs.module_archive": p.Config.Outputs.ModuleArchive, "outputs.hak_manifest": p.Config.Outputs.HAKManifest, "outputs.hak_archive": p.Config.Outputs.HAKArchive, + "generated_assets.topdata_2da": "", "validation.profile": p.Config.Validation.Profile, "scripts.source_dir": p.Config.Scripts.SourceDir, "scripts.cache": p.Config.Scripts.Cache, @@ -496,6 +511,7 @@ func (p *Project) ValidateLayout() error { failures = append(failures, validateRelativePath("outputs.module_archive", effective.Outputs.ModuleArchive)...) failures = append(failures, validateRelativePath("outputs.hak_manifest", effective.Outputs.HAKManifest)...) failures = append(failures, validateRelativePath("outputs.hak_archive", effective.Outputs.HAKArchive)...) + failures = append(failures, validateGeneratedConfig(effective.Generated)...) failures = append(failures, validateRelativePath("scripts.cache", effective.Scripts.Cache)...) failures = append(failures, validateRelativePath("scripts.source_dir", effective.Scripts.SourceDir)...) failures = append(failures, validateRelativePath("topdata.compiled_2da_dir", effective.TopData.Compiled2DADir)...) @@ -1301,6 +1317,56 @@ func validateAutogenConfig(cfg AutogenConfig) []error { return failures } +func validateGeneratedConfig(cfg GeneratedConfig) []error { + var failures []error + seen := map[string]struct{}{} + for index, top2da := range cfg.TopData2DA { + fieldPrefix := fmt.Sprintf("generated_assets.topdata_2da[%d]", index) + id := strings.TrimSpace(top2da.ID) + if id == "" { + failures = append(failures, fmt.Errorf("%s.id is required", fieldPrefix)) + } else { + if _, exists := seen[id]; exists { + failures = append(failures, fmt.Errorf("%s.id %q is duplicated", fieldPrefix, id)) + } + seen[id] = struct{}{} + } + if strings.TrimSpace(top2da.Source) == "" { + failures = append(failures, fmt.Errorf("%s.source is required", fieldPrefix)) + } + failures = append(failures, validateTreeRootPath(fieldPrefix+".source", top2da.Source)...) + if strings.TrimSpace(top2da.Output) == "" { + failures = append(failures, fmt.Errorf("%s.output is required", fieldPrefix)) + } + failures = append(failures, validateRelativePath(fieldPrefix+".output", top2da.Output)...) + if strings.TrimSpace(top2da.PackageRoot) == "" { + failures = append(failures, fmt.Errorf("%s.package_root is required", fieldPrefix)) + } + failures = append(failures, validateTreeRootPath(fieldPrefix+".package_root", top2da.PackageRoot)...) + if len(top2da.IncludeDatasets) == 0 { + failures = append(failures, fmt.Errorf("%s.include_datasets must contain at least one pattern", fieldPrefix)) + } + failures = append(failures, validateGlobList(fieldPrefix+".include_datasets", top2da.IncludeDatasets)...) + switch strings.TrimSpace(top2da.Autogen.Mode) { + case "parts_rows": + case "": + failures = append(failures, fmt.Errorf("%s.autogen.mode is required", fieldPrefix)) + default: + failures = append(failures, fmt.Errorf("%s.autogen.mode %q is not supported", fieldPrefix, top2da.Autogen.Mode)) + } + if strings.TrimSpace(top2da.Autogen.Root) == "" { + failures = append(failures, fmt.Errorf("%s.autogen.root is required", fieldPrefix)) + } + failures = append(failures, validateTreeRootPath(fieldPrefix+".autogen.root", top2da.Autogen.Root)...) + if len(top2da.Autogen.Include) == 0 { + failures = append(failures, fmt.Errorf("%s.autogen.include must contain at least one glob", fieldPrefix)) + } + failures = append(failures, validateGlobList(fieldPrefix+".autogen.include", top2da.Autogen.Include)...) + failures = append(failures, validateAutogenDeriveConfig(fieldPrefix+".autogen.derive", top2da.Autogen.Derive)...) + } + return failures +} + func validateMusicConfig(cfg MusicConfig) []error { var failures []error if cfg.Defaults != nil { diff --git a/internal/project/project_test.go b/internal/project/project_test.go index c4e80ee..1d2c5db 100644 --- a/internal/project/project_test.go +++ b/internal/project/project_test.go @@ -859,6 +859,81 @@ func TestValidateLayoutRejectsInvalidAutogenConsumerConfig(t *testing.T) { } } +func TestValidateLayoutAcceptsGeneratedTopData2DAConfig(t *testing.T) { + root := t.TempDir() + mkdirAll(t, filepath.Join(root, "assets")) + mkdirAll(t, filepath.Join(root, "build")) + proj := Project{ + Root: root, + Config: Config{ + Module: ModuleConfig{Name: "Test", ResRef: "testmod"}, + Paths: PathConfig{Assets: "assets"}, + Generated: GeneratedConfig{ + TopData2DA: []GeneratedTopData2DAConfig{ + { + ID: "parts", + Source: "topdata", + Output: "{paths.cache}/generated-assets/parts-2da", + IncludeDatasets: []string{"parts/**"}, + PackageRoot: "part", + Autogen: AutogenConsumerConfig{ + ID: "parts", + Mode: "parts_rows", + Root: "part", + Include: []string{"**/*.mdl"}, + Derive: AutogenDeriveConfig{Kind: "trailing_numeric_suffix", GroupFrom: "first_path_segment"}, + }, + }, + }, + }, + }, + } + + if err := proj.ValidateLayout(); err != nil { + t.Fatalf("ValidateLayout returned error: %v", err) + } + effective := proj.EffectiveConfig() + if got, want := effective.Generated.TopData2DA[0].Output, ".cache/generated-assets/parts-2da"; got != want { + t.Fatalf("expected expanded output %q, got %q", want, got) + } +} + +func TestValidateLayoutRejectsEscapingGeneratedTopData2DAConfig(t *testing.T) { + root := t.TempDir() + mkdirAll(t, filepath.Join(root, "assets")) + mkdirAll(t, filepath.Join(root, "build")) + proj := Project{ + Root: root, + Config: Config{ + Module: ModuleConfig{Name: "Test", ResRef: "testmod"}, + Paths: PathConfig{Assets: "assets"}, + Generated: GeneratedConfig{ + TopData2DA: []GeneratedTopData2DAConfig{ + { + ID: "parts", + Source: "../topdata", + Output: "{paths.cache}/generated-assets/parts-2da", + IncludeDatasets: []string{"parts/**"}, + PackageRoot: "part", + Autogen: AutogenConsumerConfig{ + ID: "parts", + Mode: "parts_rows", + Root: "part", + Include: []string{"**/*.mdl"}, + Derive: AutogenDeriveConfig{Kind: "trailing_numeric_suffix", GroupFrom: "first_path_segment"}, + }, + }, + }, + }, + }, + } + + err := proj.ValidateLayout() + if err == nil || !strings.Contains(err.Error(), "generated_assets.topdata_2da[0].source") { + t.Fatalf("expected generated topdata source validation error, got %v", err) + } +} + func TestValidateLayoutRejectsDuplicateHAKNames(t *testing.T) { root := t.TempDir() mkdirAll(t, filepath.Join(root, "src")) diff --git a/internal/topdata/AUTO-INCLUDE_EXISTING_PART_MODELS_IN_2DA_GENERATION_CONTRACT.md b/internal/topdata/AUTO-INCLUDE_EXISTING_PART_MODELS_IN_2DA_GENERATION_CONTRACT.md index 274cbe7..80a0efd 100644 --- a/internal/topdata/AUTO-INCLUDE_EXISTING_PART_MODELS_IN_2DA_GENERATION_CONTRACT.md +++ b/internal/topdata/AUTO-INCLUDE_EXISTING_PART_MODELS_IN_2DA_GENERATION_CONTRACT.md @@ -2,15 +2,15 @@ ## Status Snapshot -Current state as of 2026-05-13: +Current state as of 2026-05-17: -- Implemented. -- The toolkit resolves released parts manifests through the configured autogen - release source and covers this behavior with `parts_manifest_test.go` plus - broader native topdata tests. -- Remaining work is operational hardening only: keep manifest contracts stable - across `assets/` releases and maintain regression coverage as autogen config - evolves. +- Superseded for current Shadows Over Westgate builds. +- The module no longer consumes a released parts manifest. Parts data is owned by + `assets/`, generated as HAK-side 2DA assets from `assets/topdata/data/parts` + plus a direct scan of `assets/content/part/**/*.mdl`, and packaged in + `sow_part`. +- This file remains as historical context for the legacy manifest-backed + consumer path. ## Objective @@ -22,14 +22,14 @@ This behavior is required for parity with the current asset set and must not dep ## Source of Truth -The native builder must obtain existing model information from the released -`sow-parts-manifest.json` artifact published by the **`sow-assets` repository**. +The superseded manifest-backed native builder obtained existing model +information from a released artifact published by the **`sow-assets` repository**. Resolution rules: - if `topdata.assets` is configured to a local path, use that exact path - otherwise derive the **`sow-assets` repository** from the module repo's Gitea origin - and fetch only the released manifest from the `parts-manifest-current` release + and fetch only the configured released manifest Normal builds must not clone or scan the `sow-assets` repo tree. They may cache the downloaded manifest locally for reuse, but the source of truth remains the released @@ -122,9 +122,9 @@ That means: ## Acceptance Criteria -Implementation is correct only if all of the following are true: +The superseded implementation was correct only if all of the following were true: -1. The builder fetches the released `sow-parts-manifest.json` artifact for part models. +1. The builder fetches the configured released manifest artifact for part models. 2. Only the supported part categories are considered. 3. Manifest IDs derived from trailing numeric suffixes generate corresponding 2DA rows. 4. The numeric suffix is used as the row ID. diff --git a/internal/topdata/generated_assets.go b/internal/topdata/generated_assets.go new file mode 100644 index 0000000..1cfe1da --- /dev/null +++ b/internal/topdata/generated_assets.go @@ -0,0 +1,194 @@ +package topdata + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" +) + +type Generated2DAAsset struct { + Rel string + SourcePath string +} + +func BuildGenerated2DAAssets(p *project.Project, progress func(string)) ([]Generated2DAAsset, error) { + if progress == nil { + progress = func(string) {} + } + configs := p.EffectiveConfig().Generated.TopData2DA + if len(configs) == 0 { + return nil, nil + } + + results := make([]Generated2DAAsset, 0) + for _, cfg := range configs { + progress(fmt.Sprintf("Building generated 2DA assets for %s...", cfg.ID)) + generated, err := buildGenerated2DAAssetGroup(p, cfg, progress) + if err != nil { + return nil, err + } + results = append(results, generated...) + } + return results, nil +} + +func buildGenerated2DAAssetGroup(p *project.Project, cfg project.GeneratedTopData2DAConfig, progress func(string)) ([]Generated2DAAsset, error) { + sourceDir := filepath.Join(p.Root, filepath.FromSlash(strings.TrimSpace(cfg.Source))) + dataDir := filepath.Join(sourceDir, "data") + outputDir := filepath.Join(p.Root, filepath.FromSlash(strings.TrimSpace(cfg.Output))) + + datasets, err := discoverNativeDatasets(dataDir) + if err != nil { + return nil, fmt.Errorf("discover generated topdata 2DA datasets %s: %w", cfg.ID, err) + } + selected := make([]nativeDataset, 0, len(datasets)) + for _, dataset := range datasets { + if generatedDatasetIncluded(dataset.Name, cfg.IncludeDatasets) { + selected = append(selected, dataset) + } + } + if len(selected) == 0 { + return nil, fmt.Errorf("generated topdata 2DA config %s matched no datasets under %s", cfg.ID, dataDir) + } + + collected := make([]nativeCollectedDataset, 0, len(selected)) + for _, dataset := range selected { + collectedDataset, err := collectNativeDataset(dataset) + if err != nil { + return nil, err + } + collected = append(collected, collectedDataset) + } + + consumer := cfg.Autogen + consumer.LocalOverrideRoot = p.AssetsDir() + collected, err = applyAutogenConsumersForGeneratedAssets(collected, consumer, progress) + if err != nil { + return nil, err + } + collected, err = applyPartOverrides(sourceDir, collected) + if err != nil { + return nil, err + } + if err := rejectGenerated2DATLKValues(cfg.ID, collected); err != nil { + return nil, err + } + + tableRegistry, err := newResolvedTableRegistry(collected) + if err != nil { + return nil, err + } + keyToID, rowByKey := generated2DAGlobalRows(collected) + + if err := os.RemoveAll(outputDir); err != nil { + return nil, fmt.Errorf("clean generated topdata 2DA output %s: %w", outputDir, err) + } + if err := os.MkdirAll(outputDir, 0o755); err != nil { + return nil, fmt.Errorf("create generated topdata 2DA output %s: %w", outputDir, err) + } + + results := make([]Generated2DAAsset, 0, len(collected)) + for _, dataset := range collected { + compiled, err := resolveNativeDataset(dataset, keyToID, rowByKey, tableRegistry, nil) + if err != nil { + return nil, err + } + outputPath := filepath.Join(outputDir, dataset.Dataset.OutputName) + if err := write2DA(compiled, outputPath, dataset.Dataset.Kind == nativeDatasetBase); err != nil { + return nil, err + } + results = append(results, Generated2DAAsset{ + Rel: filepath.ToSlash(filepath.Join(cfg.PackageRoot, dataset.Dataset.OutputName)), + SourcePath: outputPath, + }) + } + return results, nil +} + +func applyAutogenConsumersForGeneratedAssets(collected []nativeCollectedDataset, consumer project.AutogenConsumerConfig, progress func(string)) ([]nativeCollectedDataset, error) { + if strings.TrimSpace(consumer.Mode) == "" { + return collected, nil + } + if progress == nil { + progress = func(string) {} + } + if !autogenConsumerTargetsCollectedDataset(collected, consumer) { + return collected, nil + } + if progress != nil { + progress(fmt.Sprintf("Scanning local generated 2DA autogen input for %s from %s...", consumer.ID, consumer.LocalOverrideRoot)) + } + entries, err := scanLocalAutogenEntries(consumer.LocalOverrideRoot, consumer) + if err != nil { + return nil, err + } + switch consumer.Mode { + case "parts_rows": + return augmentWithAutogeneratedParts(collected, autogenPartsInventory(entries)), nil + default: + return nil, fmt.Errorf("unsupported generated topdata 2DA autogen mode %q", consumer.Mode) + } +} + +func generatedDatasetIncluded(name string, patterns []string) bool { + name = filepath.ToSlash(strings.TrimSpace(name)) + for _, pattern := range patterns { + pattern = filepath.ToSlash(strings.TrimSpace(pattern)) + if pattern == "" { + continue + } + if pattern == name { + return true + } + if strings.HasSuffix(pattern, "/**") { + prefix := strings.TrimSuffix(pattern, "/**") + if name == prefix || strings.HasPrefix(name, prefix+"/") { + return true + } + } + } + return false +} + +func generated2DAGlobalRows(collected []nativeCollectedDataset) (map[string]int, map[string]map[string]any) { + keyToID := map[string]int{} + rowByKey := map[string]map[string]any{} + for _, dataset := range collected { + for key, rowID := range dataset.LockData { + keyToID[key] = rowID + } + for _, row := range dataset.Rows { + key, _ := row["key"].(string) + if key == "" { + continue + } + if rowID, ok := row["id"].(int); ok { + keyToID[key] = rowID + rowByKey[key] = row + } + } + } + return keyToID, rowByKey +} + +func rejectGenerated2DATLKValues(configID string, collected []nativeCollectedDataset) error { + for _, dataset := range collected { + for _, row := range dataset.Rows { + for field, value := range row { + if field == "id" || field == "key" { + continue + } + allowBare := columnMatchesSpec(dataset.Dataset.Spec, field) + if _, ok, err := parseTLKPayload(value, allowBare); err != nil { + return fmt.Errorf("generated topdata 2DA config %s dataset %s field %s has invalid TLK payload: %w", configID, dataset.Dataset.Name, field, err) + } else if ok { + return fmt.Errorf("generated topdata 2DA config %s dataset %s field %s uses TLK-backed text; 2DA-only generated asset builds do not generate TLK output", configID, dataset.Dataset.Name, field) + } + } + } + } + return nil +} diff --git a/internal/topdata/parts_discovery.go b/internal/topdata/parts_discovery.go index 6476dba..27da121 100644 --- a/internal/topdata/parts_discovery.go +++ b/internal/topdata/parts_discovery.go @@ -364,8 +364,7 @@ func applyPartOverrides(sourceDir string, collected []nativeCollectedDataset) ([ if category == "" { continue } - overridePath := filepath.Join(sourceDir, "data", "parts", "overrides", category+".json") - overrides, err := loadPartOverrides(overridePath) + overrides, err := loadPartOverridesForCategory(sourceDir, category) if err != nil { return nil, err } @@ -386,11 +385,11 @@ func applyPartOverrides(sourceDir string, collected []nativeCollectedDataset) ([ for index, override := range overrides { rawID, ok := override["id"] if !ok { - return nil, fmt.Errorf("%s: override %d is missing id", overridePath, index) + return nil, fmt.Errorf("parts/%s override %d is missing id", category, index) } rowID, err := asInt(rawID) if err != nil { - return nil, fmt.Errorf("%s: override %d id is not numeric", overridePath, index) + return nil, fmt.Errorf("parts/%s override %d id is not numeric", category, index) } row, ok := rowByID[rowID] if !ok { @@ -416,6 +415,75 @@ func applyPartOverrides(sourceDir string, collected []nativeCollectedDataset) ([ return result, nil } +func loadPartOverridesForCategory(sourceDir, category string) ([]map[string]any, error) { + overrides := []map[string]any{} + + legacyPath := filepath.Join(sourceDir, "data", "parts", "overrides", category+".json") + legacyOverrides, err := loadPartOverrides(legacyPath) + if err != nil { + return nil, err + } + overrides = append(overrides, legacyOverrides...) + + moduleDir := filepath.Join(sourceDir, "data", "parts", "modules") + modulePaths, err := partModuleOverridePaths(moduleDir, category) + if err != nil { + return nil, err + } + for _, path := range modulePaths { + moduleOverrides, err := loadPartOverrides(path) + if err != nil { + return nil, err + } + overrides = append(overrides, moduleOverrides...) + } + return overrides, nil +} + +func partModuleOverridePaths(moduleDir, category string) ([]string, error) { + info, err := os.Stat(moduleDir) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + if !info.IsDir() { + return nil, fmt.Errorf("parts module override path %s is not a directory", moduleDir) + } + + var paths []string + err = filepath.WalkDir(moduleDir, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + if strings.ToLower(filepath.Ext(path)) != ".json" { + return nil + } + if partModuleOverrideMatchesCategory(path, category) { + paths = append(paths, path) + } + return nil + }) + if err != nil { + return nil, err + } + slices.Sort(paths) + return paths, nil +} + +func partModuleOverrideMatchesCategory(path, category string) bool { + stem := strings.TrimSuffix(strings.ToLower(filepath.Base(path)), strings.ToLower(filepath.Ext(path))) + category = strings.ToLower(strings.TrimSpace(category)) + return stem == category || + strings.HasPrefix(stem, category+"_") || + strings.HasSuffix(stem, "_"+category) || + strings.Contains(stem, "_"+category+"_") +} + func normalizePartOverrideValue(value any) any { switch typed := value.(type) { case float64: