ShadowsOverWestgate/sow-tools
7
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases 38 Activity

Dev Testing + AutoGen Activation

Browse Source
This commit is contained in:
archvillainette
2026-04-25 08:57:10 +02:00
parent b6fff4bed4
commit 4009931fd0
5 changed files with 329 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/app/app.go
+3
View File
@@ -280,6 +280,9 @@ func runBuildHAKs(ctx context) error {
if result.Manifest != "" {
fmt.Fprintf(ctx.stdout, "hak manifest: %s\n", result.Manifest)
}
if len(result.AutogenManifestPaths) > 0 {
fmt.Fprintf(ctx.stdout, "autogen manifests: %d\n", len(result.AutogenManifestPaths))
}
return nil
}
internal/pipeline/build.go
+35 -8
View File
@@ -23,14 +23,15 @@ import (
)
type BuildResult struct {
ModulePath string
HAKPaths []string
Manifest string
Resources int
HAKAssets int
TopPackageHAK string
TopPackageTLK string
TopPackageFiles int
ModulePath string
HAKPaths []string
Manifest string
AutogenManifestPaths []string
Resources int
HAKAssets int
TopPackageHAK string
TopPackageTLK string
TopPackageFiles int
}
type BuildManifest struct {
@@ -195,6 +196,10 @@ func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bo
if err != nil {
return BuildResult{}, err
}
autogenManifests, err := topdata.ProduceAutogenManifests(p)
if err != nil {
return BuildResult{}, err
}
var previousManifest *BuildManifest
if sourceManifest == nil || writeArchives {
previousManifest, err = loadPreviousBuildManifest(p.HAKManifestPath())
@@ -204,6 +209,12 @@ func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bo
}
result := BuildResult{HAKAssets: len(assetResources)}
if err := writeAutogenManifestOutputs(progress, autogenManifests); err != nil {
return BuildResult{}, err
}
for _, manifest := range autogenManifests {
result.AutogenManifestPaths = append(result.AutogenManifestPaths, manifest.OutputPath)
}
if len(assetResources) == 0 {
progressf(progress, "Cleaning previous generated HAKs...")
if err := cleanupGeneratedHAKs(p); err != nil {
@@ -318,6 +329,22 @@ func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bo
return result, nil
}
func writeAutogenManifestOutputs(progress ProgressFunc, manifests []topdata.ProducedAutogenManifest) error {
if len(manifests) == 0 {
return nil
}
for _, manifest := range manifests {
progressf(progress, fmt.Sprintf("Writing autogen manifest %s...", filepath.Base(manifest.OutputPath)))
if err := os.MkdirAll(filepath.Dir(manifest.OutputPath), 0o755); err != nil {
return fmt.Errorf("create autogen manifest dir: %w", err)
}
if err := os.WriteFile(manifest.OutputPath, manifest.Payload, 0o644); err != nil {
return fmt.Errorf("write autogen manifest %s: %w", manifest.AssetName, err)
}
}
return nil
}
func filterHAKChunksByName(chunks []hakChunk, archiveNames []string) ([]hakChunk, error) {
if len(archiveNames) == 0 {
return chunks, nil
internal/pipeline/pipeline_test.go
+111
View File
@@ -1451,6 +1451,117 @@ func TestPlanHAKsWritesManifestWithoutArchives(t *testing.T) {
}
}
func TestBuildHAKsWritesConfiguredAutogenManifests(t *testing.T) {
root := t.TempDir()
mustMkdir(t, filepath.Join(root, "src"))
mustMkdir(t, filepath.Join(root, "assets", "part", "belt"))
mustMkdir(t, filepath.Join(root, "assets", "vfxs", "head_accessories"))
mustMkdir(t, filepath.Join(root, "assets", "vfxs", "head_features", "hair"))
mustMkdir(t, filepath.Join(root, "build"))
mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{
"module": {
"name": "Test Module",
"resref": "testmod",
"hak_order": ["group:core"]
},
"paths": {
"source": "src",
"assets": "assets",
"build": "build"
},
"autogen": {
"producers": [
{
"id": "parts",
"root": "part",
"include": ["**/*.mdl"],
"derive": {
"kind": "trailing_numeric_suffix",
"group_from": "first_path_segment"
},
"manifest": {
"release_tag": "parts-manifest-current",
"asset_name": "sow-parts-manifest.json",
"cache_name": "sow-parts-manifest.json"
}
},
{
"id": "head_visualeffects",
"root": "vfxs",
"include": ["head_accessories/**/*.mdl", "head_features/**/*.mdl"],
"derive": {
"kind": "model_stem",
"group_from": "first_path_segment"
},
"manifest": {
"release_tag": "head-vfx-manifest-current",
"asset_name": "sow-head-vfx-manifest.json",
"cache_name": "sow-head-vfx-manifest.json"
}
}
]
},
"haks": [
{
"name": "core",
"priority": 1,
"max_bytes": 0,
"split": false,
"include": ["part/**", "vfxs/**"]
}
]
}
`)
mustWriteFile(t, filepath.Join(root, "assets", "part", "belt", "pfa0_belt018.mdl"), "belt")
mustWriteFile(t, filepath.Join(root, "assets", "vfxs", "head_accessories", "hfx_bandana.mdl"), "bandana")
mustWriteFile(t, filepath.Join(root, "assets", "vfxs", "head_features", "hair", "hfx_hair_bangs.mdl"), "hair")
p, err := project.Load(root)
if err != nil {
t.Fatalf("load project: %v", err)
}
if err := p.ValidateLayout(); err != nil {
t.Fatalf("validate layout: %v", err)
}
if err := p.Scan(); err != nil {
t.Fatalf("scan: %v", err)
}
result, err := BuildHAKs(p)
if err != nil {
t.Fatalf("build haks: %v", err)
}
if len(result.AutogenManifestPaths) != 2 {
t.Fatalf("expected 2 autogen manifests, got %d", len(result.AutogenManifestPaths))
}
partsRaw, err := os.ReadFile(filepath.Join(root, "build", "sow-parts-manifest.json"))
if err != nil {
t.Fatalf("read parts manifest: %v", err)
}
if !strings.Contains(string(partsRaw), `"id": "parts"`) || !strings.Contains(string(partsRaw), `"row_id": 18`) {
t.Fatalf("unexpected parts manifest contents:\n%s", string(partsRaw))
}
headRaw, err := os.ReadFile(filepath.Join(root, "build", "sow-head-vfx-manifest.json"))
if err != nil {
t.Fatalf("read head visualeffects manifest: %v", err)
}
text := string(headRaw)
for _, want := range []string{
`"id": "head_visualeffects"`,
`"group": "head_accessories"`,
`"model_stem": "hfx_bandana"`,
`"group": "head_features"`,
`"model_stem": "hfx_hair_bangs"`,
} {
if !strings.Contains(text, want) {
t.Fatalf("expected head visualeffects manifest to contain %q, got:\n%s", want, text)
}
}
}
func TestBuildSkipsEmptyHAKGroupsInModuleOrder(t *testing.T) {
root := t.TempDir()
mustMkdir(t, filepath.Join(root, "src"))
internal/topdata/autogen.go
+68 -2
View File
@@ -32,6 +32,13 @@ type autogenManifestEntry struct {
RowID int `json:"row_id,omitempty"`
}
type ProducedAutogenManifest struct {
ID string
AssetName string
OutputPath string
Payload []byte
}
func applyAutogenConsumers(p *project.Project, collected []nativeCollectedDataset, progress func(string)) ([]nativeCollectedDataset, error) {
if len(p.Config.Autogen.Consumers) == 0 {
return collected, nil
@@ -152,6 +159,10 @@ func resolveAutogenLocalOverrideRoot(p *project.Project, consumer project.Autoge
func scanLocalAutogenEntries(overrideRoot string, consumer project.AutogenConsumerConfig) ([]autogenManifestEntry, error) {
scanRoot := filepath.Join(overrideRoot, consumer.Root)
return scanConfiguredAutogenEntries(scanRoot, consumer.Include, consumer.Derive)
}
func scanConfiguredAutogenEntries(scanRoot string, include []string, derive project.AutogenDeriveConfig) ([]autogenManifestEntry, error) {
info, err := os.Stat(scanRoot)
if err != nil {
if os.IsNotExist(err) {
@@ -176,10 +187,10 @@ func scanLocalAutogenEntries(overrideRoot string, consumer project.AutogenConsum
return err
}
rel = filepath.ToSlash(rel)
if !matchesAutogenInclude(rel, consumer.Include) {
if !matchesAutogenInclude(rel, include) {
return nil
}
entry, ok := deriveAutogenManifestEntry(rel, consumer.Derive)
entry, ok := deriveAutogenManifestEntry(rel, derive)
if !ok {
return nil
}
@@ -196,6 +207,61 @@ func scanLocalAutogenEntries(overrideRoot string, consumer project.AutogenConsum
return entries, nil
}
func ProduceAutogenManifests(p *project.Project) ([]ProducedAutogenManifest, error) {
if len(p.Config.Autogen.Producers) == 0 {
return nil, nil
}
assetsRoot := strings.TrimSpace(p.AssetsDir())
if assetsRoot == "" {
return nil, fmt.Errorf("autogen producers require paths.assets to be configured")
}
info, err := os.Stat(assetsRoot)
if err != nil {
return nil, fmt.Errorf("stat assets root %s: %w", assetsRoot, err)
}
if !info.IsDir() {
return nil, fmt.Errorf("assets root %s is not a directory", assetsRoot)
}
repo, ref := autogenManifestSourceMetadata(p.Root)
results := make([]ProducedAutogenManifest, 0, len(p.Config.Autogen.Producers))
for _, producer := range p.Config.Autogen.Producers {
scanRoot := filepath.Join(assetsRoot, producer.Root)
entries, err := scanConfiguredAutogenEntries(scanRoot, producer.Include, producer.Derive)
if err != nil {
return nil, fmt.Errorf("scan autogen producer %s: %w", producer.ID, err)
}
if len(entries) == 0 {
return nil, fmt.Errorf("autogen producer %s discovered no matching entries under %s", producer.ID, scanRoot)
}
payload, err := formatAutogenManifest(p.Root, producer, repo, ref, entries)
if err != nil {
return nil, fmt.Errorf("format autogen producer %s manifest: %w", producer.ID, err)
}
payload = append(payload, '\n')
results = append(results, ProducedAutogenManifest{
ID: producer.ID,
AssetName: producer.Manifest.AssetName,
OutputPath: filepath.Join(p.BuildDir(), producer.Manifest.AssetName),
Payload: payload,
})
}
return results, nil
}
func autogenManifestSourceMetadata(root string) (repo string, ref string) {
if remote, err := gitOutput(root, "remote", "get-url", "origin"); err == nil {
if spec, ok := parseRepoSpec(strings.TrimSpace(remote)); ok {
repo = spec.Owner + "/" + spec.Repo
}
}
if head, err := gitOutput(root, "rev-parse", "HEAD"); err == nil {
ref = strings.TrimSpace(head)
}
return repo, ref
}
func matchesAutogenInclude(rel string, include []string) bool {
rel = filepath.ToSlash(rel)
for _, pattern := range include {
internal/topdata/topdata_test.go
+112
View File
@@ -9189,6 +9189,118 @@ func TestBuildNativeWithReleasedPartsManifest(t *testing.T) {
}
}
func TestBuildNativeWithReleasedHeadVisualeffectsManifest(t *testing.T) {
root := t.TempDir()
projRoot := filepath.Join(root, "project")
mkdirAll(t, filepath.Join(projRoot, "src"))
mkdirAll(t, filepath.Join(projRoot, "assets"))
mkdirAll(t, filepath.Join(projRoot, "build"))
mkdirAll(t, filepath.Join(projRoot, "topdata", "data", "visualeffects"))
writeFile(t, filepath.Join(projRoot, "topdata", "base_dialog.json"), "{}\n")
writeFile(t, filepath.Join(projRoot, "topdata", "data", "visualeffects", "base.json"), `{
"output": "visualeffects.2da",
"columns": ["Label", "Type_FD", "OrientWithGround", "Imp_HeadCon_Node", "OrientWithObject"],
"rows": [
{"key": "visualeffects:existing", "id": 17, "Label": "EXISTING", "Type_FD": "F", "OrientWithGround": "0", "Imp_HeadCon_Node": "****", "OrientWithObject": "0"}
]
}`+"\n")
writeFile(t, filepath.Join(projRoot, "topdata", "data", "visualeffects", "lock.json"), `{
"visualeffects:existing": 17
}`+"\n")
mkdirAll(t, filepath.Join(projRoot, "reference"))
writeFile(t, filepath.Join(projRoot, "reference", "build.py"), "print('ok')\n")
manifestJSON := `{
"id": "head_visualeffects",
"repo": "ShadowsOverWestgate/sow-assets",
"ref": "test-manifest",
"generated_at": "2026-04-25T00:00:00Z",
"entries": [
{
"group": "head_accessories",
"model_stem": "hfx_bandana",
"source": "head_accessories/hfx_bandana.mdl"
},
{
"group": "head_features",
"model_stem": "hfx_hair_bangs",
"source": "head_features/hair/hfx_hair_bangs.mdl"
}
]
}` + "\n"
var server *httptest.Server
server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/api/v1/repos/ShadowsOverWestgate/sow-assets/releases/tags/head-vfx-manifest-current":
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"assets":[{"name":"sow-head-vfx-manifest.json","browser_download_url":"` + server.URL + `/downloads/sow-head-vfx-manifest.json"}]}`))
case "/downloads/sow-head-vfx-manifest.json":
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(manifestJSON))
default:
http.NotFound(w, r)
}
}))
defer server.Close()
runGitTest(t, "", "init", "-b", "main", projRoot)
runGitTest(t, projRoot, "remote", "add", "origin", server.URL+"/ShadowsOverWestgate/sow-module.git")
proj := &project.Project{
Root: projRoot,
Config: project.Config{
Module: project.ModuleConfig{Name: "Test", ResRef: "test"},
Paths: project.PathConfig{Source: "src", Assets: "assets", Build: "build"},
TopData: project.TopDataConfig{
Source: "topdata",
Build: "build/topdata",
},
Autogen: project.AutogenConfig{
Consumers: []project.AutogenConsumerConfig{
{
ID: "head_visualeffects",
Producer: "head_visualeffects",
Dataset: "visualeffects",
Mode: "head_visualeffects",
Root: "vfxs",
Include: []string{"head_accessories/**/*.mdl", "head_features/**/*.mdl"},
Derive: project.AutogenDeriveConfig{
Kind: "model_stem",
GroupFrom: "first_path_segment",
},
Manifest: project.AutogenManifestConfig{
ReleaseTag: "head-vfx-manifest-current",
AssetName: "sow-head-vfx-manifest.json",
CacheName: "sow-head-vfx-manifest.json",
},
},
},
},
},
}
result, err := BuildNative(proj, nil)
if err != nil {
t.Fatalf("BuildNative failed: %v", err)
}
visualeffectsBytes, err := os.ReadFile(filepath.Join(result.Output2DADir, "visualeffects.2da"))
if err != nil {
t.Fatalf("read visualeffects.2da: %v", err)
}
text := string(visualeffectsBytes)
for _, want := range []string{
"0\theadaccessory_BANDANA\tD\t0\thfx_bandana\t1",
"1\theadfeature_HAIR_BANGS\tD\t0\thfx_hair_bangs\t1",
"17\tEXISTING\tF\t0\t****\t0",
} {
if !strings.Contains(text, want) {
t.Fatalf("expected visualeffects output to contain %q, got:\n%s", want, text)
}
}
}
func TestBuildNativeRejectsGitRepoTopDataAssetsOverride(t *testing.T) {
root := testProjectRoot(t)
mkdirAll(t, filepath.Join(root, "topdata", "data", "parts"))