ShadowsOverWestgate/sow-tools
7
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases 38 Activity

Native Workflow Support

Browse Source
This commit is contained in:
archvillainette
2026-04-09 11:01:39 +02:00
parent dc93feb176
commit 4f8bfc133a
9 changed files with 912 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/pipeline/build.go
+45 -8
View File
@@ -16,15 +16,19 @@ import (
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf"
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/gff"
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/topdata"
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/validator"
)
type BuildResult struct {
ModulePath string
HAKPaths []string
Manifest string
Resources int
HAKAssets int
ModulePath string
HAKPaths []string
Manifest string
Resources int
HAKAssets int
TopPackageHAK string
TopPackageTLK string
TopPackageFiles int
}
type BuildManifest struct {
@@ -87,8 +91,26 @@ func buildModule(p *project.Project, progress ProgressFunc) (BuildResult, error)
return BuildResult{}, err
}
var topPackage topdata.PackageResult
var err error
if p.HasTopData() {
progressf(progress, "Building native top package...")
topPackage, err = topdata.BuildPackage(p, func(message string) {
progressf(progress, "Top package: "+message)
})
if err != nil {
return BuildResult{}, err
}
}
progressf(progress, "Resolving module HAK order...")
moduleHakOrder, err := plannedModuleHAKOrder(p)
if err != nil {
return BuildResult{}, err
}
progressf(progress, "Collecting module resources...")
moduleResources, err := collectModuleResources(p, nil)
moduleResources, err := collectModuleResources(p, moduleHakOrder)
if err != nil {
return BuildResult{}, err
}
@@ -106,8 +128,11 @@ func buildModule(p *project.Project, progress ProgressFunc) (BuildResult, error)
}
return BuildResult{
ModulePath: outputPath,
Resources: len(moduleResources),
ModulePath: outputPath,
Resources: len(moduleResources),
TopPackageHAK: topPackage.OutputHAKPath,
TopPackageTLK: topPackage.OutputTLKPath,
TopPackageFiles: topPackage.HAKResources,
}, nil
}
@@ -841,6 +866,18 @@ func cleanupGeneratedHAKs(buildDir, moduleResRef string) error {
return nil
}
func plannedModuleHAKOrder(p *project.Project) ([]string, error) {
assetResources, err := collectAssetResources(p)
if err != nil {
return nil, err
}
chunks, err := planHAKChunks(p, assetResources)
if err != nil {
return nil, err
}
return resolveModuleHAKOrder(p, chunks)
}
func resolveModuleHAKOrder(p *project.Project, chunks []hakChunk) ([]string, error) {
if len(p.Config.Module.HAKOrder) == 0 {
return nil, nil
internal/pipeline/compare.go
+74 -1
View File
@@ -5,6 +5,7 @@ import (
"encoding/json"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"slices"
@@ -14,6 +15,7 @@ import (
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf"
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/gff"
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/topdata"
)
type CompareResult struct {
@@ -112,6 +114,10 @@ func Compare(p *project.Project) (CompareResult, error) {
func expectedResources(p *project.Project) (map[string]resourceExpectation, error) {
out := map[string]resourceExpectation{}
moduleHakOrder, err := plannedModuleHAKOrder(p)
if err != nil {
return nil, err
}
for _, rel := range p.Inventory.SourceFiles {
abs := filepath.Join(p.SourceDir(), filepath.FromSlash(rel))
@@ -128,6 +134,9 @@ func expectedResources(p *project.Project) (map[string]resourceExpectation, erro
if err := json.Unmarshal(raw, &document); err != nil {
return nil, fmt.Errorf("parse gff json %s: %w", abs, err)
}
if extension == ".ifo" && strings.EqualFold(name, "module") && len(moduleHakOrder) > 0 {
setModuleHAKList(&document, moduleHakOrder)
}
canonical, err := json.Marshal(document)
if err != nil {
@@ -195,6 +204,12 @@ func ensureBuildIsCurrent(p *project.Project, modulePath string, hakPaths []stri
archivePaths := make([]string, 0, 1+len(hakPaths))
archivePaths = append(archivePaths, modulePath)
archivePaths = append(archivePaths, hakPaths...)
if p.HasTopData() {
archivePaths = append(archivePaths,
filepath.Join(p.BuildDir(), topdata.PackageHAKFileName),
filepath.Join(p.BuildDir(), topdata.PackageTLKFileName),
)
}
oldestArchive := time.Time{}
for _, path := range archivePaths {
@@ -217,6 +232,13 @@ func ensureBuildIsCurrent(p *project.Project, modulePath string, hakPaths []stri
for _, rel := range p.Inventory.AssetFiles {
sourcePaths = append(sourcePaths, filepath.Join(p.AssetsDir(), filepath.FromSlash(rel)))
}
if p.HasTopData() {
topSources, err := topPackageSourcePaths(p)
if err != nil {
return err
}
sourcePaths = append(sourcePaths, topSources...)
}
newestSource := time.Time{}
newestPath := ""
@@ -308,6 +330,57 @@ func manifestHAKPaths(buildDir string) ([]string, error) {
if err != nil {
return nil, fmt.Errorf("scan hak archives: %w", err)
}
slices.Sort(paths)
filtered := make([]string, 0, len(paths))
for _, path := range paths {
if filepath.Base(path) == topdata.PackageHAKFileName {
continue
}
filtered = append(filtered, path)
}
slices.Sort(filtered)
return filtered, nil
}
func topPackageSourcePaths(p *project.Project) ([]string, error) {
sourceDir := p.TopDataSourceDir()
paths := make([]string, 0)
for _, candidate := range []string{
filepath.Join(sourceDir, ".tlk_state.json"),
filepath.Join(sourceDir, "base_dialog.json"),
} {
if _, err := os.Stat(candidate); err == nil {
paths = append(paths, candidate)
} else if err != nil && !errors.Is(err, os.ErrNotExist) {
return nil, fmt.Errorf("stat topdata source %s: %w", candidate, err)
}
}
for _, dir := range []string{
filepath.Join(sourceDir, "data"),
filepath.Join(sourceDir, "assets"),
} {
info, err := os.Stat(dir)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
continue
}
return nil, fmt.Errorf("stat topdata path %s: %w", dir, err)
}
if !info.IsDir() {
continue
}
err = filepath.WalkDir(dir, func(path string, d fs.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
}
if d.IsDir() {
return nil
}
paths = append(paths, path)
return nil
})
if err != nil {
return nil, fmt.Errorf("scan topdata path %s: %w", dir, err)
}
}
return paths, nil
}
internal/pipeline/pipeline_test.go
+256 -1
View File
@@ -1,6 +1,7 @@
package pipeline
import (
"bytes"
"encoding/json"
"os"
"path/filepath"
@@ -9,6 +10,7 @@ import (
"time"
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf"
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/gff"
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
)
@@ -32,7 +34,7 @@ func TestBuildThenExtract(t *testing.T) {
`)
mustMkdir(t, filepath.Join(root, "src", "module"))
mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{
mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{
"file_type": "IFO ",
"file_version": "V3.2",
"root": {
@@ -427,6 +429,259 @@ printf 'compiled:%s\n' "$(basename "$src")" > "$out"
}
}
func TestBuildModuleBuildsTopPackageAndInjectsHAKList(t *testing.T) {
root := t.TempDir()
mustMkdir(t, filepath.Join(root, "src", "module"))
mustMkdir(t, filepath.Join(root, "assets"))
mustMkdir(t, filepath.Join(root, "build"))
mustMkdir(t, filepath.Join(root, "topdata", "data", "repadjust"))
mustMkdir(t, filepath.Join(root, "topdata", "assets", "gui"))
mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{
"module": {
"name": "Test Module",
"resref": "testmod",
"hak_order": ["sow_top"]
},
"paths": {
"source": "src",
"assets": "assets",
"build": "build"
},
"topdata": {
"source": "topdata",
"build": "build/topdata"
}
}
`)
mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{
"file_type": "IFO ",
"file_version": "V3.2",
"root": {
"struct_type": 0,
"fields": [
{"label": "Mod_Name", "type": "CExoString", "value": "Test Module"},
{"label": "Mod_CustomTlk", "type": "CExoString", "value": "sow_tlk"}
]
}
}
`)
mustWriteFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n")
mustWriteFile(t, filepath.Join(root, "topdata", "data", "repadjust", "base.json"), `{
"output": "repadjust.2da",
"columns": ["Label"],
"rows": [{"id": 0, "Label": "TEST_LABEL"}]
}
`)
mustWriteFile(t, filepath.Join(root, "topdata", "assets", "gui", "testicon.tga"), "icon-data")
p, err := project.Load(root)
if err != nil {
t.Fatalf("load project: %v", err)
}
if err := p.ValidateLayout(); err != nil {
t.Fatalf("validate layout: %v", err)
}
if err := p.Scan(); err != nil {
t.Fatalf("scan: %v", err)
}
result, err := BuildModule(p)
if err != nil {
t.Fatalf("build module: %v", err)
}
if _, err := os.Stat(filepath.Join(root, "build", "sow_top.hak")); err != nil {
t.Fatalf("expected sow_top.hak: %v", err)
}
if _, err := os.Stat(filepath.Join(root, "build", "sow_tlk.tlk")); err != nil {
t.Fatalf("expected sow_tlk.tlk: %v", err)
}
if result.TopPackageHAK == "" || result.TopPackageTLK == "" {
t.Fatalf("expected top package paths in build result: %#v", result)
}
input, err := os.Open(result.ModulePath)
if err != nil {
t.Fatalf("open module: %v", err)
}
defer input.Close()
archive, err := erf.Read(input)
if err != nil {
t.Fatalf("read module archive: %v", err)
}
var ifoData []byte
for _, resource := range archive.Resources {
ext, _ := erf.ExtensionForResourceType(resource.Type)
if strings.ToLower(resource.Name)+"."+ext == "module.ifo" {
ifoData = resource.Data
break
}
}
if ifoData == nil {
t.Fatalf("expected module.ifo in built archive")
}
document, err := gff.Read(bytes.NewReader(ifoData))
if err != nil {
t.Fatalf("decode module.ifo: %v", err)
}
foundHak := false
for _, field := range document.Root.Fields {
if field.Label != "Mod_HakList" {
continue
}
list, ok := field.Value.(gff.ListValue)
if !ok {
t.Fatalf("expected Mod_HakList list, got %T", field.Value)
}
for _, item := range list {
for _, nested := range item.Fields {
if nested.Label == "Mod_Hak" && nested.Value == gff.StringValue("sow_top") {
foundHak = true
}
}
}
}
if !foundHak {
t.Fatalf("expected built module to reference sow_top")
}
}
func TestCompareFailsWhenTopPackageIsStale(t *testing.T) {
root := t.TempDir()
mustMkdir(t, filepath.Join(root, "src", "module"))
mustMkdir(t, filepath.Join(root, "assets"))
mustMkdir(t, filepath.Join(root, "build"))
mustMkdir(t, filepath.Join(root, "topdata", "data", "repadjust"))
mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{
"module": {
"name": "Test Module",
"resref": "testmod",
"hak_order": ["sow_top"]
},
"paths": {
"source": "src",
"assets": "assets",
"build": "build"
},
"topdata": {
"source": "topdata",
"build": "build/topdata"
}
}
`)
mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{
"file_type": "IFO ",
"file_version": "V3.2",
"root": {
"struct_type": 0,
"fields": [
{"label": "Mod_Name", "type": "CExoString", "value": "Test Module"},
{"label": "Mod_CustomTlk", "type": "CExoString", "value": "sow_tlk"}
]
}
}
`)
mustWriteFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n")
topdataBase := filepath.Join(root, "topdata", "data", "repadjust", "base.json")
mustWriteFile(t, topdataBase, `{
"output": "repadjust.2da",
"columns": ["Label"],
"rows": [{"id": 0, "Label": "TEST_LABEL"}]
}
`)
p, err := project.Load(root)
if err != nil {
t.Fatalf("load project: %v", err)
}
if err := p.ValidateLayout(); err != nil {
t.Fatalf("validate layout: %v", err)
}
if err := p.Scan(); err != nil {
t.Fatalf("scan: %v", err)
}
if _, err := BuildModule(p); err != nil {
t.Fatalf("build module: %v", err)
}
newTime := time.Now().Add(2 * time.Second)
if err := os.Chtimes(topdataBase, newTime, newTime); err != nil {
t.Fatalf("touch topdata source: %v", err)
}
if _, err := Compare(p); err == nil {
t.Fatalf("expected compare to fail when top package is stale")
}
}
func TestCompareNormalizesModuleHakListFromConfig(t *testing.T) {
root := t.TempDir()
mustMkdir(t, filepath.Join(root, "src", "module"))
mustMkdir(t, filepath.Join(root, "assets"))
mustMkdir(t, filepath.Join(root, "build"))
mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{
"module": {
"name": "Test Module",
"resref": "testmod",
"hak_order": ["manual_top"]
},
"paths": {
"source": "src",
"assets": "assets",
"build": "build"
}
}`)
mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{
"file_type": "IFO ",
"file_version": "V3.2",
"root": {
"struct_type": 0,
"fields": [
{
"label": "Mod_Name",
"type": "CExoString",
"value": "Test Module"
},
{
"label": "Mod_HakList",
"type": "List",
"value": [
{
"struct_type": 8,
"fields": [
{
"label": "Mod_Hak",
"type": "CExoString",
"value": "stale_old_hak"
}
]
}
]
}
]
}
}`)
p, err := project.Load(root)
if err != nil {
t.Fatalf("load project: %v", err)
}
if err := p.ValidateLayout(); err != nil {
t.Fatalf("validate layout: %v", err)
}
if err := p.Scan(); err != nil {
t.Fatalf("scan: %v", err)
}
if _, err := BuildModule(p); err != nil {
t.Fatalf("build module: %v", err)
}
if _, err := Compare(p); err != nil {
t.Fatalf("compare project: %v", err)
}
}
func TestBuildOrdersMultipleHAKGroups(t *testing.T) {
root := t.TempDir()
mustMkdir(t, filepath.Join(root, "src"))
internal/pipeline/zz_debug_tmp_test.go
+41
View File
@@ -0,0 +1,41 @@
package pipeline
import (
"bytes"
"encoding/json"
"fmt"
"os"
"path/filepath"
"testing"
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf"
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/gff"
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
)
func TestDebugModuleIFO(t *testing.T) {
root := "/home/vicky/Projects/sow-ShadowsOverWestgate/repositories/sow-module"
p, err := project.Load(root)
if err != nil { t.Fatal(err) }
if err := p.Scan(); err != nil { t.Fatal(err) }
exp, err := expectedResources(p)
if err != nil { t.Fatal(err) }
gotArchive, err := readArchive(filepath.Join(p.BuildDir(), p.Config.Module.ResRef+".mod"))
if err != nil { t.Fatal(err) }
got := archiveIndex(gotArchive)
e := exp["module.ifo"].Bytes
a := got["module.ifo"].Data
fmt.Printf("EQUAL=%v\n", bytes.Equal(e,a))
fmt.Printf("EXPECTED=%s\n", string(e))
fmt.Printf("ACTUAL=%s\n", string(a))
var edoc gff.Document
sourceRaw, _ := os.ReadFile(filepath.Join(root, "src/module/module.ifo.json"))
_ = json.Unmarshal(sourceRaw, &edoc)
fmt.Printf("SOURCE=%s\n", string(sourceRaw))
mod, _ := erf.Read(bytes.NewReader(mustRead(filepath.Join(p.BuildDir(), p.Config.Module.ResRef+".mod"))))
_ = mod
}
func mustRead(path string) []byte { b, _ := os.ReadFile(path); return b }