ShadowsOverWestgate/sow-tools
7
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases 38 Activity

Initial sow-tools import

Browse Source
This commit is contained in:
archvillainette
2026-04-02 17:19:23 +02:00
commit b7f188779a
1370 changed files with 7357 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/app/app.go
+307
View File
@@ -0,0 +1,307 @@
package app
import (
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/pipeline"
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/validator"
)
type command struct {
name string
description string
run func(context) error
}
type context struct {
stdout io.Writer
stderr io.Writer
cwd string
}
var commands = []command{
{
name: "build",
description: "Build module and HAK outputs into build/.",
run: runBuild,
},
{
name: "build-module",
description: "Build only the module archive from src/ into build/.",
run: runBuildModule,
},
{
name: "build-haks",
description: "Build only HAK archives and manifest from assets/ into build/.",
run: runBuildHAKs,
},
{
name: "extract",
description: "Extract a module archive back into the src/ tree.",
run: runExtract,
},
{
name: "validate",
description: "Validate project layout, config, and source inventory.",
run: runValidate,
},
{
name: "compare",
description: "Compare current source content against the built archives.",
run: runCompare,
},
{
name: "apply-hak-manifest",
description: "Apply a generated HAK manifest to src/module/module.ifo.json.",
run: runApplyHAKManifest,
},
}
func Run(args []string) (int, error) {
ctx, err := newContext()
if err != nil {
return 1, err
}
if len(args) == 0 {
printUsage(ctx.stdout)
return 0, nil
}
switch args[0] {
case "-h", "--help", "help":
printUsage(ctx.stdout)
return 0, nil
default:
for _, cmd := range commands {
if cmd.name == args[0] {
if err := cmd.run(ctx); err != nil {
return 1, err
}
return 0, nil
}
}
}
return 1, fmt.Errorf("unknown command %q", args[0])
}
func newContext() (context, error) {
cwd, err := os.Getwd()
if err != nil {
return context{}, fmt.Errorf("resolve working directory: %w", err)
}
return context{
stdout: os.Stdout,
stderr: os.Stderr,
cwd: cwd,
}, nil
}
func runBuild(ctx context) error {
p, err := loadProject(ctx.cwd)
if err != nil {
return err
}
result, err := pipeline.Build(p)
if err != nil {
return err
}
fmt.Fprintf(ctx.stdout, "project: %s\n", p.Config.Module.Name)
fmt.Fprintf(ctx.stdout, "module: %s\n", result.ModulePath)
fmt.Fprintf(ctx.stdout, "resources: %d\n", result.Resources)
if len(result.HAKPaths) > 0 {
fmt.Fprintf(ctx.stdout, "hak archives: %d\n", len(result.HAKPaths))
fmt.Fprintf(ctx.stdout, "hak assets: %d\n", result.HAKAssets)
fmt.Fprintf(ctx.stdout, "hak manifest: %s\n", result.Manifest)
}
return nil
}
func runBuildModule(ctx context) error {
p, err := loadProject(ctx.cwd)
if err != nil {
return err
}
result, err := pipeline.BuildModule(p)
if err != nil {
return err
}
fmt.Fprintf(ctx.stdout, "project: %s\n", p.Config.Module.Name)
fmt.Fprintf(ctx.stdout, "module: %s\n", result.ModulePath)
fmt.Fprintf(ctx.stdout, "resources: %d\n", result.Resources)
return nil
}
func runBuildHAKs(ctx context) error {
p, err := loadProject(ctx.cwd)
if err != nil {
return err
}
result, err := pipeline.BuildHAKs(p)
if err != nil {
return err
}
fmt.Fprintf(ctx.stdout, "project: %s\n", p.Config.Module.Name)
fmt.Fprintf(ctx.stdout, "hak archives: %d\n", len(result.HAKPaths))
fmt.Fprintf(ctx.stdout, "hak assets: %d\n", result.HAKAssets)
if result.Manifest != "" {
fmt.Fprintf(ctx.stdout, "hak manifest: %s\n", result.Manifest)
}
return nil
}
func runExtract(ctx context) error {
p, err := loadProject(ctx.cwd)
if err != nil {
return err
}
result, err := pipeline.Extract(p)
if err != nil {
return err
}
fmt.Fprintf(ctx.stdout, "project: %s\n", p.Config.Module.Name)
fmt.Fprintf(ctx.stdout, "module: %s\n", result.ModulePath)
if len(result.HAKPaths) > 0 {
fmt.Fprintf(ctx.stdout, "hak archives: %d\n", len(result.HAKPaths))
}
fmt.Fprintf(ctx.stdout, "written: %d\n", result.Written)
fmt.Fprintf(ctx.stdout, "skipped: %d\n", result.Skipped)
return nil
}
func runValidate(ctx context) error {
p, err := loadProject(ctx.cwd)
if err != nil {
return err
}
validationReport := validator.ValidateProject(p)
if validationReport.HasErrors() {
for _, diagnostic := range validationReport.Diagnostics {
if diagnostic.Severity == validator.SeverityWarning {
fprintfTarget := ctx.stderr
fmt.Fprintf(fprintfTarget, "warning: %s: %s\n", diagnostic.Path, diagnostic.Message)
continue
}
fmt.Fprintf(ctx.stderr, "error: %s: %s\n", diagnostic.Path, diagnostic.Message)
}
return fmt.Errorf("validation failed with %d error(s)", validationReport.ErrorCount())
}
for _, diagnostic := range validationReport.Diagnostics {
if diagnostic.Severity == validator.SeverityWarning {
fmt.Fprintf(ctx.stderr, "warning: %s: %s\n", diagnostic.Path, diagnostic.Message)
}
}
inventoryReport := p.Inventory.Report()
fmt.Fprintf(ctx.stdout, "project: %s\n", p.Config.Module.Name)
fmt.Fprintf(ctx.stdout, "root: %s\n", p.Root)
fmt.Fprintf(ctx.stdout, "source files: %d\n", inventoryReport.SourceFiles)
fmt.Fprintf(ctx.stdout, "script files: %d\n", inventoryReport.ScriptFiles)
fmt.Fprintf(ctx.stdout, "asset files: %d\n", inventoryReport.AssetFiles)
fmt.Fprintf(ctx.stdout, "module file types: %s\n", strings.Join(inventoryReport.Extensions, ", "))
if warnings := validationReport.WarningCount(); warnings > 0 {
fmt.Fprintf(ctx.stdout, "warnings: %d\n", warnings)
}
fmt.Fprintf(ctx.stdout, "validation: ok\n")
return nil
}
func runCompare(ctx context) error {
p, err := loadProject(ctx.cwd)
if err != nil {
return err
}
result, err := pipeline.Compare(p)
if err != nil {
return err
}
fmt.Fprintf(ctx.stdout, "project: %s\n", p.Config.Module.Name)
fmt.Fprintf(ctx.stdout, "module: %s\n", result.ModulePath)
if len(result.HAKPaths) > 0 {
fmt.Fprintf(ctx.stdout, "hak archives: %d\n", len(result.HAKPaths))
}
fmt.Fprintf(ctx.stdout, "checked resources: %d\n", result.Checked)
fmt.Fprintf(ctx.stdout, "compare: ok\n")
return nil
}
func runApplyHAKManifest(ctx context) error {
p, err := loadProject(ctx.cwd)
if err != nil {
return err
}
manifestPath := filepath.Join(p.BuildDir(), "haks.json")
if len(os.Args) > 2 {
manifestPath = os.Args[2]
if !filepath.IsAbs(manifestPath) {
manifestPath = filepath.Join(ctx.cwd, manifestPath)
}
}
result, err := pipeline.ApplyHAKManifest(p, manifestPath)
if err != nil {
return err
}
fmt.Fprintf(ctx.stdout, "project: %s\n", p.Config.Module.Name)
fmt.Fprintf(ctx.stdout, "manifest: %s\n", result.ManifestPath)
fmt.Fprintf(ctx.stdout, "module source: %s\n", result.ModuleSource)
fmt.Fprintf(ctx.stdout, "hak entries: %d\n", result.HAKCount)
return nil
}
func loadProject(cwd string) (*project.Project, error) {
root, err := project.FindRoot(cwd)
if err != nil {
return nil, err
}
p, err := project.Load(root)
if err != nil {
return nil, err
}
var failures []error
if err := p.ValidateLayout(); err != nil {
failures = append(failures, err)
}
if err := p.Scan(); err != nil {
failures = append(failures, err)
}
if len(failures) > 0 {
return nil, errors.Join(failures...)
}
return p, nil
}
func printUsage(w io.Writer) {
exe := filepath.Base(os.Args[0])
fmt.Fprintf(w, "%s <command>\n\n", exe)
fmt.Fprintln(w, "Commands:")
for _, cmd := range commands {
fmt.Fprintf(w, " %-8s %s\n", cmd.name, cmd.description)
}
}
internal/erf/erf.go
+297
View File
@@ -0,0 +1,297 @@
package erf
import (
"bytes"
"encoding/binary"
"fmt"
"io"
"sort"
"strings"
)
const (
headerSize = 160
versionV10 = "V1.0"
strRefNone = 0xFFFFFFFF
typeMOD = "MOD "
typeHAK = "HAK "
)
type Archive struct {
FileType string
Version string
Resources []Resource
}
type Resource struct {
Name string
Type uint16
Data []byte
}
type header struct {
FileType [4]byte
Version [4]byte
LanguageCount uint32
LocalizedStringSize uint32
EntryCount uint32
LocalizedStringOffset uint32
KeyListOffset uint32
ResourceListOffset uint32
BuildYear uint32
BuildDay uint32
DescriptionStrRef uint32
Reserved [116]byte
}
type keyEntry struct {
ResRef [16]byte
ResourceID uint32
ResourceType uint16
Unused uint16
}
type resourceEntry struct {
Offset uint32
Size uint32
}
var extensionTypes = map[string]uint16{
"res": 0x0000,
"bmp": 0x0001,
"mve": 0x0002,
"tga": 0x0003,
"wav": 0x0004,
"plt": 0x0006,
"ini": 0x0007,
"txt": 0x000A,
"mdl": 0x07D2,
"nss": 0x07D9,
"ncs": 0x07DA,
"are": 0x07DC,
"set": 0x07DD,
"ifo": 0x07DE,
"bic": 0x07DF,
"wok": 0x07E1,
"2da": 0x07E2,
"tlk": 0x07E3,
"txi": 0x07E6,
"git": 0x07E7,
"uti": 0x07E9,
"utt": 0x07EA,
"utc": 0x07EB,
"dlg": 0x07ED,
"itp": 0x07EE,
"uts": 0x07F0,
"dds": 0x07F1,
"fac": 0x07F6,
"gff": 0x07F7,
"ute": 0x07F9,
"utd": 0x07FB,
"utp": 0x07FC,
"dfa": 0x07FD,
"gic": 0x07FE,
"gui": 0x07FF,
"utm": 0x0800,
"dwk": 0x0804,
"pwk": 0x0805,
"jrl": 0x0808,
"utw": 0x080A,
"ssf": 0x080C,
"hak": 0x080D,
"nwm": 0x080E,
"bik": 0x080F,
"ndb": 0x0810,
"ptm": 0x0811,
"ptt": 0x0812,
"ltr": 0x0813,
"shd": 0x0815,
"mtr": 0x0818,
"lod": 0x081E,
}
var typeExtensions map[uint16]string
func init() {
typeExtensions = make(map[uint16]string, len(extensionTypes))
for ext, resourceType := range extensionTypes {
if _, exists := typeExtensions[resourceType]; !exists {
typeExtensions[resourceType] = ext
}
}
}
func New(fileType string, resources []Resource) Archive {
normalized := make([]Resource, len(resources))
copy(normalized, resources)
sort.Slice(normalized, func(i, j int) bool {
if normalized[i].Name == normalized[j].Name {
return normalized[i].Type < normalized[j].Type
}
return normalized[i].Name < normalized[j].Name
})
return Archive{
FileType: padFour(fileType),
Version: versionV10,
Resources: normalized,
}
}
func ArchiveSize(resources []Resource) int64 {
return int64(headerSize + len(resources)*24 + len(resources)*8 + totalResourceBytes(resources))
}
func Write(w io.Writer, archive Archive) error {
if archive.Version == "" {
archive.Version = versionV10
}
if archive.FileType == "" {
archive.FileType = typeMOD
}
keys := make([]keyEntry, 0, len(archive.Resources))
entries := make([]resourceEntry, 0, len(archive.Resources))
dataOffset := uint32(headerSize + len(archive.Resources)*24 + len(archive.Resources)*8)
data := bytes.Buffer{}
for index, resource := range archive.Resources {
if len(resource.Name) > 16 {
return fmt.Errorf("resource %q exceeds 16-byte resref limit", resource.Name)
}
entry := resourceEntry{
Offset: dataOffset + uint32(data.Len()),
Size: uint32(len(resource.Data)),
}
entries = append(entries, entry)
var key keyEntry
copy(key.ResRef[:], []byte(resource.Name))
key.ResourceID = uint32(index)
key.ResourceType = resource.Type
keys = append(keys, key)
if _, err := data.Write(resource.Data); err != nil {
return err
}
}
hdr := header{
LanguageCount: 0,
LocalizedStringSize: 0,
EntryCount: uint32(len(archive.Resources)),
LocalizedStringOffset: headerSize,
KeyListOffset: headerSize,
ResourceListOffset: uint32(headerSize + len(keys)*24),
BuildYear: 0,
BuildDay: 0,
DescriptionStrRef: strRefNone,
}
copy(hdr.FileType[:], []byte(padFour(archive.FileType)))
copy(hdr.Version[:], []byte(padFour(archive.Version)))
var buf bytes.Buffer
if err := binary.Write(&buf, binary.LittleEndian, hdr); err != nil {
return err
}
if err := binary.Write(&buf, binary.LittleEndian, keys); err != nil {
return err
}
if err := binary.Write(&buf, binary.LittleEndian, entries); err != nil {
return err
}
if _, err := buf.Write(data.Bytes()); err != nil {
return err
}
_, err := w.Write(buf.Bytes())
return err
}
func Read(r io.Reader) (Archive, error) {
data, err := io.ReadAll(r)
if err != nil {
return Archive{}, fmt.Errorf("read erf: %w", err)
}
if len(data) < headerSize {
return Archive{}, fmt.Errorf("erf file too small: %d bytes", len(data))
}
var hdr header
if err := binary.Read(bytes.NewReader(data[:headerSize]), binary.LittleEndian, &hdr); err != nil {
return Archive{}, fmt.Errorf("decode erf header: %w", err)
}
keyStart := int(hdr.KeyListOffset)
keyEnd := keyStart + int(hdr.EntryCount)*24
if keyEnd > len(data) {
return Archive{}, fmt.Errorf("erf key list exceeds file bounds")
}
keys := make([]keyEntry, hdr.EntryCount)
if err := binary.Read(bytes.NewReader(data[keyStart:keyEnd]), binary.LittleEndian, &keys); err != nil {
return Archive{}, fmt.Errorf("decode key list: %w", err)
}
resourceStart := int(hdr.ResourceListOffset)
resourceEnd := resourceStart + int(hdr.EntryCount)*8
if resourceEnd > len(data) {
return Archive{}, fmt.Errorf("erf resource list exceeds file bounds")
}
entries := make([]resourceEntry, hdr.EntryCount)
if err := binary.Read(bytes.NewReader(data[resourceStart:resourceEnd]), binary.LittleEndian, &entries); err != nil {
return Archive{}, fmt.Errorf("decode resource list: %w", err)
}
resources := make([]Resource, 0, hdr.EntryCount)
for index, key := range keys {
entry := entries[index]
start := int(entry.Offset)
end := start + int(entry.Size)
if end > len(data) {
return Archive{}, fmt.Errorf("resource %d exceeds file bounds", index)
}
resref := string(bytes.TrimRight(key.ResRef[:], "\x00"))
payload := make([]byte, entry.Size)
copy(payload, data[start:end])
resources = append(resources, Resource{
Name: resref,
Type: key.ResourceType,
Data: payload,
})
}
return Archive{
FileType: string(hdr.FileType[:]),
Version: string(hdr.Version[:]),
Resources: resources,
}, nil
}
func ResourceTypeForExtension(extension string) (uint16, bool) {
resourceType, ok := extensionTypes[strings.TrimPrefix(strings.ToLower(extension), ".")]
return resourceType, ok
}
func ExtensionForResourceType(resourceType uint16) (string, bool) {
extension, ok := typeExtensions[resourceType]
return extension, ok
}
func padFour(value string) string {
value = strings.ToUpper(value)
if len(value) >= 4 {
return value[:4]
}
return value + strings.Repeat(" ", 4-len(value))
}
func totalResourceBytes(resources []Resource) int {
total := 0
for _, resource := range resources {
total += len(resource.Data)
}
return total
}
internal/erf/erf_test.go
+30
View File
@@ -0,0 +1,30 @@
package erf
import (
"bytes"
"testing"
)
func TestArchiveRoundTrip(t *testing.T) {
archive := New("MOD ", []Resource{
{Name: "module", Type: 0x07DE, Data: []byte("ifo")},
{Name: "start", Type: 0x07DC, Data: []byte("are")},
})
var buf bytes.Buffer
if err := Write(&buf, archive); err != nil {
t.Fatalf("write: %v", err)
}
decoded, err := Read(bytes.NewReader(buf.Bytes()))
if err != nil {
t.Fatalf("read: %v", err)
}
if len(decoded.Resources) != 2 {
t.Fatalf("expected 2 resources, got %d", len(decoded.Resources))
}
if decoded.Resources[0].Name != "module" || string(decoded.Resources[0].Data) != "ifo" {
t.Fatalf("unexpected first resource: %#v", decoded.Resources[0])
}
}
internal/gff/binary.go
+675
View File
@@ -0,0 +1,675 @@
package gff
import (
"bytes"
"encoding/binary"
"fmt"
"io"
"math"
)
const (
headerSize = 56
)
type header struct {
FileType [4]byte
FileVersion [4]byte
StructOffset uint32
StructCount uint32
FieldOffset uint32
FieldCount uint32
LabelOffset uint32
LabelCount uint32
FieldDataOffset uint32
FieldDataCount uint32
FieldIndicesOffset uint32
FieldIndicesCount uint32
ListIndicesOffset uint32
ListIndicesCount uint32
}
type rawStruct struct {
Type uint32
DataOrOffset uint32
FieldCount uint32
}
type rawField struct {
Type uint32
LabelIndex uint32
DataOrOffset uint32
}
func Read(r io.Reader) (Document, error) {
data, err := io.ReadAll(r)
if err != nil {
return Document{}, fmt.Errorf("read gff: %w", err)
}
if len(data) < headerSize {
return Document{}, fmt.Errorf("gff file too small: %d bytes", len(data))
}
var hdr header
if err := binary.Read(bytes.NewReader(data[:headerSize]), binary.LittleEndian, &hdr); err != nil {
return Document{}, fmt.Errorf("decode header: %w", err)
}
reader := binaryReader{data: data, hdr: hdr}
rawStructs, err := reader.readStructs()
if err != nil {
return Document{}, err
}
rawFields, err := reader.readFields()
if err != nil {
return Document{}, err
}
labels, err := reader.readLabels()
if err != nil {
return Document{}, err
}
root, err := reader.decodeStruct(0, rawStructs, rawFields, labels)
if err != nil {
return Document{}, err
}
return Document{
FileType: string(hdr.FileType[:]),
FileVersion: string(hdr.FileVersion[:]),
Root: root,
}, nil
}
func Write(w io.Writer, doc Document) error {
encoder := newBinaryEncoder(doc)
data, err := encoder.encode()
if err != nil {
return err
}
_, err = w.Write(data)
return err
}
type binaryReader struct {
data []byte
hdr header
}
func (r binaryReader) readStructs() ([]rawStruct, error) {
return readTable[rawStruct](r.data, r.hdr.StructOffset, r.hdr.StructCount)
}
func (r binaryReader) readFields() ([]rawField, error) {
return readTable[rawField](r.data, r.hdr.FieldOffset, r.hdr.FieldCount)
}
func (r binaryReader) readLabels() ([]string, error) {
start := int(r.hdr.LabelOffset)
end := start + int(r.hdr.LabelCount)*16
if end > len(r.data) {
return nil, fmt.Errorf("label table exceeds file bounds")
}
labels := make([]string, 0, r.hdr.LabelCount)
for offset := start; offset < end; offset += 16 {
chunk := r.data[offset : offset+16]
n := bytes.IndexByte(chunk, 0)
if n == -1 {
n = len(chunk)
}
labels = append(labels, string(chunk[:n]))
}
return labels, nil
}
func (r binaryReader) decodeStruct(index uint32, structs []rawStruct, fields []rawField, labels []string) (Struct, error) {
if index >= uint32(len(structs)) {
return Struct{}, fmt.Errorf("struct index %d out of range", index)
}
raw := structs[index]
fieldIndices, err := r.fieldIndices(raw)
if err != nil {
return Struct{}, err
}
out := Struct{
Type: raw.Type,
Fields: make([]Field, 0, len(fieldIndices)),
}
for _, fieldIndex := range fieldIndices {
if fieldIndex >= uint32(len(fields)) {
return Struct{}, fmt.Errorf("field index %d out of range", fieldIndex)
}
field, err := r.decodeField(fields[fieldIndex], structs, fields, labels)
if err != nil {
return Struct{}, err
}
out.Fields = append(out.Fields, field)
}
return out, nil
}
func (r binaryReader) fieldIndices(s rawStruct) ([]uint32, error) {
switch s.FieldCount {
case 0:
return nil, nil
case 1:
return []uint32{s.DataOrOffset}, nil
default:
start := int(r.hdr.FieldIndicesOffset + s.DataOrOffset)
end := start + int(s.FieldCount)*4
if end > len(r.data) {
return nil, fmt.Errorf("field indices exceed file bounds")
}
out := make([]uint32, s.FieldCount)
reader := bytes.NewReader(r.data[start:end])
if err := binary.Read(reader, binary.LittleEndian, &out); err != nil {
return nil, fmt.Errorf("decode field indices: %w", err)
}
return out, nil
}
}
func (r binaryReader) decodeField(raw rawField, structs []rawStruct, fields []rawField, labels []string) (Field, error) {
if raw.LabelIndex >= uint32(len(labels)) {
return Field{}, fmt.Errorf("label index %d out of range", raw.LabelIndex)
}
fieldType := FieldType(raw.Type)
value, err := r.decodeValue(fieldType, raw.DataOrOffset, structs, fields, labels)
if err != nil {
return Field{}, fmt.Errorf("decode field %q: %w", labels[raw.LabelIndex], err)
}
return Field{
Label: labels[raw.LabelIndex],
Type: fieldType,
Value: value,
}, nil
}
func (r binaryReader) decodeValue(fieldType FieldType, data uint32, structs []rawStruct, fields []rawField, labels []string) (Value, error) {
switch fieldType {
case TypeByte:
return ByteValue(uint8(data)), nil
case TypeChar:
return CharValue(int8(data)), nil
case TypeWord:
return WordValue(uint16(data)), nil
case TypeShort:
return ShortValue(int16(data)), nil
case TypeDWord:
return DWordValue(data), nil
case TypeInt:
return IntValue(int32(data)), nil
case TypeFloat:
return FloatValue(math.Float32frombits(data)), nil
case TypeDWord64:
raw, err := r.sliceFieldData(data, 8)
if err != nil {
return nil, err
}
return DWord64Value(binary.LittleEndian.Uint64(raw)), nil
case TypeInt64:
raw, err := r.sliceFieldData(data, 8)
if err != nil {
return nil, err
}
return Int64Value(int64(binary.LittleEndian.Uint64(raw))), nil
case TypeDouble:
raw, err := r.sliceFieldData(data, 8)
if err != nil {
return nil, err
}
return DoubleValue(math.Float64frombits(binary.LittleEndian.Uint64(raw))), nil
case TypeCExoString:
raw, err := r.prefixedFieldData(data)
if err != nil {
return nil, err
}
return StringValue(string(raw)), nil
case TypeResRef:
raw, err := r.prefixedByteFieldData(data, 1)
if err != nil {
return nil, err
}
return ResRefValue(string(raw)), nil
case TypeCExoLocString:
raw, err := r.locStringFieldData(data)
if err != nil {
return nil, err
}
return decodeLocString(raw)
case TypeVoid:
raw, err := r.prefixedFieldData(data)
if err != nil {
return nil, err
}
return VoidValue(raw), nil
case TypeStruct:
return r.decodeStruct(data, structs, fields, labels)
case TypeList:
return r.decodeList(data, structs, fields, labels)
case TypeOrientation:
raw, err := r.sliceFieldData(data, 16)
if err != nil {
return nil, err
}
return Orientation{
X: math.Float32frombits(binary.LittleEndian.Uint32(raw[0:4])),
Y: math.Float32frombits(binary.LittleEndian.Uint32(raw[4:8])),
Z: math.Float32frombits(binary.LittleEndian.Uint32(raw[8:12])),
W: math.Float32frombits(binary.LittleEndian.Uint32(raw[12:16])),
}, nil
case TypeVector:
raw, err := r.sliceFieldData(data, 12)
if err != nil {
return nil, err
}
return Vector{
X: math.Float32frombits(binary.LittleEndian.Uint32(raw[0:4])),
Y: math.Float32frombits(binary.LittleEndian.Uint32(raw[4:8])),
Z: math.Float32frombits(binary.LittleEndian.Uint32(raw[8:12])),
}, nil
default:
return nil, fmt.Errorf("unsupported field type %d", fieldType)
}
}
func (r binaryReader) decodeList(offset uint32, structs []rawStruct, fields []rawField, labels []string) (ListValue, error) {
start := int(r.hdr.ListIndicesOffset + offset)
if start+4 > len(r.data) {
return nil, fmt.Errorf("list data exceeds file bounds")
}
count := binary.LittleEndian.Uint32(r.data[start : start+4])
start += 4
end := start + int(count)*4
if end > len(r.data) {
return nil, fmt.Errorf("list indices exceed file bounds")
}
list := make(ListValue, 0, count)
for pos := start; pos < end; pos += 4 {
index := binary.LittleEndian.Uint32(r.data[pos : pos+4])
child, err := r.decodeStruct(index, structs, fields, labels)
if err != nil {
return nil, err
}
list = append(list, child)
}
return list, nil
}
func (r binaryReader) sliceFieldData(offset uint32, size int) ([]byte, error) {
start := int(r.hdr.FieldDataOffset + offset)
end := start + size
if end > len(r.data) {
return nil, fmt.Errorf("field data exceeds file bounds")
}
return r.data[start:end], nil
}
func (r binaryReader) prefixedFieldData(offset uint32) ([]byte, error) {
sizeRaw, err := r.sliceFieldData(offset, 4)
if err != nil {
return nil, err
}
size := int(binary.LittleEndian.Uint32(sizeRaw))
return r.sliceFieldData(offset+4, size)
}
func (r binaryReader) prefixedByteFieldData(offset uint32, prefixBytes uint32) ([]byte, error) {
header, err := r.sliceFieldData(offset, int(prefixBytes))
if err != nil {
return nil, err
}
size := int(header[0])
return r.sliceFieldData(offset+prefixBytes, size)
}
func (r binaryReader) locStringFieldData(offset uint32) ([]byte, error) {
sizeRaw, err := r.sliceFieldData(offset, 4)
if err != nil {
return nil, err
}
size := int(binary.LittleEndian.Uint32(sizeRaw))
return r.sliceFieldData(offset, size+4)
}
func readTable[T any](data []byte, offset uint32, count uint32) ([]T, error) {
if count == 0 {
return nil, nil
}
var row T
size := binary.Size(row)
start := int(offset)
end := start + int(count)*size
if end > len(data) {
return nil, fmt.Errorf("table exceeds file bounds")
}
out := make([]T, count)
reader := bytes.NewReader(data[start:end])
if err := binary.Read(reader, binary.LittleEndian, &out); err != nil {
return nil, fmt.Errorf("decode table: %w", err)
}
return out, nil
}
func decodeLocString(data []byte) (LocString, error) {
if len(data) < 12 {
return LocString{}, fmt.Errorf("locstring too small")
}
totalSize := binary.LittleEndian.Uint32(data[0:4])
if totalSize+4 != uint32(len(data)) {
return LocString{}, fmt.Errorf("locstring size mismatch")
}
stringRef := binary.LittleEndian.Uint32(data[4:8])
count := binary.LittleEndian.Uint32(data[8:12])
pos := 12
entries := make([]LocStringEntry, 0, count)
for i := uint32(0); i < count; i++ {
if pos+8 > len(data) {
return LocString{}, fmt.Errorf("locstring entry header truncated")
}
id := binary.LittleEndian.Uint32(data[pos : pos+4])
length := binary.LittleEndian.Uint32(data[pos+4 : pos+8])
pos += 8
if pos+int(length) > len(data) {
return LocString{}, fmt.Errorf("locstring entry %d truncated", i)
}
entries = append(entries, LocStringEntry{
ID: id,
Value: string(data[pos : pos+int(length)]),
})
pos += int(length)
}
return LocString{
StringRef: stringRef,
Entries: entries,
}, nil
}
type binaryEncoder struct {
document Document
structs []rawStruct
fields []rawField
labels []string
labelIndex map[string]uint32
fieldData bytes.Buffer
fieldIndices []uint32
listIndices []uint32
}
func newBinaryEncoder(doc Document) *binaryEncoder {
return &binaryEncoder{
document: doc,
labelIndex: map[string]uint32{},
}
}
func (e *binaryEncoder) encode() ([]byte, error) {
if err := e.addStruct(e.document.Root); err != nil {
return nil, err
}
var buf bytes.Buffer
hdr := header{}
copy(hdr.FileType[:], []byte(padFour(e.document.FileType)))
copy(hdr.FileVersion[:], []byte(padFour(defaultString(e.document.FileVersion, "V3.2"))))
hdr.StructOffset = headerSize
hdr.StructCount = uint32(len(e.structs))
hdr.FieldOffset = hdr.StructOffset + uint32(len(e.structs))*12
hdr.FieldCount = uint32(len(e.fields))
hdr.LabelOffset = hdr.FieldOffset + uint32(len(e.fields))*12
hdr.LabelCount = uint32(len(e.labels))
hdr.FieldDataOffset = hdr.LabelOffset + uint32(len(e.labels))*16
hdr.FieldDataCount = uint32(e.fieldData.Len())
hdr.FieldIndicesOffset = hdr.FieldDataOffset + uint32(e.fieldData.Len())
hdr.FieldIndicesCount = uint32(len(e.fieldIndices) * 4)
hdr.ListIndicesOffset = hdr.FieldIndicesOffset + uint32(len(e.fieldIndices))*4
hdr.ListIndicesCount = uint32(len(e.listIndices) * 4)
if err := binary.Write(&buf, binary.LittleEndian, hdr); err != nil {
return nil, err
}
if err := binary.Write(&buf, binary.LittleEndian, e.structs); err != nil {
return nil, err
}
if err := binary.Write(&buf, binary.LittleEndian, e.fields); err != nil {
return nil, err
}
for _, label := range e.labels {
var raw [16]byte
copy(raw[:], []byte(label))
if _, err := buf.Write(raw[:]); err != nil {
return nil, err
}
}
if _, err := buf.Write(e.fieldData.Bytes()); err != nil {
return nil, err
}
if err := binary.Write(&buf, binary.LittleEndian, e.fieldIndices); err != nil {
return nil, err
}
if err := binary.Write(&buf, binary.LittleEndian, e.listIndices); err != nil {
return nil, err
}
return buf.Bytes(), nil
}
func (e *binaryEncoder) addStruct(s Struct) error {
index := uint32(len(e.structs))
e.structs = append(e.structs, rawStruct{Type: s.Type})
fieldIndices := make([]uint32, 0, len(s.Fields))
for _, field := range s.Fields {
fieldIndex, err := e.addField(field)
if err != nil {
return err
}
fieldIndices = append(fieldIndices, fieldIndex)
}
entry := &e.structs[index]
entry.FieldCount = uint32(len(fieldIndices))
switch len(fieldIndices) {
case 0:
entry.DataOrOffset = 0
case 1:
entry.DataOrOffset = fieldIndices[0]
default:
entry.DataOrOffset = uint32(len(e.fieldIndices) * 4)
e.fieldIndices = append(e.fieldIndices, fieldIndices...)
}
return nil
}
func (e *binaryEncoder) addField(field Field) (uint32, error) {
labelIndex, err := e.indexLabel(field.Label)
if err != nil {
return 0, err
}
data, err := e.encodeValue(field.Value)
if err != nil {
return 0, fmt.Errorf("encode field %q: %w", field.Label, err)
}
entry := rawField{
Type: uint32(field.Type),
LabelIndex: labelIndex,
DataOrOffset: data,
}
index := uint32(len(e.fields))
e.fields = append(e.fields, entry)
return index, nil
}
func (e *binaryEncoder) encodeValue(value Value) (uint32, error) {
switch typed := value.(type) {
case ByteValue:
return uint32(uint8(typed)), nil
case CharValue:
return uint32(uint8(typed)), nil
case WordValue:
return uint32(uint16(typed)), nil
case ShortValue:
return uint32(uint16(typed)), nil
case DWordValue:
return uint32(typed), nil
case IntValue:
return uint32(int32(typed)), nil
case FloatValue:
return math.Float32bits(float32(typed)), nil
case DWord64Value:
return e.writeFieldData(func(buf *bytes.Buffer) error {
return binary.Write(buf, binary.LittleEndian, uint64(typed))
}), nil
case Int64Value:
return e.writeFieldData(func(buf *bytes.Buffer) error {
return binary.Write(buf, binary.LittleEndian, int64(typed))
}), nil
case DoubleValue:
return e.writeFieldData(func(buf *bytes.Buffer) error {
return binary.Write(buf, binary.LittleEndian, math.Float64bits(float64(typed)))
}), nil
case StringValue:
return e.writeLengthPrefixed([]byte(typed)), nil
case ResRefValue:
if len(typed) > 255 {
return 0, fmt.Errorf("resref exceeds 255 bytes")
}
return e.writeFieldData(func(buf *bytes.Buffer) error {
if err := buf.WriteByte(byte(len(typed))); err != nil {
return err
}
_, err := buf.Write([]byte(typed))
return err
}), nil
case LocString:
return e.writeFieldData(func(buf *bytes.Buffer) error {
payload := bytes.Buffer{}
if err := binary.Write(&payload, binary.LittleEndian, typed.StringRef); err != nil {
return err
}
if err := binary.Write(&payload, binary.LittleEndian, uint32(len(typed.Entries))); err != nil {
return err
}
for _, entry := range typed.Entries {
if err := binary.Write(&payload, binary.LittleEndian, entry.ID); err != nil {
return err
}
if err := binary.Write(&payload, binary.LittleEndian, uint32(len(entry.Value))); err != nil {
return err
}
if _, err := payload.Write([]byte(entry.Value)); err != nil {
return err
}
}
if err := binary.Write(buf, binary.LittleEndian, uint32(payload.Len())); err != nil {
return err
}
if _, err := buf.Write(payload.Bytes()); err != nil {
return err
}
return nil
}), nil
case VoidValue:
return e.writeLengthPrefixed([]byte(typed)), nil
case Struct:
index := uint32(len(e.structs))
if err := e.addStruct(typed); err != nil {
return 0, err
}
return index, nil
case ListValue:
return e.writeList(typed)
case Orientation:
return e.writeFieldData(func(buf *bytes.Buffer) error {
for _, component := range []float32{typed.X, typed.Y, typed.Z, typed.W} {
if err := binary.Write(buf, binary.LittleEndian, math.Float32bits(component)); err != nil {
return err
}
}
return nil
}), nil
case Vector:
return e.writeFieldData(func(buf *bytes.Buffer) error {
for _, component := range []float32{typed.X, typed.Y, typed.Z} {
if err := binary.Write(buf, binary.LittleEndian, math.Float32bits(component)); err != nil {
return err
}
}
return nil
}), nil
default:
return 0, fmt.Errorf("unsupported value type %T", value)
}
}
func (e *binaryEncoder) writeList(list ListValue) (uint32, error) {
indices := make([]uint32, 0, len(list))
for _, item := range list {
index := uint32(len(e.structs))
if err := e.addStruct(item); err != nil {
return 0, err
}
indices = append(indices, index)
}
offset := uint32(len(e.listIndices) * 4)
e.listIndices = append(e.listIndices, uint32(len(list)))
e.listIndices = append(e.listIndices, indices...)
return offset, nil
}
func (e *binaryEncoder) writeFieldData(write func(*bytes.Buffer) error) uint32 {
offset := uint32(e.fieldData.Len())
if err := write(&e.fieldData); err != nil {
panic(err)
}
return offset
}
func (e *binaryEncoder) writeLengthPrefixed(data []byte) uint32 {
return e.writeFieldData(func(buf *bytes.Buffer) error {
if err := binary.Write(buf, binary.LittleEndian, uint32(len(data))); err != nil {
return err
}
_, err := buf.Write(data)
return err
})
}
func (e *binaryEncoder) indexLabel(label string) (uint32, error) {
if len(label) > 16 {
return 0, fmt.Errorf("label %q exceeds 16 bytes", label)
}
if index, ok := e.labelIndex[label]; ok {
return index, nil
}
index := uint32(len(e.labels))
e.labels = append(e.labels, label)
e.labelIndex[label] = index
return index, nil
}
func padFour(value string) string {
if len(value) >= 4 {
return value[:4]
}
return value + string(bytes.Repeat([]byte(" "), 4-len(value)))
}
func defaultString(value, fallback string) string {
if value == "" {
return fallback
}
return value
}
internal/gff/binary_test.go
+55
View File
@@ -0,0 +1,55 @@
package gff
import (
"bytes"
"encoding/json"
"testing"
)
func TestRoundTripBinaryAndJSON(t *testing.T) {
original := Document{
FileType: "UTC ",
FileVersion: "V3.2",
Root: Struct{
Type: 0,
Fields: []Field{
NewField("Tag", StringValue("test_creature")),
NewField("TemplateResRef", ResRefValue("nw_test")),
NewField("HP", IntValue(12)),
NewField("Position", Vector{X: 1.25, Y: 2.5, Z: 3.75}),
NewField("Inventory", ListValue{
{
Type: 1,
Fields: []Field{
NewField("Slot", DWordValue(0)),
NewField("ResRef", ResRefValue("itm_sword")),
},
},
}),
},
},
}
var binaryBuf bytes.Buffer
if err := Write(&binaryBuf, original); err != nil {
t.Fatalf("write binary: %v", err)
}
decoded, err := Read(bytes.NewReader(binaryBuf.Bytes()))
if err != nil {
t.Fatalf("read binary: %v", err)
}
left, err := json.Marshal(original)
if err != nil {
t.Fatalf("marshal original json: %v", err)
}
right, err := json.Marshal(decoded)
if err != nil {
t.Fatalf("marshal decoded json: %v", err)
}
if string(left) != string(right) {
t.Fatalf("roundtrip mismatch\noriginal: %s\ndecoded: %s", left, right)
}
}
internal/gff/json.go
+317
View File
@@ -0,0 +1,317 @@
package gff
import (
"encoding/base64"
"encoding/json"
"fmt"
)
type jsonDocument struct {
FileType string `json:"file_type"`
FileVersion string `json:"file_version"`
Root jsonStruct `json:"root"`
}
type jsonStruct struct {
StructType uint32 `json:"struct_type"`
Fields []jsonField `json:"fields"`
}
type jsonField struct {
Label string `json:"label"`
Type string `json:"type"`
Value json.RawMessage `json:"value"`
}
type jsonLocString struct {
StringRef uint32 `json:"string_ref"`
Entries []LocStringEntry `json:"entries"`
}
func (s Struct) MarshalJSON() ([]byte, error) {
payload, err := marshalStruct(s)
if err != nil {
return nil, err
}
return json.Marshal(payload)
}
func (s *Struct) UnmarshalJSON(data []byte) error {
var payload jsonStruct
if err := json.Unmarshal(data, &payload); err != nil {
return err
}
decoded, err := unmarshalStruct(payload)
if err != nil {
return err
}
*s = decoded
return nil
}
func (d Document) MarshalJSON() ([]byte, error) {
root, err := marshalStruct(d.Root)
if err != nil {
return nil, err
}
return json.Marshal(jsonDocument{
FileType: d.FileType,
FileVersion: d.FileVersion,
Root: root,
})
}
func (d *Document) UnmarshalJSON(data []byte) error {
var payload jsonDocument
if err := json.Unmarshal(data, &payload); err != nil {
return err
}
root, err := unmarshalStruct(payload.Root)
if err != nil {
return err
}
d.FileType = payload.FileType
d.FileVersion = payload.FileVersion
d.Root = root
return nil
}
func marshalStruct(in Struct) (jsonStruct, error) {
fields := make([]jsonField, 0, len(in.Fields))
for _, field := range in.Fields {
payload, err := marshalValue(field.Value)
if err != nil {
return jsonStruct{}, fmt.Errorf("marshal field %q: %w", field.Label, err)
}
fields = append(fields, jsonField{
Label: field.Label,
Type: field.Type.String(),
Value: payload,
})
}
return jsonStruct{
StructType: in.Type,
Fields: fields,
}, nil
}
func unmarshalStruct(in jsonStruct) (Struct, error) {
fields := make([]Field, 0, len(in.Fields))
for _, field := range in.Fields {
ft, err := parseFieldType(field.Type)
if err != nil {
return Struct{}, fmt.Errorf("field %q: %w", field.Label, err)
}
value, err := unmarshalValue(ft, field.Value)
if err != nil {
return Struct{}, fmt.Errorf("field %q: %w", field.Label, err)
}
fields = append(fields, Field{
Label: field.Label,
Type: ft,
Value: value,
})
}
return Struct{
Type: in.StructType,
Fields: fields,
}, nil
}
func marshalValue(value Value) (json.RawMessage, error) {
switch typed := value.(type) {
case ByteValue:
return json.Marshal(uint8(typed))
case CharValue:
return json.Marshal(int8(typed))
case WordValue:
return json.Marshal(uint16(typed))
case ShortValue:
return json.Marshal(int16(typed))
case DWordValue:
return json.Marshal(uint32(typed))
case IntValue:
return json.Marshal(int32(typed))
case DWord64Value:
return json.Marshal(uint64(typed))
case Int64Value:
return json.Marshal(int64(typed))
case FloatValue:
return json.Marshal(float32(typed))
case DoubleValue:
return json.Marshal(float64(typed))
case StringValue:
return json.Marshal(string(typed))
case ResRefValue:
return json.Marshal(string(typed))
case LocString:
return json.Marshal(jsonLocString(typed))
case VoidValue:
return json.Marshal(base64.StdEncoding.EncodeToString([]byte(typed)))
case Struct:
payload, err := marshalStruct(typed)
if err != nil {
return nil, err
}
return json.Marshal(payload)
case ListValue:
payload := make([]jsonStruct, 0, len(typed))
for _, item := range typed {
entry, err := marshalStruct(item)
if err != nil {
return nil, err
}
payload = append(payload, entry)
}
return json.Marshal(payload)
case Orientation:
return json.Marshal(typed)
case Vector:
return json.Marshal(typed)
default:
return nil, fmt.Errorf("unsupported value type %T", value)
}
}
func unmarshalValue(ft FieldType, data []byte) (Value, error) {
switch ft {
case TypeByte:
var out uint8
if err := json.Unmarshal(data, &out); err != nil {
return nil, err
}
return ByteValue(out), nil
case TypeChar:
var out int8
if err := json.Unmarshal(data, &out); err != nil {
return nil, err
}
return CharValue(out), nil
case TypeWord:
var out uint16
if err := json.Unmarshal(data, &out); err != nil {
return nil, err
}
return WordValue(out), nil
case TypeShort:
var out int16
if err := json.Unmarshal(data, &out); err != nil {
return nil, err
}
return ShortValue(out), nil
case TypeDWord:
var out uint32
if err := json.Unmarshal(data, &out); err != nil {
return nil, err
}
return DWordValue(out), nil
case TypeInt:
var out int32
if err := json.Unmarshal(data, &out); err != nil {
return nil, err
}
return IntValue(out), nil
case TypeDWord64:
var out uint64
if err := json.Unmarshal(data, &out); err != nil {
return nil, err
}
return DWord64Value(out), nil
case TypeInt64:
var out int64
if err := json.Unmarshal(data, &out); err != nil {
return nil, err
}
return Int64Value(out), nil
case TypeFloat:
var out float32
if err := json.Unmarshal(data, &out); err != nil {
return nil, err
}
return FloatValue(out), nil
case TypeDouble:
var out float64
if err := json.Unmarshal(data, &out); err != nil {
return nil, err
}
return DoubleValue(out), nil
case TypeCExoString:
var out string
if err := json.Unmarshal(data, &out); err != nil {
return nil, err
}
return StringValue(out), nil
case TypeResRef:
var out string
if err := json.Unmarshal(data, &out); err != nil {
return nil, err
}
return ResRefValue(out), nil
case TypeCExoLocString:
var out jsonLocString
if err := json.Unmarshal(data, &out); err != nil {
return nil, err
}
return LocString(out), nil
case TypeVoid:
var out string
if err := json.Unmarshal(data, &out); err != nil {
return nil, err
}
decoded, err := base64.StdEncoding.DecodeString(out)
if err != nil {
return nil, fmt.Errorf("decode base64 void: %w", err)
}
return VoidValue(decoded), nil
case TypeStruct:
var payload jsonStruct
if err := json.Unmarshal(data, &payload); err != nil {
return nil, err
}
return unmarshalStruct(payload)
case TypeList:
var payload []jsonStruct
if err := json.Unmarshal(data, &payload); err != nil {
return nil, err
}
out := make(ListValue, 0, len(payload))
for _, item := range payload {
decoded, err := unmarshalStruct(item)
if err != nil {
return nil, err
}
out = append(out, decoded)
}
return out, nil
case TypeOrientation:
var out Orientation
if err := json.Unmarshal(data, &out); err != nil {
return nil, err
}
return out, nil
case TypeVector:
var out Vector
if err := json.Unmarshal(data, &out); err != nil {
return nil, err
}
return out, nil
default:
return nil, fmt.Errorf("unsupported field type %d", ft)
}
}
func parseFieldType(name string) (FieldType, error) {
for kind, candidate := range fieldTypeNames {
if candidate == name {
return kind, nil
}
}
return 0, fmt.Errorf("unknown field type %q", name)
}
internal/gff/types.go
+140
View File
@@ -0,0 +1,140 @@
package gff
import "fmt"
type FieldType uint32
const (
TypeByte FieldType = iota
TypeChar
TypeWord
TypeShort
TypeDWord
TypeInt
TypeDWord64
TypeInt64
TypeFloat
TypeDouble
TypeCExoString
TypeResRef
TypeCExoLocString
TypeVoid
TypeStruct
TypeList
TypeOrientation
TypeVector
)
var fieldTypeNames = map[FieldType]string{
TypeByte: "Byte",
TypeChar: "Char",
TypeWord: "Word",
TypeShort: "Short",
TypeDWord: "DWord",
TypeInt: "Int",
TypeDWord64: "DWord64",
TypeInt64: "Int64",
TypeFloat: "Float",
TypeDouble: "Double",
TypeCExoString: "CExoString",
TypeResRef: "ResRef",
TypeCExoLocString: "CExoLocString",
TypeVoid: "Void",
TypeStruct: "Struct",
TypeList: "List",
TypeOrientation: "Orientation",
TypeVector: "Vector",
}
type Document struct {
FileType string `json:"file_type"`
FileVersion string `json:"file_version"`
Root Struct `json:"root"`
}
type Struct struct {
Type uint32 `json:"struct_type"`
Fields []Field `json:"fields"`
}
type Field struct {
Label string `json:"label"`
Type FieldType `json:"-"`
Value Value `json:"-"`
}
type Value interface {
fieldType() FieldType
}
type LocString struct {
StringRef uint32 `json:"string_ref"`
Entries []LocStringEntry `json:"entries"`
}
type LocStringEntry struct {
ID uint32 `json:"id"`
Value string `json:"value"`
}
type Vector struct {
X float32 `json:"x"`
Y float32 `json:"y"`
Z float32 `json:"z"`
}
type Orientation struct {
X float32 `json:"x"`
Y float32 `json:"y"`
Z float32 `json:"z"`
W float32 `json:"w"`
}
type ByteValue uint8
type CharValue int8
type WordValue uint16
type ShortValue int16
type DWordValue uint32
type IntValue int32
type DWord64Value uint64
type Int64Value int64
type FloatValue float32
type DoubleValue float64
type StringValue string
type ResRefValue string
type VoidValue []byte
type ListValue []Struct
func (ByteValue) fieldType() FieldType { return TypeByte }
func (CharValue) fieldType() FieldType { return TypeChar }
func (WordValue) fieldType() FieldType { return TypeWord }
func (ShortValue) fieldType() FieldType { return TypeShort }
func (DWordValue) fieldType() FieldType { return TypeDWord }
func (IntValue) fieldType() FieldType { return TypeInt }
func (DWord64Value) fieldType() FieldType { return TypeDWord64 }
func (Int64Value) fieldType() FieldType { return TypeInt64 }
func (FloatValue) fieldType() FieldType { return TypeFloat }
func (DoubleValue) fieldType() FieldType { return TypeDouble }
func (StringValue) fieldType() FieldType { return TypeCExoString }
func (ResRefValue) fieldType() FieldType { return TypeResRef }
func (LocString) fieldType() FieldType { return TypeCExoLocString }
func (VoidValue) fieldType() FieldType { return TypeVoid }
func (Struct) fieldType() FieldType { return TypeStruct }
func (ListValue) fieldType() FieldType { return TypeList }
func (Orientation) fieldType() FieldType { return TypeOrientation }
func (Vector) fieldType() FieldType { return TypeVector }
func (t FieldType) String() string {
if name, ok := fieldTypeNames[t]; ok {
return name
}
return fmt.Sprintf("FieldType(%d)", t)
}
func NewField(label string, value Value) Field {
return Field{
Label: label,
Type: value.fieldType(),
Value: value,
}
}
internal/pipeline/apply_manifest.go
+62
View File
@@ -0,0 +1,62 @@
package pipeline
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/gff"
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
)
type ApplyManifestResult struct {
ManifestPath string
ModuleSource string
HAKCount int
}
func ApplyHAKManifest(p *project.Project, manifestPath string) (ApplyManifestResult, error) {
if manifestPath == "" {
manifestPath = filepath.Join(p.BuildDir(), "haks.json")
}
raw, err := os.ReadFile(manifestPath)
if err != nil {
return ApplyManifestResult{}, fmt.Errorf("read hak manifest: %w", err)
}
var manifest BuildManifest
if err := json.Unmarshal(raw, &manifest); err != nil {
return ApplyManifestResult{}, fmt.Errorf("parse hak manifest: %w", err)
}
moduleSource := filepath.Join(p.SourceDir(), "module", "module.ifo.json")
sourceRaw, err := os.ReadFile(moduleSource)
if err != nil {
return ApplyManifestResult{}, fmt.Errorf("read module ifo source: %w", err)
}
var document gff.Document
if err := json.Unmarshal(sourceRaw, &document); err != nil {
return ApplyManifestResult{}, fmt.Errorf("parse module ifo source: %w", err)
}
setModuleHAKList(&document, manifest.ModuleHAKs)
formatted, err := json.MarshalIndent(document, "", " ")
if err != nil {
return ApplyManifestResult{}, fmt.Errorf("marshal updated module ifo: %w", err)
}
formatted = append(formatted, '\n')
if err := os.WriteFile(moduleSource, formatted, 0o644); err != nil {
return ApplyManifestResult{}, fmt.Errorf("write module ifo source: %w", err)
}
return ApplyManifestResult{
ManifestPath: manifestPath,
ModuleSource: moduleSource,
HAKCount: len(manifest.ModuleHAKs),
}, nil
}
internal/pipeline/build.go
+587
View File
@@ -0,0 +1,587 @@
package pipeline
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"slices"
"strings"
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf"
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/gff"
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/validator"
)
type BuildResult struct {
ModulePath string
HAKPaths []string
Manifest string
Resources int
HAKAssets int
}
type BuildManifest struct {
ModuleHAKs []string `json:"module_haks"`
HAKs []BuildManifestHAK `json:"haks"`
}
type BuildManifestHAK struct {
Name string `json:"name"`
Group string `json:"group"`
Priority int `json:"priority"`
MaxBytes int64 `json:"max_bytes"`
SizeBytes int64 `json:"size_bytes"`
Assets []string `json:"assets"`
}
type assetResource struct {
Rel string
Resource erf.Resource
Size int64
}
type hakChunk struct {
Config project.HAKConfig
Index int
Name string
Assets []assetResource
Size int64
Resources []erf.Resource
}
func Build(p *project.Project) (BuildResult, error) {
moduleResult, err := BuildModule(p)
if err != nil {
return BuildResult{}, err
}
hakResult, err := BuildHAKs(p)
if err != nil {
return BuildResult{}, err
}
moduleResult.HAKPaths = hakResult.HAKPaths
moduleResult.Manifest = hakResult.Manifest
moduleResult.HAKAssets = hakResult.HAKAssets
return moduleResult, nil
}
func BuildModule(p *project.Project) (BuildResult, error) {
if err := validateForBuild(p); err != nil {
return BuildResult{}, err
}
moduleResources, err := collectModuleResources(p, nil)
if err != nil {
return BuildResult{}, err
}
outputPath := filepath.Join(p.BuildDir(), p.Config.Module.ResRef+".mod")
output, err := os.Create(outputPath)
if err != nil {
return BuildResult{}, fmt.Errorf("create module archive: %w", err)
}
defer output.Close()
if err := erf.Write(output, erf.New("MOD ", moduleResources)); err != nil {
return BuildResult{}, fmt.Errorf("write module archive: %w", err)
}
return BuildResult{
ModulePath: outputPath,
Resources: len(moduleResources),
}, nil
}
func BuildHAKs(p *project.Project) (BuildResult, error) {
if err := validateForBuild(p); err != nil {
return BuildResult{}, err
}
assetResources, err := collectAssetResources(p)
if err != nil {
return BuildResult{}, err
}
if err := cleanupGeneratedHAKs(p.BuildDir(), p.Config.Module.ResRef); err != nil {
return BuildResult{}, err
}
result := BuildResult{HAKAssets: len(assetResources)}
if len(assetResources) == 0 {
return result, nil
}
chunks, err := planHAKChunks(p, assetResources)
if err != nil {
return BuildResult{}, err
}
moduleHakOrder, err := resolveModuleHAKOrder(p, chunks)
if err != nil {
return BuildResult{}, err
}
manifest := BuildManifest{
ModuleHAKs: moduleHakOrder,
HAKs: make([]BuildManifestHAK, 0, len(chunks)),
}
for _, chunk := range chunks {
hakPath := filepath.Join(p.BuildDir(), chunk.Name+".hak")
hakOutput, err := os.Create(hakPath)
if err != nil {
return BuildResult{}, fmt.Errorf("create hak archive: %w", err)
}
if err := erf.Write(hakOutput, erf.New("HAK ", chunk.Resources)); err != nil {
hakOutput.Close()
return BuildResult{}, fmt.Errorf("write hak archive: %w", err)
}
if err := hakOutput.Close(); err != nil {
return BuildResult{}, fmt.Errorf("close hak archive: %w", err)
}
result.HAKPaths = append(result.HAKPaths, hakPath)
assets := make([]string, 0, len(chunk.Assets))
for _, asset := range chunk.Assets {
assets = append(assets, asset.Rel)
}
manifest.HAKs = append(manifest.HAKs, BuildManifestHAK{
Name: chunk.Name,
Group: chunk.Config.Name,
Priority: chunk.Config.Priority,
MaxBytes: chunk.Config.MaxBytes,
SizeBytes: chunk.Size,
Assets: assets,
})
}
manifestPath := filepath.Join(p.BuildDir(), "haks.json")
manifestBytes, err := json.MarshalIndent(manifest, "", " ")
if err != nil {
return BuildResult{}, fmt.Errorf("marshal hak manifest: %w", err)
}
manifestBytes = append(manifestBytes, '\n')
if err := os.WriteFile(manifestPath, manifestBytes, 0o644); err != nil {
return BuildResult{}, fmt.Errorf("write hak manifest: %w", err)
}
result.Manifest = manifestPath
return result, nil
}
func collectModuleResources(p *project.Project, moduleHakOrder []string) ([]erf.Resource, error) {
var moduleResources []erf.Resource
for _, rel := range p.Inventory.SourceFiles {
abs := filepath.Join(p.SourceDir(), filepath.FromSlash(rel))
resource, err := resourceFromJSON(abs, moduleHakOrder)
if err != nil {
return nil, err
}
moduleResources = append(moduleResources, resource)
}
for _, rel := range p.Inventory.ScriptFiles {
abs := filepath.Join(p.SourceDir(), filepath.FromSlash(rel))
resource, err := rawResource(abs)
if err != nil {
return nil, err
}
moduleResources = append(moduleResources, resource)
}
sortResources(moduleResources)
return moduleResources, nil
}
func collectAssetResources(p *project.Project) ([]assetResource, error) {
var hakResources []assetResource
for _, rel := range p.Inventory.AssetFiles {
abs := filepath.Join(p.AssetsDir(), filepath.FromSlash(rel))
resource, err := rawResource(abs)
if err != nil {
return nil, err
}
hakResources = append(hakResources, assetResource{
Rel: rel,
Resource: resource,
Size: erf.ArchiveSize([]erf.Resource{resource}),
})
}
slices.SortFunc(hakResources, func(a, b assetResource) int {
return compareResourceKeys(a.Resource, b.Resource)
})
return hakResources, nil
}
func sortResources(resources []erf.Resource) {
slices.SortFunc(resources, func(a, b erf.Resource) int {
return compareResourceKeys(a, b)
})
}
func compareResourceKeys(a, b erf.Resource) int {
if cmp := strings.Compare(a.Name, b.Name); cmp != 0 {
return cmp
}
if a.Type < b.Type {
return -1
}
if a.Type > b.Type {
return 1
}
return 0
}
func resourceFromJSON(path string, moduleHakOrder []string) (erf.Resource, error) {
name, extension, err := splitSourceName(path)
if err != nil {
return erf.Resource{}, err
}
resourceType, ok := erf.ResourceTypeForExtension(extension)
if !ok {
return erf.Resource{}, fmt.Errorf("unsupported source extension %q", extension)
}
raw, err := os.ReadFile(path)
if err != nil {
return erf.Resource{}, fmt.Errorf("read %s: %w", path, err)
}
var document gff.Document
if err := json.Unmarshal(raw, &document); err != nil {
return erf.Resource{}, fmt.Errorf("parse gff json %s: %w", path, err)
}
if document.FileType == "" {
document.FileType = strings.ToUpper(strings.TrimPrefix(extension, "."))
}
if document.FileVersion == "" {
document.FileVersion = "V3.2"
}
if extension == ".ifo" && name == "module" && len(moduleHakOrder) > 0 {
setModuleHAKList(&document, moduleHakOrder)
}
var buf bytes.Buffer
if err := gff.Write(&buf, document); err != nil {
return erf.Resource{}, fmt.Errorf("encode gff %s: %w", path, err)
}
return erf.Resource{
Name: name,
Type: resourceType,
Data: buf.Bytes(),
}, nil
}
func rawResource(path string) (erf.Resource, error) {
extension := strings.TrimPrefix(strings.ToLower(filepath.Ext(path)), ".")
resourceType, ok := erf.ResourceTypeForExtension(extension)
if !ok {
return erf.Resource{}, fmt.Errorf("unsupported resource extension %q", filepath.Ext(path))
}
name := strings.TrimSuffix(filepath.Base(path), filepath.Ext(path))
data, err := os.ReadFile(path)
if err != nil {
return erf.Resource{}, fmt.Errorf("read %s: %w", path, err)
}
return erf.Resource{
Name: name,
Type: resourceType,
Data: data,
}, nil
}
func splitSourceName(path string) (string, string, error) {
base := filepath.Base(path)
if !strings.HasSuffix(base, ".json") {
return "", "", fmt.Errorf("expected .json source file: %s", path)
}
stem := strings.TrimSuffix(base, ".json")
extension := filepath.Ext(stem)
if extension == "" {
return "", "", fmt.Errorf("source file must include target resource extension before .json: %s", path)
}
return strings.TrimSuffix(stem, extension), strings.ToLower(extension), nil
}
func planHAKChunks(p *project.Project, assets []assetResource) ([]hakChunk, error) {
if len(assets) == 0 {
return nil, nil
}
if len(p.Config.HAKs) == 0 {
chunk := hakChunk{
Config: project.HAKConfig{
Name: p.Config.Module.ResRef,
Priority: 1,
},
Index: 1,
Name: p.Config.Module.ResRef,
Size: erf.ArchiveSize(resourceSlice(assets)),
Assets: assets,
Resources: resourceSlice(assets),
}
return []hakChunk{chunk}, nil
}
configs := make([]project.HAKConfig, len(p.Config.HAKs))
copy(configs, p.Config.HAKs)
slices.SortFunc(configs, func(a, b project.HAKConfig) int {
if a.Priority != b.Priority {
if a.Priority < b.Priority {
return -1
}
return 1
}
return strings.Compare(a.Name, b.Name)
})
grouped := make([][]assetResource, len(configs))
for _, asset := range assets {
matched := false
for index, cfg := range configs {
if matchesAnyPattern(asset.Rel, cfg.Include) {
grouped[index] = append(grouped[index], asset)
matched = true
break
}
}
if !matched {
return nil, fmt.Errorf("asset %s does not match any hak include pattern", asset.Rel)
}
}
var chunks []hakChunk
for index, cfg := range configs {
groupAssets := grouped[index]
if len(groupAssets) == 0 {
continue
}
groupChunks, err := splitHAKGroup(cfg, groupAssets)
if err != nil {
return nil, err
}
chunks = append(chunks, groupChunks...)
}
return chunks, nil
}
func splitHAKGroup(cfg project.HAKConfig, assets []assetResource) ([]hakChunk, error) {
maxBytes := cfg.MaxBytes
if maxBytes == 0 {
chunk := hakChunk{
Config: cfg,
Index: 1,
Name: cfg.Name,
Assets: assets,
Resources: resourceSlice(assets),
Size: erf.ArchiveSize(resourceSlice(assets)),
}
if err := ensureUniqueChunkResources(chunk); err != nil {
return nil, err
}
return []hakChunk{chunk}, nil
}
var chunks []hakChunk
current := hakChunk{Config: cfg, Index: 1}
for _, asset := range assets {
candidateAssets := append(append([]assetResource{}, current.Assets...), asset)
candidateResources := resourceSlice(candidateAssets)
candidateSize := erf.ArchiveSize(candidateResources)
if len(current.Assets) == 0 {
if candidateSize > maxBytes {
return nil, fmt.Errorf("asset %s alone exceeds max_bytes for hak %s", asset.Rel, cfg.Name)
}
current.Assets = candidateAssets
current.Resources = candidateResources
current.Size = candidateSize
continue
}
if candidateSize <= maxBytes {
current.Assets = candidateAssets
current.Resources = candidateResources
current.Size = candidateSize
continue
}
if !cfg.Split {
return nil, fmt.Errorf("hak %s exceeds max_bytes and split is disabled", cfg.Name)
}
current.Name = chunkName(cfg.Name, current.Index, true)
if err := ensureUniqueChunkResources(current); err != nil {
return nil, err
}
chunks = append(chunks, current)
current = hakChunk{
Config: cfg,
Index: current.Index + 1,
Assets: []assetResource{asset},
Resources: []erf.Resource{asset.Resource},
Size: erf.ArchiveSize([]erf.Resource{asset.Resource}),
}
if current.Size > maxBytes {
return nil, fmt.Errorf("asset %s alone exceeds max_bytes for hak %s", asset.Rel, cfg.Name)
}
}
if len(current.Assets) > 0 {
current.Name = chunkName(cfg.Name, current.Index, cfg.Split || len(chunks) > 0)
if err := ensureUniqueChunkResources(current); err != nil {
return nil, err
}
chunks = append(chunks, current)
}
return chunks, nil
}
func chunkName(base string, index int, split bool) string {
if !split {
return base
}
return fmt.Sprintf("%s_%02d", base, index)
}
func resourceSlice(assets []assetResource) []erf.Resource {
out := make([]erf.Resource, 0, len(assets))
for _, asset := range assets {
out = append(out, asset.Resource)
}
return out
}
func matchesAnyPattern(path string, patterns []string) bool {
for _, pattern := range patterns {
if matchPathPattern(filepath.ToSlash(path), filepath.ToSlash(pattern)) {
return true
}
}
return false
}
func matchPathPattern(path, pattern string) bool {
pathSegs := strings.Split(strings.Trim(path, "/"), "/")
patternSegs := strings.Split(strings.Trim(pattern, "/"), "/")
return matchSegments(pathSegs, patternSegs)
}
func cleanupGeneratedHAKs(buildDir, moduleResRef string) error {
paths, err := manifestHAKPaths(buildDir)
if err != nil {
return err
}
for _, path := range paths {
if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
return fmt.Errorf("remove previous generated hak %s: %w", path, err)
}
}
legacy := filepath.Join(buildDir, moduleResRef+".hak")
if err := os.Remove(legacy); err != nil && !errors.Is(err, os.ErrNotExist) {
return fmt.Errorf("remove legacy generated hak %s: %w", legacy, err)
}
manifest := filepath.Join(buildDir, "haks.json")
if err := os.Remove(manifest); err != nil && !errors.Is(err, os.ErrNotExist) {
return fmt.Errorf("remove previous hak manifest: %w", err)
}
return nil
}
func resolveModuleHAKOrder(p *project.Project, chunks []hakChunk) ([]string, error) {
if len(p.Config.Module.HAKOrder) == 0 {
return nil, nil
}
byGroup := map[string][]string{}
for _, chunk := range chunks {
byGroup[chunk.Config.Name] = append(byGroup[chunk.Config.Name], chunk.Name)
}
seen := map[string]struct{}{}
var ordered []string
for _, entry := range p.Config.Module.HAKOrder {
if strings.HasPrefix(entry, "group:") {
groupName := strings.TrimPrefix(entry, "group:")
names, ok := byGroup[groupName]
if !ok {
continue
}
for _, name := range names {
if _, exists := seen[name]; exists {
continue
}
ordered = append(ordered, name)
seen[name] = struct{}{}
}
continue
}
if _, exists := seen[entry]; exists {
continue
}
ordered = append(ordered, entry)
seen[entry] = struct{}{}
}
return ordered, nil
}
func setModuleHAKList(document *gff.Document, hakNames []string) {
list := make(gff.ListValue, 0, len(hakNames))
for _, name := range hakNames {
list = append(list, gff.Struct{
Type: 8,
Fields: []gff.Field{
gff.NewField("Mod_Hak", gff.StringValue(name)),
},
})
}
for index, field := range document.Root.Fields {
if field.Label == "Mod_HakList" {
document.Root.Fields[index] = gff.NewField("Mod_HakList", list)
return
}
}
document.Root.Fields = append(document.Root.Fields, gff.NewField("Mod_HakList", list))
}
func validateForBuild(p *project.Project) error {
report := validator.ValidateProject(p)
if report.HasErrors() {
return fmt.Errorf("validation failed with %d error(s)", len(report.Diagnostics))
}
return nil
}
func matchSegments(pathSegs, patternSegs []string) bool {
if len(patternSegs) == 0 {
return len(pathSegs) == 0
}
if patternSegs[0] == "**" {
if matchSegments(pathSegs, patternSegs[1:]) {
return true
}
if len(pathSegs) > 0 {
return matchSegments(pathSegs[1:], patternSegs)
}
return false
}
if len(pathSegs) == 0 {
return false
}
ok, err := filepath.Match(patternSegs[0], pathSegs[0])
if err != nil || !ok {
return false
}
return matchSegments(pathSegs[1:], patternSegs[1:])
}
internal/pipeline/chunk_validation.go
+27
View File
@@ -0,0 +1,27 @@
package pipeline
import (
"fmt"
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf"
)
func ensureUniqueChunkResources(chunk hakChunk) error {
seen := map[string]string{}
for _, asset := range chunk.Assets {
key := fmt.Sprintf("%s:%04x", asset.Resource.Name, asset.Resource.Type)
if previous, exists := seen[key]; exists {
return fmt.Errorf("resource %s from %s conflicts with %s inside generated hak %s", chunkResourceLabel(asset), asset.Rel, previous, chunk.Name)
}
seen[key] = asset.Rel
}
return nil
}
func chunkResourceLabel(asset assetResource) string {
extension, ok := erf.ExtensionForResourceType(asset.Resource.Type)
if !ok {
return asset.Resource.Name
}
return asset.Resource.Name + "." + extension
}
internal/pipeline/compare.go
+245
View File
@@ -0,0 +1,245 @@
package pipeline
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"slices"
"strings"
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf"
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/gff"
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
)
type CompareResult struct {
ModulePath string
HAKPaths []string
Checked int
}
type resourceExpectation struct {
Key string
Kind string
Bytes []byte
}
func Compare(p *project.Project) (CompareResult, error) {
modulePath := filepath.Join(p.BuildDir(), p.Config.Module.ResRef+".mod")
moduleArchive, err := readArchive(modulePath)
if err != nil {
return CompareResult{}, err
}
var hakPaths []string
if len(p.Inventory.AssetFiles) > 0 {
var err error
hakPaths, err = manifestHAKPaths(p.BuildDir())
if err != nil {
return CompareResult{}, err
}
}
expected, err := expectedResources(p)
if err != nil {
return CompareResult{}, err
}
actual := archiveIndex(moduleArchive)
for _, hakPath := range hakPaths {
hakArchive, err := readArchive(hakPath)
if err != nil {
return CompareResult{}, err
}
for key, value := range archiveIndex(hakArchive) {
actual[key] = value
}
}
var diagnostics []error
checked := 0
expectedKeys := make([]string, 0, len(expected))
for key := range expected {
expectedKeys = append(expectedKeys, key)
}
slices.Sort(expectedKeys)
for _, key := range expectedKeys {
want := expected[key]
got, ok := actual[key]
if !ok {
diagnostics = append(diagnostics, fmt.Errorf("missing resource in built archives: %s", key))
continue
}
if !bytes.Equal(want.Bytes, got.Data) {
diagnostics = append(diagnostics, fmt.Errorf("resource content mismatch: %s", key))
continue
}
checked++
delete(actual, key)
}
if len(actual) > 0 {
extraKeys := make([]string, 0, len(actual))
for key := range actual {
extraKeys = append(extraKeys, key)
}
slices.Sort(extraKeys)
for _, key := range extraKeys {
diagnostics = append(diagnostics, fmt.Errorf("unexpected extra resource in built archives: %s", key))
}
}
result := CompareResult{
ModulePath: modulePath,
HAKPaths: hakPaths,
Checked: checked,
}
if len(diagnostics) > 0 {
return result, errors.Join(diagnostics...)
}
return result, nil
}
func expectedResources(p *project.Project) (map[string]resourceExpectation, error) {
out := map[string]resourceExpectation{}
for _, rel := range p.Inventory.SourceFiles {
abs := filepath.Join(p.SourceDir(), filepath.FromSlash(rel))
name, extension, err := splitSourceName(abs)
if err != nil {
return nil, err
}
raw, err := os.ReadFile(abs)
if err != nil {
return nil, fmt.Errorf("read %s: %w", abs, err)
}
var document gff.Document
if err := json.Unmarshal(raw, &document); err != nil {
return nil, fmt.Errorf("parse gff json %s: %w", abs, err)
}
canonical, err := json.Marshal(document)
if err != nil {
return nil, fmt.Errorf("marshal canonical gff json %s: %w", abs, err)
}
out[resourceKey(name, extension)] = resourceExpectation{
Key: resourceKey(name, extension),
Kind: "gff-json",
Bytes: canonical,
}
}
for _, rel := range p.Inventory.ScriptFiles {
abs := filepath.Join(p.SourceDir(), filepath.FromSlash(rel))
data, err := os.ReadFile(abs)
if err != nil {
return nil, fmt.Errorf("read %s: %w", abs, err)
}
name := strings.TrimSuffix(filepath.Base(abs), filepath.Ext(abs))
extension := strings.TrimPrefix(strings.ToLower(filepath.Ext(abs)), ".")
out[resourceKey(name, extension)] = resourceExpectation{
Key: resourceKey(name, extension),
Kind: "raw",
Bytes: data,
}
}
for _, rel := range p.Inventory.AssetFiles {
abs := filepath.Join(p.AssetsDir(), filepath.FromSlash(rel))
data, err := os.ReadFile(abs)
if err != nil {
return nil, fmt.Errorf("read %s: %w", abs, err)
}
name := strings.TrimSuffix(filepath.Base(abs), filepath.Ext(abs))
extension := strings.TrimPrefix(strings.ToLower(filepath.Ext(abs)), ".")
out[resourceKey(name, extension)] = resourceExpectation{
Key: resourceKey(name, extension),
Kind: "raw",
Bytes: data,
}
}
return out, nil
}
func readArchive(path string) (erf.Archive, error) {
input, err := os.Open(path)
if err != nil {
return erf.Archive{}, fmt.Errorf("open archive %s: %w", path, err)
}
defer input.Close()
archive, err := erf.Read(input)
if err != nil {
return erf.Archive{}, fmt.Errorf("read archive %s: %w", path, err)
}
return archive, nil
}
func archiveIndex(archive erf.Archive) map[string]erf.Resource {
out := map[string]erf.Resource{}
for _, resource := range archive.Resources {
extension, ok := erf.ExtensionForResourceType(resource.Type)
if !ok {
continue
}
key := resourceKey(resource.Name, extension)
switch extension {
case "utc", "utd", "ute", "uti", "utm", "utp", "uts", "utt", "utw",
"are", "dlg", "fac", "gic", "git", "ifo", "itp", "jrl":
document, err := gff.Read(bytes.NewReader(resource.Data))
if err != nil {
out[key] = resource
continue
}
canonical, err := json.Marshal(document)
if err != nil {
out[key] = resource
continue
}
resource.Data = canonical
}
out[key] = resource
}
return out
}
func resourceKey(name, extension string) string {
return name + "." + strings.TrimPrefix(strings.ToLower(extension), ".")
}
func manifestHAKPaths(buildDir string) ([]string, error) {
manifestPath := filepath.Join(buildDir, "haks.json")
raw, err := os.ReadFile(manifestPath)
if err == nil {
var manifest BuildManifest
if err := json.Unmarshal(raw, &manifest); err != nil {
return nil, fmt.Errorf("parse hak manifest: %w", err)
}
paths := make([]string, 0, len(manifest.HAKs))
for _, hak := range manifest.HAKs {
paths = append(paths, filepath.Join(buildDir, hak.Name+".hak"))
}
return paths, nil
}
if !errors.Is(err, os.ErrNotExist) {
return nil, fmt.Errorf("read hak manifest: %w", err)
}
legacy := filepath.Join(buildDir, "*.hak")
paths, err := filepath.Glob(legacy)
if err != nil {
return nil, fmt.Errorf("scan hak archives: %w", err)
}
slices.Sort(paths)
return paths, nil
}
internal/pipeline/extract.go
+192
View File
@@ -0,0 +1,192 @@
package pipeline
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"slices"
"strings"
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf"
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/gff"
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
)
type ExtractResult struct {
ModulePath string
HAKPaths []string
Written int
Skipped int
}
func Extract(p *project.Project) (ExtractResult, error) {
modulePath := filepath.Join(p.BuildDir(), p.Config.Module.ResRef+".mod")
input, err := os.Open(modulePath)
if err != nil {
return ExtractResult{}, fmt.Errorf("open module archive: %w", err)
}
defer input.Close()
archive, err := erf.Read(input)
if err != nil {
return ExtractResult{}, fmt.Errorf("read module archive: %w", err)
}
var result ExtractResult
result.ModulePath = modulePath
var failures []error
written, skipped, errs := extractArchiveResources(p, archive)
result.Written += written
result.Skipped += skipped
failures = append(failures, errs...)
hakPaths, err := filepath.Glob(filepath.Join(p.BuildDir(), "*.hak"))
if err != nil {
return result, fmt.Errorf("scan hak archives: %w", err)
}
slices.Sort(hakPaths)
for _, hakPath := range hakPaths {
input, err := os.Open(hakPath)
if err != nil {
failures = append(failures, fmt.Errorf("open hak archive %s: %w", hakPath, err))
continue
}
hakArchive, err := erf.Read(input)
input.Close()
if err != nil {
failures = append(failures, fmt.Errorf("read hak archive %s: %w", hakPath, err))
continue
}
result.HAKPaths = append(result.HAKPaths, hakPath)
written, skipped, errs := extractArchiveResources(p, hakArchive)
result.Written += written
result.Skipped += skipped
failures = append(failures, errs...)
}
if len(failures) > 0 {
return result, errors.Join(failures...)
}
return result, nil
}
func extractArchiveResources(p *project.Project, archive erf.Archive) (int, int, []error) {
var failures []error
writtenCount := 0
skippedCount := 0
for _, resource := range archive.Resources {
target, data, err := extractedFile(p, resource)
if err != nil {
failures = append(failures, err)
continue
}
written, err := writeSafely(target, data)
if err != nil {
failures = append(failures, err)
continue
}
if written {
writtenCount++
} else {
skippedCount++
}
}
return writtenCount, skippedCount, failures
}
func extractedFile(p *project.Project, resource erf.Resource) (string, []byte, error) {
extension, ok := erf.ExtensionForResourceType(resource.Type)
if !ok {
return "", nil, fmt.Errorf("unsupported resource type 0x%04X for %s", resource.Type, resource.Name)
}
switch extension {
case "nss":
return filepath.Join(p.SourceDir(), "scripts", resource.Name+".nss"), resource.Data, nil
case "utc", "utd", "ute", "uti", "utm", "utp", "uts", "utt", "utw",
"are", "dlg", "fac", "gic", "git", "ifo", "itp", "jrl":
document, err := gff.Read(bytes.NewReader(resource.Data))
if err != nil {
return "", nil, fmt.Errorf("decode gff %s.%s: %w", resource.Name, extension, err)
}
formatted, err := json.MarshalIndent(document, "", " ")
if err != nil {
return "", nil, fmt.Errorf("marshal json %s.%s: %w", resource.Name, extension, err)
}
formatted = append(formatted, '\n')
return filepath.Join(p.SourceDir(), sourceSubdir(extension), resource.Name+"."+extension+".json"), formatted, nil
default:
return filepath.Join(p.AssetsDir(), extension, resource.Name+"."+extension), resource.Data, nil
}
}
func writeSafely(path string, data []byte) (bool, error) {
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
return false, fmt.Errorf("create parent directory for %s: %w", path, err)
}
existing, err := os.ReadFile(path)
if err == nil {
if bytes.Equal(existing, data) {
return false, nil
}
return false, fmt.Errorf("refusing to overwrite existing file with different contents: %s", path)
}
if !errors.Is(err, os.ErrNotExist) {
return false, fmt.Errorf("check existing file %s: %w", path, err)
}
if err := os.WriteFile(path, data, 0o644); err != nil {
return false, fmt.Errorf("write %s: %w", path, err)
}
return true, nil
}
func sourceSubdir(extension string) string {
switch strings.ToLower(extension) {
case "are":
return "areas"
case "dlg":
return "dialogs"
case "fac":
return "factions"
case "gic":
return "instance"
case "git":
return "instance"
case "ifo":
return "module"
case "itp":
return "palettes"
case "jrl":
return "journal"
case "utc":
return filepath.Join("blueprints", "creatures")
case "utd":
return filepath.Join("blueprints", "doors")
case "ute":
return filepath.Join("blueprints", "encounters")
case "uti":
return filepath.Join("blueprints", "items")
case "utm":
return filepath.Join("blueprints", "merchants")
case "utp":
return filepath.Join("blueprints", "placeables")
case "uts":
return filepath.Join("blueprints", "sounds")
case "utt":
return filepath.Join("blueprints", "triggers")
case "utw":
return filepath.Join("blueprints", "waypoints")
default:
return extension
}
}
internal/pipeline/pipeline_test.go
+673
View File
@@ -0,0 +1,673 @@
package pipeline
import (
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf"
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
)
func TestBuildThenExtract(t *testing.T) {
root := t.TempDir()
mustMkdir(t, filepath.Join(root, "src"))
mustMkdir(t, filepath.Join(root, "assets"))
mustMkdir(t, filepath.Join(root, "build"))
mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{
"module": {
"name": "Test Module",
"resref": "testmod"
},
"paths": {
"source": "src",
"assets": "assets",
"build": "build"
}
}
`)
mustMkdir(t, filepath.Join(root, "src", "module"))
mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{
"file_type": "IFO ",
"file_version": "V3.2",
"root": {
"struct_type": 0,
"fields": [
{
"label": "Mod_Name",
"type": "CExoString",
"value": "Test Module"
}
]
}
}
`)
p, err := project.Load(root)
if err != nil {
t.Fatalf("load project: %v", err)
}
if err := p.ValidateLayout(); err != nil {
t.Fatalf("validate layout: %v", err)
}
if err := p.Scan(); err != nil {
t.Fatalf("scan: %v", err)
}
buildResult, err := Build(p)
if err != nil {
t.Fatalf("build: %v", err)
}
if buildResult.Resources != 1 {
t.Fatalf("expected 1 resource, got %d", buildResult.Resources)
}
if err := os.Remove(filepath.Join(root, "src", "module", "module.ifo.json")); err != nil {
t.Fatalf("remove source file: %v", err)
}
if err := p.Scan(); err != nil {
t.Fatalf("rescan: %v", err)
}
extractResult, err := Extract(p)
if err != nil {
t.Fatalf("extract: %v", err)
}
if extractResult.Written != 1 {
t.Fatalf("expected 1 extracted file, got %d", extractResult.Written)
}
if _, err := os.Stat(filepath.Join(root, "src", "module", "module.ifo.json")); err != nil {
t.Fatalf("expected extracted module file: %v", err)
}
if err := p.Scan(); err != nil {
t.Fatalf("scan after extract: %v", err)
}
compareResult, err := Compare(p)
if err != nil {
t.Fatalf("compare: %v", err)
}
if compareResult.Checked != 1 {
t.Fatalf("expected 1 compared resource, got %d", compareResult.Checked)
}
}
func TestExtractReadsHAKAssets(t *testing.T) {
root := t.TempDir()
mustMkdir(t, filepath.Join(root, "src"))
mustMkdir(t, filepath.Join(root, "assets"))
mustMkdir(t, filepath.Join(root, "build"))
mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{
"module": {
"name": "Test Module",
"resref": "testmod"
},
"paths": {
"source": "src",
"assets": "assets",
"build": "build"
}
}
`)
p, err := project.Load(root)
if err != nil {
t.Fatalf("load project: %v", err)
}
if err := p.ValidateLayout(); err != nil {
t.Fatalf("validate layout: %v", err)
}
if err := p.Scan(); err != nil {
t.Fatalf("scan: %v", err)
}
modFile, err := os.Create(filepath.Join(root, "build", "testmod.mod"))
if err != nil {
t.Fatalf("create mod: %v", err)
}
if err := erf.Write(modFile, erf.New("MOD ", nil)); err != nil {
t.Fatalf("write mod: %v", err)
}
if err := modFile.Close(); err != nil {
t.Fatalf("close mod: %v", err)
}
hakFile, err := os.Create(filepath.Join(root, "build", "vfx.hak"))
if err != nil {
t.Fatalf("create hak: %v", err)
}
if err := erf.Write(hakFile, erf.New("HAK ", []erf.Resource{
{Name: "test_vfx", Type: 0x07D2, Data: []byte("mdl-data")},
{Name: "test_icon", Type: 0x0006, Data: []byte("plt-data")},
})); err != nil {
t.Fatalf("write hak: %v", err)
}
if err := hakFile.Close(); err != nil {
t.Fatalf("close hak: %v", err)
}
result, err := Extract(p)
if err != nil {
t.Fatalf("extract: %v", err)
}
if len(result.HAKPaths) != 1 {
t.Fatalf("expected 1 hak path, got %d", len(result.HAKPaths))
}
if _, err := os.Stat(filepath.Join(root, "assets", "mdl", "test_vfx.mdl")); err != nil {
t.Fatalf("expected extracted mdl asset: %v", err)
}
if _, err := os.Stat(filepath.Join(root, "assets", "plt", "test_icon.plt")); err != nil {
t.Fatalf("expected extracted plt asset: %v", err)
}
}
func TestBuildSplitsConfiguredHAKs(t *testing.T) {
root := t.TempDir()
mustMkdir(t, filepath.Join(root, "src"))
mustMkdir(t, filepath.Join(root, "assets", "core"))
mustMkdir(t, filepath.Join(root, "build"))
mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{
"module": {
"name": "Test Module",
"resref": "testmod",
"hak_order": ["group:core"]
},
"paths": {
"source": "src",
"assets": "assets",
"build": "build"
},
"haks": [
{
"name": "core",
"priority": 1,
"max_bytes": 300,
"split": true,
"include": ["core/**"]
}
]
}
`)
mustMkdir(t, filepath.Join(root, "src", "module"))
mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{
"file_type": "IFO ",
"file_version": "V3.2",
"root": {
"struct_type": 0,
"fields": [
{
"label": "Mod_Name",
"type": "CExoString",
"value": "Test Module"
},
{
"label": "Mod_HakList",
"type": "List",
"value": []
}
]
}
}
`)
mustWriteFile(t, filepath.Join(root, "assets", "core", "a.tga"), strings.Repeat("a", 80))
mustWriteFile(t, filepath.Join(root, "assets", "core", "b.tga"), strings.Repeat("b", 80))
p, err := project.Load(root)
if err != nil {
t.Fatalf("load project: %v", err)
}
if err := p.ValidateLayout(); err != nil {
t.Fatalf("validate layout: %v", err)
}
if err := p.Scan(); err != nil {
t.Fatalf("scan: %v", err)
}
result, err := BuildHAKs(p)
if err != nil {
t.Fatalf("build haks: %v", err)
}
if len(result.HAKPaths) != 2 {
t.Fatalf("expected 2 hak outputs, got %d", len(result.HAKPaths))
}
if _, err := os.Stat(filepath.Join(root, "build", "core_01.hak")); err != nil {
t.Fatalf("expected first split hak: %v", err)
}
if _, err := os.Stat(filepath.Join(root, "build", "core_02.hak")); err != nil {
t.Fatalf("expected second split hak: %v", err)
}
rawManifest, err := os.ReadFile(filepath.Join(root, "build", "haks.json"))
if err != nil {
t.Fatalf("read manifest: %v", err)
}
var manifest BuildManifest
if err := json.Unmarshal(rawManifest, &manifest); err != nil {
t.Fatalf("parse manifest: %v", err)
}
if len(manifest.HAKs) != 2 {
t.Fatalf("expected 2 manifest entries, got %d", len(manifest.HAKs))
}
if len(manifest.ModuleHAKs) != 2 || manifest.ModuleHAKs[0] != "core_01" || manifest.ModuleHAKs[1] != "core_02" {
t.Fatalf("unexpected module hak order: %#v", manifest.ModuleHAKs)
}
applyResult, err := ApplyHAKManifest(p, filepath.Join(root, "build", "haks.json"))
if err != nil {
t.Fatalf("apply hak manifest: %v", err)
}
if applyResult.HAKCount != 2 {
t.Fatalf("expected 2 applied hak entries, got %d", applyResult.HAKCount)
}
updated, err := os.ReadFile(filepath.Join(root, "src", "module", "module.ifo.json"))
if err != nil {
t.Fatalf("read updated ifo: %v", err)
}
if !strings.Contains(string(updated), "\"value\": \"core_01\"") || !strings.Contains(string(updated), "\"value\": \"core_02\"") {
t.Fatalf("updated ifo did not contain expected hak entries:\n%s", string(updated))
}
}
func TestBuildOrdersMultipleHAKGroups(t *testing.T) {
root := t.TempDir()
mustMkdir(t, filepath.Join(root, "src"))
mustMkdir(t, filepath.Join(root, "src", "module"))
mustMkdir(t, filepath.Join(root, "assets", "core"))
mustMkdir(t, filepath.Join(root, "assets", "vfx"))
mustMkdir(t, filepath.Join(root, "build"))
mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{
"module": {
"name": "Test Module",
"resref": "testmod",
"hak_order": ["group:core", "manual_top", "group:vfx"]
},
"paths": {
"source": "src",
"assets": "assets",
"build": "build"
},
"haks": [
{
"name": "core",
"priority": 1,
"max_bytes": 0,
"split": false,
"include": ["core/**"]
},
{
"name": "vfx",
"priority": 2,
"max_bytes": 0,
"split": false,
"include": ["vfx/**"]
}
]
}
`)
mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{
"file_type": "IFO ",
"file_version": "V3.2",
"root": {
"struct_type": 0,
"fields": [
{
"label": "Mod_Name",
"type": "CExoString",
"value": "Test Module"
},
{
"label": "Mod_HakList",
"type": "List",
"value": []
}
]
}
}
`)
mustWriteFile(t, filepath.Join(root, "assets", "core", "a.tga"), "core-data")
mustWriteFile(t, filepath.Join(root, "assets", "vfx", "b.tga"), "vfx-data")
p, err := project.Load(root)
if err != nil {
t.Fatalf("load project: %v", err)
}
if err := p.ValidateLayout(); err != nil {
t.Fatalf("validate layout: %v", err)
}
if err := p.Scan(); err != nil {
t.Fatalf("scan: %v", err)
}
result, err := BuildHAKs(p)
if err != nil {
t.Fatalf("build haks: %v", err)
}
if len(result.HAKPaths) != 2 {
t.Fatalf("expected 2 hak outputs, got %d", len(result.HAKPaths))
}
rawManifest, err := os.ReadFile(filepath.Join(root, "build", "haks.json"))
if err != nil {
t.Fatalf("read manifest: %v", err)
}
var manifest BuildManifest
if err := json.Unmarshal(rawManifest, &manifest); err != nil {
t.Fatalf("parse manifest: %v", err)
}
if len(manifest.ModuleHAKs) != 3 {
t.Fatalf("expected 3 module hak entries, got %#v", manifest.ModuleHAKs)
}
if manifest.ModuleHAKs[0] != "core" || manifest.ModuleHAKs[1] != "manual_top" || manifest.ModuleHAKs[2] != "vfx" {
t.Fatalf("unexpected module hak order: %#v", manifest.ModuleHAKs)
}
}
func TestBuildSplitsMultipleHAKGroupsDeterministically(t *testing.T) {
root := t.TempDir()
mustMkdir(t, filepath.Join(root, "src"))
mustMkdir(t, filepath.Join(root, "src", "module"))
mustMkdir(t, filepath.Join(root, "assets", "core"))
mustMkdir(t, filepath.Join(root, "assets", "vfx"))
mustMkdir(t, filepath.Join(root, "build"))
mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{
"module": {
"name": "Test Module",
"resref": "testmod",
"hak_order": ["group:core", "manual_top", "group:vfx"]
},
"paths": {
"source": "src",
"assets": "assets",
"build": "build"
},
"haks": [
{
"name": "core",
"priority": 1,
"max_bytes": 310,
"split": true,
"include": ["core/**"]
},
{
"name": "vfx",
"priority": 2,
"max_bytes": 310,
"split": true,
"include": ["vfx/**"]
}
]
}
`)
mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{
"file_type": "IFO ",
"file_version": "V3.2",
"root": {
"struct_type": 0,
"fields": [
{
"label": "Mod_Name",
"type": "CExoString",
"value": "Test Module"
},
{
"label": "Mod_HakList",
"type": "List",
"value": []
}
]
}
}
`)
mustWriteFile(t, filepath.Join(root, "assets", "core", "a.tga"), strings.Repeat("a", 80))
mustWriteFile(t, filepath.Join(root, "assets", "core", "b.tga"), strings.Repeat("b", 80))
mustWriteFile(t, filepath.Join(root, "assets", "vfx", "c.tga"), strings.Repeat("c", 80))
mustWriteFile(t, filepath.Join(root, "assets", "vfx", "d.tga"), strings.Repeat("d", 80))
p, err := project.Load(root)
if err != nil {
t.Fatalf("load project: %v", err)
}
if err := p.ValidateLayout(); err != nil {
t.Fatalf("validate layout: %v", err)
}
if err := p.Scan(); err != nil {
t.Fatalf("scan: %v", err)
}
result, err := BuildHAKs(p)
if err != nil {
t.Fatalf("build haks: %v", err)
}
if len(result.HAKPaths) != 4 {
t.Fatalf("expected 4 hak outputs, got %d", len(result.HAKPaths))
}
expectedFiles := []string{
"core_01.hak",
"core_02.hak",
"vfx_01.hak",
"vfx_02.hak",
}
for _, name := range expectedFiles {
if _, err := os.Stat(filepath.Join(root, "build", name)); err != nil {
t.Fatalf("expected split hak %s: %v", name, err)
}
}
rawManifest, err := os.ReadFile(filepath.Join(root, "build", "haks.json"))
if err != nil {
t.Fatalf("read manifest: %v", err)
}
var manifest BuildManifest
if err := json.Unmarshal(rawManifest, &manifest); err != nil {
t.Fatalf("parse manifest: %v", err)
}
if len(manifest.HAKs) != 4 {
t.Fatalf("expected 4 manifest hak entries, got %d", len(manifest.HAKs))
}
actualManifestNames := make([]string, 0, len(manifest.HAKs))
for _, hak := range manifest.HAKs {
actualManifestNames = append(actualManifestNames, hak.Name)
}
expectedManifestNames := []string{"core_01", "core_02", "vfx_01", "vfx_02"}
if strings.Join(actualManifestNames, ",") != strings.Join(expectedManifestNames, ",") {
t.Fatalf("unexpected manifest hak order: %#v", actualManifestNames)
}
expectedModuleHAKs := []string{"core_01", "core_02", "manual_top", "vfx_01", "vfx_02"}
if strings.Join(manifest.ModuleHAKs, ",") != strings.Join(expectedModuleHAKs, ",") {
t.Fatalf("unexpected module hak order: %#v", manifest.ModuleHAKs)
}
applyResult, err := ApplyHAKManifest(p, filepath.Join(root, "build", "haks.json"))
if err != nil {
t.Fatalf("apply hak manifest: %v", err)
}
if applyResult.HAKCount != 5 {
t.Fatalf("expected 5 applied hak entries, got %d", applyResult.HAKCount)
}
updated, err := os.ReadFile(filepath.Join(root, "src", "module", "module.ifo.json"))
if err != nil {
t.Fatalf("read updated ifo: %v", err)
}
for _, name := range expectedModuleHAKs {
if !strings.Contains(string(updated), "\"value\": \""+name+"\"") {
t.Fatalf("updated ifo missing hak %s:\n%s", name, string(updated))
}
}
}
func TestBuildSkipsEmptyHAKGroupsInModuleOrder(t *testing.T) {
root := t.TempDir()
mustMkdir(t, filepath.Join(root, "src"))
mustMkdir(t, filepath.Join(root, "src", "module"))
mustMkdir(t, filepath.Join(root, "assets", "core"))
mustMkdir(t, filepath.Join(root, "assets", "vfx"))
mustMkdir(t, filepath.Join(root, "build"))
mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{
"module": {
"name": "Test Module",
"resref": "testmod",
"hak_order": ["manual_top", "group:core", "group:empty", "group:vfx"]
},
"paths": {
"source": "src",
"assets": "assets",
"build": "build"
},
"haks": [
{
"name": "core",
"priority": 1,
"max_bytes": 1024,
"split": false,
"include": ["core/**"]
},
{
"name": "empty",
"priority": 2,
"max_bytes": 1024,
"split": false,
"include": ["empty/**"]
},
{
"name": "vfx",
"priority": 3,
"max_bytes": 1024,
"split": false,
"include": ["vfx/**"]
}
]
}`)
mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{
"file_type": "IFO ",
"file_version": "V3.2",
"root": {
"struct_type": 0,
"fields": [
{
"label": "Mod_Name",
"type": "CExoString",
"value": "Test Module"
},
{
"label": "Mod_HakList",
"type": "List",
"value": []
}
]
}
}
`)
mustWriteFile(t, filepath.Join(root, "assets", "core", "core_a.tga"), "core-a")
mustWriteFile(t, filepath.Join(root, "assets", "vfx", "vfx_a.tga"), "vfx-a")
p, err := project.Load(root)
if err != nil {
t.Fatalf("load project: %v", err)
}
if err := p.ValidateLayout(); err != nil {
t.Fatalf("validate layout: %v", err)
}
if err := p.Scan(); err != nil {
t.Fatalf("scan: %v", err)
}
result, err := BuildHAKs(p)
if err != nil {
t.Fatalf("build haks: %v", err)
}
rawManifest, err := os.ReadFile(result.Manifest)
if err != nil {
t.Fatalf("read manifest: %v", err)
}
var manifest BuildManifest
if err := json.Unmarshal(rawManifest, &manifest); err != nil {
t.Fatalf("parse manifest: %v", err)
}
if got, want := strings.Join(manifest.ModuleHAKs, ","), "manual_top,core,vfx"; got != want {
t.Fatalf("unexpected module hak order: got %q want %q", got, want)
}
}
func mustMkdir(t *testing.T, path string) {
t.Helper()
if err := os.MkdirAll(path, 0o755); err != nil {
t.Fatalf("mkdir %s: %v", path, err)
}
}
func mustWriteFile(t *testing.T, path, data string) {
t.Helper()
if err := os.WriteFile(path, []byte(data), 0o644); err != nil {
t.Fatalf("write %s: %v", path, err)
}
}
func TestBuildHAKsFailsForDuplicateResourcesInSameHAK(t *testing.T) {
root := t.TempDir()
mustMkdir(t, filepath.Join(root, "src"))
mustMkdir(t, filepath.Join(root, "assets", "core", "a"))
mustMkdir(t, filepath.Join(root, "assets", "core", "b"))
mustMkdir(t, filepath.Join(root, "build"))
mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{
"module": {
"name": "Test Module",
"resref": "testmod"
},
"paths": {
"source": "src",
"assets": "assets",
"build": "build"
},
"haks": [
{
"name": "core",
"priority": 1,
"max_bytes": 0,
"split": false,
"include": ["core/**"]
}
]
}
`)
mustWriteFile(t, filepath.Join(root, "assets", "core", "a", "shared.mdl"), "one")
mustWriteFile(t, filepath.Join(root, "assets", "core", "b", "shared.mdl"), "two")
p, err := project.Load(root)
if err != nil {
t.Fatalf("load project: %v", err)
}
if err := p.ValidateLayout(); err != nil {
t.Fatalf("validate layout: %v", err)
}
if err := p.Scan(); err != nil {
t.Fatalf("scan: %v", err)
}
_, err = BuildHAKs(p)
if err == nil {
t.Fatal("expected duplicate same-hak build error")
}
if !strings.Contains(err.Error(), "conflicts") {
t.Fatalf("unexpected error: %v", err)
}
}
internal/project/project.go
+277
View File
@@ -0,0 +1,277 @@
package project
import (
"encoding/json"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"slices"
"strings"
)
const ConfigFile = "nwn-tool.json"
var SourceExtensions = []string{
".are", ".dlg", ".fac", ".git", ".ifo", ".jrl",
".utc", ".utd", ".ute", ".uti", ".utm", ".utp", ".uts", ".utt", ".utw",
}
var AssetExtensions = []string{
".2da", ".bik", ".dds", ".dwk", ".itp", ".lod", ".lyt", ".mdl", ".mdx", ".mtr", ".plt", ".pwk", ".set", ".shd", ".tga", ".txi", ".uti", ".vis", ".wav", ".wok",
}
type Project struct {
Root string
Config Config
Inventory Inventory
}
type Config struct {
Module ModuleConfig `json:"module"`
Paths PathConfig `json:"paths"`
HAKs []HAKConfig `json:"haks"`
}
type ModuleConfig struct {
Name string `json:"name"`
ResRef string `json:"resref"`
Description string `json:"description"`
HAKOrder []string `json:"hak_order"`
}
type PathConfig struct {
Source string `json:"source"`
Assets string `json:"assets"`
Build string `json:"build"`
}
type HAKConfig struct {
Name string `json:"name"`
Priority int `json:"priority"`
MaxBytes int64 `json:"max_bytes"`
Split bool `json:"split"`
Include []string `json:"include"`
}
type Inventory struct {
SourceFiles []string
ScriptFiles []string
AssetFiles []string
Extensions []string
}
type InventoryReport struct {
SourceFiles int
ScriptFiles int
AssetFiles int
Extensions []string
}
func FindRoot(start string) (string, error) {
current := start
for {
candidate := filepath.Join(current, ConfigFile)
if _, err := os.Stat(candidate); err == nil {
return current, nil
}
parent := filepath.Dir(current)
if parent == current {
return "", fmt.Errorf("could not find %s from %s", ConfigFile, start)
}
current = parent
}
}
func Load(root string) (*Project, error) {
raw, err := os.ReadFile(filepath.Join(root, ConfigFile))
if err != nil {
return nil, fmt.Errorf("read %s: %w", ConfigFile, err)
}
cfg := defaultConfig()
if err := json.Unmarshal(raw, &cfg); err != nil {
return nil, fmt.Errorf("parse %s: %w", ConfigFile, err)
}
return &Project{
Root: root,
Config: cfg,
}, nil
}
func (p *Project) ValidateLayout() error {
var failures []error
if strings.TrimSpace(p.Config.Module.Name) == "" {
failures = append(failures, errors.New("module.name is required"))
}
if strings.TrimSpace(p.Config.Module.ResRef) == "" {
failures = append(failures, errors.New("module.resref is required"))
}
if len(p.Config.Module.ResRef) > 16 {
failures = append(failures, fmt.Errorf("module.resref %q exceeds 16 characters", p.Config.Module.ResRef))
}
for index, hak := range p.Config.HAKs {
if strings.TrimSpace(hak.Name) == "" {
failures = append(failures, fmt.Errorf("haks[%d].name is required", index))
}
if hak.Priority <= 0 {
failures = append(failures, fmt.Errorf("haks[%d].priority must be greater than zero", index))
}
if hak.MaxBytes < 0 {
failures = append(failures, fmt.Errorf("haks[%d].max_bytes must be zero or greater", index))
}
if len(hak.Include) == 0 {
failures = append(failures, fmt.Errorf("haks[%d].include must have at least one pattern", index))
}
}
for label, path := range map[string]string{
"paths.source": p.SourceDir(),
"paths.assets": p.AssetsDir(),
"paths.build": p.BuildDir(),
} {
info, err := os.Stat(path)
if err != nil {
failures = append(failures, fmt.Errorf("%s: %w", label, err))
continue
}
if !info.IsDir() {
failures = append(failures, fmt.Errorf("%s must be a directory: %s", label, path))
}
}
if len(failures) > 0 {
return errors.Join(failures...)
}
return nil
}
func (p *Project) Scan() error {
sourceFiles, sourceExts, err := scanDir(p.SourceDir(), func(path string) bool {
ext := strings.ToLower(filepath.Ext(path))
return ext == ".json" || ext == ".nss"
})
if err != nil {
return fmt.Errorf("scan source tree: %w", err)
}
assetFiles, _, err := scanDir(p.AssetsDir(), func(path string) bool {
ext := strings.ToLower(filepath.Ext(path))
return slices.Contains(AssetExtensions, ext)
})
if err != nil {
return fmt.Errorf("scan assets tree: %w", err)
}
var scripts []string
var sources []string
extensionSet := map[string]struct{}{}
for _, ext := range sourceExts {
extensionSet[ext] = struct{}{}
}
for _, path := range sourceFiles {
switch strings.ToLower(filepath.Ext(path)) {
case ".nss":
scripts = append(scripts, path)
case ".json":
sources = append(sources, path)
}
}
extensions := make([]string, 0, len(extensionSet))
for ext := range extensionSet {
extensions = append(extensions, ext)
}
slices.Sort(extensions)
p.Inventory = Inventory{
SourceFiles: sources,
ScriptFiles: scripts,
AssetFiles: assetFiles,
Extensions: extensions,
}
return nil
}
func (p *Project) SourceDir() string {
return filepath.Join(p.Root, p.Config.Paths.Source)
}
func (p *Project) AssetsDir() string {
return filepath.Join(p.Root, p.Config.Paths.Assets)
}
func (p *Project) BuildDir() string {
return filepath.Join(p.Root, p.Config.Paths.Build)
}
func (i Inventory) Report() InventoryReport {
return InventoryReport{
SourceFiles: len(i.SourceFiles),
ScriptFiles: len(i.ScriptFiles),
AssetFiles: len(i.AssetFiles),
Extensions: i.Extensions,
}
}
func defaultConfig() Config {
return Config{
Paths: PathConfig{
Source: "src",
Assets: "assets",
Build: "build",
},
}
}
func scanDir(root string, include func(path string) bool) ([]string, []string, error) {
var files []string
extSet := map[string]struct{}{}
err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() {
return nil
}
if !include(path) {
return nil
}
rel, err := filepath.Rel(root, path)
if err != nil {
return err
}
rel = filepath.ToSlash(rel)
files = append(files, rel)
if strings.HasSuffix(rel, ".json") {
base := strings.TrimSuffix(filepath.Base(rel), ".json")
ext := strings.ToLower(filepath.Ext(base))
if ext != "" {
extSet[ext] = struct{}{}
}
}
return nil
})
if err != nil {
return nil, nil, err
}
slices.Sort(files)
exts := make([]string, 0, len(extSet))
for ext := range extSet {
exts = append(exts, ext)
}
slices.Sort(exts)
return files, exts, nil
}
internal/validator/validator.go
+429
View File
@@ -0,0 +1,429 @@
package validator
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"slices"
"strings"
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf"
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/gff"
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
)
type Severity string
const (
SeverityError Severity = "error"
SeverityWarning Severity = "warning"
)
type Diagnostic struct {
Path string
Message string
Severity Severity
}
type Report struct {
Diagnostics []Diagnostic
SourceCount int
}
type loadedDocument struct {
Path string
ResRef string
Extension string
Document gff.Document
}
type assetOccurrence struct {
Path string
Group string
}
type assetGroupResolver struct {
configs []project.HAKConfig
defaultGroup string
}
var requiredFields = map[string][]string{
"ifo": {"Mod_Name"},
}
var builtinScriptPrefixes = []string{
"nw_",
"x0_",
"x1_",
"x2_",
"x3_",
"ga_",
"gc_",
"gen_",
"gui_",
"nwg_",
"ta_",
}
func ValidateProject(p *project.Project) Report {
report := Report{}
resourceIndex := map[string]string{}
scriptIndex := map[string]string{}
assetIndex := map[string]string{}
assetOccurrences := map[string][]assetOccurrence{}
documents := make([]loadedDocument, 0, len(p.Inventory.SourceFiles))
resolver := newAssetGroupResolver(p)
for _, rel := range p.Inventory.SourceFiles {
abs := filepath.Join(p.SourceDir(), filepath.FromSlash(rel))
document, resref, extension, err := loadDocument(abs)
if err != nil {
report.add(rel, err.Error(), SeverityError)
continue
}
report.SourceCount++
documents = append(documents, loadedDocument{
Path: rel,
ResRef: resref,
Extension: extension,
Document: document,
})
key := resref + "." + extension
if previous, exists := resourceIndex[key]; exists {
report.add(rel, fmt.Sprintf("duplicate resource %s also defined by %s", key, previous), SeverityError)
} else {
resourceIndex[key] = rel
}
validateDocumentStructure(&report, rel, extension, document)
}
for _, rel := range p.Inventory.ScriptFiles {
base := strings.TrimSuffix(filepath.Base(rel), filepath.Ext(rel))
if previous, exists := scriptIndex[base]; exists {
report.add(rel, fmt.Sprintf("duplicate script resource %s also defined by %s", base, previous), SeverityError)
} else {
scriptIndex[base] = rel
}
}
for _, rel := range p.Inventory.AssetFiles {
base := strings.TrimSuffix(filepath.Base(rel), filepath.Ext(rel))
extension := strings.TrimPrefix(strings.ToLower(filepath.Ext(rel)), ".")
key := base + "." + extension
if _, exists := assetIndex[key]; !exists {
assetIndex[key] = rel
}
group, err := resolver.groupFor(rel)
if err != nil {
report.add(rel, err.Error(), SeverityError)
continue
}
assetOccurrences[key] = append(assetOccurrences[key], assetOccurrence{
Path: rel,
Group: group,
})
}
classifyAssetDuplicates(&report, assetOccurrences)
for _, document := range documents {
validateReferences(&report, document, resourceIndex, scriptIndex, assetIndex)
}
slices.SortFunc(report.Diagnostics, func(a, b Diagnostic) int {
if cmp := strings.Compare(a.Path, b.Path); cmp != 0 {
return cmp
}
if cmp := strings.Compare(string(a.Severity), string(b.Severity)); cmp != 0 {
return cmp
}
return strings.Compare(a.Message, b.Message)
})
return report
}
func (r *Report) add(path, message string, severity Severity) {
r.Diagnostics = append(r.Diagnostics, Diagnostic{Path: path, Message: message, Severity: severity})
}
func (r Report) HasErrors() bool {
return r.ErrorCount() > 0
}
func (r Report) ErrorCount() int {
count := 0
for _, diagnostic := range r.Diagnostics {
if diagnostic.Severity == SeverityError {
count++
}
}
return count
}
func (r Report) WarningCount() int {
count := 0
for _, diagnostic := range r.Diagnostics {
if diagnostic.Severity == SeverityWarning {
count++
}
}
return count
}
func loadDocument(path string) (gff.Document, string, string, error) {
raw, err := os.ReadFile(path)
if err != nil {
return gff.Document{}, "", "", fmt.Errorf("read file: %w", err)
}
stem := strings.TrimSuffix(filepath.Base(path), ".json")
extension := strings.TrimPrefix(strings.ToLower(filepath.Ext(stem)), ".")
if extension == "" {
return gff.Document{}, "", "", fmt.Errorf("missing resource extension before .json")
}
resref := strings.TrimSuffix(stem, "."+extension)
var document gff.Document
if err := json.Unmarshal(raw, &document); err != nil {
return gff.Document{}, "", "", fmt.Errorf("parse json: %w", err)
}
return document, resref, extension, nil
}
func validateDocumentStructure(report *Report, path, extension string, document gff.Document) {
expectedType := strings.ToUpper(extension)
if strings.TrimSpace(document.FileType) != "" && strings.TrimSpace(document.FileType) != expectedType {
report.add(path, fmt.Sprintf("file_type %q does not match expected %q", document.FileType, expectedType), SeverityError)
}
required := requiredFields[extension]
if len(required) > 0 {
fields := map[string]struct{}{}
for _, field := range document.Root.Fields {
fields[field.Label] = struct{}{}
}
for _, label := range required {
if _, ok := fields[label]; !ok {
report.add(path, fmt.Sprintf("missing required field %q", label), SeverityError)
}
}
}
validateUniqueLabels(report, path, "root", document.Root)
}
func validateUniqueLabels(report *Report, path, scope string, s gff.Struct) {
seen := map[string]struct{}{}
for _, field := range s.Fields {
if _, exists := seen[field.Label]; exists {
report.add(path, fmt.Sprintf("duplicate field label %q in %s", field.Label, scope), SeverityError)
} else {
seen[field.Label] = struct{}{}
}
switch value := field.Value.(type) {
case gff.Struct:
validateUniqueLabels(report, path, scope+"."+field.Label, value)
case gff.ListValue:
for index, item := range value {
validateUniqueLabels(report, path, fmt.Sprintf("%s.%s[%d]", scope, field.Label, index), item)
}
}
}
}
func validateReferences(report *Report, document loadedDocument, resources, scripts, assets map[string]string) {
walkFields(document.Document.Root, func(field gff.Field) {
value, ok := fieldStringValue(field.Value)
if !ok || value == "" {
return
}
switch {
case isScriptField(field.Label):
if isBuiltinScript(value) {
return
}
if _, exists := scripts[value]; !exists {
report.add(document.Path, fmt.Sprintf("missing script reference %q from field %q", value, field.Label), SeverityError)
}
case field.Label == "Conversation":
key := value + ".dlg"
if _, exists := resources[key]; !exists {
report.add(document.Path, fmt.Sprintf("missing dialog reference %q", key), SeverityError)
}
case field.Label == "Mod_Entry_Area":
key := value + ".are"
if _, exists := resources[key]; !exists {
report.add(document.Path, fmt.Sprintf("missing area reference %q", key), SeverityError)
}
case field.Label == "Model":
if !hasAsset(value, []string{"mdl"}, assets) {
report.add(document.Path, fmt.Sprintf("missing model asset for %q", value), SeverityError)
}
case field.Label == "Sound":
if !hasAsset(value, []string{"wav"}, assets) {
report.add(document.Path, fmt.Sprintf("missing sound asset for %q", value), SeverityError)
}
}
})
}
func classifyAssetDuplicates(report *Report, occurrences map[string][]assetOccurrence) {
keys := make([]string, 0, len(occurrences))
for key := range occurrences {
keys = append(keys, key)
}
slices.Sort(keys)
for _, key := range keys {
entries := occurrences[key]
if len(entries) < 2 {
continue
}
groups := make([]string, 0, len(entries))
groupSet := map[string]struct{}{}
paths := make([]string, 0, len(entries))
for _, entry := range entries {
paths = append(paths, entry.Path)
if _, exists := groupSet[entry.Group]; exists {
continue
}
groupSet[entry.Group] = struct{}{}
groups = append(groups, entry.Group)
}
slices.Sort(groups)
slices.Sort(paths)
if len(groups) == 1 {
report.add(entries[0].Path, fmt.Sprintf("duplicate asset resource %s appears multiple times in hak group %q (%s); build will fail if duplicates land in the same generated hak", key, groups[0], strings.Join(paths, ", ")), SeverityWarning)
continue
}
report.add(entries[0].Path, fmt.Sprintf("duplicate asset resource %s is split across hak groups %s (%s); later hak order overrides earlier at runtime", key, strings.Join(groups, ", "), strings.Join(paths, ", ")), SeverityWarning)
}
}
func newAssetGroupResolver(p *project.Project) assetGroupResolver {
configs := make([]project.HAKConfig, len(p.Config.HAKs))
copy(configs, p.Config.HAKs)
slices.SortFunc(configs, func(a, b project.HAKConfig) int {
if a.Priority != b.Priority {
if a.Priority < b.Priority {
return -1
}
return 1
}
return strings.Compare(a.Name, b.Name)
})
return assetGroupResolver{configs: configs, defaultGroup: p.Config.Module.ResRef}
}
func (r assetGroupResolver) groupFor(rel string) (string, error) {
if len(r.configs) == 0 {
return r.defaultGroup, nil
}
for _, cfg := range r.configs {
if matchesAnyPattern(rel, cfg.Include) {
return cfg.Name, nil
}
}
return "", fmt.Errorf("asset does not match any hak include pattern")
}
func matchesAnyPattern(path string, patterns []string) bool {
for _, pattern := range patterns {
if matchPathPattern(filepath.ToSlash(path), filepath.ToSlash(pattern)) {
return true
}
}
return false
}
func matchPathPattern(path, pattern string) bool {
pathSegs := strings.Split(strings.Trim(path, "/"), "/")
patternSegs := strings.Split(strings.Trim(pattern, "/"), "/")
return matchSegments(pathSegs, patternSegs)
}
func matchSegments(pathSegs, patternSegs []string) bool {
if len(patternSegs) == 0 {
return len(pathSegs) == 0
}
if patternSegs[0] == "**" {
if matchSegments(pathSegs, patternSegs[1:]) {
return true
}
if len(pathSegs) > 0 {
return matchSegments(pathSegs[1:], patternSegs)
}
return false
}
if len(pathSegs) == 0 {
return false
}
ok, err := filepath.Match(patternSegs[0], pathSegs[0])
if err != nil || !ok {
return false
}
return matchSegments(pathSegs[1:], patternSegs[1:])
}
func walkFields(s gff.Struct, visit func(gff.Field)) {
for _, field := range s.Fields {
visit(field)
switch value := field.Value.(type) {
case gff.Struct:
walkFields(value, visit)
case gff.ListValue:
for _, item := range value {
walkFields(item, visit)
}
}
}
}
func fieldStringValue(value gff.Value) (string, bool) {
switch typed := value.(type) {
case gff.StringValue:
return string(typed), true
case gff.ResRefValue:
return string(typed), true
default:
return "", false
}
}
func isScriptField(label string) bool {
return strings.HasPrefix(label, "On") || strings.HasSuffix(label, "Script") || strings.Contains(label, "Script")
}
func hasAsset(name string, extensions []string, assets map[string]string) bool {
for _, extension := range extensions {
if _, exists := assets[name+"."+extension]; exists {
return true
}
}
return false
}
func isBuiltinScript(name string) bool {
lower := strings.ToLower(name)
for _, prefix := range builtinScriptPrefixes {
if strings.HasPrefix(lower, prefix) {
return true
}
}
return false
}
func ResourceTypeForSourceExtension(extension string) (uint16, bool) {
return erf.ResourceTypeForExtension(extension)
}
internal/validator/validator_test.go
+69
View File
@@ -0,0 +1,69 @@
package validator
import (
"os"
"path/filepath"
"testing"
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
)
func TestValidateProjectWarnsForCrossHAKDuplicateAssets(t *testing.T) {
root := t.TempDir()
mustMkdir(t, filepath.Join(root, "src"))
mustMkdir(t, filepath.Join(root, "assets", "low"))
mustMkdir(t, filepath.Join(root, "assets", "high"))
mustMkdir(t, filepath.Join(root, "build"))
mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{
"module": {
"name": "Test Assets",
"resref": "testassets"
},
"paths": {
"source": "src",
"assets": "assets",
"build": "build"
},
"haks": [
{ "name": "low", "priority": 1, "max_bytes": 0, "split": false, "include": ["low/**"] },
{ "name": "high", "priority": 2, "max_bytes": 0, "split": false, "include": ["high/**"] }
]
}
`)
mustWriteFile(t, filepath.Join(root, "assets", "low", "shared.mdl"), "low")
mustWriteFile(t, filepath.Join(root, "assets", "high", "shared.mdl"), "high")
p, err := project.Load(root)
if err != nil {
t.Fatalf("load project: %v", err)
}
if err := p.ValidateLayout(); err != nil {
t.Fatalf("validate layout: %v", err)
}
if err := p.Scan(); err != nil {
t.Fatalf("scan: %v", err)
}
report := ValidateProject(p)
if report.HasErrors() {
t.Fatalf("expected warnings only, got errors: %#v", report.Diagnostics)
}
if report.WarningCount() != 1 {
t.Fatalf("expected 1 warning, got %d (%#v)", report.WarningCount(), report.Diagnostics)
}
}
func mustMkdir(t *testing.T, path string) {
t.Helper()
if err := os.MkdirAll(path, 0o755); err != nil {
t.Fatalf("mkdir %s: %v", path, err)
}
}
func mustWriteFile(t *testing.T, path, data string) {
t.Helper()
if err := os.WriteFile(path, []byte(data), 0o644); err != nil {
t.Fatalf("write %s: %v", path, err)
}
}