claude: fold in old sow-tools

internal/validator/validator.go

@@ -0,0 +1,546 @@
+package validator
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"slices"
+	"strings"
+
+	"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf"
+	"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/gff"
+	"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/music"
+	"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
+)
+
+type Severity string
+
+const (
+	SeverityError   Severity = "error"
+	SeverityWarning Severity = "warning"
+)
+
+type Diagnostic struct {
+	Path     string
+	Message  string
+	Severity Severity
+}
+
+type Report struct {
+	Diagnostics      []Diagnostic
+	SourceCount      int
+	DecompiledModels []string
+}
+
+type loadedDocument struct {
+	Path      string
+	ResRef    string
+	Extension string
+	Document  gff.Document
+}
+
+type assetOccurrence struct {
+	Path  string
+	Group string
+}
+
+type assetGroupResolver struct {
+	configs      []project.HAKConfig
+	defaultGroup string
+}
+
+func ValidateProject(p *project.Project) Report {
+	report := Report{}
+	resourceIndex := map[string]string{}
+	scriptIndex := map[string]string{}
+	assetIndex := map[string]string{}
+	assetOccurrences := map[string][]assetOccurrence{}
+	documents := make([]loadedDocument, 0, len(p.Inventory.SourceFiles))
+	resolver := newAssetGroupResolver(p)
+	effective := p.EffectiveConfig()
+
+	for _, rel := range p.Inventory.SourceFiles {
+		if hasUppercaseResourceName(rel) {
+			report.add(rel, "resource filenames should be lowercase", SeverityWarning)
+		}
+		abs := filepath.Join(p.SourceDir(), filepath.FromSlash(rel))
+		document, resref, extension, err := loadDocument(abs)
+		if err != nil {
+			report.add(rel, err.Error(), SeverityError)
+			continue
+		}
+		report.SourceCount++
+		documents = append(documents, loadedDocument{
+			Path:      rel,
+			ResRef:    resref,
+			Extension: extension,
+			Document:  document,
+		})
+
+		key := strings.ToLower(resref) + "." + extension
+		if previous, exists := resourceIndex[key]; exists {
+			report.add(rel, fmt.Sprintf("duplicate resource %s also defined by %s", key, previous), SeverityError)
+		} else {
+			resourceIndex[key] = rel
+		}
+
+		validateDocumentStructure(&report, rel, extension, document, effective.Validation.RequiredFields)
+	}
+
+	for _, rel := range p.Inventory.ScriptFiles {
+		if hasUppercaseResourceName(rel) {
+			report.add(rel, "resource filenames should be lowercase", SeverityWarning)
+		}
+		base := strings.ToLower(strings.TrimSuffix(filepath.Base(rel), filepath.Ext(rel)))
+		if previous, exists := scriptIndex[base]; exists {
+			report.add(rel, fmt.Sprintf("duplicate script resource %s also defined by %s", base, previous), SeverityError)
+		} else {
+			scriptIndex[base] = rel
+		}
+	}
+
+	for _, rel := range p.Inventory.AssetFiles {
+		musicSource := isMusicSourceAsset(p, rel)
+		if hasUppercaseResourceName(rel) && !musicSource {
+			report.add(rel, "resource filenames should be lowercase", SeverityWarning)
+		}
+		base := strings.ToLower(strings.TrimSuffix(filepath.Base(rel), filepath.Ext(rel)))
+		extension := strings.TrimPrefix(strings.ToLower(filepath.Ext(rel)), ".")
+		if extension == "mdl" {
+			if isTextMDL(filepath.Join(p.AssetsDir(), filepath.FromSlash(rel))) {
+				report.DecompiledModels = append(report.DecompiledModels, rel)
+			}
+		}
+		key := strings.ToLower(base) + "." + extension
+		if _, exists := assetIndex[key]; !exists {
+			assetIndex[key] = rel
+		}
+		if musicSource {
+			continue
+		}
+
+		group, err := resolver.groupFor(rel)
+		if err != nil {
+			report.add(rel, err.Error(), SeverityError)
+			continue
+		}
+
+		assetOccurrences[key] = append(assetOccurrences[key], assetOccurrence{
+			Path:  rel,
+			Group: group,
+		})
+	}
+
+	classifyAssetDuplicates(&report, assetOccurrences)
+
+	for _, document := range documents {
+		validateReferences(&report, document, resourceIndex, scriptIndex, assetIndex, effective.Validation.BuiltinScriptPrefixes)
+	}
+
+	slices.SortFunc(report.Diagnostics, func(a, b Diagnostic) int {
+		if cmp := strings.Compare(a.Path, b.Path); cmp != 0 {
+			return cmp
+		}
+		if cmp := strings.Compare(string(a.Severity), string(b.Severity)); cmp != 0 {
+			return cmp
+		}
+		return strings.Compare(a.Message, b.Message)
+	})
+	slices.Sort(report.DecompiledModels)
+
+	return report
+}
+
+func (r Report) SummaryLines(maxPaths int) []string {
+	return summarizeDiagnostics(r.Diagnostics, maxPaths)
+}
+
+func (r Report) DecompiledSummaryLine(maxPaths int) string {
+	if len(r.DecompiledModels) == 0 {
+		return ""
+	}
+	return formatSummaryLine("info", "ASCII/decompiled .mdl files detected", r.DecompiledModels, maxPaths)
+}
+
+func (r *Report) add(path, message string, severity Severity) {
+	r.Diagnostics = append(r.Diagnostics, Diagnostic{Path: path, Message: message, Severity: severity})
+}
+
+func (r Report) HasErrors() bool {
+	return r.ErrorCount() > 0
+}
+
+func (r Report) ErrorCount() int {
+	count := 0
+	for _, diagnostic := range r.Diagnostics {
+		if diagnostic.Severity == SeverityError {
+			count++
+		}
+	}
+	return count
+}
+
+func (r Report) WarningCount() int {
+	count := 0
+	for _, diagnostic := range r.Diagnostics {
+		if diagnostic.Severity == SeverityWarning {
+			count++
+		}
+	}
+	return count
+}
+
+func summarizeDiagnostics(diagnostics []Diagnostic, maxPaths int) []string {
+	type key struct {
+		severity Severity
+		message  string
+	}
+	groups := map[key][]string{}
+	order := make([]key, 0)
+	for _, diagnostic := range diagnostics {
+		k := key{severity: diagnostic.Severity, message: diagnostic.Message}
+		if _, ok := groups[k]; !ok {
+			order = append(order, k)
+		}
+		groups[k] = append(groups[k], diagnostic.Path)
+	}
+	slices.SortFunc(order, func(a, b key) int {
+		if a.severity != b.severity {
+			if a.severity == SeverityError {
+				return -1
+			}
+			return 1
+		}
+		return strings.Compare(a.message, b.message)
+	})
+	lines := make([]string, 0, len(order))
+	for _, k := range order {
+		paths := append([]string(nil), groups[k]...)
+		slices.Sort(paths)
+		paths = slices.Compact(paths)
+		lines = append(lines, formatSummaryLine(string(k.severity), k.message, paths, maxPaths))
+	}
+	return lines
+}
+
+func formatSummaryLine(prefix, message string, paths []string, maxPaths int) string {
+	if len(paths) == 0 {
+		return fmt.Sprintf("%s: %s", prefix, message)
+	}
+	if maxPaths <= 0 {
+		maxPaths = 3
+	}
+	display := paths
+	if len(display) > maxPaths {
+		display = display[:maxPaths]
+	}
+	location := strings.Join(display, ", ")
+	if len(paths) > len(display) {
+		location += fmt.Sprintf(", and %d more", len(paths)-len(display))
+	}
+	return fmt.Sprintf("%s: %s [%s]", prefix, message, location)
+}
+
+func isTextMDL(path string) bool {
+	data, err := os.ReadFile(path)
+	if err != nil || len(data) == 0 {
+		return false
+	}
+	if len(data) > 4096 {
+		data = data[:4096]
+	}
+	if strings.IndexByte(string(data), 0) >= 0 {
+		return false
+	}
+	printable := 0
+	for _, b := range data {
+		if b == '\n' || b == '\r' || b == '\t' || (b >= 32 && b <= 126) {
+			printable++
+		}
+	}
+	ratio := float64(printable) / float64(len(data))
+	lower := strings.ToLower(string(data))
+	return strings.Contains(lower, "newmodel ") || strings.Contains(lower, "donemodel") || ratio >= 0.9
+}
+
+func loadDocument(path string) (gff.Document, string, string, error) {
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return gff.Document{}, "", "", fmt.Errorf("read file: %w", err)
+	}
+
+	stem := strings.TrimSuffix(filepath.Base(path), ".json")
+	extension := strings.TrimPrefix(strings.ToLower(filepath.Ext(stem)), ".")
+	if extension == "" {
+		return gff.Document{}, "", "", fmt.Errorf("missing resource extension before .json")
+	}
+	resref := strings.ToLower(strings.TrimSuffix(stem, "."+extension))
+
+	var document gff.Document
+	if err := json.Unmarshal(raw, &document); err != nil {
+		return gff.Document{}, "", "", fmt.Errorf("parse json: %w", err)
+	}
+	return document, resref, extension, nil
+}
+
+func validateDocumentStructure(report *Report, path, extension string, document gff.Document, requiredFields map[string][]string) {
+	expectedType := strings.ToUpper(extension)
+	if strings.TrimSpace(document.FileType) != "" && strings.TrimSpace(document.FileType) != expectedType {
+		report.add(path, fmt.Sprintf("file_type %q does not match expected %q", document.FileType, expectedType), SeverityError)
+	}
+
+	required := requiredFields[extension]
+	if len(required) > 0 {
+		fields := map[string]struct{}{}
+		for _, field := range document.Root.Fields {
+			fields[field.Label] = struct{}{}
+		}
+		for _, label := range required {
+			if _, ok := fields[label]; !ok {
+				report.add(path, fmt.Sprintf("missing required field %q", label), SeverityError)
+			}
+		}
+	}
+
+	validateUniqueLabels(report, path, "root", document.Root)
+}
+
+func validateUniqueLabels(report *Report, path, scope string, s gff.Struct) {
+	seen := map[string]struct{}{}
+	for _, field := range s.Fields {
+		if _, exists := seen[field.Label]; exists {
+			report.add(path, fmt.Sprintf("duplicate field label %q in %s", field.Label, scope), SeverityError)
+		} else {
+			seen[field.Label] = struct{}{}
+		}
+
+		switch value := field.Value.(type) {
+		case gff.Struct:
+			validateUniqueLabels(report, path, scope+"."+field.Label, value)
+		case gff.ListValue:
+			for index, item := range value {
+				validateUniqueLabels(report, path, fmt.Sprintf("%s.%s[%d]", scope, field.Label, index), item)
+			}
+		}
+	}
+}
+
+func validateReferences(report *Report, document loadedDocument, resources, scripts, assets map[string]string, builtinScriptPrefixes []string) {
+	walkFields(document.Document.Root, func(field gff.Field) {
+		value, ok := fieldStringValue(field.Value)
+		if !ok || value == "" {
+			return
+		}
+
+		switch {
+		case isScriptField(field.Label):
+			if isBuiltinScript(value, builtinScriptPrefixes) {
+				return
+			}
+			if _, exists := scripts[strings.ToLower(value)]; !exists {
+				report.add(document.Path, fmt.Sprintf("missing script reference %q from field %q", value, field.Label), SeverityWarning)
+			}
+		case field.Label == "Conversation":
+			key := strings.ToLower(value) + ".dlg"
+			if _, exists := resources[key]; !exists {
+				report.add(document.Path, fmt.Sprintf("missing dialog reference %q", key), SeverityError)
+			}
+		case field.Label == "Mod_Entry_Area":
+			key := strings.ToLower(value) + ".are"
+			if _, exists := resources[key]; !exists {
+				report.add(document.Path, fmt.Sprintf("missing area reference %q", key), SeverityError)
+			}
+		case field.Label == "Model":
+			if !hasAsset(strings.ToLower(value), []string{"mdl"}, assets) {
+				report.add(document.Path, fmt.Sprintf("missing model asset for %q", value), SeverityWarning)
+			}
+		case field.Label == "Sound":
+			if !hasAsset(strings.ToLower(value), []string{"wav"}, assets) {
+				report.add(document.Path, fmt.Sprintf("missing sound asset for %q", value), SeverityWarning)
+			}
+		}
+	})
+}
+
+func classifyAssetDuplicates(report *Report, occurrences map[string][]assetOccurrence) {
+	keys := make([]string, 0, len(occurrences))
+	for key := range occurrences {
+		keys = append(keys, key)
+	}
+	slices.Sort(keys)
+
+	for _, key := range keys {
+		entries := occurrences[key]
+		if len(entries) < 2 {
+			continue
+		}
+
+		groups := make([]string, 0, len(entries))
+		groupSet := map[string]struct{}{}
+		paths := make([]string, 0, len(entries))
+		for _, entry := range entries {
+			paths = append(paths, entry.Path)
+			if _, exists := groupSet[entry.Group]; exists {
+				continue
+			}
+			groupSet[entry.Group] = struct{}{}
+			groups = append(groups, entry.Group)
+		}
+		slices.Sort(groups)
+		slices.Sort(paths)
+
+		if len(groups) == 1 {
+			report.add(entries[0].Path, fmt.Sprintf("duplicate asset resource %s appears multiple times in hak group %q (%s); build will fail if duplicates land in the same generated hak", key, groups[0], strings.Join(paths, ", ")), SeverityError)
+			continue
+		}
+
+		report.add(entries[0].Path, fmt.Sprintf("duplicate asset resource %s is split across hak groups %s (%s); later hak order overrides earlier at runtime", key, strings.Join(groups, ", "), strings.Join(paths, ", ")), SeverityWarning)
+	}
+}
+
+func newAssetGroupResolver(p *project.Project) assetGroupResolver {
+	configs := make([]project.HAKConfig, len(p.Config.HAKs))
+	copy(configs, p.Config.HAKs)
+	slices.SortFunc(configs, func(a, b project.HAKConfig) int {
+		if a.Priority != b.Priority {
+			if a.Priority < b.Priority {
+				return -1
+			}
+			return 1
+		}
+		return strings.Compare(a.Name, b.Name)
+	})
+	return assetGroupResolver{configs: configs, defaultGroup: p.Config.Module.ResRef}
+}
+
+func (r assetGroupResolver) groupFor(rel string) (string, error) {
+	if len(r.configs) == 0 {
+		return r.defaultGroup, nil
+	}
+	for _, cfg := range r.configs {
+		if matchesAnyPattern(rel, cfg.Include) {
+			return cfg.Name, nil
+		}
+	}
+	return "", fmt.Errorf("asset does not match any hak include pattern")
+}
+
+func matchesAnyPattern(path string, patterns []string) bool {
+	for _, pattern := range patterns {
+		if matchPathPattern(filepath.ToSlash(path), filepath.ToSlash(pattern)) {
+			return true
+		}
+	}
+	return false
+}
+
+func matchPathPattern(path, pattern string) bool {
+	pathSegs := strings.Split(strings.Trim(path, "/"), "/")
+	patternSegs := strings.Split(strings.Trim(pattern, "/"), "/")
+	return matchSegments(pathSegs, patternSegs)
+}
+
+func matchSegments(pathSegs, patternSegs []string) bool {
+	if len(patternSegs) == 0 {
+		return len(pathSegs) == 0
+	}
+	if patternSegs[0] == "**" {
+		if matchSegments(pathSegs, patternSegs[1:]) {
+			return true
+		}
+		if len(pathSegs) > 0 {
+			return matchSegments(pathSegs[1:], patternSegs)
+		}
+		return false
+	}
+	if len(pathSegs) == 0 {
+		return false
+	}
+	ok, err := filepath.Match(patternSegs[0], pathSegs[0])
+	if err != nil || !ok {
+		return false
+	}
+	return matchSegments(pathSegs[1:], patternSegs[1:])
+}
+
+func walkFields(s gff.Struct, visit func(gff.Field)) {
+	for _, field := range s.Fields {
+		visit(field)
+		switch value := field.Value.(type) {
+		case gff.Struct:
+			walkFields(value, visit)
+		case gff.ListValue:
+			for _, item := range value {
+				walkFields(item, visit)
+			}
+		}
+	}
+}
+
+func hasUppercaseResourceName(rel string) bool {
+	base := filepath.Base(rel)
+	return base != strings.ToLower(base)
+}
+
+func fieldStringValue(value gff.Value) (string, bool) {
+	switch typed := value.(type) {
+	case gff.StringValue:
+		return string(typed), true
+	case gff.ResRefValue:
+		return string(typed), true
+	default:
+		return "", false
+	}
+}
+
+func isScriptField(label string) bool {
+	return strings.HasPrefix(label, "On") || strings.HasSuffix(label, "Script") || strings.Contains(label, "Script")
+}
+
+func hasAsset(name string, extensions []string, assets map[string]string) bool {
+	for _, extension := range extensions {
+		if _, exists := assets[name+"."+extension]; exists {
+			return true
+		}
+	}
+	return false
+}
+
+func isMusicSourceAsset(p *project.Project, rel string) bool {
+	rel = filepath.ToSlash(strings.TrimSpace(rel))
+	ext := strings.ToLower(filepath.Ext(rel))
+	effective := p.EffectiveConfig()
+	for _, dataset := range effective.Music.Datasets {
+		if !music.PathIsUnder(dataset.Source, rel) {
+			continue
+		}
+		for _, candidate := range dataset.ConvertExtensions {
+			if ext == strings.ToLower(strings.TrimSpace(candidate)) {
+				return true
+			}
+		}
+	}
+	if music.PathIsUnder("envi/music", rel) {
+		for _, candidate := range effective.Music.ConvertExtensions {
+			if ext == strings.ToLower(strings.TrimSpace(candidate)) {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+func isBuiltinScript(name string, builtinScriptPrefixes []string) bool {
+	lower := strings.ToLower(name)
+	for _, prefix := range builtinScriptPrefixes {
+		if strings.HasPrefix(lower, prefix) {
+			return true
+		}
+	}
+	return false
+}
+
+func ResourceTypeForSourceExtension(extension string) (uint16, bool) {
+	return erf.ResourceTypeForExtension(extension)
+}