Improved Logging

internal/app/app.go

@@ -6,7 +6,6 @@ import (
 	"io"
 	"os"
 	"path/filepath"
-	"slices"
 	"strings"
 
 	"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/pipeline"
@@ -235,19 +234,11 @@ func runValidate(ctx context) error {
 	}
 
 	validationReport := validator.ValidateProject(p)
+	emitValidatorReport(ctx.stderr, validationReport)
 	if validationReport.HasErrors() {
-		emitValidationWarnings(ctx.stderr, validationReport.Diagnostics)
-		for _, diagnostic := range validationReport.Diagnostics {
-			if diagnostic.Severity == validator.SeverityWarning {
-				continue
-			}
-			fmt.Fprintf(ctx.stderr, "error: %s: %s\n", diagnostic.Path, diagnostic.Message)
-		}
 		return fmt.Errorf("validation failed with %d error(s)", validationReport.ErrorCount())
 	}
 
-	emitValidationWarnings(ctx.stderr, validationReport.Diagnostics)
-
 	inventoryReport := p.Inventory.Report()
 	fmt.Fprintf(ctx.stdout, "project: %s\n", p.Config.Module.Name)
 	fmt.Fprintf(ctx.stdout, "root: %s\n", p.Root)
@@ -262,34 +253,30 @@ func runValidate(ctx context) error {
 	return nil
 }
 
-func emitValidationWarnings(w io.Writer, diagnostics []validator.Diagnostic) {
+func emitValidatorReport(w io.Writer, report validator.Report) {
-	const lowercaseWarning = "resource filenames should be lowercase"
+	lines := report.SummaryLines(5)
+	emitSummaryLines(w, lines, 20, "validation diagnostics")
+	if line := report.DecompiledSummaryLine(12); line != "" {
+		fmt.Fprintln(w, line)
+	}
+}
 
-	lowercasePaths := make([]string, 0)
+func emitTopdataValidationReport(w io.Writer, report topdata.ValidationReport) {
-	for _, diagnostic := range diagnostics {
+	lines := report.SummaryLines(5)
-		if diagnostic.Severity != validator.SeverityWarning {
+	emitSummaryLines(w, lines, 20, "topdata validation diagnostics")
-			continue
+}
-		}
-		if diagnostic.Message == lowercaseWarning {
-			lowercasePaths = append(lowercasePaths, diagnostic.Path)
-			continue
-		}
-		fmt.Fprintf(w, "warning: %s: %s\n", diagnostic.Path, diagnostic.Message)
-	}
 
-	if len(lowercasePaths) == 0 {
+func emitSummaryLines(w io.Writer, lines []string, maxLines int, label string) {
+	if maxLines <= 0 || len(lines) <= maxLines {
+		for _, line := range lines {
+			fmt.Fprintln(w, line)
+		}
 		return
 	}
-	slices.Sort(lowercasePaths)
+	for _, line := range lines[:maxLines] {
-	examples := lowercasePaths
+		fmt.Fprintln(w, line)
-	if len(examples) > 5 {
-		examples = examples[:5]
 	}
-	message := strings.Join(examples, ", ")
+	fmt.Fprintf(w, "info: %d additional %s omitted\n", len(lines)-maxLines, label)
-	if len(lowercasePaths) > len(examples) {
-		message += fmt.Sprintf(", and %d more", len(lowercasePaths)-len(examples))
-	}
-	fmt.Fprintf(w, "warning: %d resource filenames are not lowercase: %s\n", len(lowercasePaths), message)
 }
 
 func runCompare(ctx context) error {
@@ -346,13 +333,7 @@ func runValidateTopData(ctx context) error {
 	}
 
 	report := topdata.ValidateProject(p)
-	for _, diagnostic := range report.Diagnostics {
+	emitTopdataValidationReport(ctx.stderr, report)
-		if diagnostic.Severity == topdata.SeverityWarning {
-			fmt.Fprintf(ctx.stderr, "warning: %s: %s\n", diagnostic.Path, diagnostic.Message)
-			continue
-		}
-		fmt.Fprintf(ctx.stderr, "error: %s: %s\n", diagnostic.Path, diagnostic.Message)
-	}
 	if report.HasErrors() {
 		return fmt.Errorf("topdata validation failed with %d error(s)", report.ErrorCount())
 	}

internal/pipeline/build.go

@@ -949,7 +949,11 @@ func setModuleHAKList(document *gff.Document, hakNames []string) {
 func validateForBuild(p *project.Project) error {
 	report := validator.ValidateProject(p)
 	if report.HasErrors() {
-		return fmt.Errorf("validation failed with %d error(s)", len(report.Diagnostics))
+		summary := report.SummaryLines(3)
+		if len(summary) > 0 {
+			return fmt.Errorf("validation failed with %d error(s): %s", report.ErrorCount(), summary[0])
+		}
+		return fmt.Errorf("validation failed with %d error(s)", report.ErrorCount())
 	}
 	return nil
 }

internal/topdata/native.go

@@ -141,8 +141,11 @@ func BuildNative(p *project.Project, progress func(string)) (BuildResult, error)
 
 	report := ValidateProject(p)
 	if report.HasErrors() {
-		first := report.Diagnostics[0]
+		summary := report.SummaryLines(3)
-		return BuildResult{}, fmt.Errorf("topdata validation failed with %d error(s): %s: %s", report.ErrorCount(), first.Path, first.Message)
+		if len(summary) > 0 {
+			return BuildResult{}, fmt.Errorf("topdata validation failed with %d error(s): %s", report.ErrorCount(), summary[0])
+		}
+		return BuildResult{}, fmt.Errorf("topdata validation failed with %d error(s)", report.ErrorCount())
 	}
 	return buildNativeUnchecked(p, progress)
 }
@@ -269,8 +272,14 @@ func buildNativeUnchecked(p *project.Project, progress func(string)) (BuildResul
 		globalKeyToID[key] = rowID
 	}
 
+	groupTotals := nativeCompileGroupTotals(collected)
+	currentGroup := ""
 	for _, dataset := range collected {
-		progress(fmt.Sprintf("Compiling %s -> %s", dataset.Dataset.Name, dataset.Dataset.OutputName))
+		group := nativeCompileGroup(dataset.Dataset.Name)
+		if group != currentGroup {
+			currentGroup = group
+			progress(fmt.Sprintf("Compiling %s datasets (%d tables)...", group, groupTotals[group]))
+		}
 		compiled, err := resolveNativeDataset(dataset, globalKeyToID, globalRowByKey, tableRegistry, compiler)
 		if err != nil {
 			return BuildResult{}, err
@@ -298,6 +307,27 @@ func buildNativeUnchecked(p *project.Project, progress func(string)) (BuildResul
 	}, nil
 }
 
+func nativeCompileGroup(datasetName string) string {
+	head, _, ok := strings.Cut(datasetName, "/")
+	if !ok {
+		return datasetName
+	}
+	switch head {
+	case "itemprops", "racialtypes", "damagetypes", "parts", "classes":
+		return head
+	default:
+		return head
+	}
+}
+
+func nativeCompileGroupTotals(collected []nativeCollectedDataset) map[string]int {
+	totals := make(map[string]int)
+	for _, dataset := range collected {
+		totals[nativeCompileGroup(dataset.Dataset.Name)]++
+	}
+	return totals
+}
+
 func discoverNativeDatasets(dataDir string) ([]nativeDataset, error) {
 	var datasets []nativeDataset
 	err := filepath.WalkDir(dataDir, func(path string, d fs.DirEntry, err error) error {

internal/topdata/topdata.go

@@ -36,6 +36,39 @@ type ValidationReport struct {
 	Diagnostics []Diagnostic
 }
 
+func (r ValidationReport) SummaryLines(maxPaths int) []string {
+	type key struct {
+		severity Severity
+		message  string
+	}
+	groups := map[key][]string{}
+	order := make([]key, 0)
+	for _, diagnostic := range r.Diagnostics {
+		k := key{severity: diagnostic.Severity, message: diagnostic.Message}
+		if _, ok := groups[k]; !ok {
+			order = append(order, k)
+		}
+		groups[k] = append(groups[k], diagnostic.Path)
+	}
+	slices.SortFunc(order, func(a, b key) int {
+		if a.severity != b.severity {
+			if a.severity == SeverityError {
+				return -1
+			}
+			return 1
+		}
+		return strings.Compare(a.message, b.message)
+	})
+	lines := make([]string, 0, len(order))
+	for _, k := range order {
+		paths := append([]string(nil), groups[k]...)
+		slices.Sort(paths)
+		paths = slices.Compact(paths)
+		lines = append(lines, formatDiagnosticSummaryLine(string(k.severity), k.message, paths, maxPaths))
+	}
+	return lines
+}
+
 func (r ValidationReport) HasErrors() bool {
 	for _, diagnostic := range r.Diagnostics {
 		if diagnostic.Severity == SeverityError {
@@ -65,6 +98,24 @@ func (r ValidationReport) WarningCount() int {
 	return count
 }
 
+func formatDiagnosticSummaryLine(prefix, message string, paths []string, maxPaths int) string {
+	if len(paths) == 0 {
+		return fmt.Sprintf("%s: %s", prefix, message)
+	}
+	if maxPaths <= 0 {
+		maxPaths = 3
+	}
+	display := paths
+	if len(display) > maxPaths {
+		display = display[:maxPaths]
+	}
+	location := strings.Join(display, ", ")
+	if len(paths) > len(display) {
+		location += fmt.Sprintf(", and %d more", len(paths)-len(display))
+	}
+	return fmt.Sprintf("%s: %s [%s]", prefix, message, location)
+}
+
 type BuildResult struct {
 	Mode         string
 	Output2DADir string

internal/validator/validator.go

@@ -29,6 +29,7 @@ type Diagnostic struct {
 type Report struct {
 	Diagnostics      []Diagnostic
 	SourceCount      int
+	DecompiledModels []string
 }
 
 type loadedDocument struct {
@@ -121,6 +122,11 @@ func ValidateProject(p *project.Project) Report {
 		}
 		base := strings.ToLower(strings.TrimSuffix(filepath.Base(rel), filepath.Ext(rel)))
 		extension := strings.TrimPrefix(strings.ToLower(filepath.Ext(rel)), ".")
+		if extension == "mdl" {
+			if isTextMDL(filepath.Join(p.AssetsDir(), filepath.FromSlash(rel))) {
+				report.DecompiledModels = append(report.DecompiledModels, rel)
+			}
+		}
 		key := strings.ToLower(base) + "." + extension
 		if _, exists := assetIndex[key]; !exists {
 			assetIndex[key] = rel
@@ -153,10 +159,22 @@ func ValidateProject(p *project.Project) Report {
 		}
 		return strings.Compare(a.Message, b.Message)
 	})
+	slices.Sort(report.DecompiledModels)
 
 	return report
 }
 
+func (r Report) SummaryLines(maxPaths int) []string {
+	return summarizeDiagnostics(r.Diagnostics, maxPaths)
+}
+
+func (r Report) DecompiledSummaryLine(maxPaths int) string {
+	if len(r.DecompiledModels) == 0 {
+		return ""
+	}
+	return formatSummaryLine("info", "ASCII/decompiled .mdl files detected", r.DecompiledModels, maxPaths)
+}
+
 func (r *Report) add(path, message string, severity Severity) {
 	r.Diagnostics = append(r.Diagnostics, Diagnostic{Path: path, Message: message, Severity: severity})
 }
@@ -185,6 +203,79 @@ func (r Report) WarningCount() int {
 	return count
 }
 
+func summarizeDiagnostics(diagnostics []Diagnostic, maxPaths int) []string {
+	type key struct {
+		severity Severity
+		message  string
+	}
+	groups := map[key][]string{}
+	order := make([]key, 0)
+	for _, diagnostic := range diagnostics {
+		k := key{severity: diagnostic.Severity, message: diagnostic.Message}
+		if _, ok := groups[k]; !ok {
+			order = append(order, k)
+		}
+		groups[k] = append(groups[k], diagnostic.Path)
+	}
+	slices.SortFunc(order, func(a, b key) int {
+		if a.severity != b.severity {
+			if a.severity == SeverityError {
+				return -1
+			}
+			return 1
+		}
+		return strings.Compare(a.message, b.message)
+	})
+	lines := make([]string, 0, len(order))
+	for _, k := range order {
+		paths := append([]string(nil), groups[k]...)
+		slices.Sort(paths)
+		paths = slices.Compact(paths)
+		lines = append(lines, formatSummaryLine(string(k.severity), k.message, paths, maxPaths))
+	}
+	return lines
+}
+
+func formatSummaryLine(prefix, message string, paths []string, maxPaths int) string {
+	if len(paths) == 0 {
+		return fmt.Sprintf("%s: %s", prefix, message)
+	}
+	if maxPaths <= 0 {
+		maxPaths = 3
+	}
+	display := paths
+	if len(display) > maxPaths {
+		display = display[:maxPaths]
+	}
+	location := strings.Join(display, ", ")
+	if len(paths) > len(display) {
+		location += fmt.Sprintf(", and %d more", len(paths)-len(display))
+	}
+	return fmt.Sprintf("%s: %s [%s]", prefix, message, location)
+}
+
+func isTextMDL(path string) bool {
+	data, err := os.ReadFile(path)
+	if err != nil || len(data) == 0 {
+		return false
+	}
+	if len(data) > 4096 {
+		data = data[:4096]
+	}
+	if strings.IndexByte(string(data), 0) >= 0 {
+		return false
+	}
+	printable := 0
+	for _, b := range data {
+		if b == '\n' || b == '\r' || b == '\t' || (b >= 32 && b <= 126) {
+			printable++
+		}
+	}
+	ratio := float64(printable) / float64(len(data))
+	lower := strings.ToLower(string(data))
+	return strings.Contains(lower, "newmodel ") || strings.Contains(lower, "donemodel") || ratio >= 0.9
+}
+
 func loadDocument(path string) (gff.Document, string, string, error) {
 	raw, err := os.ReadFile(path)
 	if err != nil {

internal/validator/validator_test.go

@@ -199,6 +199,52 @@ func TestValidateProjectWarnsForMissingScriptReferences(t *testing.T) {
 	}
 }
 
+func TestValidateProjectReportsDecompiledModelsWithoutFailingValidation(t *testing.T) {
+	root := t.TempDir()
+	mustMkdir(t, filepath.Join(root, "src"))
+	mustMkdir(t, filepath.Join(root, "assets", "models"))
+	mustMkdir(t, filepath.Join(root, "build"))
+
+	mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{
+  "module": {
+    "name": "Test Module",
+    "resref": "testmod"
+  },
+  "paths": {
+    "source": "src",
+    "assets": "assets",
+    "build": "build"
+  }
+}
+`)
+	mustWriteFile(t, filepath.Join(root, "assets", "models", "plain_text.mdl"), "newmodel plain_text\nsetsupermodel plain_text NULL\nbeginmodelgeom plain_text\nendmodelgeom plain_text\ndonemodel plain_text\n")
+	if err := os.WriteFile(filepath.Join(root, "assets", "models", "binary_ok.mdl"), []byte{0x00, 0x01, 0x02, 0x03}, 0o644); err != nil {
+		t.Fatalf("write binary mdl: %v", err)
+	}
+
+	p, err := project.Load(root)
+	if err != nil {
+		t.Fatalf("load project: %v", err)
+	}
+	if err := p.ValidateLayout(); err != nil {
+		t.Fatalf("validate layout: %v", err)
+	}
+	if err := p.Scan(); err != nil {
+		t.Fatalf("scan: %v", err)
+	}
+
+	report := ValidateProject(p)
+	if report.HasErrors() {
+		t.Fatalf("expected no validation errors, got %#v", report.Diagnostics)
+	}
+	if len(report.DecompiledModels) != 1 {
+		t.Fatalf("expected 1 decompiled model, got %#v", report.DecompiledModels)
+	}
+	if report.DecompiledModels[0] != "models/plain_text.mdl" {
+		t.Fatalf("unexpected decompiled model path: %#v", report.DecompiledModels)
+	}
+}
+
 func mustMkdir(t *testing.T, path string) {
 	t.Helper()
 	if err := os.MkdirAll(path, 0o755); err != nil {