From 9e13bb9babcbe68dcea3df55d9e47343eb9eb2ee Mon Sep 17 00:00:00 2001 From: vickydotbat Date: Thu, 9 Apr 2026 21:56:53 +0200 Subject: [PATCH] Improved Logging --- internal/app/app.go | 61 ++++++------------ internal/pipeline/build.go | 6 +- internal/topdata/native.go | 36 ++++++++++- internal/topdata/topdata.go | 51 +++++++++++++++ internal/validator/validator.go | 95 +++++++++++++++++++++++++++- internal/validator/validator_test.go | 46 ++++++++++++++ 6 files changed, 249 insertions(+), 46 deletions(-) diff --git a/internal/app/app.go b/internal/app/app.go index d6420c3..f2c6b9f 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -6,7 +6,6 @@ import ( "io" "os" "path/filepath" - "slices" "strings" "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/pipeline" @@ -235,19 +234,11 @@ func runValidate(ctx context) error { } validationReport := validator.ValidateProject(p) + emitValidatorReport(ctx.stderr, validationReport) if validationReport.HasErrors() { - emitValidationWarnings(ctx.stderr, validationReport.Diagnostics) - for _, diagnostic := range validationReport.Diagnostics { - if diagnostic.Severity == validator.SeverityWarning { - continue - } - fmt.Fprintf(ctx.stderr, "error: %s: %s\n", diagnostic.Path, diagnostic.Message) - } return fmt.Errorf("validation failed with %d error(s)", validationReport.ErrorCount()) } - emitValidationWarnings(ctx.stderr, validationReport.Diagnostics) - inventoryReport := p.Inventory.Report() fmt.Fprintf(ctx.stdout, "project: %s\n", p.Config.Module.Name) fmt.Fprintf(ctx.stdout, "root: %s\n", p.Root) @@ -262,34 +253,30 @@ func runValidate(ctx context) error { return nil } -func emitValidationWarnings(w io.Writer, diagnostics []validator.Diagnostic) { - const lowercaseWarning = "resource filenames should be lowercase" - - lowercasePaths := make([]string, 0) - for _, diagnostic := range diagnostics { - if diagnostic.Severity != validator.SeverityWarning { - continue - } - if diagnostic.Message == lowercaseWarning { - lowercasePaths = append(lowercasePaths, diagnostic.Path) - continue - } - fmt.Fprintf(w, "warning: %s: %s\n", diagnostic.Path, diagnostic.Message) +func emitValidatorReport(w io.Writer, report validator.Report) { + lines := report.SummaryLines(5) + emitSummaryLines(w, lines, 20, "validation diagnostics") + if line := report.DecompiledSummaryLine(12); line != "" { + fmt.Fprintln(w, line) } +} - if len(lowercasePaths) == 0 { +func emitTopdataValidationReport(w io.Writer, report topdata.ValidationReport) { + lines := report.SummaryLines(5) + emitSummaryLines(w, lines, 20, "topdata validation diagnostics") +} + +func emitSummaryLines(w io.Writer, lines []string, maxLines int, label string) { + if maxLines <= 0 || len(lines) <= maxLines { + for _, line := range lines { + fmt.Fprintln(w, line) + } return } - slices.Sort(lowercasePaths) - examples := lowercasePaths - if len(examples) > 5 { - examples = examples[:5] + for _, line := range lines[:maxLines] { + fmt.Fprintln(w, line) } - message := strings.Join(examples, ", ") - if len(lowercasePaths) > len(examples) { - message += fmt.Sprintf(", and %d more", len(lowercasePaths)-len(examples)) - } - fmt.Fprintf(w, "warning: %d resource filenames are not lowercase: %s\n", len(lowercasePaths), message) + fmt.Fprintf(w, "info: %d additional %s omitted\n", len(lines)-maxLines, label) } func runCompare(ctx context) error { @@ -346,13 +333,7 @@ func runValidateTopData(ctx context) error { } report := topdata.ValidateProject(p) - for _, diagnostic := range report.Diagnostics { - if diagnostic.Severity == topdata.SeverityWarning { - fmt.Fprintf(ctx.stderr, "warning: %s: %s\n", diagnostic.Path, diagnostic.Message) - continue - } - fmt.Fprintf(ctx.stderr, "error: %s: %s\n", diagnostic.Path, diagnostic.Message) - } + emitTopdataValidationReport(ctx.stderr, report) if report.HasErrors() { return fmt.Errorf("topdata validation failed with %d error(s)", report.ErrorCount()) } diff --git a/internal/pipeline/build.go b/internal/pipeline/build.go index 96b9e25..601862d 100644 --- a/internal/pipeline/build.go +++ b/internal/pipeline/build.go @@ -949,7 +949,11 @@ func setModuleHAKList(document *gff.Document, hakNames []string) { func validateForBuild(p *project.Project) error { report := validator.ValidateProject(p) if report.HasErrors() { - return fmt.Errorf("validation failed with %d error(s)", len(report.Diagnostics)) + summary := report.SummaryLines(3) + if len(summary) > 0 { + return fmt.Errorf("validation failed with %d error(s): %s", report.ErrorCount(), summary[0]) + } + return fmt.Errorf("validation failed with %d error(s)", report.ErrorCount()) } return nil } diff --git a/internal/topdata/native.go b/internal/topdata/native.go index 328c7e7..aac2fa7 100644 --- a/internal/topdata/native.go +++ b/internal/topdata/native.go @@ -141,8 +141,11 @@ func BuildNative(p *project.Project, progress func(string)) (BuildResult, error) report := ValidateProject(p) if report.HasErrors() { - first := report.Diagnostics[0] - return BuildResult{}, fmt.Errorf("topdata validation failed with %d error(s): %s: %s", report.ErrorCount(), first.Path, first.Message) + summary := report.SummaryLines(3) + if len(summary) > 0 { + return BuildResult{}, fmt.Errorf("topdata validation failed with %d error(s): %s", report.ErrorCount(), summary[0]) + } + return BuildResult{}, fmt.Errorf("topdata validation failed with %d error(s)", report.ErrorCount()) } return buildNativeUnchecked(p, progress) } @@ -269,8 +272,14 @@ func buildNativeUnchecked(p *project.Project, progress func(string)) (BuildResul globalKeyToID[key] = rowID } + groupTotals := nativeCompileGroupTotals(collected) + currentGroup := "" for _, dataset := range collected { - progress(fmt.Sprintf("Compiling %s -> %s", dataset.Dataset.Name, dataset.Dataset.OutputName)) + group := nativeCompileGroup(dataset.Dataset.Name) + if group != currentGroup { + currentGroup = group + progress(fmt.Sprintf("Compiling %s datasets (%d tables)...", group, groupTotals[group])) + } compiled, err := resolveNativeDataset(dataset, globalKeyToID, globalRowByKey, tableRegistry, compiler) if err != nil { return BuildResult{}, err @@ -298,6 +307,27 @@ func buildNativeUnchecked(p *project.Project, progress func(string)) (BuildResul }, nil } +func nativeCompileGroup(datasetName string) string { + head, _, ok := strings.Cut(datasetName, "/") + if !ok { + return datasetName + } + switch head { + case "itemprops", "racialtypes", "damagetypes", "parts", "classes": + return head + default: + return head + } +} + +func nativeCompileGroupTotals(collected []nativeCollectedDataset) map[string]int { + totals := make(map[string]int) + for _, dataset := range collected { + totals[nativeCompileGroup(dataset.Dataset.Name)]++ + } + return totals +} + func discoverNativeDatasets(dataDir string) ([]nativeDataset, error) { var datasets []nativeDataset err := filepath.WalkDir(dataDir, func(path string, d fs.DirEntry, err error) error { diff --git a/internal/topdata/topdata.go b/internal/topdata/topdata.go index 46ebe41..a02257a 100644 --- a/internal/topdata/topdata.go +++ b/internal/topdata/topdata.go @@ -36,6 +36,39 @@ type ValidationReport struct { Diagnostics []Diagnostic } +func (r ValidationReport) SummaryLines(maxPaths int) []string { + type key struct { + severity Severity + message string + } + groups := map[key][]string{} + order := make([]key, 0) + for _, diagnostic := range r.Diagnostics { + k := key{severity: diagnostic.Severity, message: diagnostic.Message} + if _, ok := groups[k]; !ok { + order = append(order, k) + } + groups[k] = append(groups[k], diagnostic.Path) + } + slices.SortFunc(order, func(a, b key) int { + if a.severity != b.severity { + if a.severity == SeverityError { + return -1 + } + return 1 + } + return strings.Compare(a.message, b.message) + }) + lines := make([]string, 0, len(order)) + for _, k := range order { + paths := append([]string(nil), groups[k]...) + slices.Sort(paths) + paths = slices.Compact(paths) + lines = append(lines, formatDiagnosticSummaryLine(string(k.severity), k.message, paths, maxPaths)) + } + return lines +} + func (r ValidationReport) HasErrors() bool { for _, diagnostic := range r.Diagnostics { if diagnostic.Severity == SeverityError { @@ -65,6 +98,24 @@ func (r ValidationReport) WarningCount() int { return count } +func formatDiagnosticSummaryLine(prefix, message string, paths []string, maxPaths int) string { + if len(paths) == 0 { + return fmt.Sprintf("%s: %s", prefix, message) + } + if maxPaths <= 0 { + maxPaths = 3 + } + display := paths + if len(display) > maxPaths { + display = display[:maxPaths] + } + location := strings.Join(display, ", ") + if len(paths) > len(display) { + location += fmt.Sprintf(", and %d more", len(paths)-len(display)) + } + return fmt.Sprintf("%s: %s [%s]", prefix, message, location) +} + type BuildResult struct { Mode string Output2DADir string diff --git a/internal/validator/validator.go b/internal/validator/validator.go index 6f80e93..cb9cdbc 100644 --- a/internal/validator/validator.go +++ b/internal/validator/validator.go @@ -27,8 +27,9 @@ type Diagnostic struct { } type Report struct { - Diagnostics []Diagnostic - SourceCount int + Diagnostics []Diagnostic + SourceCount int + DecompiledModels []string } type loadedDocument struct { @@ -121,6 +122,11 @@ func ValidateProject(p *project.Project) Report { } base := strings.ToLower(strings.TrimSuffix(filepath.Base(rel), filepath.Ext(rel))) extension := strings.TrimPrefix(strings.ToLower(filepath.Ext(rel)), ".") + if extension == "mdl" { + if isTextMDL(filepath.Join(p.AssetsDir(), filepath.FromSlash(rel))) { + report.DecompiledModels = append(report.DecompiledModels, rel) + } + } key := strings.ToLower(base) + "." + extension if _, exists := assetIndex[key]; !exists { assetIndex[key] = rel @@ -153,10 +159,22 @@ func ValidateProject(p *project.Project) Report { } return strings.Compare(a.Message, b.Message) }) + slices.Sort(report.DecompiledModels) return report } +func (r Report) SummaryLines(maxPaths int) []string { + return summarizeDiagnostics(r.Diagnostics, maxPaths) +} + +func (r Report) DecompiledSummaryLine(maxPaths int) string { + if len(r.DecompiledModels) == 0 { + return "" + } + return formatSummaryLine("info", "ASCII/decompiled .mdl files detected", r.DecompiledModels, maxPaths) +} + func (r *Report) add(path, message string, severity Severity) { r.Diagnostics = append(r.Diagnostics, Diagnostic{Path: path, Message: message, Severity: severity}) } @@ -185,6 +203,79 @@ func (r Report) WarningCount() int { return count } +func summarizeDiagnostics(diagnostics []Diagnostic, maxPaths int) []string { + type key struct { + severity Severity + message string + } + groups := map[key][]string{} + order := make([]key, 0) + for _, diagnostic := range diagnostics { + k := key{severity: diagnostic.Severity, message: diagnostic.Message} + if _, ok := groups[k]; !ok { + order = append(order, k) + } + groups[k] = append(groups[k], diagnostic.Path) + } + slices.SortFunc(order, func(a, b key) int { + if a.severity != b.severity { + if a.severity == SeverityError { + return -1 + } + return 1 + } + return strings.Compare(a.message, b.message) + }) + lines := make([]string, 0, len(order)) + for _, k := range order { + paths := append([]string(nil), groups[k]...) + slices.Sort(paths) + paths = slices.Compact(paths) + lines = append(lines, formatSummaryLine(string(k.severity), k.message, paths, maxPaths)) + } + return lines +} + +func formatSummaryLine(prefix, message string, paths []string, maxPaths int) string { + if len(paths) == 0 { + return fmt.Sprintf("%s: %s", prefix, message) + } + if maxPaths <= 0 { + maxPaths = 3 + } + display := paths + if len(display) > maxPaths { + display = display[:maxPaths] + } + location := strings.Join(display, ", ") + if len(paths) > len(display) { + location += fmt.Sprintf(", and %d more", len(paths)-len(display)) + } + return fmt.Sprintf("%s: %s [%s]", prefix, message, location) +} + +func isTextMDL(path string) bool { + data, err := os.ReadFile(path) + if err != nil || len(data) == 0 { + return false + } + if len(data) > 4096 { + data = data[:4096] + } + if strings.IndexByte(string(data), 0) >= 0 { + return false + } + printable := 0 + for _, b := range data { + if b == '\n' || b == '\r' || b == '\t' || (b >= 32 && b <= 126) { + printable++ + } + } + ratio := float64(printable) / float64(len(data)) + lower := strings.ToLower(string(data)) + return strings.Contains(lower, "newmodel ") || strings.Contains(lower, "donemodel") || ratio >= 0.9 +} + func loadDocument(path string) (gff.Document, string, string, error) { raw, err := os.ReadFile(path) if err != nil { diff --git a/internal/validator/validator_test.go b/internal/validator/validator_test.go index 994fa52..a7030c2 100644 --- a/internal/validator/validator_test.go +++ b/internal/validator/validator_test.go @@ -199,6 +199,52 @@ func TestValidateProjectWarnsForMissingScriptReferences(t *testing.T) { } } +func TestValidateProjectReportsDecompiledModelsWithoutFailingValidation(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src")) + mustMkdir(t, filepath.Join(root, "assets", "models")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{ + "module": { + "name": "Test Module", + "resref": "testmod" + }, + "paths": { + "source": "src", + "assets": "assets", + "build": "build" + } +} +`) + mustWriteFile(t, filepath.Join(root, "assets", "models", "plain_text.mdl"), "newmodel plain_text\nsetsupermodel plain_text NULL\nbeginmodelgeom plain_text\nendmodelgeom plain_text\ndonemodel plain_text\n") + if err := os.WriteFile(filepath.Join(root, "assets", "models", "binary_ok.mdl"), []byte{0x00, 0x01, 0x02, 0x03}, 0o644); err != nil { + t.Fatalf("write binary mdl: %v", err) + } + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + + report := ValidateProject(p) + if report.HasErrors() { + t.Fatalf("expected no validation errors, got %#v", report.Diagnostics) + } + if len(report.DecompiledModels) != 1 { + t.Fatalf("expected 1 decompiled model, got %#v", report.DecompiledModels) + } + if report.DecompiledModels[0] != "models/plain_text.mdl" { + t.Fatalf("unexpected decompiled model path: %#v", report.DecompiledModels) + } +} + func mustMkdir(t *testing.T, path string) { t.Helper() if err := os.MkdirAll(path, 0o755); err != nil {