From 338b4abf3babfe54c763cb83d66d7695ab20675e Mon Sep 17 00:00:00 2001 From: vickydotbat Date: Sat, 18 Apr 2026 08:57:33 +0200 Subject: [PATCH] Optimization Pass --- internal/app/app.go | 33 ++- internal/topdata/native.go | 31 ++- internal/topdata/top_package.go | 135 ++++++++++ internal/topdata/topdata.go | 406 ++++++++++++++++++++++++++++++- internal/topdata/topdata_test.go | 160 ++++++++++++ 5 files changed, 746 insertions(+), 19 deletions(-) diff --git a/internal/app/app.go b/internal/app/app.go index 4fabd0d..e4a24a4 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -181,8 +181,8 @@ func runBuildModule(ctx context) error { type buildHAKOptions struct { filteredHAKs []string filteredArchives []string - sourceManifest string - planOnly bool + sourceManifest string + planOnly bool } func runBuildHAKs(ctx context) error { @@ -436,7 +436,12 @@ func runBuildTopData(ctx context) error { return err } - result, err := topdata.BuildAndPackage(p, func(message string) { + opts, err := parseBuildTopDataArgs("build-topdata", ctx.args[1:]) + if err != nil { + return err + } + + result, err := topdata.BuildAndPackageWithOptions(p, opts, func(message string) { fmt.Fprintf(ctx.stdout, "[build-topdata] %s\n", message) }) if err != nil { @@ -467,7 +472,12 @@ func runBuildTopPackage(ctx context) error { return err } - result, err := topdata.BuildAndPackage(p, func(message string) { + opts, err := parseBuildTopDataArgs("build-top-package", ctx.args[1:]) + if err != nil { + return err + } + + result, err := topdata.BuildAndPackageWithOptions(p, opts, func(message string) { fmt.Fprintf(ctx.stdout, "[build-top-package] %s\n", message) }) if err != nil { @@ -492,6 +502,21 @@ func runBuildTopPackage(ctx context) error { return nil } +func parseBuildTopDataArgs(commandName string, args []string) (topdata.BuildAndPackageOptions, error) { + opts := topdata.BuildAndPackageOptions{} + for _, arg := range args { + switch arg { + case "--force": + opts.Force = true + case "-h", "--help": + return opts, fmt.Errorf("usage: %s [--force]", commandName) + default: + return opts, fmt.Errorf("unknown %s argument %q", commandName, arg) + } + } + return opts, nil +} + func runCompareTopData(ctx context) error { p, err := loadProject(ctx.cwd) if err != nil { diff --git a/internal/topdata/native.go b/internal/topdata/native.go index 9e452cb..adee590 100644 --- a/internal/topdata/native.go +++ b/internal/topdata/native.go @@ -2063,6 +2063,9 @@ func featSourceID(value any) (int, error) { } func (c *featGeneratedContext) resolveGeneratedFeatIdentityBySource(familyKey, slug string, rawSource any) (string, int, bool, error) { + if ref, ok := rawSource.(string); ok && strings.HasPrefix(ref, "feat:") { + return c.resolveGeneratedFeatIdentityFromRef(familyKey, ref) + } if rawSource != nil && rawSource != nullValue { if parsedID, err := featSourceID(rawSource); err == nil && parsedID > 0 { if generatedKey, ok := c.generatedFeatKeyForID(familyKey, parsedID); ok { @@ -2073,18 +2076,7 @@ func (c *featGeneratedContext) resolveGeneratedFeatIdentityBySource(familyKey, s } if rawMap, ok := rawSource.(map[string]any); ok { if ref, ok := rawMap["id"].(string); ok && strings.HasPrefix(ref, "feat:") { - if generatedKey, rowID, ok := c.generatedFeatKeyFromCanonicalAlias(ref); ok { - return generatedKey, rowID, true, nil - } - if rowID, hasID, err := c.featID(ref, nil); err != nil { - return "", 0, false, err - } else if hasID { - if generatedKey, ok := c.generatedFeatKeyForID(familyKey, rowID); ok { - return generatedKey, rowID, true, nil - } - } - rowID, hasID, err := c.featID(ref, nil) - return ref, rowID, hasID, err + return c.resolveGeneratedFeatIdentityFromRef(familyKey, ref) } } featKey := c.preferredGeneratedFeatKey(familyKey, slug) @@ -2092,6 +2084,21 @@ func (c *featGeneratedContext) resolveGeneratedFeatIdentityBySource(familyKey, s return featKey, rowID, hasID, err } +func (c *featGeneratedContext) resolveGeneratedFeatIdentityFromRef(familyKey, ref string) (string, int, bool, error) { + if generatedKey, rowID, ok := c.generatedFeatKeyFromCanonicalAlias(ref); ok { + return generatedKey, rowID, true, nil + } + if rowID, hasID, err := c.featID(ref, nil); err != nil { + return "", 0, false, err + } else if hasID { + if generatedKey, ok := c.generatedFeatKeyForID(familyKey, rowID); ok { + return generatedKey, rowID, true, nil + } + } + rowID, hasID, err := c.featID(ref, nil) + return ref, rowID, hasID, err +} + func (c *featGeneratedContext) resolveGeneratedFeatIdentity(spec familyExpansionSpec, slug string, sourceRow map[string]any) (string, int, bool, error) { if spec.IdentitySource == "child_source_value" { rawSource, ok := lookupField(sourceRow, spec.ChildSource.Column) diff --git a/internal/topdata/top_package.go b/internal/topdata/top_package.go index 038807c..608736d 100644 --- a/internal/topdata/top_package.go +++ b/internal/topdata/top_package.go @@ -56,9 +56,41 @@ func BuildPackage(p *project.Project, progress func(string)) (PackageResult, err } func BuildAndPackage(p *project.Project, progress func(string)) (PackageResult, error) { + return BuildAndPackageWithOptions(p, BuildAndPackageOptions{}, progress) +} + +type BuildAndPackageOptions struct { + Force bool +} + +func BuildAndPackageWithOptions(p *project.Project, opts BuildAndPackageOptions, progress func(string)) (PackageResult, error) { if progress == nil { progress = func(string) {} } + if !opts.Force { + incrementalResult, ok, err := currentTopPackageResult(p) + if err != nil { + return PackageResult{}, err + } + if ok { + progress("Topdata outputs are current; skipping native build and top package refresh.") + return incrementalResult, nil + } + + nativeResult, ok, err := currentCompiledBuildResult(p) + if err != nil { + return PackageResult{}, err + } + if ok { + progress("Compiled topdata outputs are current; refreshing sow_top.hak and sow_tlk.tlk only.") + return packageBuiltTopData(p, nativeResult, progress) + } + } + + return buildAndPackageFull(p, progress) +} + +func buildAndPackageFull(p *project.Project, progress func(string)) (PackageResult, error) { nativeResult, err := BuildNative(p, progress) if err != nil { return PackageResult{}, err @@ -323,6 +355,109 @@ func packagedBuildResult(p *project.Project) (BuildResult, error) { }, nil } +func currentCompiledBuildResult(p *project.Project) (BuildResult, bool, error) { + nativeResult, err := packagedBuildResult(p) + if err == nil { + return nativeResult, true, nil + } + if strings.Contains(err.Error(), "run build-topdata first") { + return BuildResult{}, false, nil + } + return BuildResult{}, false, err +} + +func currentTopPackageResult(p *project.Project) (PackageResult, bool, error) { + nativeResult, ok, err := currentCompiledBuildResult(p) + if err != nil || !ok { + return PackageResult{}, ok, err + } + + outputHAK := filepath.Join(p.BuildDir(), PackageHAKFileName) + outputTLK := filepath.Join(p.BuildDir(), PackageTLKFileName) + hakInfo, err := os.Stat(outputHAK) + if err != nil { + if os.IsNotExist(err) { + return PackageResult{}, false, nil + } + return PackageResult{}, false, fmt.Errorf("stat top package hak %s: %w", outputHAK, err) + } + tlkInfo, err := os.Stat(outputTLK) + if err != nil { + if os.IsNotExist(err) { + return PackageResult{}, false, nil + } + return PackageResult{}, false, fmt.Errorf("stat top package tlk %s: %w", outputTLK, err) + } + + newestSource, _, err := newestTopDataSource(p.TopDataSourceDir()) + if err != nil { + return PackageResult{}, false, err + } + newestCompiled, err := newestFileModTime(nativeResult.Output2DADir, ".2da") + if err != nil { + return PackageResult{}, false, err + } + compiledTLKInfo, err := os.Stat(compiledTLKOutputPath(p)) + if err != nil { + return PackageResult{}, false, fmt.Errorf("stat compiled tlk output %s: %w", compiledTLKOutputPath(p), err) + } + if compiledTLKInfo.ModTime().After(newestCompiled) { + newestCompiled = compiledTLKInfo.ModTime() + } + newestInput := newestSource + if newestCompiled.After(newestInput) { + newestInput = newestCompiled + } + + oldestPackage := hakInfo.ModTime() + if tlkInfo.ModTime().Before(oldestPackage) { + oldestPackage = tlkInfo.ModTime() + } + if !newestInput.IsZero() && newestInput.After(oldestPackage) { + return PackageResult{}, false, nil + } + + resources, assetFiles, err := collectTopPackageResources(p, nativeResult.Output2DADir) + if err != nil { + return PackageResult{}, false, err + } + return PackageResult{ + Mode: "incremental-noop", + Output2DADir: nativeResult.Output2DADir, + OutputTLKDir: nativeResult.OutputTLKDir, + OutputHAKPath: outputHAK, + OutputTLKPath: outputTLK, + Files2DA: nativeResult.Files2DA, + FilesTLK: nativeResult.FilesTLK, + HAKResources: len(resources), + AssetFiles: assetFiles, + WikiOutputDir: nativeResult.WikiOutputDir, + WikiPages: nativeResult.WikiPages, + WikiStatus: nativeResult.WikiStatus, + }, true, nil +} + +func newestFileModTime(dir, extension string) (time.Time, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return time.Time{}, fmt.Errorf("read output dir %s: %w", dir, err) + } + newest := time.Time{} + for _, entry := range entries { + if entry.IsDir() || !strings.EqualFold(filepath.Ext(entry.Name()), extension) { + continue + } + info, err := entry.Info() + if err != nil { + return time.Time{}, fmt.Errorf("stat output %s: %w", filepath.Join(dir, entry.Name()), err) + } + if newest.IsZero() || info.ModTime().After(newest) { + newest = info.ModTime() + } + } + return newest, nil +} + func countCompiledFiles(dir, extension string) (int, time.Time, error) { entries, err := os.ReadDir(dir) if err != nil { diff --git a/internal/topdata/topdata.go b/internal/topdata/topdata.go index 6925829..98d3e37 100644 --- a/internal/topdata/topdata.go +++ b/internal/topdata/topdata.go @@ -265,6 +265,7 @@ func ValidateProject(p *project.Project) ValidationReport { validateTopPackageAssets(sourceDir, dataDir, &report) validateNativeOutputCatalog(dataDir, &report) validateNativeLockAllocation(dataDir, &report) + validateGeneratedFeatFamilies(dataDir, &report) validateItempropsRegistryGraph(dataDir, &report) if _, err := os.Stat(statePath); err == nil { report.Files++ @@ -289,6 +290,7 @@ func validateJSONFile(path, domain string, report *ValidationReport) error { } var payload any + validateDuplicateJSONKeys(path, raw, report) if err := json.Unmarshal(raw, &payload); err != nil { report.Diagnostics = append(report.Diagnostics, Diagnostic{ Severity: SeverityError, @@ -321,6 +323,62 @@ func validateJSONFile(path, domain string, report *ValidationReport) error { return nil } +func validateDuplicateJSONKeys(path string, raw []byte, report *ValidationReport) { + decoder := json.NewDecoder(bytes.NewReader(raw)) + if err := scanJSONValueForDuplicateKeys(decoder, path, report); err != nil { + return + } +} + +func scanJSONValueForDuplicateKeys(decoder *json.Decoder, path string, report *ValidationReport) error { + token, err := decoder.Token() + if err != nil { + return err + } + switch delim := token.(type) { + case json.Delim: + switch delim { + case '{': + seen := map[string]struct{}{} + for decoder.More() { + rawKey, err := decoder.Token() + if err != nil { + return err + } + key, ok := rawKey.(string) + if !ok { + return fmt.Errorf("expected object key") + } + if _, ok := seen[key]; ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("duplicate JSON object key %q", key), + }) + } + seen[key] = struct{}{} + if err := scanJSONValueForDuplicateKeys(decoder, path, report); err != nil { + return err + } + } + _, err := decoder.Token() + return err + case '[': + for decoder.More() { + if err := scanJSONValueForDuplicateKeys(decoder, path, report); err != nil { + return err + } + } + _, err := decoder.Token() + return err + default: + return nil + } + default: + return nil + } +} + func validateDataObject(path string, obj map[string]any, report *ValidationReport) { base := filepath.Base(path) slashed := filepath.ToSlash(path) @@ -689,10 +747,16 @@ func validateLegacyDialogEntries(path string, obj map[string]any, report *Valida } func validateLockObject(path string, obj map[string]any, report *ValidationReport) { + ids := map[int]string{} + checkDuplicateIDs := !strings.Contains(filepath.ToSlash(path), "/data/itemprops/registry/lock.json") for key, value := range obj { + var rowID int switch value.(type) { case float64: // JSON numbers land here. + if parsed, err := asInt(value); err == nil { + rowID = parsed + } default: report.Diagnostics = append(report.Diagnostics, Diagnostic{ Severity: SeverityError, @@ -700,6 +764,19 @@ func validateLockObject(path string, obj map[string]any, report *ValidationRepor Message: fmt.Sprintf("lock entry %q must be numeric", key), }) } + if checkDuplicateIDs { + if rowID <= 0 { + continue + } + if other, ok := ids[rowID]; ok && other != key { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("lock id collision between %q and %q at id %d", other, key, rowID), + }) + } + ids[rowID] = key + } } } @@ -1030,6 +1107,304 @@ func validateNativeLockAllocation(dataDir string, report *ValidationReport) { } } +type requiredFeatFamily struct { + FamilyKey string + Dataset string + Column string + Predicate string +} + +func validateGeneratedFeatFamilies(dataDir string, report *ValidationReport) { + featDataset, ok, err := discoverNamedNativeDataset(dataDir, "feat") + if err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: dataDir, + Message: fmt.Sprintf("discover feat dataset for generated family validation: %v", err), + }) + return + } + if !ok { + return + } + lockData, err := loadLockfile(featDataset.LockPath) + if err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: featDataset.LockPath, + Message: fmt.Sprintf("load feat lockfile for generated family validation: %v", err), + }) + return + } + ctx, err := newFeatGeneratedContext(featDataset, lockData) + if err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: featDataset.GeneratedDir, + Message: fmt.Sprintf("prepare generated family validation: %v", err), + }) + return + } + specs, specPaths, err := collectGeneratedFamilySpecs(featDataset.GeneratedDir) + if err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: featDataset.GeneratedDir, + Message: fmt.Sprintf("load generated family specs: %v", err), + }) + return + } + for familyKey, spec := range specs { + ctx.familySpecs[familyKey] = spec + } + + required := []requiredFeatFamily{ + {FamilyKey: "skillfocus", Dataset: "skills", Predicate: "accessible"}, + {FamilyKey: "greaterskillfocus", Dataset: "skills", Predicate: "accessible"}, + {FamilyKey: "weaponfocus", Dataset: "baseitems", Column: "WeaponFocusFeat"}, + {FamilyKey: "weaponspecialization", Dataset: "baseitems", Column: "WeaponSpecializationFeat"}, + {FamilyKey: "improvedcritical", Dataset: "baseitems", Column: "WeaponImprovedCriticalFeat"}, + {FamilyKey: "overwhelmingcritical", Dataset: "baseitems", Column: "EpicWeaponOverwhelmingCriticalFeat"}, + {FamilyKey: "greaterweaponfocus", Dataset: "baseitems", Column: "EpicWeaponFocusFeat"}, + {FamilyKey: "greaterweaponspecialization", Dataset: "baseitems", Column: "EpicWeaponSpecializationFeat"}, + {FamilyKey: "weaponofchoice", Dataset: "baseitems", Column: "WeaponOfChoiceFeat"}, + } + for _, requirement := range required { + spec, ok := specs[requirement.FamilyKey] + if !ok { + continue + } + path := specPaths[requirement.FamilyKey] + if spec.ChildSource.Dataset != requirement.Dataset || + spec.ChildSource.Column != requirement.Column || + spec.ChildSource.Predicate != requirement.Predicate { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("generated feat family %q must use child source dataset=%q column=%q predicate=%q", requirement.FamilyKey, requirement.Dataset, requirement.Column, requirement.Predicate), + }) + continue + } + validateGeneratedFeatFamilyCompleteness(path, spec, ctx, report) + } + validateRequiredProductionGeneratedFamilies(featDataset.GeneratedDir, specs, report) +} + +func validateRequiredProductionGeneratedFamilies(path string, specs map[string]familyExpansionSpec, report *ValidationReport) { + weaponFamilies := []string{ + "weaponfocus", + "weaponspecialization", + "improvedcritical", + "overwhelmingcritical", + "greaterweaponfocus", + "greaterweaponspecialization", + } + for _, familyKey := range weaponFamilies { + if _, ok := specs[familyKey]; !ok { + return + } + } + if _, ok := specs["weaponofchoice"]; !ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: "missing generated feat family \"weaponofchoice\"", + }) + } +} + +func discoverNamedNativeDataset(dataDir, name string) (nativeDataset, bool, error) { + datasets, err := discoverNativeDatasets(dataDir) + if err != nil { + return nativeDataset{}, false, err + } + for _, dataset := range datasets { + if dataset.Name == name { + return dataset, true, nil + } + } + return nativeDataset{}, false, nil +} + +func collectGeneratedFamilySpecs(generatedDir string) (map[string]familyExpansionSpec, map[string]string, error) { + paths, err := collectModulePaths(generatedDir) + if err != nil { + return nil, nil, err + } + specs := map[string]familyExpansionSpec{} + pathsByFamily := map[string]string{} + for _, path := range paths { + obj, err := loadJSONObject(path) + if err != nil { + return nil, nil, err + } + if !isFamilyExpansionObject(obj) { + continue + } + spec, err := parseFamilyExpansionSpec(path, obj) + if err != nil { + return nil, nil, err + } + if previous, ok := pathsByFamily[spec.FamilyKey]; ok { + return nil, nil, fmt.Errorf("generated family %q is declared by both %s and %s", spec.FamilyKey, previous, path) + } + specs[spec.FamilyKey] = spec + pathsByFamily[spec.FamilyKey] = path + } + return specs, pathsByFamily, nil +} + +func validateGeneratedFeatFamilyCompleteness(path string, spec familyExpansionSpec, ctx *featGeneratedContext, report *ValidationReport) { + sourceRows, err := ctx.datasetRows(spec.ChildSource.Dataset) + if err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: err.Error(), + }) + return + } + moduleData, err := buildFamilyExpansionGeneratedModule(path, map[string]any{ + "family": spec.Family, + "family_key": spec.FamilyKey, + "template": spec.Template, + "name_prefix": spec.NamePrefix, + "label_prefix": spec.LabelPrefix, + "constant_prefix": spec.ConstantPrefix, + "template_fields": validationStringSliceToAny(spec.TemplateFields), + "default_fields": spec.DefaultFields, + "apply_after_modules": spec.ApplyAfterModules, + "child_ref_field": spec.ChildRefField, + "identity_source": spec.IdentitySource, + "allow_existing_only": spec.AllowExistingOnly, + "auto_prereq_fields": stringMapToAny(spec.AutoPrereqFields), + "child_source": map[string]any{ + "dataset": spec.ChildSource.Dataset, + "column": spec.ChildSource.Column, + "predicate": spec.ChildSource.Predicate, + }, + "overrides": map[string]any{}, + }, ctx) + if err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: err.Error(), + }) + return + } + expected := map[string]struct{}{} + familyAllowlist := spec.AllowExistingOnly && ctx.familyHasExistingRows(spec.FamilyKey) + for sourceKey, row := range sourceRows { + include, err := familyExpansionSourceEligible(spec, row) + if err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: err.Error(), + }) + return + } + if include { + if familyAllowlist { + slug := strings.TrimPrefix(sourceKey, spec.ChildSource.Dataset+":") + featKey, _, _, err := ctx.resolveGeneratedFeatIdentity(spec, slug, row) + if err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: err.Error(), + }) + return + } + if !ctx.featKeyExists(featKey) { + continue + } + } + expected[sourceKey] = struct{}{} + } + } + actual := map[string]string{} + overrides, _ := moduleData["overrides"].([]any) + for index, raw := range overrides { + override, ok := raw.(map[string]any) + if !ok { + continue + } + source, ok := familySourceFromMeta(override["meta"]) + if !ok || source == "" { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("generated family %q row %d is missing family source metadata", spec.FamilyKey, index), + }) + continue + } + if previousKey, ok := actual[source]; ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("generated family %q emits source %q more than once (%s and row %d)", spec.FamilyKey, source, previousKey, index), + }) + } + key, _ := override["key"].(string) + actual[source] = key + } + for source := range expected { + if _, ok := actual[source]; !ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("generated family %q is missing source %q", spec.FamilyKey, source), + }) + } + } + for source := range actual { + if _, ok := expected[source]; !ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("generated family %q includes unexpected source %q", spec.FamilyKey, source), + }) + } + } +} + +func familySourceFromMeta(raw any) (string, bool) { + meta, ok := raw.(map[string]any) + if !ok { + return "", false + } + family, ok := meta["family"].(map[string]any) + if !ok { + return "", false + } + source, ok := family["source"].(string) + return source, ok +} + +func validationStringSliceToAny(items []string) []any { + if items == nil { + return nil + } + out := make([]any, 0, len(items)) + for _, item := range items { + out = append(out, item) + } + return out +} + +func stringMapToAny(items map[string]string) map[string]any { + if items == nil { + return nil + } + out := make(map[string]any, len(items)) + for key, value := range items { + out[key] = value + } + return out +} + func validateTopPackageAssets(sourceDir, dataDir string, report *ValidationReport) { assetsDir := filepath.Join(sourceDir, "assets") generated2DADir := filepath.Join(assetsDir, "2da") @@ -1237,6 +1612,8 @@ func restoreLockfiles(snapshot lockfileSnapshot) { func validateRowCollection(path string, obj map[string]any, rows []any, report *ValidationReport) { columns := extractValidationColumns(obj) + declaredKeys := map[string]int{} + checkDuplicateKeys := filepath.Base(path) != "base.json" rowHasIdentity := func(row map[string]any, fallbackID int) (string, bool) { if key, ok := row["key"].(string); ok && strings.TrimSpace(key) != "" { return key, true @@ -1252,6 +1629,18 @@ func validateRowCollection(path string, obj map[string]any, rows []any, report * if !ok { continue } + if checkDuplicateKeys { + if key, ok := row["key"].(string); ok && strings.TrimSpace(key) != "" { + if previous, ok := declaredKeys[key]; ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("row %d repeats key %q already declared by row %d", index, key, previous), + }) + } + declaredKeys[key] = index + } + } for key := range row { if isAuthoringOnlyField(key) && !preserveLegacyWikiColumnForPath(path, key) { report.Diagnostics = append(report.Diagnostics, Diagnostic{ @@ -1604,7 +1993,7 @@ func compareNativeSelfCheck(p *project.Project, actual2DA, actualTLK string, pro if !slices.Equal(actual2DAFiles, expected2DAFiles) { mismatch += symmetricDiffCount(actual2DAFiles, expected2DAFiles) } - comparedTLK, tlkMismatch, err := compareExactDirFiles(actualTLK, nativeResult.OutputTLKDir) + comparedTLK, tlkMismatch, err := compareExactDirFilesByExtension(actualTLK, nativeResult.OutputTLKDir, ".tlk") if err != nil { return 0, 0, 0, 0, err } @@ -1732,6 +2121,10 @@ func ioDiscardProgress(progress func(string)) *progressWriter { } func listRelativeFiles(root string) ([]string, error) { + return listRelativeFilesByExtension(root, "") +} + +func listRelativeFilesByExtension(root, extension string) ([]string, error) { files := make([]string, 0) if _, err := os.Stat(root); err != nil { if os.IsNotExist(err) { @@ -1746,6 +2139,9 @@ func listRelativeFiles(root string) ([]string, error) { if d.IsDir() { return nil } + if extension != "" && !strings.EqualFold(filepath.Ext(path), extension) { + return nil + } rel, err := filepath.Rel(root, path) if err != nil { return err @@ -1784,11 +2180,15 @@ func symmetricDiffCount(left, right []string) int { } func compareExactDirFiles(actualRoot, expectedRoot string) (int, int, error) { - actualFiles, err := listRelativeFiles(actualRoot) + return compareExactDirFilesByExtension(actualRoot, expectedRoot, "") +} + +func compareExactDirFilesByExtension(actualRoot, expectedRoot, extension string) (int, int, error) { + actualFiles, err := listRelativeFilesByExtension(actualRoot, extension) if err != nil { return 0, 0, err } - expectedFiles, err := listRelativeFiles(expectedRoot) + expectedFiles, err := listRelativeFilesByExtension(expectedRoot, extension) if err != nil { return 0, 0, err } diff --git a/internal/topdata/topdata_test.go b/internal/topdata/topdata_test.go index 08c0e81..ab05d17 100644 --- a/internal/topdata/topdata_test.go +++ b/internal/topdata/topdata_test.go @@ -44,6 +44,26 @@ func TestValidateProjectAcceptsInlineTLKSchema(t *testing.T) { } } +func TestValidateProjectRejectsDuplicateJSONKeys(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "repadjust")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "repadjust", "base.json"), `{ + "output": "repadjust.2da", + "output": "other.2da", + "columns": ["Label"], + "rows": [{"id": 0, "Label": "TEST_LABEL"}] +}`+"\n") + + report := ValidateProject(testProject(root)) + if !report.HasErrors() { + t.Fatal("expected duplicate JSON key validation error") + } + if !diagnosticsContain(report.Diagnostics, "duplicate JSON object key") { + t.Fatalf("expected duplicate JSON object key diagnostic, got %#v", report.Diagnostics) + } +} + func TestResolvedTableRegistryRegistersConsistentTables(t *testing.T) { collected := []nativeCollectedDataset{ { @@ -9085,6 +9105,72 @@ func TestBuildAndPackageIncludesCompiled2DAAndTopAssets(t *testing.T) { } } +func TestBuildAndPackageNoOpsWhenOutputsAreCurrent(t *testing.T) { + root := topPackageTestProject(t) + proj := testProject(root) + proj.Config.TopData.ReferenceBuilder = "" + + result, err := BuildAndPackage(proj, nil) + if err != nil { + t.Fatalf("initial BuildAndPackage failed: %v", err) + } + + sourceTime := time.Now().Add(-4 * time.Hour) + outputTime := time.Now().Add(-2 * time.Hour) + setTopDataSourceTimes(t, root, sourceTime) + setBuildOutputTimes(t, result, outputTime) + + beforeHAK := mustStat(t, result.OutputHAKPath).ModTime() + beforeTLK := mustStat(t, result.OutputTLKPath).ModTime() + incremental, err := BuildAndPackage(proj, nil) + if err != nil { + t.Fatalf("incremental BuildAndPackage failed: %v", err) + } + if incremental.Mode != "incremental-noop" { + t.Fatalf("expected incremental-noop mode, got %q", incremental.Mode) + } + if after := mustStat(t, result.OutputHAKPath).ModTime(); !after.Equal(beforeHAK) { + t.Fatalf("expected hak mtime to be unchanged, got %s want %s", after, beforeHAK) + } + if after := mustStat(t, result.OutputTLKPath).ModTime(); !after.Equal(beforeTLK) { + t.Fatalf("expected tlk mtime to be unchanged, got %s want %s", after, beforeTLK) + } +} + +func TestBuildAndPackageRefreshesPackageWhenCompiledOutputsAreCurrent(t *testing.T) { + root := topPackageTestProject(t) + proj := testProject(root) + proj.Config.TopData.ReferenceBuilder = "" + + result, err := BuildAndPackage(proj, nil) + if err != nil { + t.Fatalf("initial BuildAndPackage failed: %v", err) + } + + sourceTime := time.Now().Add(-6 * time.Hour) + compiledTime := time.Now().Add(-4 * time.Hour) + packageTime := time.Now().Add(-8 * time.Hour) + setTopDataSourceTimes(t, root, sourceTime) + setCompiledOutputTimes(t, result, compiledTime) + setFileTime(t, result.OutputHAKPath, packageTime) + + compiled2DA := filepath.Join(result.Output2DADir, "repadjust.2da") + before2DA := mustStat(t, compiled2DA).ModTime() + refreshed, err := BuildAndPackage(proj, nil) + if err != nil { + t.Fatalf("package refresh BuildAndPackage failed: %v", err) + } + if refreshed.Mode != "native" { + t.Fatalf("expected native mode from packaged compiled output, got %q", refreshed.Mode) + } + if after := mustStat(t, compiled2DA).ModTime(); !after.Equal(before2DA) { + t.Fatalf("expected compiled 2da mtime to be unchanged, got %s want %s", after, before2DA) + } + if after := mustStat(t, result.OutputHAKPath).ModTime(); !after.After(packageTime) { + t.Fatalf("expected hak to be refreshed after %s, got %s", packageTime, after) + } +} + func TestNativeCompileGroupStatsCountSourceFragmentsForBaseDatasets(t *testing.T) { root := testProjectRoot(t) modulesDir := filepath.Join(root, "placeables", "modules") @@ -9136,6 +9222,80 @@ func TestNativeCompileGroupStatsCountSourceFragmentsForBaseDatasets(t *testing.T } } +func topPackageTestProject(t *testing.T) string { + t.Helper() + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "repadjust")) + mkdirAll(t, filepath.Join(root, "topdata", "assets", "gui")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "repadjust", "base.json"), `{ + "output": "repadjust.2da", + "columns": ["Label"], + "rows": [{"id": 0, "Label": "TEST_LABEL"}] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "assets", "gui", "testicon.png"), "icon-data") + return root +} + +func setTopDataSourceTimes(t *testing.T, root string, modTime time.Time) { + t.Helper() + setTreeFileTimes(t, filepath.Join(root, "topdata"), modTime) +} + +func setBuildOutputTimes(t *testing.T, result PackageResult, modTime time.Time) { + t.Helper() + setCompiledOutputTimes(t, result, modTime) + setFileTime(t, result.OutputHAKPath, modTime) + setFileTime(t, result.OutputTLKPath, modTime) +} + +func setCompiledOutputTimes(t *testing.T, result PackageResult, modTime time.Time) { + t.Helper() + setTreeFileTimes(t, result.Output2DADir, modTime) + setTreeFileTimes(t, result.OutputTLKDir, modTime) +} + +func setTreeFileTimes(t *testing.T, root string, modTime time.Time) { + t.Helper() + err := filepath.WalkDir(root, func(path string, d os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if d.IsDir() { + return nil + } + return os.Chtimes(path, modTime, modTime) + }) + if err != nil { + t.Fatalf("set file times under %s: %v", root, err) + } +} + +func setFileTime(t *testing.T, path string, modTime time.Time) { + t.Helper() + if err := os.Chtimes(path, modTime, modTime); err != nil { + t.Fatalf("set file time %s: %v", path, err) + } +} + +func mustStat(t *testing.T, path string) os.FileInfo { + t.Helper() + info, err := os.Stat(path) + if err != nil { + t.Fatalf("stat %s: %v", path, err) + } + return info +} + +func diagnosticsContain(diagnostics []Diagnostic, text string) bool { + for _, diagnostic := range diagnostics { + if strings.Contains(diagnostic.Message, text) { + return true + } + } + return false +} + func TestValidateProjectAcceptsPNGTopPackageAsset(t *testing.T) { root := testProjectRoot(t) mkdirAll(t, filepath.Join(root, "topdata", "data", "repadjust"))