From 505afb5ac5105cef999b75a7a1be46f81474d791 Mon Sep 17 00:00:00 2001 From: vickydotbat Date: Fri, 15 May 2026 09:41:54 +0200 Subject: [PATCH] Wiki pipeline audit Changes Made - page-index.json now includes generated meta/status pages, not just entity pages. - Wiki page count now comes from the final page index, so wiki pages: matches generated HTML files. - Page IDs now normalize /, \, and _ to : to prevent duplicate index entries for slash-bearing keys. - Added page-index validation for duplicate page_id and duplicate output_path. --- internal/topdata/wiki_native.go | 74 ++++++++++++++++++++++++++-- internal/topdata/wiki_native_test.go | 63 ++++++++++++++++++++--- 2 files changed, 128 insertions(+), 9 deletions(-) diff --git a/internal/topdata/wiki_native.go b/internal/topdata/wiki_native.go index 965dfdb..f10dec3 100644 --- a/internal/topdata/wiki_native.go +++ b/internal/topdata/wiki_native.go @@ -148,7 +148,6 @@ func buildWiki(p *project.Project, nativeResult BuildResult, force bool, progres pageTitles := map[string]string{} pageStatuses := map[string]string{} pageIndex := wikiPageIndex{Version: wikiGeneratorVersion} - pageCount := 0 writePage := func(pageID, content, title, status string) error { path := filepath.Join(outputDir, wikiPageIDToRelPath(pageID)) @@ -177,7 +176,6 @@ func buildWiki(p *project.Project, nativeResult BuildResult, force bool, progres EditPolicy: wikiEditPolicy(namespace), StalePolicy: p.EffectiveConfig().TopData.Wiki.StalePages.Default, }) - pageCount++ return nil } @@ -187,6 +185,9 @@ func buildWiki(p *project.Project, nativeResult BuildResult, force bool, progres if err := generateStatusPages(outputDir, pageStatuses, pageTitles, p.EffectiveConfig().TopData.Wiki.ManagedNamespaces, loadWikiManualSections(p)); err != nil { return wikiResult{}, err } + if err := indexUnregisteredWikiPages(rootDir, outputDir, p.EffectiveConfig().TopData.Wiki.StalePages.Default, &pageIndex); err != nil { + return wikiResult{}, err + } if err := saveWikiPageIndex(filepath.Join(rootDir, "page-index.json"), pageIndex); err != nil { return wikiResult{}, err } @@ -194,6 +195,7 @@ func buildWiki(p *project.Project, nativeResult BuildResult, force bool, progres if err := os.MkdirAll(rootDir, 0o755); err != nil { return wikiResult{}, fmt.Errorf("create wiki root dir: %w", err) } + pageCount := len(pageIndex.Pages) state = wikiStateDocument{Version: wikiGeneratorVersion, Digest: digest, Pages: pageCount} if err := saveWikiState(statePath, state); err != nil { return wikiResult{}, err @@ -243,6 +245,9 @@ func loadWikiManualSections(p *project.Project) []wikiManualSection { } func saveWikiPageIndex(path string, index wikiPageIndex) error { + if err := validateWikiPageIndex(index); err != nil { + return err + } slices.SortFunc(index.Pages, func(a, b wikiPageIndexEntry) int { return strings.Compare(a.PageID, b.PageID) }) @@ -256,6 +261,68 @@ func saveWikiPageIndex(path string, index wikiPageIndex) error { return os.WriteFile(path, append(raw, '\n'), 0o644) } +func validateWikiPageIndex(index wikiPageIndex) error { + pageIDs := map[string]string{} + outputPaths := map[string]string{} + for _, entry := range index.Pages { + if previous, ok := pageIDs[entry.PageID]; ok { + return fmt.Errorf("duplicate wiki page-index page_id %q for %s and %s", entry.PageID, previous, entry.OutputPath) + } + if previous, ok := outputPaths[entry.OutputPath]; ok { + return fmt.Errorf("duplicate wiki page-index output_path %q for %s and %s", entry.OutputPath, previous, entry.PageID) + } + pageIDs[entry.PageID] = entry.OutputPath + outputPaths[entry.OutputPath] = entry.PageID + } + return nil +} + +func indexUnregisteredWikiPages(rootDir, outputDir, stalePolicy string, index *wikiPageIndex) error { + seen := map[string]struct{}{} + for _, entry := range index.Pages { + seen[entry.PageID] = struct{}{} + } + return filepath.WalkDir(outputDir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || !strings.EqualFold(filepath.Ext(path), ".html") { + return nil + } + relToPages, err := filepath.Rel(outputDir, path) + if err != nil { + return err + } + pageID := strings.TrimSuffix(filepath.ToSlash(relToPages), filepath.Ext(relToPages)) + pageID = strings.ReplaceAll(pageID, "/", ":") + if _, ok := seen[pageID]; ok { + return nil + } + raw, err := os.ReadFile(path) + if err != nil { + return err + } + namespace := strings.SplitN(pageID, ":", 2)[0] + relToRoot, err := filepath.Rel(rootDir, path) + if err != nil { + relToRoot = filepath.Base(path) + } + index.Pages = append(index.Pages, wikiPageIndexEntry{ + PageID: pageID, + Namespace: namespace, + SourceDataset: categoryFromWikiNamespace(namespace), + SourceKey: pageID, + Title: extractHTMLTitle(string(raw), pageID), + Hash: computeManagedHash(string(raw)), + OutputPath: filepath.ToSlash(relToRoot), + EditPolicy: wikiEditPolicy(namespace), + StalePolicy: stalePolicy, + }) + seen[pageID] = struct{}{} + return nil + }) +} + func loadWikiState(path string) (wikiStateDocument, error) { raw, err := os.ReadFile(path) if err != nil { @@ -1482,7 +1549,8 @@ func wikiPageIDForKey(key string) string { if namespace == "" { return "" } - return namespace + ":" + strings.ReplaceAll(value, "_", ":") + value = strings.NewReplacer("/", ":", "\\", ":", "_", ":").Replace(value) + return namespace + ":" + value } func wikiPageIDToRelPath(pageID string) string { diff --git a/internal/topdata/wiki_native_test.go b/internal/topdata/wiki_native_test.go index e0dea1b..02af3a8 100644 --- a/internal/topdata/wiki_native_test.go +++ b/internal/topdata/wiki_native_test.go @@ -1,6 +1,7 @@ package topdata import ( + "encoding/json" "os" "path/filepath" "strings" @@ -39,8 +40,9 @@ func TestBuildNativeGeneratesAndSkipsWikiPages(t *testing.T) { if err != nil { t.Fatalf("BuildNative failed: %v", err) } - if result.WikiPages != 1 { - t.Fatalf("expected one generated entity page, got %d", result.WikiPages) + generatedHTMLCount := countGeneratedWikiHTMLFiles(t, filepath.Join(root, ".cache", "wiki", "pages")) + if result.WikiPages != generatedHTMLCount { + t.Fatalf("expected wiki page count to match generated HTML files, got result=%d files=%d", result.WikiPages, generatedHTMLCount) } pagePath := filepath.Join(root, ".cache", "wiki", "pages", "skills", "athletics.html") got, err := os.ReadFile(pagePath) @@ -72,10 +74,20 @@ func TestBuildNativeGeneratesAndSkipsWikiPages(t *testing.T) { if err != nil { t.Fatalf("read page index: %v", err) } + var pageIndex wikiPageIndex + if err := json.Unmarshal(indexRaw, &pageIndex); err != nil { + t.Fatalf("parse page index: %v", err) + } + if len(pageIndex.Pages) != generatedHTMLCount { + t.Fatalf("expected page-index entries to match generated HTML files, got index=%d files=%d", len(pageIndex.Pages), generatedHTMLCount) + } indexText := string(indexRaw) if !strings.Contains(indexText, `"page_id": "skills:athletics"`) || !strings.Contains(indexText, `"output_path": "pages/skills/athletics.html"`) || !strings.Contains(indexText, `"edit_policy": "preserve_manual_sections"`) { t.Fatalf("expected deterministic page-index metadata, got:\n%s", indexText) } + if !strings.Contains(indexText, `"page_id": "meta:wikistatus"`) || !strings.Contains(indexText, `"output_path": "pages/meta/wikistatus.html"`) || !strings.Contains(indexText, `"edit_policy": "generated_only"`) { + t.Fatalf("expected status pages in page-index metadata, got:\n%s", indexText) + } if _, err := os.Stat(filepath.Join(root, ".cache", "wiki", "state.json")); err != nil { t.Fatalf("expected state file after first build: %v", err) } @@ -95,11 +107,29 @@ func TestBuildNativeGeneratesAndSkipsWikiPages(t *testing.T) { if err != nil { t.Fatalf("second BuildNative failed: %v", err) } - if second.WikiPages != 1 { - t.Fatalf("expected a single generated skill wiki page after rebuild, got %d", second.WikiPages) + if second.WikiPages != generatedHTMLCount { + t.Fatalf("expected generated wiki page count after rebuild to remain stable, got %d want %d", second.WikiPages, generatedHTMLCount) } } +func countGeneratedWikiHTMLFiles(t *testing.T, root string) int { + t.Helper() + count := 0 + err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if !d.IsDir() && strings.EqualFold(filepath.Ext(path), ".html") { + count++ + } + return nil + }) + if err != nil { + t.Fatalf("count generated wiki HTML files: %v", err) + } + return count +} + func TestWikiClassNamePrefersFullNameInsteadOfShortCode(t *testing.T) { ctx := &wikiContext{} got := ctx.resolveRowName("classes", map[string]any{ @@ -111,6 +141,26 @@ func TestWikiClassNamePrefersFullNameInsteadOfShortCode(t *testing.T) { } } +func TestWikiPageIDForKeyNormalizesSlashAndUnderscoreSeparators(t *testing.T) { + got := wikiPageIDForKey("feat:special/attacks_bull_rush") + if want := "feat:special:attacks:bull:rush"; got != want { + t.Fatalf("expected normalized page ID %q, got %q", want, got) + } +} + +func TestSaveWikiPageIndexRejectsDuplicateOutputPaths(t *testing.T) { + err := saveWikiPageIndex(filepath.Join(t.TempDir(), "page-index.json"), wikiPageIndex{ + Version: wikiGeneratorVersion, + Pages: []wikiPageIndexEntry{ + {PageID: "feat:special:attacks:bull:rush", OutputPath: "pages/feat/special/attacks/bull/rush.html"}, + {PageID: "feat:special/attacks:bull:rush", OutputPath: "pages/feat/special/attacks/bull/rush.html"}, + }, + }) + if err == nil || !strings.Contains(err.Error(), "duplicate wiki page-index output_path") { + t.Fatalf("expected duplicate output_path validation error, got %v", err) + } +} + func TestBuildAndPackageBuildsWikiOnlyWhenRequested(t *testing.T) { root := testProjectRoot(t) mkdirAll(t, filepath.Join(root, "topdata", "data", "skills")) @@ -153,8 +203,9 @@ func TestBuildAndPackageBuildsWikiOnlyWhenRequested(t *testing.T) { if err != nil { t.Fatalf("BuildAndPackageWithOptions with wiki failed: %v", err) } - if result.WikiPages != 1 { - t.Fatalf("expected one generated wiki page when requested, got %d", result.WikiPages) + generatedHTMLCount := countGeneratedWikiHTMLFiles(t, filepath.Join(root, ".cache", "wiki", "pages")) + if result.WikiPages != generatedHTMLCount { + t.Fatalf("expected wiki page count to match generated HTML files when requested, got result=%d files=%d", result.WikiPages, generatedHTMLCount) } if _, err := os.Stat(filepath.Join(root, ".cache", "wiki", "state.json")); err != nil { t.Fatalf("expected wiki state after explicit wiki build: %v", err)