ShadowsOverWestgate/sow-tools
7
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases 38 Activity

Wiki pipeline audit

Browse Source 
Changes Made

- page-index.json now includes generated meta/status pages, not just entity pages.
- Wiki page count now comes from the final page index, so wiki pages: matches generated HTML files.
- Page IDs now normalize /, \, and _ to : to prevent duplicate index entries for slash-bearing keys.
- Added page-index validation for duplicate page_id and duplicate output_path.
This commit is contained in:
archvillainette
2026-05-15 09:41:54 +02:00
parent 9b2084344d
commit 505afb5ac5
2 changed files with 128 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/topdata/wiki_native.go
+71 -3
View File
@@ -148,7 +148,6 @@ func buildWiki(p *project.Project, nativeResult BuildResult, force bool, progres
pageTitles := map[string]string{}
pageStatuses := map[string]string{}
pageIndex := wikiPageIndex{Version: wikiGeneratorVersion}
pageCount := 0
writePage := func(pageID, content, title, status string) error {
path := filepath.Join(outputDir, wikiPageIDToRelPath(pageID))
@@ -177,7 +176,6 @@ func buildWiki(p *project.Project, nativeResult BuildResult, force bool, progres
EditPolicy: wikiEditPolicy(namespace),
StalePolicy: p.EffectiveConfig().TopData.Wiki.StalePages.Default,
})
pageCount++
return nil
}
@@ -187,6 +185,9 @@ func buildWiki(p *project.Project, nativeResult BuildResult, force bool, progres
if err := generateStatusPages(outputDir, pageStatuses, pageTitles, p.EffectiveConfig().TopData.Wiki.ManagedNamespaces, loadWikiManualSections(p)); err != nil {
return wikiResult{}, err
}
if err := indexUnregisteredWikiPages(rootDir, outputDir, p.EffectiveConfig().TopData.Wiki.StalePages.Default, &pageIndex); err != nil {
return wikiResult{}, err
}
if err := saveWikiPageIndex(filepath.Join(rootDir, "page-index.json"), pageIndex); err != nil {
return wikiResult{}, err
}
@@ -194,6 +195,7 @@ func buildWiki(p *project.Project, nativeResult BuildResult, force bool, progres
if err := os.MkdirAll(rootDir, 0o755); err != nil {
return wikiResult{}, fmt.Errorf("create wiki root dir: %w", err)
}
pageCount := len(pageIndex.Pages)
state = wikiStateDocument{Version: wikiGeneratorVersion, Digest: digest, Pages: pageCount}
if err := saveWikiState(statePath, state); err != nil {
return wikiResult{}, err
@@ -243,6 +245,9 @@ func loadWikiManualSections(p *project.Project) []wikiManualSection {
}
func saveWikiPageIndex(path string, index wikiPageIndex) error {
if err := validateWikiPageIndex(index); err != nil {
return err
}
slices.SortFunc(index.Pages, func(a, b wikiPageIndexEntry) int {
return strings.Compare(a.PageID, b.PageID)
})
@@ -256,6 +261,68 @@ func saveWikiPageIndex(path string, index wikiPageIndex) error {
return os.WriteFile(path, append(raw, '\n'), 0o644)
}
func validateWikiPageIndex(index wikiPageIndex) error {
pageIDs := map[string]string{}
outputPaths := map[string]string{}
for _, entry := range index.Pages {
if previous, ok := pageIDs[entry.PageID]; ok {
return fmt.Errorf("duplicate wiki page-index page_id %q for %s and %s", entry.PageID, previous, entry.OutputPath)
}
if previous, ok := outputPaths[entry.OutputPath]; ok {
return fmt.Errorf("duplicate wiki page-index output_path %q for %s and %s", entry.OutputPath, previous, entry.PageID)
}
pageIDs[entry.PageID] = entry.OutputPath
outputPaths[entry.OutputPath] = entry.PageID
}
return nil
}
func indexUnregisteredWikiPages(rootDir, outputDir, stalePolicy string, index *wikiPageIndex) error {
seen := map[string]struct{}{}
for _, entry := range index.Pages {
seen[entry.PageID] = struct{}{}
}
return filepath.WalkDir(outputDir, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() || !strings.EqualFold(filepath.Ext(path), ".html") {
return nil
}
relToPages, err := filepath.Rel(outputDir, path)
if err != nil {
return err
}
pageID := strings.TrimSuffix(filepath.ToSlash(relToPages), filepath.Ext(relToPages))
pageID = strings.ReplaceAll(pageID, "/", ":")
if _, ok := seen[pageID]; ok {
return nil
}
raw, err := os.ReadFile(path)
if err != nil {
return err
}
namespace := strings.SplitN(pageID, ":", 2)[0]
relToRoot, err := filepath.Rel(rootDir, path)
if err != nil {
relToRoot = filepath.Base(path)
}
index.Pages = append(index.Pages, wikiPageIndexEntry{
PageID: pageID,
Namespace: namespace,
SourceDataset: categoryFromWikiNamespace(namespace),
SourceKey: pageID,
Title: extractHTMLTitle(string(raw), pageID),
Hash: computeManagedHash(string(raw)),
OutputPath: filepath.ToSlash(relToRoot),
EditPolicy: wikiEditPolicy(namespace),
StalePolicy: stalePolicy,
})
seen[pageID] = struct{}{}
return nil
})
}
func loadWikiState(path string) (wikiStateDocument, error) {
raw, err := os.ReadFile(path)
if err != nil {
@@ -1482,7 +1549,8 @@ func wikiPageIDForKey(key string) string {
if namespace == "" {
return ""
}
return namespace + ":" + strings.ReplaceAll(value, "_", ":")
value = strings.NewReplacer("/", ":", "\\", ":", "_", ":").Replace(value)
return namespace + ":" + value
}
func wikiPageIDToRelPath(pageID string) string {
internal/topdata/wiki_native_test.go
+57 -6
View File
@@ -1,6 +1,7 @@
package topdata
import (
"encoding/json"
"os"
"path/filepath"
"strings"
@@ -39,8 +40,9 @@ func TestBuildNativeGeneratesAndSkipsWikiPages(t *testing.T) {
if err != nil {
t.Fatalf("BuildNative failed: %v", err)
}
if result.WikiPages != 1 {
t.Fatalf("expected one generated entity page, got %d", result.WikiPages)
generatedHTMLCount := countGeneratedWikiHTMLFiles(t, filepath.Join(root, ".cache", "wiki", "pages"))
if result.WikiPages != generatedHTMLCount {
t.Fatalf("expected wiki page count to match generated HTML files, got result=%d files=%d", result.WikiPages, generatedHTMLCount)
}
pagePath := filepath.Join(root, ".cache", "wiki", "pages", "skills", "athletics.html")
got, err := os.ReadFile(pagePath)
@@ -72,10 +74,20 @@ func TestBuildNativeGeneratesAndSkipsWikiPages(t *testing.T) {
if err != nil {
t.Fatalf("read page index: %v", err)
}
var pageIndex wikiPageIndex
if err := json.Unmarshal(indexRaw, &pageIndex); err != nil {
t.Fatalf("parse page index: %v", err)
}
if len(pageIndex.Pages) != generatedHTMLCount {
t.Fatalf("expected page-index entries to match generated HTML files, got index=%d files=%d", len(pageIndex.Pages), generatedHTMLCount)
}
indexText := string(indexRaw)
if !strings.Contains(indexText, `"page_id": "skills:athletics"`) || !strings.Contains(indexText, `"output_path": "pages/skills/athletics.html"`) || !strings.Contains(indexText, `"edit_policy": "preserve_manual_sections"`) {
t.Fatalf("expected deterministic page-index metadata, got:\n%s", indexText)
}
if !strings.Contains(indexText, `"page_id": "meta:wikistatus"`) || !strings.Contains(indexText, `"output_path": "pages/meta/wikistatus.html"`) || !strings.Contains(indexText, `"edit_policy": "generated_only"`) {
t.Fatalf("expected status pages in page-index metadata, got:\n%s", indexText)
}
if _, err := os.Stat(filepath.Join(root, ".cache", "wiki", "state.json")); err != nil {
t.Fatalf("expected state file after first build: %v", err)
}
@@ -95,11 +107,29 @@ func TestBuildNativeGeneratesAndSkipsWikiPages(t *testing.T) {
if err != nil {
t.Fatalf("second BuildNative failed: %v", err)
}
if second.WikiPages != 1 {
t.Fatalf("expected a single generated skill wiki page after rebuild, got %d", second.WikiPages)
if second.WikiPages != generatedHTMLCount {
t.Fatalf("expected generated wiki page count after rebuild to remain stable, got %d want %d", second.WikiPages, generatedHTMLCount)
}
}
func countGeneratedWikiHTMLFiles(t *testing.T, root string) int {
t.Helper()
count := 0
err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
if err != nil {
return err
}
if !d.IsDir() && strings.EqualFold(filepath.Ext(path), ".html") {
count++
}
return nil
})
if err != nil {
t.Fatalf("count generated wiki HTML files: %v", err)
}
return count
}
func TestWikiClassNamePrefersFullNameInsteadOfShortCode(t *testing.T) {
ctx := &wikiContext{}
got := ctx.resolveRowName("classes", map[string]any{
@@ -111,6 +141,26 @@ func TestWikiClassNamePrefersFullNameInsteadOfShortCode(t *testing.T) {
}
}
func TestWikiPageIDForKeyNormalizesSlashAndUnderscoreSeparators(t *testing.T) {
got := wikiPageIDForKey("feat:special/attacks_bull_rush")
if want := "feat:special:attacks:bull:rush"; got != want {
t.Fatalf("expected normalized page ID %q, got %q", want, got)
}
}
func TestSaveWikiPageIndexRejectsDuplicateOutputPaths(t *testing.T) {
err := saveWikiPageIndex(filepath.Join(t.TempDir(), "page-index.json"), wikiPageIndex{
Version: wikiGeneratorVersion,
Pages: []wikiPageIndexEntry{
{PageID: "feat:special:attacks:bull:rush", OutputPath: "pages/feat/special/attacks/bull/rush.html"},
{PageID: "feat:special/attacks:bull:rush", OutputPath: "pages/feat/special/attacks/bull/rush.html"},
},
})
if err == nil || !strings.Contains(err.Error(), "duplicate wiki page-index output_path") {
t.Fatalf("expected duplicate output_path validation error, got %v", err)
}
}
func TestBuildAndPackageBuildsWikiOnlyWhenRequested(t *testing.T) {
root := testProjectRoot(t)
mkdirAll(t, filepath.Join(root, "topdata", "data", "skills"))
@@ -153,8 +203,9 @@ func TestBuildAndPackageBuildsWikiOnlyWhenRequested(t *testing.T) {
if err != nil {
t.Fatalf("BuildAndPackageWithOptions with wiki failed: %v", err)
}
if result.WikiPages != 1 {
t.Fatalf("expected one generated wiki page when requested, got %d", result.WikiPages)
generatedHTMLCount := countGeneratedWikiHTMLFiles(t, filepath.Join(root, ".cache", "wiki", "pages"))
if result.WikiPages != generatedHTMLCount {
t.Fatalf("expected wiki page count to match generated HTML files when requested, got result=%d files=%d", result.WikiPages, generatedHTMLCount)
}
if _, err := os.Stat(filepath.Join(root, ".cache", "wiki", "state.json")); err != nil {
t.Fatalf("expected wiki state after explicit wiki build: %v", err)