fix(wiki): compare drift against what NodeBB stored, not what we rendered (#104)

Fixes #103. ## What was wrong `crucible wiki deploy` decided a page had drifted by hashing NodeBB's stored copy and comparing it to the hash of the text Crucible rendered. Those match only if NodeBB gives our HTML back byte for byte. It does not, so every `sow-topdata` tag deploy failed: ``` local pages: 1200, updated: 1, skipped: 1176, drifted: 23 remote managed wiki content drifted; rerun with --force to overwrite ``` Nobody had edited those pages. The operator's only way out was to leave `--force` on, which removes the protection the guard exists for. ## What changed - New manifest field `remote_hash`: the managed-region hash of the post NodeBB hands back right after we write it. Drift compares against that, so it means "the live page changed after we last wrote it". - A post with no `sourceContent` is not drift. It predates `sourceContent` sync, reads back as rendered HTML, and belongs to the existing `SourceContentSynced` repair — which the drift refusal used to block. - The error names the drifted pages, capped at ten plus a count. - Deleted the dead `wikiDeployPlan.RemoteHash` field. Old manifests keep the previous comparison until each page is next written, so no re-seed is needed. Cost: one extra post read per page written. Pages that skip are still never fetched. ## Tests `internal/topdata/wiki_deploy_test.go`, same seam as the rest of the file (`DeployWikiWithOptions` against the fake NodeBB): normalized remote copy is not drift, a hand-edited page still is and the error names it, `--force` overwrites it and re-records the hash, a post without `sourceContent` is repaired instead of refused, and an old-format manifest deploys and gains a `remote_hash`. `make check` passes. 🤖 Generated with [Claude Code](https://claude.com/claude-code)Reviewed-on: #104 Co-authored-by: vickydotbat <vickydotbat@tutamail.com>
2026-08-07 21:46:36 +00:00
parent 166ffb545e
commit 3f1e33a39a
2 changed files with 320 additions and 12 deletions
@@ -741,6 +741,10 @@ func TestDeployWikiCreatesNodeBBTopicAndWritesManifest(t *testing.T) {
 			_ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pages": []any{}, "hasMore": false}})
 			return
 		}
+		if r.Method == http.MethodGet && r.URL.Path == "/api/v3/posts/42" {
+			respondWikiPost(w, 42, 11, generated)
+			return
+		}
 		if r.Method != http.MethodPost || r.URL.Path != "/api/v3/topics" {
 			t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path)
 		}
@@ -796,6 +800,9 @@ func TestDeployWikiCreatesNodeBBTopicAndWritesManifest(t *testing.T) {
 	if !entry.SourceContentSynced {
 		t.Fatalf("expected created manifest entry to record sourceContent sync")
 	}
+	if entry.RemoteHash != computeManagedHash(generated) {
+		t.Fatalf("expected created manifest entry to record the read-back remote hash, got %#v", entry)
+	}
 }

 func TestParseNodeBBPostPrefersSourceContentForWikiHTML(t *testing.T) {
@@ -958,6 +965,10 @@ func TestDeployWikiCreatesNodeBBTopicWithoutFallbackForDefaultThreeCharacterTitl
 			_ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pages": []any{}, "hasMore": false}})
 			return
 		}
+		if r.Method == http.MethodGet && r.URL.Path == "/api/v3/posts/42" {
+			respondWikiPost(w, 42, 11, generated)
+			return
+		}
 		if r.Method != http.MethodPost || r.URL.Path != "/api/v3/topics" {
 			t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path)
 		}
@@ -1017,6 +1028,10 @@ func TestDeployWikiCreatesNodeBBTopicWithFallbackForTitleShorterThanConfiguredMi
 			_ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pages": []any{}, "hasMore": false}})
 			return
 		}
+		if r.Method == http.MethodGet && r.URL.Path == "/api/v3/posts/42" {
+			respondWikiPost(w, 42, 11, generated)
+			return
+		}
 		if r.Method != http.MethodPost || r.URL.Path != "/api/v3/topics" {
 			t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path)
 		}
@@ -2081,6 +2096,8 @@ func TestDeployWikiPurgesTrackedStaleTopicsBeforeCreatingReplacementPages(t *tes
 		case r.Method == http.MethodGet && r.URL.Path == "/api/v3/plugins/westgate-wiki/namespace/3/pages":
 			w.Header().Set("Content-Type", "application/json")
 			_ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pages": []any{}, "hasMore": false}})
+		case r.Method == http.MethodGet && r.URL.Path == "/api/v3/posts/99":
+			respondWikiPost(w, 99, 11, generated)
 		case r.Method == http.MethodPost && r.URL.Path == "/api/v3/topics":
 			calls = append(calls, "create")
 			if strings.Join(calls, ",") != "tombstone:7,hard-purge:7,create" {
@@ -2165,6 +2182,8 @@ func TestDeployWikiResetManagedNamespacesPurgesRemotePagesBeforeCreatingFreshMan
 					"hasMore": false,
 				},
 			})
+		case r.Method == http.MethodGet && r.URL.Path == "/api/v3/posts/99":
+			respondWikiPost(w, 99, 11, generated)
 		case r.Method == http.MethodPost && r.URL.Path == "/api/v3/topics":
 			calls = append(calls, "create")
 			if strings.Join(calls, ",") != resetCalls+",create" {
@@ -2399,6 +2418,234 @@ func respondWikiEditLock(t *testing.T, w http.ResponseWriter, r *http.Request, e
 // goes through here, so a deployer that reaches for the core topic API fails in
 // CI the way it fails in production rather than passing against a fake that is
 // more permissive than the real thing.
+// driftScenario is one managed page whose generated text changed since the last
+// deploy, so the deployer has to read the live post and rule on drift. The fake
+// NodeBB serves whatever it was last told to store, which is what a real deploy
+// reads back after a write.
+type driftScenario struct {
+	sourceDir    string
+	manifestPath string
+	endpoint     string
+	generated    string
+	stored       string
+	updateCalls  int
+	lastWritten  string
+}
+
+// newDriftScenario writes the local page and the manifest entry. remoteStored is
+// the body NodeBB hands back; hasSourceContent says whether the post carries the
+// sourceContent the wiki plugin stores.
+func newDriftScenario(t *testing.T, entry wikiDeployManifestPage, remoteStored string, hasSourceContent bool) (*driftScenario, *httptest.Server) {
+	t.Helper()
+	root := t.TempDir()
+	sourceDir := filepath.Join(root, "pages")
+	if err := os.MkdirAll(filepath.Join(sourceDir, "skills"), 0755); err != nil {
+		t.Fatalf("create source dir: %v", err)
+	}
+	generated := `<!-- sow-topdata-wiki:page=skills:athletics -->
+<!-- sow-topdata-wiki:managed:start hash="sha256:local" -->
+<h1>Athletics</h1>
+<p>Generated athletics page</p>
+<!-- sow-topdata-wiki:managed:end -->
+`
+	if err := os.WriteFile(filepath.Join(sourceDir, "skills", "athletics.html"), []byte(generated), 0644); err != nil {
+		t.Fatalf("write source page: %v", err)
+	}
+	manifestPath := filepath.Join(root, "deploy-manifest.json")
+	entry.TID, entry.PID, entry.CID = 7, 42, 3
+	entry.SourceContentSynced = true
+	if err := saveDeployManifest(manifestPath, wikiDeployManifest{
+		Version: "nodebb-v1",
+		Pages:   map[string]wikiDeployManifestPage{"skills:athletics": entry},
+	}); err != nil {
+		t.Fatalf("write deploy manifest: %v", err)
+	}
+
+	scenario := &driftScenario{sourceDir: sourceDir, manifestPath: manifestPath, generated: generated, stored: remoteStored}
+	server := newFakeNodeBB(t, func(w http.ResponseWriter, r *http.Request) {
+		switch {
+		case r.Method == http.MethodGet && r.URL.Path == "/api/v3/posts/42":
+			post := map[string]any{"pid": 42, "tid": 7, "content": scenario.stored}
+			if hasSourceContent {
+				post["sourceContent"] = scenario.stored
+			}
+			w.Header().Set("Content-Type", "application/json")
+			_ = json.NewEncoder(w).Encode(map[string]any{"response": post})
+		case r.Method == http.MethodPut && r.URL.Path == "/api/v3/plugins/westgate-wiki/edit-lock":
+			respondWikiEditLock(t, w, r, 7, "drift-lock")
+		case r.Method == http.MethodPut && r.URL.Path == "/api/v3/posts/42":
+			var req struct {
+				Content string `json:"content"`
+			}
+			if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+				t.Fatalf("decode update request: %v", err)
+			}
+			scenario.updateCalls++
+			scenario.lastWritten = req.Content
+			// NodeBB owns the stored body from here on, and the read-back that
+			// follows has to see it.
+			scenario.stored = req.Content
+			hasSourceContent = true
+			w.Header().Set("Content-Type", "application/json")
+			_ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pid": 42, "tid": 7}})
+		default:
+			t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path)
+		}
+	})
+	return scenario, server
+}
+
+func (s *driftScenario) deploy(t *testing.T, force bool) (DeployResult, error) {
+	t.Helper()
+	return DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{
+		SourceDir:    s.sourceDir,
+		Endpoint:     s.endpoint,
+		Token:        "nodebb-token",
+		ManifestPath: s.manifestPath,
+		Force:        force,
+	}, nil)
+}
+
+func TestDeployWikiDoesNotTreatNodeBBNormalizationAsDrift(t *testing.T) {
+	// NodeBB stores its own copy of the page, so what it hands back never has
+	// to be byte-equal to the text we rendered. Only a change since our last
+	// write is drift.
+	stored := `<!-- sow-topdata-wiki:page=skills:athletics -->
+<!-- sow-topdata-wiki:managed:start hash="sha256:local" -->
+<h1>Athletics</h1><p>Old generated text</p>
+<!-- sow-topdata-wiki:managed:end -->
+`
+	scenario, server := newDriftScenario(t, wikiDeployManifestPage{
+		Hash:       "sha256:whatever-we-rendered-last-time",
+		RemoteHash: computeManagedHash(stored),
+	}, stored, true)
+	defer server.Close()
+	scenario.endpoint = server.URL
+
+	result, err := scenario.deploy(t, false)
+	if err != nil {
+		t.Fatalf("expected normalized remote copy to deploy cleanly, got: %v", err)
+	}
+	if result.Drifted != 0 || result.Updated != 1 || scenario.updateCalls != 1 {
+		t.Fatalf("expected a clean update, got %#v (updateCalls=%d)", result, scenario.updateCalls)
+	}
+	entry := loadDeployManifest(scenario.manifestPath).Pages["skills:athletics"]
+	if entry.RemoteHash != computeManagedHash(scenario.lastWritten) {
+		t.Fatalf("expected manifest to record the read-back remote hash, got %#v", entry)
+	}
+}
+
+func TestDeployWikiReportsEditedPageAsDriftedAndNamesIt(t *testing.T) {
+	stored := `<!-- sow-topdata-wiki:page=skills:athletics -->
+<!-- sow-topdata-wiki:managed:start hash="sha256:local" -->
+<h1>Athletics</h1>
+<p>A person rewrote this by hand.</p>
+<!-- sow-topdata-wiki:managed:end -->
+`
+	scenario, server := newDriftScenario(t, wikiDeployManifestPage{
+		Hash:       "sha256:whatever-we-rendered-last-time",
+		RemoteHash: "sha256:what-we-wrote-last-time",
+	}, stored, true)
+	defer server.Close()
+	scenario.endpoint = server.URL
+
+	result, err := scenario.deploy(t, false)
+	if err == nil {
+		t.Fatalf("expected a hand-edited page to block the deploy, got %#v", result)
+	}
+	if !strings.Contains(err.Error(), "skills:athletics") {
+		t.Fatalf("expected the drift error to name the page, got: %v", err)
+	}
+	if result.Drifted != 1 || scenario.updateCalls != 0 {
+		t.Fatalf("expected one drifted page and no write, got %#v (updateCalls=%d)", result, scenario.updateCalls)
+	}
+}
+
+func TestDeployWikiForceOverwritesDriftedPageAndRerecordsRemoteHash(t *testing.T) {
+	stored := `<!-- sow-topdata-wiki:page=skills:athletics -->
+<!-- sow-topdata-wiki:managed:start hash="sha256:local" -->
+<h1>Athletics</h1>
+<p>A person rewrote this by hand.</p>
+<!-- sow-topdata-wiki:managed:end -->
+`
+	scenario, server := newDriftScenario(t, wikiDeployManifestPage{
+		Hash:       "sha256:whatever-we-rendered-last-time",
+		RemoteHash: "sha256:what-we-wrote-last-time",
+	}, stored, true)
+	defer server.Close()
+	scenario.endpoint = server.URL
+
+	result, err := scenario.deploy(t, true)
+	if err != nil {
+		t.Fatalf("expected --force to overwrite the drifted page, got: %v", err)
+	}
+	if result.Drifted != 1 || result.Updated != 1 || scenario.updateCalls != 1 {
+		t.Fatalf("expected the drifted page overwritten once, got %#v (updateCalls=%d)", result, scenario.updateCalls)
+	}
+	entry := loadDeployManifest(scenario.manifestPath).Pages["skills:athletics"]
+	if entry.RemoteHash != computeManagedHash(scenario.lastWritten) {
+		t.Fatalf("expected the forced write to re-record the remote hash, got %#v", entry)
+	}
+}
+
+func TestDeployWikiTreatsPostWithoutSourceContentAsUnsyncedNotDrifted(t *testing.T) {
+	// A post written before the deployer stored sourceContent reads back as
+	// NodeBB's rendered HTML, which can never match anything we wrote. That is
+	// a page waiting for the sourceContent repair, not a hand edit.
+	rendered := "&lt;h1&gt;Athletics&lt;/h1&gt;&lt;p&gt;Old generated text&lt;/p&gt;"
+	scenario, server := newDriftScenario(t, wikiDeployManifestPage{
+		Hash: "sha256:whatever-we-rendered-last-time",
+	}, rendered, false)
+	defer server.Close()
+	scenario.endpoint = server.URL
+
+	result, err := scenario.deploy(t, false)
+	if err != nil {
+		t.Fatalf("expected an unsynced page to deploy, got: %v", err)
+	}
+	if result.Drifted != 0 || result.Updated != 1 || scenario.updateCalls != 1 {
+		t.Fatalf("expected one repair update and no drift, got %#v (updateCalls=%d)", result, scenario.updateCalls)
+	}
+}
+
+func TestDeployWikiFallsBackToLocalHashForManifestWithoutRemoteHash(t *testing.T) {
+	// Manifest written before remote_hash existed: the remote still matches the
+	// recorded local hash, so nothing drifted, and this run records the remote
+	// hash for the next one.
+	stored := `<!-- sow-topdata-wiki:page=skills:athletics -->
+<!-- sow-topdata-wiki:managed:start hash="sha256:old" -->
+<h1>Athletics</h1>
+<p>Old generated text</p>
+<!-- sow-topdata-wiki:managed:end -->
+`
+	scenario, server := newDriftScenario(t, wikiDeployManifestPage{
+		Hash: computeManagedHash(stored),
+	}, stored, true)
+	defer server.Close()
+	scenario.endpoint = server.URL
+
+	result, err := scenario.deploy(t, false)
+	if err != nil {
+		t.Fatalf("expected an old-format manifest to deploy, got: %v", err)
+	}
+	if result.Drifted != 0 || result.Updated != 1 {
+		t.Fatalf("expected a clean update, got %#v", result)
+	}
+	entry := loadDeployManifest(scenario.manifestPath).Pages["skills:athletics"]
+	if entry.RemoteHash == "" {
+		t.Fatalf("expected the deploy to record a remote hash, got %#v", entry)
+	}
+}
+
+// respondWikiPost answers a post read the way the wiki plugin does: the stored
+// source HTML in sourceContent, alongside the body NodeBB renders from it.
+func respondWikiPost(w http.ResponseWriter, pid, tid int, content string) {
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{
+		"pid": pid, "tid": tid, "content": content, "sourceContent": content,
+	}})
+}
+
 func newFakeNodeBB(t *testing.T, handler http.HandlerFunc) *httptest.Server {
 	t.Helper()
 	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {