Fix Autogen Lockfile Overrides

internal/topdata/autogen.go

@@ -32,6 +32,11 @@ type autogenManifestEntry struct {
 	RowID     int    `json:"row_id,omitempty"`
 }
 
+type autogenManifestAssetMetadata struct {
+	DownloadURL string
+	UpdatedAt   time.Time
+}
+
 type ProducedAutogenManifest struct {
 	ID         string
 	AssetName  string
@@ -331,10 +336,19 @@ func resolveReleasedAutogenManifestEntries(p *project.Project, consumer project.
 				progress(fmt.Sprintf("Ignoring cached autogen manifest %s: %v", consumer.Manifest.CacheName, err))
 			}
 		} else if fresh {
-			if progress != nil {
+			if current, remoteTime, err := releasedAutogenManifestCacheCurrent(p, consumer, cachePath, manifest); err != nil {
-				progress(fmt.Sprintf("Using cached released autogen manifest %s...", cachePath))
+				if progress != nil {
+					progress(fmt.Sprintf("Using cached released autogen manifest %s (remote check failed: %v)...", cachePath, err))
+				}
+				return manifest.Entries, nil
+			} else if current {
+				if progress != nil {
+					progress(fmt.Sprintf("Using cached released autogen manifest %s...", cachePath))
+				}
+				return manifest.Entries, nil
+			} else if progress != nil {
+				progress(fmt.Sprintf("Refreshing released autogen manifest %s; remote asset changed at %s.", consumer.Manifest.CacheName, remoteTime.Format(time.RFC3339)))
 			}
-			return manifest.Entries, nil
 		}
 	}
 
@@ -368,30 +382,103 @@ func resolveReleasedAutogenManifestEntries(p *project.Project, consumer project.
 	return manifest.Entries, nil
 }
 
+func releasedAutogenManifestCacheCurrent(p *project.Project, consumer project.AutogenConsumerConfig, cachePath string, manifest *autogenManifest) (bool, time.Time, error) {
+	spec, err := deriveSowAssetsRepoSpec(p.Root)
+	if err != nil {
+		return false, time.Time{}, err
+	}
+	asset, err := resolveAutogenManifestAssetMetadata(spec, consumer.Manifest.ReleaseTag, consumer.Manifest.AssetName)
+	if err != nil {
+		return false, time.Time{}, err
+	}
+	if asset.UpdatedAt.IsZero() {
+		return true, time.Time{}, nil
+	}
+	return !asset.UpdatedAt.After(localAutogenManifestComparableTime(cachePath, manifest)), asset.UpdatedAt, nil
+}
+
+func localAutogenManifestComparableTime(cachePath string, manifest *autogenManifest) time.Time {
+	best := time.Time{}
+	if info, err := os.Stat(cachePath); err == nil {
+		best = info.ModTime()
+	}
+	if generatedAt, err := time.Parse(time.RFC3339, strings.TrimSpace(manifest.GeneratedAt)); err == nil && generatedAt.After(best) {
+		best = generatedAt
+	}
+	return best
+}
+
+func newestReleasedAutogenManifestInput(p *project.Project, consumer project.AutogenConsumerConfig, now time.Time, cachePath string) (time.Time, string, error) {
+	if strings.TrimSpace(os.Getenv("SOW_AUTOGEN_MANIFEST_REFRESH")) != "" {
+		return now, cachePath, nil
+	}
+
+	info, err := os.Stat(cachePath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return now, cachePath, nil
+		}
+		return time.Time{}, "", fmt.Errorf("stat autogen manifest cache %s: %w", cachePath, err)
+	}
+	if autogenManifestCacheMaxAge > 0 && now.Sub(info.ModTime()) > autogenManifestCacheMaxAge {
+		return now, cachePath, nil
+	}
+
+	raw, err := os.ReadFile(cachePath)
+	if err != nil {
+		return info.ModTime(), cachePath, nil
+	}
+	var manifest autogenManifest
+	if err := json.Unmarshal(raw, &manifest); err != nil {
+		return info.ModTime(), cachePath, nil
+	}
+
+	current, remoteTime, err := releasedAutogenManifestCacheCurrent(p, consumer, cachePath, &manifest)
+	if err != nil || current || remoteTime.IsZero() {
+		return info.ModTime(), cachePath, nil
+	}
+	return remoteTime, consumer.Manifest.AssetName, nil
+}
+
 func resolveAutogenManifestAssetURL(spec giteaRepoSpec, releaseTag, assetName string) (string, error) {
+	asset, err := resolveAutogenManifestAssetMetadata(spec, releaseTag, assetName)
+	if err != nil {
+		return "", err
+	}
+	return asset.DownloadURL, nil
+}
+
+func resolveAutogenManifestAssetMetadata(spec giteaRepoSpec, releaseTag, assetName string) (autogenManifestAssetMetadata, error) {
 	apiURL := fmt.Sprintf("%s/api/v1/repos/%s/%s/releases/tags/%s", strings.TrimRight(spec.BaseURL, "/"), releasePathEscape(spec.Owner), releasePathEscape(spec.Repo), releasePathEscape(releaseTag))
 	var release struct {
 		Assets []struct {
 			Name               string `json:"name"`
 			BrowserDownloadURL string `json:"browser_download_url"`
 			DownloadURL        string `json:"download_url"`
+			UpdatedAt          string `json:"updated_at"`
 		} `json:"assets"`
 	}
 	if err := fetchJSON(apiURL, &release); err != nil {
-		return "", fmt.Errorf("fetch autogen manifest release metadata: %w", err)
+		return autogenManifestAssetMetadata{}, fmt.Errorf("fetch autogen manifest release metadata: %w", err)
 	}
 	for _, asset := range release.Assets {
 		if asset.Name != assetName {
 			continue
 		}
+		metadata := autogenManifestAssetMetadata{}
+		if updatedAt, err := time.Parse(time.RFC3339, strings.TrimSpace(asset.UpdatedAt)); err == nil {
+			metadata.UpdatedAt = updatedAt
+		}
 		if asset.DownloadURL != "" {
-			return asset.DownloadURL, nil
+			metadata.DownloadURL = asset.DownloadURL
+			return metadata, nil
 		}
 		if asset.BrowserDownloadURL != "" {
-			return asset.BrowserDownloadURL, nil
+			metadata.DownloadURL = asset.BrowserDownloadURL
+			return metadata, nil
 		}
 	}
-	return "", fmt.Errorf("autogen manifest asset %s not found in release %s/%s:%s", assetName, spec.Owner, spec.Repo, releaseTag)
+	return autogenManifestAssetMetadata{}, fmt.Errorf("autogen manifest asset %s not found in release %s/%s:%s", assetName, spec.Owner, spec.Repo, releaseTag)
 }
 
 func releasePathEscape(value string) string {
@@ -529,11 +616,26 @@ func augmentWithAutogeneratedHeadVisualeffects(collected []nativeCollectedDatase
 
 		rows := make([]map[string]any, len(dataset.Rows))
 		rowByKey := make(map[string]map[string]any, len(dataset.Rows))
-		usedIDs := make(map[int]struct{}, len(dataset.Rows)+len(dataset.LockData))
+		historicalLockData := make(map[string]int, len(dataset.LockData))
+		for key, rowID := range dataset.LockData {
+			historicalLockData[key] = rowID
+		}
+		if dataset.Dataset.LockPath != "" {
+			if existingLockData, err := loadLockfile(dataset.Dataset.LockPath); err == nil {
+				for key, rowID := range existingLockData {
+					if _, ok := historicalLockData[key]; !ok {
+						historicalLockData[key] = rowID
+					}
+				}
+			}
+		}
+		usedIDs := make(map[int]struct{}, len(dataset.Rows)+len(historicalLockData))
 		lockData := make(map[string]int, len(dataset.LockData))
+		for _, rowID := range historicalLockData {
+			usedIDs[rowID] = struct{}{}
+		}
 		for key, rowID := range dataset.LockData {
 			lockData[key] = rowID
-			usedIDs[rowID] = struct{}{}
 		}
 		for index, row := range dataset.Rows {
 			cloned := cloneRowMap(row)
@@ -566,7 +668,11 @@ func augmentWithAutogeneratedHeadVisualeffects(collected []nativeCollectedDatase
 
 			rowID, ok := lockData[key]
 			if !ok {
-				rowID = allocateNextID()
+				if preservedRowID, preserved := historicalLockData[key]; preserved {
+					rowID = preservedRowID
+				} else {
+					rowID = allocateNextID()
+				}
 				lockData[key] = rowID
 			}
 			newRow := createDefaultHeadVisualeffectRow(dataset.Columns, rowID, key, label, entry.ModelStem)

internal/topdata/top_package.go

@@ -319,25 +319,13 @@ func newestAutogenInput(p *project.Project, now time.Time) (time.Time, string, e
 		}
 
 		cachePath := filepath.Join(p.Root, ".cache", consumer.Manifest.CacheName)
-		info, err := os.Stat(cachePath)
+		candidateTime, candidatePath, err := newestReleasedAutogenManifestInput(p, consumer, now, cachePath)
 		if err != nil {
-			if os.IsNotExist(err) {
+			return time.Time{}, "", err
-				if now.After(newest) {
-					newest = now
-					newestPath = cachePath
-				}
-				continue
-			}
-			return time.Time{}, "", fmt.Errorf("stat autogen manifest cache %s: %w", cachePath, err)
-		}
-
-		candidateTime := info.ModTime()
-		if autogenManifestCacheMaxAge > 0 && now.Sub(info.ModTime()) > autogenManifestCacheMaxAge {
-			candidateTime = now
 		}
 		if candidateTime.After(newest) {
 			newest = candidateTime
-			newestPath = cachePath
+			newestPath = candidatePath
 		}
 	}
 	return newest, newestPath, nil

internal/topdata/topdata_test.go

@@ -9195,6 +9195,7 @@ func TestBuildNativeWithReleasedHeadVisualeffectsManifest(t *testing.T) {
 	mkdirAll(t, filepath.Join(projRoot, "src"))
 	mkdirAll(t, filepath.Join(projRoot, "assets"))
 	mkdirAll(t, filepath.Join(projRoot, "build"))
+	mkdirAll(t, filepath.Join(projRoot, ".cache"))
 	mkdirAll(t, filepath.Join(projRoot, "topdata", "data", "visualeffects"))
 	writeFile(t, filepath.Join(projRoot, "topdata", "base_dialog.json"), "{}\n")
 	writeFile(t, filepath.Join(projRoot, "topdata", "data", "visualeffects", "base.json"), `{
@@ -9301,6 +9302,191 @@ func TestBuildNativeWithReleasedHeadVisualeffectsManifest(t *testing.T) {
 	}
 }
 
+func TestBuildNativePrefersRemoteReleasedHeadVisualeffectsManifestOverFreshCache(t *testing.T) {
+	root := t.TempDir()
+	projRoot := filepath.Join(root, "project")
+	mkdirAll(t, filepath.Join(projRoot, "src"))
+	mkdirAll(t, filepath.Join(projRoot, "assets"))
+	mkdirAll(t, filepath.Join(projRoot, "build"))
+	mkdirAll(t, filepath.Join(projRoot, ".cache"))
+	mkdirAll(t, filepath.Join(projRoot, "topdata", "data", "visualeffects"))
+	writeFile(t, filepath.Join(projRoot, "topdata", "base_dialog.json"), "{}\n")
+	writeFile(t, filepath.Join(projRoot, "topdata", "data", "visualeffects", "base.json"), `{
+  "output": "visualeffects.2da",
+  "columns": ["Label", "Type_FD", "OrientWithGround", "Imp_HeadCon_Node", "OrientWithObject"],
+  "rows": [
+    {"key": "visualeffects:existing", "id": 17, "Label": "EXISTING", "Type_FD": "F", "OrientWithGround": "0", "Imp_HeadCon_Node": "****", "OrientWithObject": "0"}
+  ]
+}`+"\n")
+	writeFile(t, filepath.Join(projRoot, "topdata", "data", "visualeffects", "lock.json"), `{
+  "visualeffects:existing": 17,
+  "visualeffects:headaccessory_bandana": 42
+}`+"\n")
+	mkdirAll(t, filepath.Join(projRoot, "reference"))
+	writeFile(t, filepath.Join(projRoot, "reference", "build.py"), "print('ok')\n")
+
+	cachePath := filepath.Join(projRoot, ".cache", "sow-head-vfx-manifest.json")
+	writeFile(t, cachePath, `{
+  "id": "head_visualeffects",
+  "generated_at": "2026-04-25T00:00:00Z",
+  "entries": [
+    {
+      "group": "head_features",
+      "model_stem": "hfx_hair_bangs",
+      "source": "head_features/hair/hfx_hair_bangs.mdl"
+    }
+  ]
+}`+"\n")
+	cacheTime := time.Date(2026, 4, 25, 10, 0, 0, 0, time.UTC)
+	setFileTime(t, cachePath, cacheTime)
+
+	manifestJSON := `{
+  "id": "head_visualeffects",
+  "repo": "ShadowsOverWestgate/sow-assets",
+  "ref": "remote-manifest",
+  "generated_at": "2026-04-25T12:00:00Z",
+  "entries": [
+    {
+      "group": "head_accessories",
+      "model_stem": "hfx_bandana",
+      "source": "head_accessories/hfx_bandana.mdl"
+    }
+  ]
+}` + "\n"
+
+	var server *httptest.Server
+	server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/api/v1/repos/ShadowsOverWestgate/sow-assets/releases/tags/head-vfx-manifest-current":
+			w.Header().Set("Content-Type", "application/json")
+			_, _ = w.Write([]byte(`{"assets":[{"name":"sow-head-vfx-manifest.json","updated_at":"2026-04-25T12:00:00Z","browser_download_url":"` + server.URL + `/downloads/sow-head-vfx-manifest.json"}]}`))
+		case "/downloads/sow-head-vfx-manifest.json":
+			w.Header().Set("Content-Type", "application/json")
+			_, _ = w.Write([]byte(manifestJSON))
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+	defer server.Close()
+
+	runGitTest(t, "", "init", "-b", "main", projRoot)
+	runGitTest(t, projRoot, "remote", "add", "origin", server.URL+"/ShadowsOverWestgate/sow-module.git")
+
+	proj := &project.Project{
+		Root: projRoot,
+		Config: project.Config{
+			Module: project.ModuleConfig{Name: "Test", ResRef: "test"},
+			Paths:  project.PathConfig{Source: "src", Assets: "assets", Build: "build"},
+			TopData: project.TopDataConfig{
+				Source: "topdata",
+				Build:  "build/topdata",
+			},
+			Autogen: project.AutogenConfig{
+				Consumers: []project.AutogenConsumerConfig{
+					{
+						ID:       "head_visualeffects",
+						Producer: "head_visualeffects",
+						Dataset:  "visualeffects",
+						Mode:     "head_visualeffects",
+						Root:     "vfxs",
+						Include:  []string{"head_accessories/**/*.mdl", "head_features/**/*.mdl"},
+						Derive: project.AutogenDeriveConfig{
+							Kind:      "model_stem",
+							GroupFrom: "first_path_segment",
+						},
+						Manifest: project.AutogenManifestConfig{
+							ReleaseTag: "head-vfx-manifest-current",
+							AssetName:  "sow-head-vfx-manifest.json",
+							CacheName:  "sow-head-vfx-manifest.json",
+						},
+					},
+				},
+			},
+		},
+	}
+
+	result, err := BuildNative(proj, nil)
+	if err != nil {
+		t.Fatalf("BuildNative failed: %v", err)
+	}
+
+	visualeffectsBytes, err := os.ReadFile(filepath.Join(result.Output2DADir, "visualeffects.2da"))
+	if err != nil {
+		t.Fatalf("read visualeffects.2da: %v", err)
+	}
+	text := string(visualeffectsBytes)
+	if !strings.Contains(text, "42\theadaccessory_BANDANA\tD\t0\thfx_bandana\t1") {
+		t.Fatalf("expected remote manifest entry with preserved lock id, got:\n%s", text)
+	}
+	if strings.Contains(text, "HAIR_BANGS") {
+		t.Fatalf("expected fresh remote manifest to replace cached manifest contents, got:\n%s", text)
+	}
+}
+
+func TestBuildPackageInvalidatesFreshAutogenCacheWhenReleaseAssetIsNewer(t *testing.T) {
+	root := topPackageTestProject(t)
+	proj := testProject(root)
+	proj.Config.TopData.ReferenceBuilder = ""
+	proj.Config.Autogen.Consumers = nil
+
+	result, err := BuildAndPackage(proj, nil)
+	if err != nil {
+		t.Fatalf("initial BuildAndPackage failed: %v", err)
+	}
+
+	sourceTime := time.Now().Add(-6 * time.Hour)
+	outputTime := time.Now().Add(-4 * time.Hour)
+	setTopDataSourceTimes(t, root, sourceTime)
+	setCompiledOutputTimes(t, result, outputTime)
+
+	cachePath := filepath.Join(root, ".cache", "sow-head-vfx-manifest.json")
+	writeFile(t, cachePath, `{"id":"head_visualeffects","generated_at":"2026-04-25T10:00:00Z","entries":[{"group":"head_accessories","model_stem":"hfx_bandana","source":"head_accessories/hfx_bandana.mdl"}]}`+"\n")
+	setFileTime(t, cachePath, time.Now())
+
+	var server *httptest.Server
+	server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/api/v1/repos/ShadowsOverWestgate/sow-assets/releases/tags/head-vfx-manifest-current":
+			w.Header().Set("Content-Type", "application/json")
+			_, _ = w.Write([]byte(`{"assets":[{"name":"sow-head-vfx-manifest.json","updated_at":"3026-04-25T12:00:00Z","browser_download_url":"` + server.URL + `/downloads/sow-head-vfx-manifest.json"}]}`))
+		case "/downloads/sow-head-vfx-manifest.json":
+			w.Header().Set("Content-Type", "application/json")
+			_, _ = w.Write([]byte(`{"id":"head_visualeffects","entries":[{"group":"head_accessories","model_stem":"hfx_bandana","source":"head_accessories/hfx_bandana.mdl"}]}`))
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+	defer server.Close()
+
+	runGitTest(t, "", "init", "-b", "main", root)
+	runGitTest(t, root, "remote", "add", "origin", server.URL+"/ShadowsOverWestgate/sow-module.git")
+
+	proj.Config.Autogen.Consumers = []project.AutogenConsumerConfig{
+		{
+			ID:       "head_visualeffects",
+			Producer: "head_visualeffects",
+			Dataset:  "visualeffects",
+			Mode:     "head_visualeffects",
+			Optional: true,
+			Root:     "vfxs",
+			Include:  []string{"head_accessories/**/*.mdl", "head_features/**/*.mdl"},
+			Derive: project.AutogenDeriveConfig{
+				Kind:      "model_stem",
+				GroupFrom: "first_path_segment",
+			},
+			Manifest: project.AutogenManifestConfig{
+				ReleaseTag: "head-vfx-manifest-current",
+				AssetName:  "sow-head-vfx-manifest.json",
+				CacheName:  "sow-head-vfx-manifest.json",
+			},
+		},
+	}
+
+	if _, err := BuildPackage(proj, nil); err == nil || !strings.Contains(err.Error(), "older than source file") {
+		t.Fatalf("expected newer remote autogen manifest to invalidate compiled output, got %v", err)
+	}
+}
+
 func TestBuildNativeRejectsGitRepoTopDataAssetsOverride(t *testing.T) {
 	root := testProjectRoot(t)
 	mkdirAll(t, filepath.Join(root, "topdata", "data", "parts"))