diff --git a/internal/topdata/autogen.go b/internal/topdata/autogen.go index 05913f0..a7fdfe8 100644 --- a/internal/topdata/autogen.go +++ b/internal/topdata/autogen.go @@ -32,6 +32,11 @@ type autogenManifestEntry struct { RowID int `json:"row_id,omitempty"` } +type autogenManifestAssetMetadata struct { + DownloadURL string + UpdatedAt time.Time +} + type ProducedAutogenManifest struct { ID string AssetName string @@ -331,10 +336,19 @@ func resolveReleasedAutogenManifestEntries(p *project.Project, consumer project. progress(fmt.Sprintf("Ignoring cached autogen manifest %s: %v", consumer.Manifest.CacheName, err)) } } else if fresh { - if progress != nil { - progress(fmt.Sprintf("Using cached released autogen manifest %s...", cachePath)) + if current, remoteTime, err := releasedAutogenManifestCacheCurrent(p, consumer, cachePath, manifest); err != nil { + if progress != nil { + progress(fmt.Sprintf("Using cached released autogen manifest %s (remote check failed: %v)...", cachePath, err)) + } + return manifest.Entries, nil + } else if current { + if progress != nil { + progress(fmt.Sprintf("Using cached released autogen manifest %s...", cachePath)) + } + return manifest.Entries, nil + } else if progress != nil { + progress(fmt.Sprintf("Refreshing released autogen manifest %s; remote asset changed at %s.", consumer.Manifest.CacheName, remoteTime.Format(time.RFC3339))) } - return manifest.Entries, nil } } @@ -368,30 +382,103 @@ func resolveReleasedAutogenManifestEntries(p *project.Project, consumer project. return manifest.Entries, nil } +func releasedAutogenManifestCacheCurrent(p *project.Project, consumer project.AutogenConsumerConfig, cachePath string, manifest *autogenManifest) (bool, time.Time, error) { + spec, err := deriveSowAssetsRepoSpec(p.Root) + if err != nil { + return false, time.Time{}, err + } + asset, err := resolveAutogenManifestAssetMetadata(spec, consumer.Manifest.ReleaseTag, consumer.Manifest.AssetName) + if err != nil { + return false, time.Time{}, err + } + if asset.UpdatedAt.IsZero() { + return true, time.Time{}, nil + } + return !asset.UpdatedAt.After(localAutogenManifestComparableTime(cachePath, manifest)), asset.UpdatedAt, nil +} + +func localAutogenManifestComparableTime(cachePath string, manifest *autogenManifest) time.Time { + best := time.Time{} + if info, err := os.Stat(cachePath); err == nil { + best = info.ModTime() + } + if generatedAt, err := time.Parse(time.RFC3339, strings.TrimSpace(manifest.GeneratedAt)); err == nil && generatedAt.After(best) { + best = generatedAt + } + return best +} + +func newestReleasedAutogenManifestInput(p *project.Project, consumer project.AutogenConsumerConfig, now time.Time, cachePath string) (time.Time, string, error) { + if strings.TrimSpace(os.Getenv("SOW_AUTOGEN_MANIFEST_REFRESH")) != "" { + return now, cachePath, nil + } + + info, err := os.Stat(cachePath) + if err != nil { + if os.IsNotExist(err) { + return now, cachePath, nil + } + return time.Time{}, "", fmt.Errorf("stat autogen manifest cache %s: %w", cachePath, err) + } + if autogenManifestCacheMaxAge > 0 && now.Sub(info.ModTime()) > autogenManifestCacheMaxAge { + return now, cachePath, nil + } + + raw, err := os.ReadFile(cachePath) + if err != nil { + return info.ModTime(), cachePath, nil + } + var manifest autogenManifest + if err := json.Unmarshal(raw, &manifest); err != nil { + return info.ModTime(), cachePath, nil + } + + current, remoteTime, err := releasedAutogenManifestCacheCurrent(p, consumer, cachePath, &manifest) + if err != nil || current || remoteTime.IsZero() { + return info.ModTime(), cachePath, nil + } + return remoteTime, consumer.Manifest.AssetName, nil +} + func resolveAutogenManifestAssetURL(spec giteaRepoSpec, releaseTag, assetName string) (string, error) { + asset, err := resolveAutogenManifestAssetMetadata(spec, releaseTag, assetName) + if err != nil { + return "", err + } + return asset.DownloadURL, nil +} + +func resolveAutogenManifestAssetMetadata(spec giteaRepoSpec, releaseTag, assetName string) (autogenManifestAssetMetadata, error) { apiURL := fmt.Sprintf("%s/api/v1/repos/%s/%s/releases/tags/%s", strings.TrimRight(spec.BaseURL, "/"), releasePathEscape(spec.Owner), releasePathEscape(spec.Repo), releasePathEscape(releaseTag)) var release struct { Assets []struct { Name string `json:"name"` BrowserDownloadURL string `json:"browser_download_url"` DownloadURL string `json:"download_url"` + UpdatedAt string `json:"updated_at"` } `json:"assets"` } if err := fetchJSON(apiURL, &release); err != nil { - return "", fmt.Errorf("fetch autogen manifest release metadata: %w", err) + return autogenManifestAssetMetadata{}, fmt.Errorf("fetch autogen manifest release metadata: %w", err) } for _, asset := range release.Assets { if asset.Name != assetName { continue } + metadata := autogenManifestAssetMetadata{} + if updatedAt, err := time.Parse(time.RFC3339, strings.TrimSpace(asset.UpdatedAt)); err == nil { + metadata.UpdatedAt = updatedAt + } if asset.DownloadURL != "" { - return asset.DownloadURL, nil + metadata.DownloadURL = asset.DownloadURL + return metadata, nil } if asset.BrowserDownloadURL != "" { - return asset.BrowserDownloadURL, nil + metadata.DownloadURL = asset.BrowserDownloadURL + return metadata, nil } } - return "", fmt.Errorf("autogen manifest asset %s not found in release %s/%s:%s", assetName, spec.Owner, spec.Repo, releaseTag) + return autogenManifestAssetMetadata{}, fmt.Errorf("autogen manifest asset %s not found in release %s/%s:%s", assetName, spec.Owner, spec.Repo, releaseTag) } func releasePathEscape(value string) string { @@ -529,11 +616,26 @@ func augmentWithAutogeneratedHeadVisualeffects(collected []nativeCollectedDatase rows := make([]map[string]any, len(dataset.Rows)) rowByKey := make(map[string]map[string]any, len(dataset.Rows)) - usedIDs := make(map[int]struct{}, len(dataset.Rows)+len(dataset.LockData)) + historicalLockData := make(map[string]int, len(dataset.LockData)) + for key, rowID := range dataset.LockData { + historicalLockData[key] = rowID + } + if dataset.Dataset.LockPath != "" { + if existingLockData, err := loadLockfile(dataset.Dataset.LockPath); err == nil { + for key, rowID := range existingLockData { + if _, ok := historicalLockData[key]; !ok { + historicalLockData[key] = rowID + } + } + } + } + usedIDs := make(map[int]struct{}, len(dataset.Rows)+len(historicalLockData)) lockData := make(map[string]int, len(dataset.LockData)) + for _, rowID := range historicalLockData { + usedIDs[rowID] = struct{}{} + } for key, rowID := range dataset.LockData { lockData[key] = rowID - usedIDs[rowID] = struct{}{} } for index, row := range dataset.Rows { cloned := cloneRowMap(row) @@ -566,7 +668,11 @@ func augmentWithAutogeneratedHeadVisualeffects(collected []nativeCollectedDatase rowID, ok := lockData[key] if !ok { - rowID = allocateNextID() + if preservedRowID, preserved := historicalLockData[key]; preserved { + rowID = preservedRowID + } else { + rowID = allocateNextID() + } lockData[key] = rowID } newRow := createDefaultHeadVisualeffectRow(dataset.Columns, rowID, key, label, entry.ModelStem) diff --git a/internal/topdata/top_package.go b/internal/topdata/top_package.go index 9ed819f..76e4093 100644 --- a/internal/topdata/top_package.go +++ b/internal/topdata/top_package.go @@ -319,25 +319,13 @@ func newestAutogenInput(p *project.Project, now time.Time) (time.Time, string, e } cachePath := filepath.Join(p.Root, ".cache", consumer.Manifest.CacheName) - info, err := os.Stat(cachePath) + candidateTime, candidatePath, err := newestReleasedAutogenManifestInput(p, consumer, now, cachePath) if err != nil { - if os.IsNotExist(err) { - if now.After(newest) { - newest = now - newestPath = cachePath - } - continue - } - return time.Time{}, "", fmt.Errorf("stat autogen manifest cache %s: %w", cachePath, err) - } - - candidateTime := info.ModTime() - if autogenManifestCacheMaxAge > 0 && now.Sub(info.ModTime()) > autogenManifestCacheMaxAge { - candidateTime = now + return time.Time{}, "", err } if candidateTime.After(newest) { newest = candidateTime - newestPath = cachePath + newestPath = candidatePath } } return newest, newestPath, nil diff --git a/internal/topdata/topdata_test.go b/internal/topdata/topdata_test.go index 43cd050..e6c5e98 100644 --- a/internal/topdata/topdata_test.go +++ b/internal/topdata/topdata_test.go @@ -9195,6 +9195,7 @@ func TestBuildNativeWithReleasedHeadVisualeffectsManifest(t *testing.T) { mkdirAll(t, filepath.Join(projRoot, "src")) mkdirAll(t, filepath.Join(projRoot, "assets")) mkdirAll(t, filepath.Join(projRoot, "build")) + mkdirAll(t, filepath.Join(projRoot, ".cache")) mkdirAll(t, filepath.Join(projRoot, "topdata", "data", "visualeffects")) writeFile(t, filepath.Join(projRoot, "topdata", "base_dialog.json"), "{}\n") writeFile(t, filepath.Join(projRoot, "topdata", "data", "visualeffects", "base.json"), `{ @@ -9301,6 +9302,191 @@ func TestBuildNativeWithReleasedHeadVisualeffectsManifest(t *testing.T) { } } +func TestBuildNativePrefersRemoteReleasedHeadVisualeffectsManifestOverFreshCache(t *testing.T) { + root := t.TempDir() + projRoot := filepath.Join(root, "project") + mkdirAll(t, filepath.Join(projRoot, "src")) + mkdirAll(t, filepath.Join(projRoot, "assets")) + mkdirAll(t, filepath.Join(projRoot, "build")) + mkdirAll(t, filepath.Join(projRoot, ".cache")) + mkdirAll(t, filepath.Join(projRoot, "topdata", "data", "visualeffects")) + writeFile(t, filepath.Join(projRoot, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(projRoot, "topdata", "data", "visualeffects", "base.json"), `{ + "output": "visualeffects.2da", + "columns": ["Label", "Type_FD", "OrientWithGround", "Imp_HeadCon_Node", "OrientWithObject"], + "rows": [ + {"key": "visualeffects:existing", "id": 17, "Label": "EXISTING", "Type_FD": "F", "OrientWithGround": "0", "Imp_HeadCon_Node": "****", "OrientWithObject": "0"} + ] +}`+"\n") + writeFile(t, filepath.Join(projRoot, "topdata", "data", "visualeffects", "lock.json"), `{ + "visualeffects:existing": 17, + "visualeffects:headaccessory_bandana": 42 +}`+"\n") + mkdirAll(t, filepath.Join(projRoot, "reference")) + writeFile(t, filepath.Join(projRoot, "reference", "build.py"), "print('ok')\n") + + cachePath := filepath.Join(projRoot, ".cache", "sow-head-vfx-manifest.json") + writeFile(t, cachePath, `{ + "id": "head_visualeffects", + "generated_at": "2026-04-25T00:00:00Z", + "entries": [ + { + "group": "head_features", + "model_stem": "hfx_hair_bangs", + "source": "head_features/hair/hfx_hair_bangs.mdl" + } + ] +}`+"\n") + cacheTime := time.Date(2026, 4, 25, 10, 0, 0, 0, time.UTC) + setFileTime(t, cachePath, cacheTime) + + manifestJSON := `{ + "id": "head_visualeffects", + "repo": "ShadowsOverWestgate/sow-assets", + "ref": "remote-manifest", + "generated_at": "2026-04-25T12:00:00Z", + "entries": [ + { + "group": "head_accessories", + "model_stem": "hfx_bandana", + "source": "head_accessories/hfx_bandana.mdl" + } + ] +}` + "\n" + + var server *httptest.Server + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/v1/repos/ShadowsOverWestgate/sow-assets/releases/tags/head-vfx-manifest-current": + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"assets":[{"name":"sow-head-vfx-manifest.json","updated_at":"2026-04-25T12:00:00Z","browser_download_url":"` + server.URL + `/downloads/sow-head-vfx-manifest.json"}]}`)) + case "/downloads/sow-head-vfx-manifest.json": + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(manifestJSON)) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + runGitTest(t, "", "init", "-b", "main", projRoot) + runGitTest(t, projRoot, "remote", "add", "origin", server.URL+"/ShadowsOverWestgate/sow-module.git") + + proj := &project.Project{ + Root: projRoot, + Config: project.Config{ + Module: project.ModuleConfig{Name: "Test", ResRef: "test"}, + Paths: project.PathConfig{Source: "src", Assets: "assets", Build: "build"}, + TopData: project.TopDataConfig{ + Source: "topdata", + Build: "build/topdata", + }, + Autogen: project.AutogenConfig{ + Consumers: []project.AutogenConsumerConfig{ + { + ID: "head_visualeffects", + Producer: "head_visualeffects", + Dataset: "visualeffects", + Mode: "head_visualeffects", + Root: "vfxs", + Include: []string{"head_accessories/**/*.mdl", "head_features/**/*.mdl"}, + Derive: project.AutogenDeriveConfig{ + Kind: "model_stem", + GroupFrom: "first_path_segment", + }, + Manifest: project.AutogenManifestConfig{ + ReleaseTag: "head-vfx-manifest-current", + AssetName: "sow-head-vfx-manifest.json", + CacheName: "sow-head-vfx-manifest.json", + }, + }, + }, + }, + }, + } + + result, err := BuildNative(proj, nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + + visualeffectsBytes, err := os.ReadFile(filepath.Join(result.Output2DADir, "visualeffects.2da")) + if err != nil { + t.Fatalf("read visualeffects.2da: %v", err) + } + text := string(visualeffectsBytes) + if !strings.Contains(text, "42\theadaccessory_BANDANA\tD\t0\thfx_bandana\t1") { + t.Fatalf("expected remote manifest entry with preserved lock id, got:\n%s", text) + } + if strings.Contains(text, "HAIR_BANGS") { + t.Fatalf("expected fresh remote manifest to replace cached manifest contents, got:\n%s", text) + } +} + +func TestBuildPackageInvalidatesFreshAutogenCacheWhenReleaseAssetIsNewer(t *testing.T) { + root := topPackageTestProject(t) + proj := testProject(root) + proj.Config.TopData.ReferenceBuilder = "" + proj.Config.Autogen.Consumers = nil + + result, err := BuildAndPackage(proj, nil) + if err != nil { + t.Fatalf("initial BuildAndPackage failed: %v", err) + } + + sourceTime := time.Now().Add(-6 * time.Hour) + outputTime := time.Now().Add(-4 * time.Hour) + setTopDataSourceTimes(t, root, sourceTime) + setCompiledOutputTimes(t, result, outputTime) + + cachePath := filepath.Join(root, ".cache", "sow-head-vfx-manifest.json") + writeFile(t, cachePath, `{"id":"head_visualeffects","generated_at":"2026-04-25T10:00:00Z","entries":[{"group":"head_accessories","model_stem":"hfx_bandana","source":"head_accessories/hfx_bandana.mdl"}]}`+"\n") + setFileTime(t, cachePath, time.Now()) + + var server *httptest.Server + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/v1/repos/ShadowsOverWestgate/sow-assets/releases/tags/head-vfx-manifest-current": + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"assets":[{"name":"sow-head-vfx-manifest.json","updated_at":"3026-04-25T12:00:00Z","browser_download_url":"` + server.URL + `/downloads/sow-head-vfx-manifest.json"}]}`)) + case "/downloads/sow-head-vfx-manifest.json": + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"id":"head_visualeffects","entries":[{"group":"head_accessories","model_stem":"hfx_bandana","source":"head_accessories/hfx_bandana.mdl"}]}`)) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + runGitTest(t, "", "init", "-b", "main", root) + runGitTest(t, root, "remote", "add", "origin", server.URL+"/ShadowsOverWestgate/sow-module.git") + + proj.Config.Autogen.Consumers = []project.AutogenConsumerConfig{ + { + ID: "head_visualeffects", + Producer: "head_visualeffects", + Dataset: "visualeffects", + Mode: "head_visualeffects", + Optional: true, + Root: "vfxs", + Include: []string{"head_accessories/**/*.mdl", "head_features/**/*.mdl"}, + Derive: project.AutogenDeriveConfig{ + Kind: "model_stem", + GroupFrom: "first_path_segment", + }, + Manifest: project.AutogenManifestConfig{ + ReleaseTag: "head-vfx-manifest-current", + AssetName: "sow-head-vfx-manifest.json", + CacheName: "sow-head-vfx-manifest.json", + }, + }, + } + + if _, err := BuildPackage(proj, nil); err == nil || !strings.Contains(err.Error(), "older than source file") { + t.Fatalf("expected newer remote autogen manifest to invalidate compiled output, got %v", err) + } +} + func TestBuildNativeRejectsGitRepoTopDataAssetsOverride(t *testing.T) { root := testProjectRoot(t) mkdirAll(t, filepath.Join(root, "topdata", "data", "parts"))