Refresh Manifests

internal/topdata/top_package.go

@@ -282,6 +282,111 @@ func newestTopDataSource(p *project.Project) (time.Time, string, error) {
 	return newest, newestPath, nil
 }
 
+func newestTopDataInput(p *project.Project, now time.Time) (time.Time, string, error) {
+	newest, newestPath, err := newestTopDataSource(p)
+	if err != nil {
+		return time.Time{}, "", err
+	}
+	autogenTime, autogenPath, err := newestAutogenInput(p, now)
+	if err != nil {
+		return time.Time{}, "", err
+	}
+	if autogenTime.After(newest) {
+		return autogenTime, autogenPath, nil
+	}
+	return newest, newestPath, nil
+}
+
+func newestAutogenInput(p *project.Project, now time.Time) (time.Time, string, error) {
+	newest := time.Time{}
+	newestPath := ""
+	for _, consumer := range p.Config.Autogen.Consumers {
+		overrideRoot, err := resolveAutogenLocalOverrideRoot(p, consumer)
+		if err != nil {
+			return time.Time{}, "", err
+		}
+		if overrideRoot != "" {
+			candidateRoot := filepath.Join(overrideRoot, consumer.Root)
+			candidateTime, candidatePath, err := newestMatchingAutogenOverrideInput(candidateRoot, consumer.Include)
+			if err != nil {
+				return time.Time{}, "", err
+			}
+			if candidateTime.After(newest) {
+				newest = candidateTime
+				newestPath = candidatePath
+			}
+			continue
+		}
+
+		cachePath := filepath.Join(p.Root, ".cache", consumer.Manifest.CacheName)
+		info, err := os.Stat(cachePath)
+		if err != nil {
+			if os.IsNotExist(err) {
+				if now.After(newest) {
+					newest = now
+					newestPath = cachePath
+				}
+				continue
+			}
+			return time.Time{}, "", fmt.Errorf("stat autogen manifest cache %s: %w", cachePath, err)
+		}
+
+		candidateTime := info.ModTime()
+		if autogenManifestCacheMaxAge > 0 && now.Sub(info.ModTime()) > autogenManifestCacheMaxAge {
+			candidateTime = now
+		}
+		if candidateTime.After(newest) {
+			newest = candidateTime
+			newestPath = cachePath
+		}
+	}
+	return newest, newestPath, nil
+}
+
+func newestMatchingAutogenOverrideInput(scanRoot string, include []string) (time.Time, string, error) {
+	info, err := os.Stat(scanRoot)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return time.Time{}, "", nil
+		}
+		return time.Time{}, "", fmt.Errorf("stat autogen override root %s: %w", scanRoot, err)
+	}
+	if !info.IsDir() {
+		return time.Time{}, "", fmt.Errorf("autogen override root %s is not a directory", scanRoot)
+	}
+
+	newest := time.Time{}
+	newestPath := ""
+	err = filepath.WalkDir(scanRoot, func(path string, d fs.DirEntry, walkErr error) error {
+		if walkErr != nil {
+			return walkErr
+		}
+		if d.IsDir() {
+			return nil
+		}
+		rel, err := filepath.Rel(scanRoot, path)
+		if err != nil {
+			return err
+		}
+		if !matchesAutogenInclude(filepath.ToSlash(rel), include) {
+			return nil
+		}
+		info, err := d.Info()
+		if err != nil {
+			return err
+		}
+		if info.ModTime().After(newest) {
+			newest = info.ModTime()
+			newestPath = path
+		}
+		return nil
+	})
+	if err != nil {
+		return time.Time{}, "", fmt.Errorf("scan autogen override %s: %w", scanRoot, err)
+	}
+	return newest, newestPath, nil
+}
+
 func topPackageResourceFromPath(path string) (erf.Resource, error) {
 	extension := strings.TrimPrefix(strings.ToLower(filepath.Ext(path)), ".")
 	resourceType, ok := erf.HAKResourceTypeForExtension(extension)
@@ -397,7 +502,7 @@ func packagedBuildResult(p *project.Project) (BuildResult, error) {
 		return BuildResult{}, fmt.Errorf("topdata build output missing: run build-topdata first")
 	}
 
-	newestSource, newestPath, err := newestTopDataSource(p)
+	newestSource, newestPath, err := newestTopDataInput(p, time.Now())
 	if err != nil {
 		return BuildResult{}, err
 	}
@@ -452,7 +557,7 @@ func currentTopPackageResult(p *project.Project) (PackageResult, bool, error) {
 		return PackageResult{}, false, fmt.Errorf("stat top package tlk %s: %w", outputTLK, err)
 	}
 
-	newestSource, _, err := newestTopDataSource(p)
+	newestSource, _, err := newestTopDataInput(p, time.Now())
 	if err != nil {
 		return PackageResult{}, false, err
 	}

internal/topdata/topdata_test.go

@@ -9490,6 +9490,7 @@ func TestBuildAndPackageNoOpsWhenOutputsAreCurrent(t *testing.T) {
 	root := topPackageTestProject(t)
 	proj := testProject(root)
 	proj.Config.TopData.ReferenceBuilder = ""
+	proj.Config.Autogen.Consumers = nil
 
 	result, err := BuildAndPackage(proj, nil)
 	if err != nil {
@@ -9497,7 +9498,7 @@ func TestBuildAndPackageNoOpsWhenOutputsAreCurrent(t *testing.T) {
 	}
 
 	sourceTime := time.Now().Add(-4 * time.Hour)
-	outputTime := time.Now().Add(-2 * time.Hour)
+	outputTime := time.Now().Add(-10 * time.Minute)
 	setTopDataSourceTimes(t, root, sourceTime)
 	setBuildOutputTimes(t, result, outputTime)
 
@@ -9518,10 +9519,56 @@ func TestBuildAndPackageNoOpsWhenOutputsAreCurrent(t *testing.T) {
 	}
 }
 
+func TestBuildPackageFailsWhenAutogenManifestCacheIsStale(t *testing.T) {
+	root := topPackageTestProject(t)
+	proj := testProject(root)
+	proj.Config.TopData.ReferenceBuilder = ""
+	proj.Config.Autogen.Consumers = nil
+
+	result, err := BuildAndPackage(proj, nil)
+	if err != nil {
+		t.Fatalf("initial BuildAndPackage failed: %v", err)
+	}
+
+	sourceTime := time.Now().Add(-6 * time.Hour)
+	outputTime := time.Now().Add(-4 * time.Hour)
+	setTopDataSourceTimes(t, root, sourceTime)
+	setCompiledOutputTimes(t, result, outputTime)
+
+	proj.Config.Autogen.Consumers = []project.AutogenConsumerConfig{
+		{
+			ID:       "head_visualeffects",
+			Producer: "head_visualeffects",
+			Dataset:  "visualeffects",
+			Mode:     "head_visualeffects",
+			Optional: true,
+			Root:     "vfxs",
+			Include:  []string{"head_accessories/**/*.mdl", "head_features/**/*.mdl"},
+			Derive: project.AutogenDeriveConfig{
+				Kind:      "model_stem",
+				GroupFrom: "first_path_segment",
+			},
+			Manifest: project.AutogenManifestConfig{
+				ReleaseTag: "head-vfx-manifest-current",
+				AssetName:  "sow-head-vfx-manifest.json",
+				CacheName:  "sow-head-vfx-manifest.json",
+			},
+		},
+	}
+	cachePath := filepath.Join(root, ".cache", "sow-head-vfx-manifest.json")
+	writeFile(t, cachePath, `{"id":"head_visualeffects","entries":[{"group":"head_accessories","model_stem":"hfx_bandana","source":"head_accessories/hfx_bandana.mdl"}]}`+"\n")
+	setFileTime(t, cachePath, time.Now().Add(-2*autogenManifestCacheMaxAge))
+
+	if _, err := BuildPackage(proj, nil); err == nil || !strings.Contains(err.Error(), "older than source file") {
+		t.Fatalf("expected stale autogen manifest cache to invalidate compiled output, got %v", err)
+	}
+}
+
 func TestBuildAndPackageRefreshesPackageWhenCompiledOutputsAreCurrent(t *testing.T) {
 	root := topPackageTestProject(t)
 	proj := testProject(root)
 	proj.Config.TopData.ReferenceBuilder = ""
+	proj.Config.Autogen.Consumers = nil
 
 	result, err := BuildAndPackage(proj, nil)
 	if err != nil {
@@ -9754,6 +9801,7 @@ func TestBuildPackageIgnoresTemplateSourcesForFreshness(t *testing.T) {
 
 	proj := testProject(root)
 	proj.Config.TopData.ReferenceBuilder = ""
+	proj.Config.Autogen.Consumers = nil
 
 	if _, err := BuildNative(proj, nil); err != nil {
 		t.Fatalf("BuildNative failed: %v", err)

internal/topdata/wiki_native_test.go

@@ -163,6 +163,7 @@ func TestBuildNativeRegeneratesWikiWhenTLKTextChanges(t *testing.T) {
 
 	proj := testProject(root)
 	proj.Config.TopData.ReferenceBuilder = ""
+	proj.Config.Autogen.Consumers = nil
 
 	if _, err := BuildNative(proj, nil); err != nil {
 		t.Fatalf("initial BuildNative failed: %v", err)
@@ -225,6 +226,7 @@ func TestBuildPackageIgnoresWikiSourcesForFreshness(t *testing.T) {
 
 	proj := testProject(root)
 	proj.Config.TopData.ReferenceBuilder = ""
+	proj.Config.Autogen.Consumers = nil
 
 	if _, err := BuildNative(proj, nil); err != nil {
 		t.Fatalf("BuildNative failed: %v", err)