Fix Autogen Lockfile Overrides

2026-04-25 23:36:12 +02:00
parent 88843a1ec1
commit d893785c0c
3 changed files with 305 additions and 25 deletions
@@ -32,6 +32,11 @@ type autogenManifestEntry struct {
 	RowID     int    `json:"row_id,omitempty"`
 }

+type autogenManifestAssetMetadata struct {
+	DownloadURL string
+	UpdatedAt   time.Time
+}
+
 type ProducedAutogenManifest struct {
 	ID         string
 	AssetName  string
@@ -331,10 +336,19 @@ func resolveReleasedAutogenManifestEntries(p *project.Project, consumer project.
 				progress(fmt.Sprintf("Ignoring cached autogen manifest %s: %v", consumer.Manifest.CacheName, err))
 			}
 		} else if fresh {
-			if progress != nil {
-				progress(fmt.Sprintf("Using cached released autogen manifest %s...", cachePath))
+			if current, remoteTime, err := releasedAutogenManifestCacheCurrent(p, consumer, cachePath, manifest); err != nil {
+				if progress != nil {
+					progress(fmt.Sprintf("Using cached released autogen manifest %s (remote check failed: %v)...", cachePath, err))
+				}
+				return manifest.Entries, nil
+			} else if current {
+				if progress != nil {
+					progress(fmt.Sprintf("Using cached released autogen manifest %s...", cachePath))
+				}
+				return manifest.Entries, nil
+			} else if progress != nil {
+				progress(fmt.Sprintf("Refreshing released autogen manifest %s; remote asset changed at %s.", consumer.Manifest.CacheName, remoteTime.Format(time.RFC3339)))
 			}
-			return manifest.Entries, nil
 		}
 	}

@@ -368,30 +382,103 @@ func resolveReleasedAutogenManifestEntries(p *project.Project, consumer project.
 	return manifest.Entries, nil
 }

+func releasedAutogenManifestCacheCurrent(p *project.Project, consumer project.AutogenConsumerConfig, cachePath string, manifest *autogenManifest) (bool, time.Time, error) {
+	spec, err := deriveSowAssetsRepoSpec(p.Root)
+	if err != nil {
+		return false, time.Time{}, err
+	}
+	asset, err := resolveAutogenManifestAssetMetadata(spec, consumer.Manifest.ReleaseTag, consumer.Manifest.AssetName)
+	if err != nil {
+		return false, time.Time{}, err
+	}
+	if asset.UpdatedAt.IsZero() {
+		return true, time.Time{}, nil
+	}
+	return !asset.UpdatedAt.After(localAutogenManifestComparableTime(cachePath, manifest)), asset.UpdatedAt, nil
+}
+
+func localAutogenManifestComparableTime(cachePath string, manifest *autogenManifest) time.Time {
+	best := time.Time{}
+	if info, err := os.Stat(cachePath); err == nil {
+		best = info.ModTime()
+	}
+	if generatedAt, err := time.Parse(time.RFC3339, strings.TrimSpace(manifest.GeneratedAt)); err == nil && generatedAt.After(best) {
+		best = generatedAt
+	}
+	return best
+}
+
+func newestReleasedAutogenManifestInput(p *project.Project, consumer project.AutogenConsumerConfig, now time.Time, cachePath string) (time.Time, string, error) {
+	if strings.TrimSpace(os.Getenv("SOW_AUTOGEN_MANIFEST_REFRESH")) != "" {
+		return now, cachePath, nil
+	}
+
+	info, err := os.Stat(cachePath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return now, cachePath, nil
+		}
+		return time.Time{}, "", fmt.Errorf("stat autogen manifest cache %s: %w", cachePath, err)
+	}
+	if autogenManifestCacheMaxAge > 0 && now.Sub(info.ModTime()) > autogenManifestCacheMaxAge {
+		return now, cachePath, nil
+	}
+
+	raw, err := os.ReadFile(cachePath)
+	if err != nil {
+		return info.ModTime(), cachePath, nil
+	}
+	var manifest autogenManifest
+	if err := json.Unmarshal(raw, &manifest); err != nil {
+		return info.ModTime(), cachePath, nil
+	}
+
+	current, remoteTime, err := releasedAutogenManifestCacheCurrent(p, consumer, cachePath, &manifest)
+	if err != nil || current || remoteTime.IsZero() {
+		return info.ModTime(), cachePath, nil
+	}
+	return remoteTime, consumer.Manifest.AssetName, nil
+}
+
 func resolveAutogenManifestAssetURL(spec giteaRepoSpec, releaseTag, assetName string) (string, error) {
+	asset, err := resolveAutogenManifestAssetMetadata(spec, releaseTag, assetName)
+	if err != nil {
+		return "", err
+	}
+	return asset.DownloadURL, nil
+}
+
+func resolveAutogenManifestAssetMetadata(spec giteaRepoSpec, releaseTag, assetName string) (autogenManifestAssetMetadata, error) {
 	apiURL := fmt.Sprintf("%s/api/v1/repos/%s/%s/releases/tags/%s", strings.TrimRight(spec.BaseURL, "/"), releasePathEscape(spec.Owner), releasePathEscape(spec.Repo), releasePathEscape(releaseTag))
 	var release struct {
 		Assets []struct {
 			Name               string `json:"name"`
 			BrowserDownloadURL string `json:"browser_download_url"`
 			DownloadURL        string `json:"download_url"`
+			UpdatedAt          string `json:"updated_at"`
 		} `json:"assets"`
 	}
 	if err := fetchJSON(apiURL, &release); err != nil {
-		return "", fmt.Errorf("fetch autogen manifest release metadata: %w", err)
+		return autogenManifestAssetMetadata{}, fmt.Errorf("fetch autogen manifest release metadata: %w", err)
 	}
 	for _, asset := range release.Assets {
 		if asset.Name != assetName {
 			continue
 		}
+		metadata := autogenManifestAssetMetadata{}
+		if updatedAt, err := time.Parse(time.RFC3339, strings.TrimSpace(asset.UpdatedAt)); err == nil {
+			metadata.UpdatedAt = updatedAt
+		}
 		if asset.DownloadURL != "" {
-			return asset.DownloadURL, nil
+			metadata.DownloadURL = asset.DownloadURL
+			return metadata, nil
 		}
 		if asset.BrowserDownloadURL != "" {
-			return asset.BrowserDownloadURL, nil
+			metadata.DownloadURL = asset.BrowserDownloadURL
+			return metadata, nil
 		}
 	}
-	return "", fmt.Errorf("autogen manifest asset %s not found in release %s/%s:%s", assetName, spec.Owner, spec.Repo, releaseTag)
+	return autogenManifestAssetMetadata{}, fmt.Errorf("autogen manifest asset %s not found in release %s/%s:%s", assetName, spec.Owner, spec.Repo, releaseTag)
 }

 func releasePathEscape(value string) string {
@@ -529,11 +616,26 @@ func augmentWithAutogeneratedHeadVisualeffects(collected []nativeCollectedDatase

 		rows := make([]map[string]any, len(dataset.Rows))
 		rowByKey := make(map[string]map[string]any, len(dataset.Rows))
-		usedIDs := make(map[int]struct{}, len(dataset.Rows)+len(dataset.LockData))
+		historicalLockData := make(map[string]int, len(dataset.LockData))
+		for key, rowID := range dataset.LockData {
+			historicalLockData[key] = rowID
+		}
+		if dataset.Dataset.LockPath != "" {
+			if existingLockData, err := loadLockfile(dataset.Dataset.LockPath); err == nil {
+				for key, rowID := range existingLockData {
+					if _, ok := historicalLockData[key]; !ok {
+						historicalLockData[key] = rowID
+					}
+				}
+			}
+		}
+		usedIDs := make(map[int]struct{}, len(dataset.Rows)+len(historicalLockData))
 		lockData := make(map[string]int, len(dataset.LockData))
+		for _, rowID := range historicalLockData {
+			usedIDs[rowID] = struct{}{}
+		}
 		for key, rowID := range dataset.LockData {
 			lockData[key] = rowID
-			usedIDs[rowID] = struct{}{}
 		}
 		for index, row := range dataset.Rows {
 			cloned := cloneRowMap(row)
@@ -566,7 +668,11 @@ func augmentWithAutogeneratedHeadVisualeffects(collected []nativeCollectedDatase

 			rowID, ok := lockData[key]
 			if !ok {
-				rowID = allocateNextID()
+				if preservedRowID, preserved := historicalLockData[key]; preserved {
+					rowID = preservedRowID
+				} else {
+					rowID = allocateNextID()
+				}
 				lockData[key] = rowID
 			}
 			newRow := createDefaultHeadVisualeffectRow(dataset.Columns, rowID, key, label, entry.ModelStem)