package topdata import ( "bytes" "encoding/json" "fmt" "os" "path/filepath" "slices" "strconv" "strings" "time" "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" ) const autogenManifestCacheMaxAge = time.Hour type autogenManifest struct { ID string `json:"id"` Repo string `json:"repo"` Ref string `json:"ref"` GeneratedAt string `json:"generated_at"` Entries []autogenManifestEntry `json:"entries"` } type autogenManifestEntry struct { Source string `json:"source"` Group string `json:"group,omitempty"` Subgroup string `json:"subgroup,omitempty"` ModelStem string `json:"model_stem,omitempty"` RowID int `json:"row_id,omitempty"` } type autogenManifestAssetMetadata struct { DownloadURL string UpdatedAt time.Time } type ProducedAutogenManifest struct { ID string AssetName string OutputPath string Payload []byte } func applyAutogenConsumers(p *project.Project, collected []nativeCollectedDataset, progress func(string)) ([]nativeCollectedDataset, error) { if len(p.Config.Autogen.Consumers) == 0 { return collected, nil } if progress == nil { progress = func(string) {} } result := append([]nativeCollectedDataset(nil), collected...) for _, consumer := range p.Config.Autogen.Consumers { if !autogenConsumerTargetsCollectedDataset(result, consumer) { continue } entries, err := resolveAutogenConsumerEntries(p, consumer, progress) if err != nil { if consumer.Optional && isIgnorableOptionalAutogenError(err) { if progress != nil { progress(fmt.Sprintf("Skipping optional autogen consumer %s: %v", consumer.ID, err)) } continue } return nil, err } if len(entries) == 0 { continue } switch consumer.Mode { case "parts_rows": result = augmentWithAutogeneratedParts(result, autogenPartsInventory(entries)) case "head_visualeffects": result, err = augmentWithAutogeneratedHeadVisualeffects(result, entries) if err != nil { return nil, err } default: return nil, fmt.Errorf("unsupported autogen consumer mode %q", consumer.Mode) } } return result, nil } func autogenConsumerTargetsCollectedDataset(collected []nativeCollectedDataset, consumer project.AutogenConsumerConfig) bool { for _, dataset := range collected { switch consumer.Mode { case "parts_rows": if isAutogenEligiblePartsDataset(dataset) { return true } case "head_visualeffects": if dataset.Dataset.Name == "visualeffects" { return true } } } return false } func isIgnorableOptionalAutogenError(err error) bool { if err == nil { return false } message := err.Error() return strings.Contains(message, "HTTP 404") || strings.Contains(message, "not found") || strings.Contains(message, "entries is empty") } func resolveAutogenConsumerEntries(p *project.Project, consumer project.AutogenConsumerConfig, progress func(string)) ([]autogenManifestEntry, error) { overrideRoot, err := resolveAutogenLocalOverrideRoot(p, consumer) if err != nil { return nil, err } if overrideRoot != "" { if progress != nil { progress(fmt.Sprintf("Scanning local autogen override for %s from %s...", consumer.ID, overrideRoot)) } return scanLocalAutogenEntries(overrideRoot, consumer) } return resolveReleasedAutogenManifestEntries(p, consumer, progress) } func resolveAutogenLocalOverrideRoot(p *project.Project, consumer project.AutogenConsumerConfig) (string, error) { root := strings.TrimSpace(consumer.LocalOverrideRoot) if root != "" { if filepath.IsAbs(root) { return root, nil } return filepath.Join(p.Root, root), nil } spec := strings.TrimSpace(p.Config.TopData.Assets) if spec == "" { return "", nil } if looksLikeGitRepoSpec(spec) { return "", fmt.Errorf("topdata.assets no longer supports git repo specs for autogen discovery; use a local path override or the default released manifest") } path := p.TopDataAssetsDir() if path == "" { return "", nil } if _, err := os.Stat(path); err != nil { return "", fmt.Errorf("configured topdata.assets path %s is not accessible: %w", path, err) } candidate := filepath.Join(path, consumer.Root) info, err := os.Stat(candidate) if err != nil { if os.IsNotExist(err) { return "", nil } return "", fmt.Errorf("configured topdata.assets override root %s is not accessible: %w", candidate, err) } if !info.IsDir() { return "", fmt.Errorf("configured topdata.assets override root %s is not a directory", candidate) } return path, nil } func scanLocalAutogenEntries(overrideRoot string, consumer project.AutogenConsumerConfig) ([]autogenManifestEntry, error) { scanRoot := filepath.Join(overrideRoot, consumer.Root) return scanConfiguredAutogenEntries(scanRoot, consumer.Include, consumer.Derive) } func scanConfiguredAutogenEntries(scanRoot string, include []string, derive project.AutogenDeriveConfig) ([]autogenManifestEntry, error) { info, err := os.Stat(scanRoot) if err != nil { if os.IsNotExist(err) { return nil, nil } return nil, fmt.Errorf("stat autogen override root %s: %w", scanRoot, err) } if !info.IsDir() { return nil, fmt.Errorf("autogen override root %s is not a directory", scanRoot) } var entries []autogenManifestEntry err = filepath.Walk(scanRoot, func(path string, info os.FileInfo, walkErr error) error { if walkErr != nil { return walkErr } if info.IsDir() { return nil } rel, err := filepath.Rel(scanRoot, path) if err != nil { return err } rel = filepath.ToSlash(rel) if !matchesAutogenInclude(rel, include) { return nil } entry, ok := deriveAutogenManifestEntry(rel, derive) if !ok { return nil } entries = append(entries, entry) return nil }) if err != nil { return nil, fmt.Errorf("scan autogen override %s: %w", scanRoot, err) } slices.SortFunc(entries, func(a, b autogenManifestEntry) int { return strings.Compare(a.Source, b.Source) }) return entries, nil } func ProduceAutogenManifests(p *project.Project) ([]ProducedAutogenManifest, error) { if len(p.Config.Autogen.Producers) == 0 { return nil, nil } assetsRoot := strings.TrimSpace(p.AssetsDir()) if assetsRoot == "" { return nil, fmt.Errorf("autogen producers require paths.assets to be configured") } info, err := os.Stat(assetsRoot) if err != nil { return nil, fmt.Errorf("stat assets root %s: %w", assetsRoot, err) } if !info.IsDir() { return nil, fmt.Errorf("assets root %s is not a directory", assetsRoot) } repo, ref := autogenManifestSourceMetadata(p.Root) results := make([]ProducedAutogenManifest, 0, len(p.Config.Autogen.Producers)) for _, producer := range p.Config.Autogen.Producers { scanRoot := filepath.Join(assetsRoot, producer.Root) entries, err := scanConfiguredAutogenEntries(scanRoot, producer.Include, producer.Derive) if err != nil { return nil, fmt.Errorf("scan autogen producer %s: %w", producer.ID, err) } if len(entries) == 0 { return nil, fmt.Errorf("autogen producer %s discovered no matching entries under %s", producer.ID, scanRoot) } payload, err := formatAutogenManifest(p.Root, producer, repo, ref, entries) if err != nil { return nil, fmt.Errorf("format autogen producer %s manifest: %w", producer.ID, err) } payload = append(payload, '\n') results = append(results, ProducedAutogenManifest{ ID: producer.ID, AssetName: producer.Manifest.AssetName, OutputPath: filepath.Join(p.BuildDir(), producer.Manifest.AssetName), Payload: payload, }) } return results, nil } func autogenManifestSourceMetadata(root string) (repo string, ref string) { if remote, err := gitOutput(root, "remote", "get-url", "origin"); err == nil { if spec, ok := parseRepoSpec(strings.TrimSpace(remote)); ok { repo = spec.Owner + "/" + spec.Repo } } if head, err := gitOutput(root, "rev-parse", "HEAD"); err == nil { ref = strings.TrimSpace(head) } return repo, ref } func matchesAutogenInclude(rel string, include []string) bool { rel = filepath.ToSlash(rel) for _, pattern := range include { pattern = filepath.ToSlash(strings.TrimSpace(pattern)) if pattern == "" { continue } switch { case pattern == "**/*.mdl": if strings.HasSuffix(strings.ToLower(rel), ".mdl") { return true } case strings.HasSuffix(pattern, "/**/*.mdl"): prefix := strings.TrimSuffix(pattern, "/**/*.mdl") if strings.HasPrefix(rel, prefix+"/") && strings.HasSuffix(strings.ToLower(rel), ".mdl") { return true } case strings.HasSuffix(pattern, "/*.mdl"): prefix := strings.TrimSuffix(pattern, "/*.mdl") dir := filepath.ToSlash(filepath.Dir(rel)) if dir == prefix && strings.HasSuffix(strings.ToLower(rel), ".mdl") { return true } case rel == pattern: return true } } return false } func deriveAutogenManifestEntry(rel string, derive project.AutogenDeriveConfig) (autogenManifestEntry, bool) { rel = filepath.ToSlash(rel) stem := strings.TrimSuffix(filepath.Base(rel), filepath.Ext(rel)) entry := autogenManifestEntry{ Source: rel, ModelStem: stem, } parts := strings.Split(rel, "/") if derive.GroupFrom == "first_path_segment" && len(parts) > 1 { entry.Group = parts[0] if len(parts) > 2 { entry.Subgroup = strings.Join(parts[1:len(parts)-1], "/") } } switch derive.Kind { case "trailing_numeric_suffix": rowID, ok := extractTrailingNumericSuffix(stem) if !ok { return autogenManifestEntry{}, false } entry.RowID = rowID case "model_stem": default: return autogenManifestEntry{}, false } return entry, true } func resolveReleasedAutogenManifestEntries(p *project.Project, consumer project.AutogenConsumerConfig, progress func(string)) ([]autogenManifestEntry, error) { cachePath := p.AutogenCachePath(consumer.Manifest.CacheName) if strings.TrimSpace(os.Getenv("SOW_AUTOGEN_MANIFEST_REFRESH")) == "" { manifest, fresh, err := readFreshAutogenManifestCache(cachePath, autogenManifestCacheMaxAge) if err != nil { if progress != nil { progress(fmt.Sprintf("Ignoring cached autogen manifest %s: %v", consumer.Manifest.CacheName, err)) } } else if fresh { if current, remoteTime, err := releasedAutogenManifestCacheCurrent(p, consumer, cachePath, manifest); err != nil { if progress != nil { progress(fmt.Sprintf("Using cached released autogen manifest %s (remote check failed: %v)...", cachePath, err)) } return manifest.Entries, nil } else if current { if progress != nil { progress(fmt.Sprintf("Using cached released autogen manifest %s...", cachePath)) } return manifest.Entries, nil } else if progress != nil { progress(fmt.Sprintf("Refreshing released autogen manifest %s; remote asset changed at %s.", consumer.Manifest.CacheName, remoteTime.Format(time.RFC3339))) } } } spec, err := deriveSowAssetsRepoSpec(p.Root) if err != nil { return nil, err } manifestURL, err := resolveAutogenManifestAssetURL(spec, consumer.Manifest.ReleaseTag, consumer.Manifest.AssetName) if err != nil { if consumer.Mode == "parts_rows" { return nil, fmt.Errorf("%s", strings.Replace(err.Error(), "autogen manifest", "parts manifest", 1)) } return nil, err } if progress != nil { progress(fmt.Sprintf("Fetching released autogen manifest from %s...", manifestURL)) } manifest, err := fetchAutogenManifest(manifestURL) if err != nil { if consumer.Mode == "parts_rows" { return nil, fmt.Errorf("%s", strings.Replace(err.Error(), "autogen manifest", "parts manifest", 1)) } return nil, err } if manifest.ID != "" && manifest.ID != consumer.Producer { return nil, fmt.Errorf("autogen manifest %s declared id %q, expected %q", consumer.Manifest.AssetName, manifest.ID, consumer.Producer) } if err := writeAutogenManifestCache(cachePath, manifest); err != nil { return nil, err } return manifest.Entries, nil } func releasedAutogenManifestCacheCurrent(p *project.Project, consumer project.AutogenConsumerConfig, cachePath string, manifest *autogenManifest) (bool, time.Time, error) { spec, err := deriveSowAssetsRepoSpec(p.Root) if err != nil { return false, time.Time{}, err } asset, err := resolveAutogenManifestAssetMetadata(spec, consumer.Manifest.ReleaseTag, consumer.Manifest.AssetName) if err != nil { return false, time.Time{}, err } if asset.UpdatedAt.IsZero() { return true, time.Time{}, nil } return !asset.UpdatedAt.After(localAutogenManifestComparableTime(cachePath, manifest)), asset.UpdatedAt, nil } func localAutogenManifestComparableTime(cachePath string, manifest *autogenManifest) time.Time { best := time.Time{} if info, err := os.Stat(cachePath); err == nil { best = info.ModTime() } if generatedAt, err := time.Parse(time.RFC3339, strings.TrimSpace(manifest.GeneratedAt)); err == nil && generatedAt.After(best) { best = generatedAt } return best } func newestReleasedAutogenManifestInput(p *project.Project, consumer project.AutogenConsumerConfig, now time.Time, cachePath string) (time.Time, string, error) { if strings.TrimSpace(os.Getenv("SOW_AUTOGEN_MANIFEST_REFRESH")) != "" { return now, cachePath, nil } info, err := os.Stat(cachePath) if err != nil { if os.IsNotExist(err) { return now, cachePath, nil } return time.Time{}, "", fmt.Errorf("stat autogen manifest cache %s: %w", cachePath, err) } if autogenManifestCacheMaxAge > 0 && now.Sub(info.ModTime()) > autogenManifestCacheMaxAge { return now, cachePath, nil } raw, err := os.ReadFile(cachePath) if err != nil { return info.ModTime(), cachePath, nil } var manifest autogenManifest if err := json.Unmarshal(raw, &manifest); err != nil { return info.ModTime(), cachePath, nil } current, remoteTime, err := releasedAutogenManifestCacheCurrent(p, consumer, cachePath, &manifest) if err != nil || current || remoteTime.IsZero() { return info.ModTime(), cachePath, nil } return remoteTime, consumer.Manifest.AssetName, nil } func resolveAutogenManifestAssetURL(spec giteaRepoSpec, releaseTag, assetName string) (string, error) { asset, err := resolveAutogenManifestAssetMetadata(spec, releaseTag, assetName) if err != nil { return "", err } return asset.DownloadURL, nil } func resolveAutogenManifestAssetMetadata(spec giteaRepoSpec, releaseTag, assetName string) (autogenManifestAssetMetadata, error) { apiURL := fmt.Sprintf("%s/api/v1/repos/%s/%s/releases/tags/%s", strings.TrimRight(spec.BaseURL, "/"), releasePathEscape(spec.Owner), releasePathEscape(spec.Repo), releasePathEscape(releaseTag)) var release struct { Assets []struct { Name string `json:"name"` BrowserDownloadURL string `json:"browser_download_url"` DownloadURL string `json:"download_url"` UpdatedAt string `json:"updated_at"` } `json:"assets"` } if err := fetchJSON(apiURL, &release); err != nil { return autogenManifestAssetMetadata{}, fmt.Errorf("fetch autogen manifest release metadata: %w", err) } for _, asset := range release.Assets { if asset.Name != assetName { continue } metadata := autogenManifestAssetMetadata{} if updatedAt, err := time.Parse(time.RFC3339, strings.TrimSpace(asset.UpdatedAt)); err == nil { metadata.UpdatedAt = updatedAt } if asset.DownloadURL != "" { metadata.DownloadURL = asset.DownloadURL return metadata, nil } if asset.BrowserDownloadURL != "" { metadata.DownloadURL = asset.BrowserDownloadURL return metadata, nil } } return autogenManifestAssetMetadata{}, fmt.Errorf("autogen manifest asset %s not found in release %s/%s:%s", assetName, spec.Owner, spec.Repo, releaseTag) } func releasePathEscape(value string) string { replacer := strings.NewReplacer("/", "%2F") return replacer.Replace(value) } func fetchAutogenManifest(manifestURL string) (*autogenManifest, error) { var manifest autogenManifest if err := fetchJSON(manifestURL, &manifest); err != nil { return nil, fmt.Errorf("download autogen manifest: %w", err) } if len(manifest.Entries) == 0 { var legacy partsManifest if err := fetchJSON(manifestURL, &legacy); err == nil && len(legacy.Categories) > 0 { return legacyPartsManifestToAutogen(&legacy), nil } } if len(manifest.Entries) == 0 { return nil, fmt.Errorf("download autogen manifest: entries is empty") } return &manifest, nil } func readFreshAutogenManifestCache(path string, maxAge time.Duration) (*autogenManifest, bool, error) { info, err := os.Stat(path) if err != nil { if os.IsNotExist(err) { return nil, false, nil } return nil, false, err } if maxAge > 0 && time.Since(info.ModTime()) > maxAge { return nil, false, nil } raw, err := os.ReadFile(path) if err != nil { return nil, false, err } var manifest autogenManifest if err := json.Unmarshal(raw, &manifest); err != nil { return nil, false, err } if len(manifest.Entries) == 0 { var legacy partsManifest if err := json.Unmarshal(raw, &legacy); err == nil && len(legacy.Categories) > 0 { return legacyPartsManifestToAutogen(&legacy), true, nil } } if len(manifest.Entries) == 0 { return nil, false, fmt.Errorf("entries is empty") } return &manifest, true, nil } func legacyPartsManifestToAutogen(legacy *partsManifest) *autogenManifest { entries := make([]autogenManifestEntry, 0) for _, category := range supportedPartCategories { for _, rowID := range legacy.Categories[category] { entries = append(entries, autogenManifestEntry{ Group: category, RowID: rowID, }) } } slices.SortFunc(entries, func(a, b autogenManifestEntry) int { if cmp := strings.Compare(a.Group, b.Group); cmp != 0 { return cmp } return a.RowID - b.RowID }) return &autogenManifest{ ID: "parts", Repo: legacy.Repo, Ref: legacy.Ref, GeneratedAt: legacy.GeneratedAt, Entries: entries, } } func writeAutogenManifestCache(path string, manifest *autogenManifest) error { if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { return fmt.Errorf("create autogen manifest cache dir: %w", err) } payload, err := json.MarshalIndent(manifest, "", " ") if err != nil { return fmt.Errorf("marshal autogen manifest cache: %w", err) } payload = append(payload, '\n') current, err := os.ReadFile(path) if err == nil && bytes.Equal(current, payload) { now := time.Now() if err := os.Chtimes(path, now, now); err != nil { return fmt.Errorf("refresh autogen manifest cache timestamp: %w", err) } return nil } if err != nil && !os.IsNotExist(err) { return fmt.Errorf("read autogen manifest cache: %w", err) } tmpPath := path + ".tmp" if err := os.WriteFile(tmpPath, payload, 0o644); err != nil { return fmt.Errorf("write autogen manifest cache: %w", err) } if err := os.Rename(tmpPath, path); err != nil { return fmt.Errorf("replace autogen manifest cache: %w", err) } return nil } func autogenPartsInventory(entries []autogenManifestEntry) map[string]map[int]struct{} { result := make(map[string]map[int]struct{}, len(supportedPartCategories)) for _, category := range supportedPartCategories { result[category] = map[int]struct{}{} } for _, entry := range entries { if !isSupportedPartCategory(entry.Group) || entry.RowID <= 0 { continue } result[entry.Group][entry.RowID] = struct{}{} } return result } func augmentWithAutogeneratedHeadVisualeffects(collected []nativeCollectedDataset, entries []autogenManifestEntry) ([]nativeCollectedDataset, error) { if len(entries) == 0 { return collected, nil } result := append([]nativeCollectedDataset(nil), collected...) for i, dataset := range result { if dataset.Dataset.Name != "visualeffects" { continue } rows := make([]map[string]any, len(dataset.Rows)) rowByKey := make(map[string]map[string]any, len(dataset.Rows)) historicalLockData := make(map[string]int, len(dataset.LockData)) for key, rowID := range dataset.LockData { historicalLockData[key] = rowID } if dataset.Dataset.LockPath != "" { if existingLockData, err := loadLockfile(dataset.Dataset.LockPath); err == nil { for key, rowID := range existingLockData { if _, ok := historicalLockData[key]; !ok { historicalLockData[key] = rowID } } } } usedIDs := make(map[int]struct{}, len(dataset.Rows)+len(historicalLockData)) lockData := make(map[string]int, len(dataset.LockData)) for _, rowID := range historicalLockData { usedIDs[rowID] = struct{}{} } for key, rowID := range dataset.LockData { lockData[key] = rowID } for index, row := range dataset.Rows { cloned := cloneRowMap(row) rows[index] = cloned if key, ok := cloned["key"].(string); ok && key != "" { rowByKey[key] = cloned } if rowID, ok := cloned["id"].(int); ok { usedIDs[rowID] = struct{}{} } } nextID := nextAvailableAutogenID(usedIDs) allocateNextID := func() int { rowID := nextID usedIDs[rowID] = struct{}{} nextID = nextAvailableAutogenID(usedIDs) return rowID } for _, entry := range entries { key, label, ok := headVisualeffectIdentity(entry) if !ok { continue } if existing, exists := rowByKey[key]; exists { applyDiscoveredHeadVisualeffectDefaults(existing, dataset.Columns, entry.ModelStem, label) continue } rowID, ok := lockData[key] if !ok { if preservedRowID, preserved := historicalLockData[key]; preserved { rowID = preservedRowID } else { rowID = allocateNextID() } lockData[key] = rowID } newRow := createDefaultHeadVisualeffectRow(dataset.Columns, rowID, key, label, entry.ModelStem) rows = append(rows, newRow) rowByKey[key] = newRow } slices.SortFunc(rows, func(a, b map[string]any) int { return a["id"].(int) - b["id"].(int) }) result[i].Rows = rows result[i].LockData = lockData result[i].LockModified = !lockDataEqual(dataset.LockData, lockData) } return result, nil } func nextAvailableAutogenID(used map[int]struct{}) int { for rowID := 0; ; rowID++ { if _, ok := used[rowID]; !ok { return rowID } } } func headVisualeffectIdentity(entry autogenManifestEntry) (string, string, bool) { groupPrefix := map[string]string{ "head_accessories": "headaccessory", "head_features": "headfeature", } prefix, ok := groupPrefix[entry.Group] if !ok || strings.TrimSpace(entry.ModelStem) == "" { return "", "", false } stem := strings.TrimPrefix(entry.ModelStem, "hfx_") stem = strings.TrimPrefix(stem, "vfx_") stem = strings.TrimSpace(stem) if stem == "" { return "", "", false } return "visualeffects:" + prefix + "_" + stem, prefix + "_" + strings.ToUpper(stem), true } func createDefaultHeadVisualeffectRow(columns []string, rowID int, key, label, modelStem string) map[string]any { row := map[string]any{ "id": rowID, "key": key, } for _, column := range columns { row[column] = nullValue } row["Label"] = label row["Type_FD"] = "D" row["OrientWithGround"] = "0" row["Imp_HeadCon_Node"] = modelStem row["OrientWithObject"] = "1" return row } func applyDiscoveredHeadVisualeffectDefaults(row map[string]any, columns []string, modelStem, label string) { if isNullLikeValue(row["Label"]) { row["Label"] = label } if isNullLikeValue(row["Type_FD"]) { row["Type_FD"] = "D" } if isNullLikeValue(row["OrientWithGround"]) { row["OrientWithGround"] = "0" } if isNullLikeValue(row["Imp_HeadCon_Node"]) { row["Imp_HeadCon_Node"] = modelStem } if isNullLikeValue(row["OrientWithObject"]) { row["OrientWithObject"] = "1" } for _, column := range columns { if _, ok := row[column]; !ok { row[column] = nullValue } } } func formatAutogenManifest(root string, producer project.AutogenProducerConfig, repo, ref string, entries []autogenManifestEntry) ([]byte, error) { manifest := autogenManifest{ ID: producer.ID, Repo: repo, Ref: ref, GeneratedAt: time.Now().UTC().Format(time.RFC3339), Entries: entries, } return json.MarshalIndent(manifest, "", " ") } func parseAutogenManifestEntryList(raw []byte) ([]autogenManifestEntry, error) { var manifest autogenManifest if err := json.Unmarshal(raw, &manifest); err != nil { return nil, err } return manifest.Entries, nil } func normalizeAutogenNumericValue(value any) string { switch typed := value.(type) { case string: return typed case int: return strconv.Itoa(typed) case float64: return strconv.Itoa(int(typed)) default: return fmt.Sprintf("%v", value) } }