diff --git a/internal/project/project.go b/internal/project/project.go index 7ae8daf..4e41195 100644 --- a/internal/project/project.go +++ b/internal/project/project.go @@ -34,6 +34,7 @@ type Config struct { HAKs []HAKConfig `json:"haks"` TopData TopDataConfig `json:"topdata"` Extract ExtractConfig `json:"extract"` + Autogen AutogenConfig `json:"autogen"` } type ModuleConfig struct { @@ -72,6 +73,43 @@ type ExtractConfig struct { DeleteModuleArchiveAfterSuccess bool `json:"delete_module_archive_after_success,omitempty"` } +type AutogenConfig struct { + Producers []AutogenProducerConfig `json:"producers"` + Consumers []AutogenConsumerConfig `json:"consumers"` +} + +type AutogenProducerConfig struct { + ID string `json:"id"` + Root string `json:"root"` + Include []string `json:"include"` + Derive AutogenDeriveConfig `json:"derive"` + Manifest AutogenManifestConfig `json:"manifest"` +} + +type AutogenConsumerConfig struct { + ID string `json:"id"` + Producer string `json:"producer"` + Dataset string `json:"dataset"` + Mode string `json:"mode"` + Optional bool `json:"optional,omitempty"` + Root string `json:"root"` + Include []string `json:"include"` + Derive AutogenDeriveConfig `json:"derive"` + Manifest AutogenManifestConfig `json:"manifest"` + LocalOverrideRoot string `json:"local_override_root,omitempty"` +} + +type AutogenDeriveConfig struct { + Kind string `json:"kind"` + GroupFrom string `json:"group_from,omitempty"` +} + +type AutogenManifestConfig struct { + ReleaseTag string `json:"release_tag"` + AssetName string `json:"asset_name"` + CacheName string `json:"cache_name"` +} + type Inventory struct { SourceFiles []string ScriptFiles []string @@ -141,6 +179,7 @@ func (p *Project) ValidateLayout() error { failures = append(failures, err) } } + failures = append(failures, validateAutogenConfig(p.Config.Autogen)...) for index, hak := range p.Config.HAKs { if strings.TrimSpace(hak.Name) == "" { @@ -447,6 +486,100 @@ func defaultConfig() Config { } } +func validateAutogenConfig(cfg AutogenConfig) []error { + var failures []error + producerIDs := map[string]struct{}{} + consumerIDs := map[string]struct{}{} + + for index, producer := range cfg.Producers { + fieldPrefix := fmt.Sprintf("autogen.producers[%d]", index) + id := strings.TrimSpace(producer.ID) + if id == "" { + failures = append(failures, fmt.Errorf("%s.id is required", fieldPrefix)) + } else { + if _, exists := producerIDs[id]; exists { + failures = append(failures, fmt.Errorf("%s.id %q is duplicated", fieldPrefix, id)) + } + producerIDs[id] = struct{}{} + } + if strings.TrimSpace(producer.Root) == "" { + failures = append(failures, fmt.Errorf("%s.root is required", fieldPrefix)) + } + if len(producer.Include) == 0 { + failures = append(failures, fmt.Errorf("%s.include must contain at least one glob", fieldPrefix)) + } + failures = append(failures, validateAutogenDeriveConfig(fieldPrefix+".derive", producer.Derive)...) + failures = append(failures, validateAutogenManifestConfig(fieldPrefix+".manifest", producer.Manifest)...) + } + + for index, consumer := range cfg.Consumers { + fieldPrefix := fmt.Sprintf("autogen.consumers[%d]", index) + id := strings.TrimSpace(consumer.ID) + if id == "" { + failures = append(failures, fmt.Errorf("%s.id is required", fieldPrefix)) + } else { + if _, exists := consumerIDs[id]; exists { + failures = append(failures, fmt.Errorf("%s.id %q is duplicated", fieldPrefix, id)) + } + consumerIDs[id] = struct{}{} + } + if strings.TrimSpace(consumer.Producer) == "" { + failures = append(failures, fmt.Errorf("%s.producer is required", fieldPrefix)) + } + if strings.TrimSpace(consumer.Dataset) == "" { + failures = append(failures, fmt.Errorf("%s.dataset is required", fieldPrefix)) + } + switch strings.TrimSpace(consumer.Mode) { + case "parts_rows", "head_visualeffects": + case "": + failures = append(failures, fmt.Errorf("%s.mode is required", fieldPrefix)) + default: + failures = append(failures, fmt.Errorf("%s.mode %q is not supported", fieldPrefix, consumer.Mode)) + } + if strings.TrimSpace(consumer.Root) == "" { + failures = append(failures, fmt.Errorf("%s.root is required", fieldPrefix)) + } + if len(consumer.Include) == 0 { + failures = append(failures, fmt.Errorf("%s.include must contain at least one glob", fieldPrefix)) + } + failures = append(failures, validateAutogenDeriveConfig(fieldPrefix+".derive", consumer.Derive)...) + failures = append(failures, validateAutogenManifestConfig(fieldPrefix+".manifest", consumer.Manifest)...) + } + + return failures +} + +func validateAutogenDeriveConfig(fieldPrefix string, cfg AutogenDeriveConfig) []error { + var failures []error + switch strings.TrimSpace(cfg.Kind) { + case "trailing_numeric_suffix", "model_stem": + case "": + failures = append(failures, fmt.Errorf("%s.kind is required", fieldPrefix)) + default: + failures = append(failures, fmt.Errorf("%s.kind %q is not supported", fieldPrefix, cfg.Kind)) + } + switch strings.TrimSpace(cfg.GroupFrom) { + case "", "first_path_segment": + default: + failures = append(failures, fmt.Errorf("%s.group_from %q is not supported", fieldPrefix, cfg.GroupFrom)) + } + return failures +} + +func validateAutogenManifestConfig(fieldPrefix string, cfg AutogenManifestConfig) []error { + var failures []error + if strings.TrimSpace(cfg.ReleaseTag) == "" { + failures = append(failures, fmt.Errorf("%s.release_tag is required", fieldPrefix)) + } + if err := validateOutputFileName(fieldPrefix+".asset_name", cfg.AssetName, ".json"); err != nil { + failures = append(failures, err) + } + if err := validateOutputFileName(fieldPrefix+".cache_name", cfg.CacheName, ".json"); err != nil { + failures = append(failures, err) + } + return failures +} + func scanDir(root string, include func(path string) bool) ([]string, []string, error) { var files []string extSet := map[string]struct{}{} diff --git a/internal/project/project_test.go b/internal/project/project_test.go index 3c56d35..3eba387 100644 --- a/internal/project/project_test.go +++ b/internal/project/project_test.go @@ -196,6 +196,52 @@ func TestValidateLayoutRejectsInvalidTopDataPackageOutputNames(t *testing.T) { } } +func TestValidateLayoutRejectsInvalidAutogenConsumerConfig(t *testing.T) { + root := t.TempDir() + mkdirAll(t, filepath.Join(root, "src")) + mkdirAll(t, filepath.Join(root, "build")) + + proj := &Project{ + Root: root, + Config: Config{ + Module: ModuleConfig{Name: "Test", ResRef: "test"}, + Paths: PathConfig{Source: "src", Build: "build"}, + Autogen: AutogenConfig{ + Consumers: []AutogenConsumerConfig{ + { + ID: "head_visualeffects", + Producer: "head_visualeffects", + Dataset: "visualeffects", + Mode: "unsupported", + Root: "vfxs", + Include: []string{"head_accessories/**/*.mdl"}, + Derive: AutogenDeriveConfig{Kind: "model_stem", GroupFrom: "first_path_segment"}, + Manifest: AutogenManifestConfig{ + ReleaseTag: "head-vfx-manifest-current", + AssetName: "nested/sow-head-vfx-manifest.json", + CacheName: "sow-head-vfx-manifest.txt", + }, + }, + }, + }, + }, + } + + err := proj.ValidateLayout() + if err == nil { + t.Fatal("expected autogen validation error") + } + if !strings.Contains(err.Error(), "autogen.consumers[0].mode") { + t.Fatalf("expected autogen mode validation error, got %v", err) + } + if !strings.Contains(err.Error(), "autogen.consumers[0].manifest.asset_name") { + t.Fatalf("expected autogen asset_name validation error, got %v", err) + } + if !strings.Contains(err.Error(), "autogen.consumers[0].manifest.cache_name") { + t.Fatalf("expected autogen cache_name validation error, got %v", err) + } +} + func TestScanAllowsMissingAssetsDir(t *testing.T) { root := t.TempDir() mkdirAll(t, filepath.Join(root, "src")) diff --git a/internal/topdata/autogen.go b/internal/topdata/autogen.go new file mode 100644 index 0000000..3616dd1 --- /dev/null +++ b/internal/topdata/autogen.go @@ -0,0 +1,616 @@ +package topdata + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "time" + + "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" +) + +const autogenManifestCacheMaxAge = time.Hour + +type autogenManifest struct { + ID string `json:"id"` + Repo string `json:"repo"` + Ref string `json:"ref"` + GeneratedAt string `json:"generated_at"` + Entries []autogenManifestEntry `json:"entries"` +} + +type autogenManifestEntry struct { + Source string `json:"source"` + Group string `json:"group,omitempty"` + Subgroup string `json:"subgroup,omitempty"` + ModelStem string `json:"model_stem,omitempty"` + RowID int `json:"row_id,omitempty"` +} + +func applyAutogenConsumers(p *project.Project, collected []nativeCollectedDataset, progress func(string)) ([]nativeCollectedDataset, error) { + if len(p.Config.Autogen.Consumers) == 0 { + return collected, nil + } + if progress == nil { + progress = func(string) {} + } + + result := append([]nativeCollectedDataset(nil), collected...) + for _, consumer := range p.Config.Autogen.Consumers { + if !autogenConsumerTargetsCollectedDataset(result, consumer) { + continue + } + entries, err := resolveAutogenConsumerEntries(p, consumer, progress) + if err != nil { + if consumer.Optional && isIgnorableOptionalAutogenError(err) { + if progress != nil { + progress(fmt.Sprintf("Skipping optional autogen consumer %s: %v", consumer.ID, err)) + } + continue + } + return nil, err + } + if len(entries) == 0 { + continue + } + switch consumer.Mode { + case "parts_rows": + result = augmentWithAutogeneratedParts(result, autogenPartsInventory(entries)) + case "head_visualeffects": + result, err = augmentWithAutogeneratedHeadVisualeffects(result, entries) + if err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("unsupported autogen consumer mode %q", consumer.Mode) + } + } + return result, nil +} + +func autogenConsumerTargetsCollectedDataset(collected []nativeCollectedDataset, consumer project.AutogenConsumerConfig) bool { + for _, dataset := range collected { + switch consumer.Mode { + case "parts_rows": + if isAutogenEligiblePartsDataset(dataset) { + return true + } + case "head_visualeffects": + if dataset.Dataset.Name == "visualeffects" { + return true + } + } + } + return false +} + +func isIgnorableOptionalAutogenError(err error) bool { + if err == nil { + return false + } + message := err.Error() + return strings.Contains(message, "HTTP 404") || + strings.Contains(message, "not found") || + strings.Contains(message, "entries is empty") +} + +func resolveAutogenConsumerEntries(p *project.Project, consumer project.AutogenConsumerConfig, progress func(string)) ([]autogenManifestEntry, error) { + overrideRoot, err := resolveAutogenLocalOverrideRoot(p, consumer) + if err != nil { + return nil, err + } + if overrideRoot != "" { + if progress != nil { + progress(fmt.Sprintf("Scanning local autogen override for %s from %s...", consumer.ID, overrideRoot)) + } + return scanLocalAutogenEntries(overrideRoot, consumer) + } + return resolveReleasedAutogenManifestEntries(p, consumer, progress) +} + +func resolveAutogenLocalOverrideRoot(p *project.Project, consumer project.AutogenConsumerConfig) (string, error) { + root := strings.TrimSpace(consumer.LocalOverrideRoot) + if root != "" { + if filepath.IsAbs(root) { + return root, nil + } + return filepath.Join(p.Root, root), nil + } + + spec := strings.TrimSpace(p.Config.TopData.Assets) + if spec == "" { + return "", nil + } + if looksLikeGitRepoSpec(spec) { + return "", fmt.Errorf("topdata.assets no longer supports git repo specs for autogen discovery; use a local path override or the default released manifest") + } + path := p.TopDataAssetsDir() + if path == "" { + return "", nil + } + if _, err := os.Stat(path); err != nil { + return "", fmt.Errorf("configured topdata.assets path %s is not accessible: %w", path, err) + } + candidate := filepath.Join(path, consumer.Root) + info, err := os.Stat(candidate) + if err != nil { + if os.IsNotExist(err) { + return "", nil + } + return "", fmt.Errorf("configured topdata.assets override root %s is not accessible: %w", candidate, err) + } + if !info.IsDir() { + return "", fmt.Errorf("configured topdata.assets override root %s is not a directory", candidate) + } + return path, nil +} + +func scanLocalAutogenEntries(overrideRoot string, consumer project.AutogenConsumerConfig) ([]autogenManifestEntry, error) { + scanRoot := filepath.Join(overrideRoot, consumer.Root) + info, err := os.Stat(scanRoot) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, fmt.Errorf("stat autogen override root %s: %w", scanRoot, err) + } + if !info.IsDir() { + return nil, fmt.Errorf("autogen override root %s is not a directory", scanRoot) + } + + var entries []autogenManifestEntry + err = filepath.Walk(scanRoot, func(path string, info os.FileInfo, walkErr error) error { + if walkErr != nil { + return walkErr + } + if info.IsDir() { + return nil + } + rel, err := filepath.Rel(scanRoot, path) + if err != nil { + return err + } + rel = filepath.ToSlash(rel) + if !matchesAutogenInclude(rel, consumer.Include) { + return nil + } + entry, ok := deriveAutogenManifestEntry(rel, consumer.Derive) + if !ok { + return nil + } + entries = append(entries, entry) + return nil + }) + if err != nil { + return nil, fmt.Errorf("scan autogen override %s: %w", scanRoot, err) + } + + slices.SortFunc(entries, func(a, b autogenManifestEntry) int { + return strings.Compare(a.Source, b.Source) + }) + return entries, nil +} + +func matchesAutogenInclude(rel string, include []string) bool { + rel = filepath.ToSlash(rel) + for _, pattern := range include { + pattern = filepath.ToSlash(strings.TrimSpace(pattern)) + if pattern == "" { + continue + } + switch { + case pattern == "**/*.mdl": + if strings.HasSuffix(strings.ToLower(rel), ".mdl") { + return true + } + case strings.HasSuffix(pattern, "/**/*.mdl"): + prefix := strings.TrimSuffix(pattern, "/**/*.mdl") + if strings.HasPrefix(rel, prefix+"/") && strings.HasSuffix(strings.ToLower(rel), ".mdl") { + return true + } + case strings.HasSuffix(pattern, "/*.mdl"): + prefix := strings.TrimSuffix(pattern, "/*.mdl") + dir := filepath.ToSlash(filepath.Dir(rel)) + if dir == prefix && strings.HasSuffix(strings.ToLower(rel), ".mdl") { + return true + } + case rel == pattern: + return true + } + } + return false +} + +func deriveAutogenManifestEntry(rel string, derive project.AutogenDeriveConfig) (autogenManifestEntry, bool) { + rel = filepath.ToSlash(rel) + stem := strings.TrimSuffix(filepath.Base(rel), filepath.Ext(rel)) + entry := autogenManifestEntry{ + Source: rel, + ModelStem: stem, + } + parts := strings.Split(rel, "/") + if derive.GroupFrom == "first_path_segment" && len(parts) > 1 { + entry.Group = parts[0] + if len(parts) > 2 { + entry.Subgroup = strings.Join(parts[1:len(parts)-1], "/") + } + } + + switch derive.Kind { + case "trailing_numeric_suffix": + rowID, ok := extractTrailingNumericSuffix(stem) + if !ok { + return autogenManifestEntry{}, false + } + entry.RowID = rowID + case "model_stem": + default: + return autogenManifestEntry{}, false + } + + return entry, true +} + +func resolveReleasedAutogenManifestEntries(p *project.Project, consumer project.AutogenConsumerConfig, progress func(string)) ([]autogenManifestEntry, error) { + cachePath := filepath.Join(p.Root, ".cache", consumer.Manifest.CacheName) + if strings.TrimSpace(os.Getenv("SOW_AUTOGEN_MANIFEST_REFRESH")) == "" { + manifest, fresh, err := readFreshAutogenManifestCache(cachePath, autogenManifestCacheMaxAge) + if err != nil { + if progress != nil { + progress(fmt.Sprintf("Ignoring cached autogen manifest %s: %v", consumer.Manifest.CacheName, err)) + } + } else if fresh { + if progress != nil { + progress(fmt.Sprintf("Using cached released autogen manifest %s...", cachePath)) + } + return manifest.Entries, nil + } + } + + spec, err := deriveSowAssetsRepoSpec(p.Root) + if err != nil { + return nil, err + } + manifestURL, err := resolveAutogenManifestAssetURL(spec, consumer.Manifest.ReleaseTag, consumer.Manifest.AssetName) + if err != nil { + if consumer.Mode == "parts_rows" { + return nil, fmt.Errorf("%s", strings.Replace(err.Error(), "autogen manifest", "parts manifest", 1)) + } + return nil, err + } + if progress != nil { + progress(fmt.Sprintf("Fetching released autogen manifest from %s...", manifestURL)) + } + manifest, err := fetchAutogenManifest(manifestURL) + if err != nil { + if consumer.Mode == "parts_rows" { + return nil, fmt.Errorf("%s", strings.Replace(err.Error(), "autogen manifest", "parts manifest", 1)) + } + return nil, err + } + if manifest.ID != "" && manifest.ID != consumer.Producer { + return nil, fmt.Errorf("autogen manifest %s declared id %q, expected %q", consumer.Manifest.AssetName, manifest.ID, consumer.Producer) + } + if err := writeAutogenManifestCache(cachePath, manifest); err != nil { + return nil, err + } + return manifest.Entries, nil +} + +func resolveAutogenManifestAssetURL(spec giteaRepoSpec, releaseTag, assetName string) (string, error) { + apiURL := fmt.Sprintf("%s/api/v1/repos/%s/%s/releases/tags/%s", strings.TrimRight(spec.BaseURL, "/"), releasePathEscape(spec.Owner), releasePathEscape(spec.Repo), releasePathEscape(releaseTag)) + var release struct { + Assets []struct { + Name string `json:"name"` + BrowserDownloadURL string `json:"browser_download_url"` + DownloadURL string `json:"download_url"` + } `json:"assets"` + } + if err := fetchJSON(apiURL, &release); err != nil { + return "", fmt.Errorf("fetch autogen manifest release metadata: %w", err) + } + for _, asset := range release.Assets { + if asset.Name != assetName { + continue + } + if asset.DownloadURL != "" { + return asset.DownloadURL, nil + } + if asset.BrowserDownloadURL != "" { + return asset.BrowserDownloadURL, nil + } + } + return "", fmt.Errorf("autogen manifest asset %s not found in release %s/%s:%s", assetName, spec.Owner, spec.Repo, releaseTag) +} + +func releasePathEscape(value string) string { + replacer := strings.NewReplacer("/", "%2F") + return replacer.Replace(value) +} + +func fetchAutogenManifest(manifestURL string) (*autogenManifest, error) { + var manifest autogenManifest + if err := fetchJSON(manifestURL, &manifest); err != nil { + return nil, fmt.Errorf("download autogen manifest: %w", err) + } + if len(manifest.Entries) == 0 { + var legacy partsManifest + if err := fetchJSON(manifestURL, &legacy); err == nil && len(legacy.Categories) > 0 { + return legacyPartsManifestToAutogen(&legacy), nil + } + } + if len(manifest.Entries) == 0 { + return nil, fmt.Errorf("download autogen manifest: entries is empty") + } + return &manifest, nil +} + +func readFreshAutogenManifestCache(path string, maxAge time.Duration) (*autogenManifest, bool, error) { + info, err := os.Stat(path) + if err != nil { + if os.IsNotExist(err) { + return nil, false, nil + } + return nil, false, err + } + if maxAge > 0 && time.Since(info.ModTime()) > maxAge { + return nil, false, nil + } + raw, err := os.ReadFile(path) + if err != nil { + return nil, false, err + } + var manifest autogenManifest + if err := json.Unmarshal(raw, &manifest); err != nil { + return nil, false, err + } + if len(manifest.Entries) == 0 { + var legacy partsManifest + if err := json.Unmarshal(raw, &legacy); err == nil && len(legacy.Categories) > 0 { + return legacyPartsManifestToAutogen(&legacy), true, nil + } + } + if len(manifest.Entries) == 0 { + return nil, false, fmt.Errorf("entries is empty") + } + return &manifest, true, nil +} + +func legacyPartsManifestToAutogen(legacy *partsManifest) *autogenManifest { + entries := make([]autogenManifestEntry, 0) + for _, category := range supportedPartCategories { + for _, rowID := range legacy.Categories[category] { + entries = append(entries, autogenManifestEntry{ + Group: category, + RowID: rowID, + }) + } + } + slices.SortFunc(entries, func(a, b autogenManifestEntry) int { + if cmp := strings.Compare(a.Group, b.Group); cmp != 0 { + return cmp + } + return a.RowID - b.RowID + }) + return &autogenManifest{ + ID: "parts", + Repo: legacy.Repo, + Ref: legacy.Ref, + GeneratedAt: legacy.GeneratedAt, + Entries: entries, + } +} + +func writeAutogenManifestCache(path string, manifest *autogenManifest) error { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return fmt.Errorf("create autogen manifest cache dir: %w", err) + } + payload, err := json.MarshalIndent(manifest, "", " ") + if err != nil { + return fmt.Errorf("marshal autogen manifest cache: %w", err) + } + payload = append(payload, '\n') + current, err := os.ReadFile(path) + if err == nil && bytes.Equal(current, payload) { + now := time.Now() + if err := os.Chtimes(path, now, now); err != nil { + return fmt.Errorf("refresh autogen manifest cache timestamp: %w", err) + } + return nil + } + if err != nil && !os.IsNotExist(err) { + return fmt.Errorf("read autogen manifest cache: %w", err) + } + tmpPath := path + ".tmp" + if err := os.WriteFile(tmpPath, payload, 0o644); err != nil { + return fmt.Errorf("write autogen manifest cache: %w", err) + } + if err := os.Rename(tmpPath, path); err != nil { + return fmt.Errorf("replace autogen manifest cache: %w", err) + } + return nil +} + +func autogenPartsInventory(entries []autogenManifestEntry) map[string]map[int]struct{} { + result := make(map[string]map[int]struct{}, len(supportedPartCategories)) + for _, category := range supportedPartCategories { + result[category] = map[int]struct{}{} + } + for _, entry := range entries { + if !isSupportedPartCategory(entry.Group) || entry.RowID <= 0 { + continue + } + result[entry.Group][entry.RowID] = struct{}{} + } + return result +} + +func augmentWithAutogeneratedHeadVisualeffects(collected []nativeCollectedDataset, entries []autogenManifestEntry) ([]nativeCollectedDataset, error) { + if len(entries) == 0 { + return collected, nil + } + + result := append([]nativeCollectedDataset(nil), collected...) + for i, dataset := range result { + if dataset.Dataset.Name != "visualeffects" { + continue + } + + rows := make([]map[string]any, len(dataset.Rows)) + rowByKey := make(map[string]map[string]any, len(dataset.Rows)) + usedIDs := make(map[int]struct{}, len(dataset.Rows)+len(dataset.LockData)) + lockData := make(map[string]int, len(dataset.LockData)) + for key, rowID := range dataset.LockData { + lockData[key] = rowID + usedIDs[rowID] = struct{}{} + } + for index, row := range dataset.Rows { + cloned := cloneRowMap(row) + rows[index] = cloned + if key, ok := cloned["key"].(string); ok && key != "" { + rowByKey[key] = cloned + } + if rowID, ok := cloned["id"].(int); ok { + usedIDs[rowID] = struct{}{} + } + } + + nextID := nextAvailableAutogenID(usedIDs) + allocateNextID := func() int { + rowID := nextID + usedIDs[rowID] = struct{}{} + nextID = nextAvailableAutogenID(usedIDs) + return rowID + } + + for _, entry := range entries { + key, label, ok := headVisualeffectIdentity(entry) + if !ok { + continue + } + if existing, exists := rowByKey[key]; exists { + applyDiscoveredHeadVisualeffectDefaults(existing, dataset.Columns, entry.ModelStem, label) + continue + } + + rowID, ok := lockData[key] + if !ok { + rowID = allocateNextID() + lockData[key] = rowID + } + newRow := createDefaultHeadVisualeffectRow(dataset.Columns, rowID, key, label, entry.ModelStem) + rows = append(rows, newRow) + rowByKey[key] = newRow + } + + slices.SortFunc(rows, func(a, b map[string]any) int { + return a["id"].(int) - b["id"].(int) + }) + result[i].Rows = rows + result[i].LockData = lockData + result[i].LockModified = !lockDataEqual(dataset.LockData, lockData) + } + return result, nil +} + +func nextAvailableAutogenID(used map[int]struct{}) int { + for rowID := 0; ; rowID++ { + if _, ok := used[rowID]; !ok { + return rowID + } + } +} + +func headVisualeffectIdentity(entry autogenManifestEntry) (string, string, bool) { + groupPrefix := map[string]string{ + "head_accessories": "headaccessory", + "head_features": "headfeature", + } + prefix, ok := groupPrefix[entry.Group] + if !ok || strings.TrimSpace(entry.ModelStem) == "" { + return "", "", false + } + stem := strings.TrimPrefix(entry.ModelStem, "hfx_") + stem = strings.TrimPrefix(stem, "vfx_") + stem = strings.TrimSpace(stem) + if stem == "" { + return "", "", false + } + return "visualeffects:" + prefix + "_" + stem, prefix + "_" + strings.ToUpper(stem), true +} + +func createDefaultHeadVisualeffectRow(columns []string, rowID int, key, label, modelStem string) map[string]any { + row := map[string]any{ + "id": rowID, + "key": key, + } + for _, column := range columns { + row[column] = nullValue + } + row["Label"] = label + row["Type_FD"] = "D" + row["OrientWithGround"] = "0" + row["Imp_HeadCon_Node"] = modelStem + row["OrientWithObject"] = "1" + return row +} + +func applyDiscoveredHeadVisualeffectDefaults(row map[string]any, columns []string, modelStem, label string) { + if isNullLikeValue(row["Label"]) { + row["Label"] = label + } + if isNullLikeValue(row["Type_FD"]) { + row["Type_FD"] = "D" + } + if isNullLikeValue(row["OrientWithGround"]) { + row["OrientWithGround"] = "0" + } + if isNullLikeValue(row["Imp_HeadCon_Node"]) { + row["Imp_HeadCon_Node"] = modelStem + } + if isNullLikeValue(row["OrientWithObject"]) { + row["OrientWithObject"] = "1" + } + for _, column := range columns { + if _, ok := row[column]; !ok { + row[column] = nullValue + } + } +} + +func formatAutogenManifest(root string, producer project.AutogenProducerConfig, repo, ref string, entries []autogenManifestEntry) ([]byte, error) { + manifest := autogenManifest{ + ID: producer.ID, + Repo: repo, + Ref: ref, + GeneratedAt: time.Now().UTC().Format(time.RFC3339), + Entries: entries, + } + return json.MarshalIndent(manifest, "", " ") +} + +func parseAutogenManifestEntryList(raw []byte) ([]autogenManifestEntry, error) { + var manifest autogenManifest + if err := json.Unmarshal(raw, &manifest); err != nil { + return nil, err + } + return manifest.Entries, nil +} + +func normalizeAutogenNumericValue(value any) string { + switch typed := value.(type) { + case string: + return typed + case int: + return strconv.Itoa(typed) + case float64: + return strconv.Itoa(int(typed)) + default: + return fmt.Sprintf("%v", value) + } +} diff --git a/internal/topdata/native.go b/internal/topdata/native.go index fbb97fa..4e21ec8 100644 --- a/internal/topdata/native.go +++ b/internal/topdata/native.go @@ -284,16 +284,9 @@ func buildNativeUnchecked(p *project.Project, opts NativeBuildOptions, progress } } } - if hasCollectedPartsDatasets(collected) { - progress("Resolving released parts manifest...") - autogenerated, err := resolveAutogeneratedPartsInventory(p, progress) - if err != nil { - return BuildResult{}, err - } - if autogenerated != nil { - collected = augmentWithAutogeneratedParts(collected, autogenerated) - progress("Augmented parts datasets with discovered models") - } + collected, err = applyAutogenConsumers(p, collected, progress) + if err != nil { + return BuildResult{}, err } collected, err = applyPartOverrides(sourceDir, collected) if err != nil { diff --git a/internal/topdata/parts_discovery.go b/internal/topdata/parts_discovery.go index 015d126..6476dba 100644 --- a/internal/topdata/parts_discovery.go +++ b/internal/topdata/parts_discovery.go @@ -60,6 +60,19 @@ func isPartsDataset(datasetName string) bool { return false } +func isAutogenEligiblePartsDataset(dataset nativeCollectedDataset) bool { + if !isPartsDataset(dataset.Dataset.Name) { + return false + } + for _, row := range dataset.Rows { + key, ok := row["key"].(string) + if ok && strings.TrimSpace(key) != "" { + return false + } + } + return true +} + // extractPartCategory extracts the part category from a dataset name // e.g., "parts/belt" -> "belt", "parts/bicep" -> "bicep" func extractPartCategory(datasetName string) string { @@ -430,6 +443,9 @@ func augmentWithAutogeneratedParts(collected []nativeCollectedDataset, autogener copy(result, collected) for i, dataset := range collected { + if !isAutogenEligiblePartsDataset(dataset) { + continue + } category := partAssetCategoryForDataset(dataset.Dataset.Name) if !isSupportedPartCategory(category) { continue diff --git a/internal/topdata/topdata_test.go b/internal/topdata/topdata_test.go index cfe8d28..177fc03 100644 --- a/internal/topdata/topdata_test.go +++ b/internal/topdata/topdata_test.go @@ -5479,6 +5479,121 @@ with open(os.path.join(tlk, "sow_tlk.tlk"), "wb") as f: } } +func TestApplyAutogenConsumersAugmentsPartsFromLocalOverride(t *testing.T) { + root := testProjectRoot(t) + overrideRoot := filepath.Join(root, "autogen-assets") + mkdirAll(t, filepath.Join(overrideRoot, "part", "belt")) + writeFile(t, filepath.Join(overrideRoot, "part", "belt", "pfa0_belt018.mdl"), "mdl\n") + + p := testProject(root) + p.Config.Autogen.Consumers = []project.AutogenConsumerConfig{ + { + ID: "parts", + Producer: "parts", + Dataset: "parts", + Mode: "parts_rows", + Root: "part", + Include: []string{"**/*.mdl"}, + Derive: project.AutogenDeriveConfig{Kind: "trailing_numeric_suffix", GroupFrom: "first_path_segment"}, + Manifest: project.AutogenManifestConfig{ReleaseTag: "parts-manifest-current", AssetName: "sow-parts-manifest.json", CacheName: "sow-parts-manifest.json"}, + LocalOverrideRoot: "autogen-assets", + }, + } + + collected := []nativeCollectedDataset{ + { + Dataset: nativeDataset{Name: "parts/belt"}, + Columns: []string{"COSTMODIFIER", "ACBONUS"}, + Rows: []map[string]any{ + {"id": 0, "COSTMODIFIER": "0", "ACBONUS": "0.00"}, + }, + }, + } + + got, err := applyAutogenConsumers(p, collected, nil) + if err != nil { + t.Fatalf("applyAutogenConsumers failed: %v", err) + } + if len(got) != 1 || len(got[0].Rows) != 2 { + t.Fatalf("expected autogenerated part row, got %#v", got) + } + row := got[0].Rows[1] + if row["id"] != 18 || row["COSTMODIFIER"] != "0" || row["ACBONUS"] != "0.00" { + t.Fatalf("unexpected autogenerated part row: %#v", row) + } +} + +func TestApplyAutogenConsumersAugmentsHeadVisualeffectsFromLocalOverride(t *testing.T) { + root := testProjectRoot(t) + overrideRoot := filepath.Join(root, "autogen-assets") + mkdirAll(t, filepath.Join(overrideRoot, "vfxs", "head_accessories")) + mkdirAll(t, filepath.Join(overrideRoot, "vfxs", "head_features", "hair")) + writeFile(t, filepath.Join(overrideRoot, "vfxs", "head_accessories", "hfx_bandana.mdl"), "mdl\n") + writeFile(t, filepath.Join(overrideRoot, "vfxs", "head_features", "hair", "hfx_hair_bangs.mdl"), "mdl\n") + + p := testProject(root) + p.Config.Autogen.Consumers = []project.AutogenConsumerConfig{ + { + ID: "head_visualeffects", + Producer: "head_visualeffects", + Dataset: "visualeffects", + Mode: "head_visualeffects", + Root: "vfxs", + Include: []string{"head_accessories/**/*.mdl", "head_features/**/*.mdl"}, + Derive: project.AutogenDeriveConfig{Kind: "model_stem", GroupFrom: "first_path_segment"}, + Manifest: project.AutogenManifestConfig{ReleaseTag: "head-vfx-manifest-current", AssetName: "sow-head-vfx-manifest.json", CacheName: "sow-head-vfx-manifest.json"}, + LocalOverrideRoot: "autogen-assets", + }, + } + + collected := []nativeCollectedDataset{ + { + Dataset: nativeDataset{Name: "visualeffects", Kind: nativeDatasetBase}, + Columns: []string{"Label", "Type_FD", "OrientWithGround", "Imp_HeadCon_Node", "OrientWithObject"}, + Rows: nil, + LockData: map[string]int{ + "visualeffects:existing": 17, + }, + }, + } + + got, err := applyAutogenConsumers(p, collected, nil) + if err != nil { + t.Fatalf("applyAutogenConsumers failed: %v", err) + } + if len(got) != 1 || len(got[0].Rows) != 2 { + t.Fatalf("expected autogenerated visualeffects rows, got %#v", got) + } + + rowByKey := map[string]map[string]any{} + for _, row := range got[0].Rows { + rowByKey[row["key"].(string)] = row + } + + bandana := rowByKey["visualeffects:headaccessory_bandana"] + if bandana == nil { + t.Fatalf("expected head accessory row, got %#v", got[0].Rows) + } + if bandana["Label"] != "headaccessory_BANDANA" || bandana["Imp_HeadCon_Node"] != "hfx_bandana" || bandana["OrientWithObject"] != "1" { + t.Fatalf("unexpected head accessory defaults: %#v", bandana) + } + + hair := rowByKey["visualeffects:headfeature_hair_bangs"] + if hair == nil { + t.Fatalf("expected head feature row, got %#v", got[0].Rows) + } + if hair["Label"] != "headfeature_HAIR_BANGS" || hair["Imp_HeadCon_Node"] != "hfx_hair_bangs" || hair["Type_FD"] != "D" { + t.Fatalf("unexpected head feature defaults: %#v", hair) + } + + if _, ok := got[0].LockData["visualeffects:headaccessory_bandana"]; !ok { + t.Fatalf("expected head accessory lock id, got %#v", got[0].LockData) + } + if _, ok := got[0].LockData["visualeffects:headfeature_hair_bangs"]; !ok { + t.Fatalf("expected head feature lock id, got %#v", got[0].LockData) + } +} + func TestNormalizeProjectImportsLegacyLoadscreens(t *testing.T) { root := testProjectRoot(t) mkdirAll(t, filepath.Join(root, "topdata", "data")) @@ -9028,6 +9143,27 @@ func TestBuildNativeWithReleasedPartsManifest(t *testing.T) { Source: "topdata", Build: "build/topdata", }, + Autogen: project.AutogenConfig{ + Consumers: []project.AutogenConsumerConfig{ + { + ID: "parts", + Producer: "parts", + Dataset: "parts", + Mode: "parts_rows", + Root: "part", + Include: []string{"**/*.mdl"}, + Derive: project.AutogenDeriveConfig{ + Kind: "trailing_numeric_suffix", + GroupFrom: "first_path_segment", + }, + Manifest: project.AutogenManifestConfig{ + ReleaseTag: "parts-manifest-current", + AssetName: "sow-parts-manifest.json", + CacheName: "sow-parts-manifest.json", + }, + }, + }, + }, }, } @@ -9105,6 +9241,27 @@ func TestBuildNativeFailsWhenReleasedPartsManifestUnavailable(t *testing.T) { Source: "topdata", Build: "build/topdata", }, + Autogen: project.AutogenConfig{ + Consumers: []project.AutogenConsumerConfig{ + { + ID: "parts", + Producer: "parts", + Dataset: "parts", + Mode: "parts_rows", + Root: "part", + Include: []string{"**/*.mdl"}, + Derive: project.AutogenDeriveConfig{ + Kind: "trailing_numeric_suffix", + GroupFrom: "first_path_segment", + }, + Manifest: project.AutogenManifestConfig{ + ReleaseTag: "parts-manifest-current", + AssetName: "sow-parts-manifest.json", + CacheName: "sow-parts-manifest.json", + }, + }, + }, + }, }, } @@ -9564,6 +9721,45 @@ func testProject(root string) *project.Project { ReferenceBuilder: "reference", Assets: ".", }, + Autogen: project.AutogenConfig{ + Consumers: []project.AutogenConsumerConfig{ + { + ID: "parts", + Producer: "parts", + Dataset: "parts", + Mode: "parts_rows", + Root: "part", + Include: []string{"**/*.mdl"}, + Derive: project.AutogenDeriveConfig{ + Kind: "trailing_numeric_suffix", + GroupFrom: "first_path_segment", + }, + Manifest: project.AutogenManifestConfig{ + ReleaseTag: "parts-manifest-current", + AssetName: "sow-parts-manifest.json", + CacheName: "sow-parts-manifest.json", + }, + }, + { + ID: "head_visualeffects", + Producer: "head_visualeffects", + Dataset: "visualeffects", + Mode: "head_visualeffects", + Optional: true, + Root: "vfxs", + Include: []string{"head_accessories/**/*.mdl", "head_features/**/*.mdl"}, + Derive: project.AutogenDeriveConfig{ + Kind: "model_stem", + GroupFrom: "first_path_segment", + }, + Manifest: project.AutogenManifestConfig{ + ReleaseTag: "head-vfx-manifest-current", + AssetName: "sow-head-vfx-manifest.json", + CacheName: "sow-head-vfx-manifest.json", + }, + }, + }, + }, }, } }