feat(topdata): parts autogen from CDN part.yml channel
Phase 2 of the parts-row autogen design: Crucible resolves parts 2DA rows from the immutable part.yml published with the asset HAK release, over the anonymous Bunny CDN channel, with the parts consumer required (no fail-open). - generalize resolveCDNChannelManifest: derive manifest basename + provenance label from config; drop hardcoded assets/vfxs.yml so part.yml resolves too - add filterPartsCDNChannelEntries + mode dispatch implementing the inventory contract (restype mdl under part/<cat>/, trailing-digit row id, dedup l/r/race/gender variants, sort by source, zero accepted rows = hard fail) - support hand category + parts/hand dataset mapping (12 datasets total) - native pipeline: single augment -> normalize -> override sequence with the configured parts_rows policy; overrides may update but never synthesize a row - reject multiple parts_rows consumers in project validation - tests: parts filter contract, category coverage, offline basename, orphan override reject, and a parity guard proving a bare CDN-only parts build Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
+104
-17
@@ -299,23 +299,34 @@ func resolveCDNChannelManifest(p *project.Project, consumer project.AutogenConsu
|
||||
progress = func(string) {}
|
||||
}
|
||||
|
||||
// 1. Offline / air-gapped override: a vfxs.yml path or a manifest-repo
|
||||
// checkout root containing assets/vfxs.yml. Skips the network entirely.
|
||||
// Manifest basename + provenance label are derived from config, not hardcoded,
|
||||
// so the same resolver serves vfxs.yml (accessory VFX) and part.yml (parts).
|
||||
manifestBase := filepath.Base(filepath.FromSlash(strings.TrimSpace(src.ManifestPath)))
|
||||
if manifestBase == "." || manifestBase == "" || manifestBase == string(filepath.Separator) {
|
||||
manifestBase = "manifest.yml"
|
||||
}
|
||||
label := strings.TrimSpace(consumer.ID)
|
||||
if label == "" {
|
||||
label = "autogen"
|
||||
}
|
||||
|
||||
// 1. Offline / air-gapped override: a manifest file path or a manifest-repo
|
||||
// checkout root containing assets/<manifestBase>. Skips the network entirely.
|
||||
if envName := strings.TrimSpace(src.OfflineOverrideEnv); envName != "" {
|
||||
if override := strings.TrimSpace(os.Getenv(envName)); override != "" {
|
||||
vfxsPath := override
|
||||
manifestPath := override
|
||||
if info, err := os.Stat(override); err == nil && info.IsDir() {
|
||||
vfxsPath = filepath.Join(override, "assets", "vfxs.yml")
|
||||
manifestPath = filepath.Join(override, "assets", manifestBase)
|
||||
}
|
||||
raw, err := os.ReadFile(vfxsPath)
|
||||
raw, err := os.ReadFile(manifestPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("offline vfxs override %s: %w", vfxsPath, err)
|
||||
return nil, fmt.Errorf("offline %s override %s: %w", manifestBase, manifestPath, err)
|
||||
}
|
||||
entries, err := filterCDNChannelEntries(raw, consumer)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
progress(fmt.Sprintf("Using offline vfxs override for %s from %s...", consumer.ID, vfxsPath))
|
||||
progress(fmt.Sprintf("Using offline %s override for %s from %s...", manifestBase, consumer.ID, manifestPath))
|
||||
return &autogenManifest{ID: consumer.Producer, Ref: "local", Entries: entries}, nil
|
||||
}
|
||||
}
|
||||
@@ -359,13 +370,13 @@ func resolveCDNChannelManifest(p *project.Project, consumer project.AutogenConsu
|
||||
if tag == "" {
|
||||
return nil, unavailableAutogenManifest(fmt.Errorf("channel %q absent in channels.json (%s)", channel, channelsURL))
|
||||
}
|
||||
progress(fmt.Sprintf("Accessory VFX: channel %s -> tag %s", channel, tag))
|
||||
progress(fmt.Sprintf("%s: channel %s -> tag %s", label, channel, tag))
|
||||
|
||||
// 5. vfxs.yml for the tag.
|
||||
// 5. per-tag manifest for the tag.
|
||||
manifestURL := join(src.ManifestPath, tag)
|
||||
vstatus, vbody, err := httpGetStatus(manifestURL)
|
||||
if err != nil {
|
||||
return nil, unavailableAutogenManifest(fmt.Errorf("vfxs.yml unreachable %s: %w", manifestURL, err))
|
||||
return nil, unavailableAutogenManifest(fmt.Errorf("%s unreachable %s: %w", manifestBase, manifestURL, err))
|
||||
}
|
||||
switch vstatus {
|
||||
case http.StatusOK:
|
||||
@@ -377,27 +388,103 @@ func resolveCDNChannelManifest(p *project.Project, consumer project.AutogenConsu
|
||||
if marker := strings.TrimSpace(src.ReleaseMarkerPath); marker != "" {
|
||||
markerURL := join(marker, tag)
|
||||
if mstatus, _, merr := httpGetStatus(markerURL); merr == nil && mstatus == http.StatusOK {
|
||||
return nil, fmt.Errorf("release %s has %s but no vfxs.yml (%s) — accessory rows would silently vanish; backfill/republish vfxs.yml for %s", tag, marker, manifestURL, tag)
|
||||
return nil, fmt.Errorf("release %s has %s but no %s (%s) — rows would silently vanish; backfill/republish %s for %s", tag, marker, manifestBase, manifestURL, manifestBase, tag)
|
||||
}
|
||||
}
|
||||
return nil, unavailableAutogenManifest(fmt.Errorf("vfxs.yml 404 %s (no published release for %s)", manifestURL, tag))
|
||||
return nil, unavailableAutogenManifest(fmt.Errorf("%s 404 %s (no published release for %s)", manifestBase, manifestURL, tag))
|
||||
default:
|
||||
return nil, unavailableAutogenManifest(fmt.Errorf("vfxs.yml HTTP %d %s", vstatus, manifestURL))
|
||||
return nil, unavailableAutogenManifest(fmt.Errorf("%s HTTP %d %s", manifestBase, vstatus, manifestURL))
|
||||
}
|
||||
|
||||
entries, err := filterCDNChannelEntries(vbody, consumer)
|
||||
if err != nil {
|
||||
return nil, err // malformed vfxs.yml = HARD fail
|
||||
return nil, err // malformed manifest = HARD fail
|
||||
}
|
||||
progress(fmt.Sprintf("Accessory VFX: resolved %d model entries from %s", len(entries), manifestURL))
|
||||
progress(fmt.Sprintf("%s: resolved %d model entries from %s", label, len(entries), manifestURL))
|
||||
return &autogenManifest{ID: consumer.Producer, Ref: tag, Entries: entries}, nil
|
||||
}
|
||||
|
||||
// filterCDNChannelEntries parses vfxs.yml and keeps restype==mdl assets under
|
||||
// filterCDNChannelEntries dispatches per-tag manifest parsing on the consumer
|
||||
// mode. parts_rows consumes part.yml (the parts inventory contract); every other
|
||||
// mode consumes vfxs.yml (the accessory-VFX contract).
|
||||
func filterCDNChannelEntries(raw []byte, consumer project.AutogenConsumerConfig) ([]autogenManifestEntry, error) {
|
||||
switch strings.TrimSpace(consumer.Mode) {
|
||||
case "parts_rows":
|
||||
return filterPartsCDNChannelEntries(raw)
|
||||
default:
|
||||
return filterVFXCDNChannelEntries(raw, consumer)
|
||||
}
|
||||
}
|
||||
|
||||
// filterPartsCDNChannelEntries parses part.yml and keeps restype==mdl assets
|
||||
// under part/<supported-category>/.../<stem><digits>.mdl. The asset category is
|
||||
// the path segment after part/; the row ID is the trailing decimal of the
|
||||
// filename stem. Body/race/gender/left-right variants dedup by (category,rowID).
|
||||
// Zero accepted rows is a HARD fail (silent-drop guard).
|
||||
func filterPartsCDNChannelEntries(raw []byte) ([]autogenManifestEntry, error) {
|
||||
var manifest struct {
|
||||
Assets *[]struct {
|
||||
Path string `yaml:"path"`
|
||||
Restype string `yaml:"restype"`
|
||||
} `yaml:"assets"`
|
||||
}
|
||||
if err := yaml.Unmarshal(raw, &manifest); err != nil {
|
||||
return nil, fmt.Errorf("malformed part.yml (cannot parse): %w", err)
|
||||
}
|
||||
if manifest.Assets == nil {
|
||||
return nil, fmt.Errorf("malformed part.yml (no assets array)")
|
||||
}
|
||||
seen := map[string]struct{}{}
|
||||
var entries []autogenManifestEntry
|
||||
for _, a := range *manifest.Assets {
|
||||
if a.Restype != "mdl" {
|
||||
continue
|
||||
}
|
||||
path := filepath.ToSlash(strings.TrimSpace(a.Path))
|
||||
if !strings.HasPrefix(path, "part/") {
|
||||
continue
|
||||
}
|
||||
rel := strings.TrimPrefix(path, "part/")
|
||||
segs := strings.Split(rel, "/")
|
||||
if len(segs) < 2 {
|
||||
continue
|
||||
}
|
||||
category := segs[0]
|
||||
if !isSupportedPartCategory(category) {
|
||||
continue // ignores _masters, cloak, head, helm, tail, wings
|
||||
}
|
||||
stem := strings.TrimSuffix(segs[len(segs)-1], ".mdl")
|
||||
match := trailingNumberRegex.FindString(stem)
|
||||
if match == "" {
|
||||
return nil, fmt.Errorf("part.yml: supported-category model %q has no trailing row number", path)
|
||||
}
|
||||
rowID, err := strconv.Atoi(match)
|
||||
if err != nil || rowID == 0 {
|
||||
return nil, fmt.Errorf("part.yml: supported-category model %q resolves to invalid row id %q", path, match)
|
||||
}
|
||||
key := category + "/" + strconv.Itoa(rowID)
|
||||
if _, dup := seen[key]; dup {
|
||||
continue
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
entries = append(entries, autogenManifestEntry{
|
||||
Source: rel, Group: category, ModelStem: stem, RowID: rowID,
|
||||
})
|
||||
}
|
||||
if len(entries) == 0 {
|
||||
return nil, fmt.Errorf("part.yml: zero supported part rows after filtering")
|
||||
}
|
||||
slices.SortFunc(entries, func(a, b autogenManifestEntry) int {
|
||||
return strings.Compare(a.Source, b.Source)
|
||||
})
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// filterVFXCDNChannelEntries parses vfxs.yml and keeps restype==mdl assets under
|
||||
// vfxs/<group>/ for the consumer's 4 accessory groups, stripping the leading
|
||||
// vfxs/ from each source path. A present-but-malformed manifest (unparseable, or
|
||||
// no assets array) is an error; an empty assets array yields zero entries.
|
||||
func filterCDNChannelEntries(raw []byte, consumer project.AutogenConsumerConfig) ([]autogenManifestEntry, error) {
|
||||
func filterVFXCDNChannelEntries(raw []byte, consumer project.AutogenConsumerConfig) ([]autogenManifestEntry, error) {
|
||||
var manifest struct {
|
||||
Assets *[]struct {
|
||||
Path string `yaml:"path"`
|
||||
|
||||
Reference in New Issue
Block a user