feat(topdata): parts autogen from CDN part.yml channel
build-binaries / build-binaries (pull_request) Successful in 2m15s
test-image / build-image (pull_request) Successful in 37s
test / test (pull_request) Successful in 1m24s

Phase 2 of the parts-row autogen design: Crucible resolves parts 2DA rows
from the immutable part.yml published with the asset HAK release, over the
anonymous Bunny CDN channel, with the parts consumer required (no fail-open).

- generalize resolveCDNChannelManifest: derive manifest basename + provenance
  label from config; drop hardcoded assets/vfxs.yml so part.yml resolves too
- add filterPartsCDNChannelEntries + mode dispatch implementing the inventory
  contract (restype mdl under part/<cat>/, trailing-digit row id, dedup
  l/r/race/gender variants, sort by source, zero accepted rows = hard fail)
- support hand category + parts/hand dataset mapping (12 datasets total)
- native pipeline: single augment -> normalize -> override sequence with the
  configured parts_rows policy; overrides may update but never synthesize a row
- reject multiple parts_rows consumers in project validation
- tests: parts filter contract, category coverage, offline basename, orphan
  override reject, and a parity guard proving a bare CDN-only parts build

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-06-26 13:18:20 +02:00
co-authored by Claude Opus 4.8
parent 332a24fd3e
commit 2586b9193e
9 changed files with 363 additions and 21 deletions
+104 -17
View File
@@ -299,23 +299,34 @@ func resolveCDNChannelManifest(p *project.Project, consumer project.AutogenConsu
progress = func(string) {}
}
// 1. Offline / air-gapped override: a vfxs.yml path or a manifest-repo
// checkout root containing assets/vfxs.yml. Skips the network entirely.
// Manifest basename + provenance label are derived from config, not hardcoded,
// so the same resolver serves vfxs.yml (accessory VFX) and part.yml (parts).
manifestBase := filepath.Base(filepath.FromSlash(strings.TrimSpace(src.ManifestPath)))
if manifestBase == "." || manifestBase == "" || manifestBase == string(filepath.Separator) {
manifestBase = "manifest.yml"
}
label := strings.TrimSpace(consumer.ID)
if label == "" {
label = "autogen"
}
// 1. Offline / air-gapped override: a manifest file path or a manifest-repo
// checkout root containing assets/<manifestBase>. Skips the network entirely.
if envName := strings.TrimSpace(src.OfflineOverrideEnv); envName != "" {
if override := strings.TrimSpace(os.Getenv(envName)); override != "" {
vfxsPath := override
manifestPath := override
if info, err := os.Stat(override); err == nil && info.IsDir() {
vfxsPath = filepath.Join(override, "assets", "vfxs.yml")
manifestPath = filepath.Join(override, "assets", manifestBase)
}
raw, err := os.ReadFile(vfxsPath)
raw, err := os.ReadFile(manifestPath)
if err != nil {
return nil, fmt.Errorf("offline vfxs override %s: %w", vfxsPath, err)
return nil, fmt.Errorf("offline %s override %s: %w", manifestBase, manifestPath, err)
}
entries, err := filterCDNChannelEntries(raw, consumer)
if err != nil {
return nil, err
}
progress(fmt.Sprintf("Using offline vfxs override for %s from %s...", consumer.ID, vfxsPath))
progress(fmt.Sprintf("Using offline %s override for %s from %s...", manifestBase, consumer.ID, manifestPath))
return &autogenManifest{ID: consumer.Producer, Ref: "local", Entries: entries}, nil
}
}
@@ -359,13 +370,13 @@ func resolveCDNChannelManifest(p *project.Project, consumer project.AutogenConsu
if tag == "" {
return nil, unavailableAutogenManifest(fmt.Errorf("channel %q absent in channels.json (%s)", channel, channelsURL))
}
progress(fmt.Sprintf("Accessory VFX: channel %s -> tag %s", channel, tag))
progress(fmt.Sprintf("%s: channel %s -> tag %s", label, channel, tag))
// 5. vfxs.yml for the tag.
// 5. per-tag manifest for the tag.
manifestURL := join(src.ManifestPath, tag)
vstatus, vbody, err := httpGetStatus(manifestURL)
if err != nil {
return nil, unavailableAutogenManifest(fmt.Errorf("vfxs.yml unreachable %s: %w", manifestURL, err))
return nil, unavailableAutogenManifest(fmt.Errorf("%s unreachable %s: %w", manifestBase, manifestURL, err))
}
switch vstatus {
case http.StatusOK:
@@ -377,27 +388,103 @@ func resolveCDNChannelManifest(p *project.Project, consumer project.AutogenConsu
if marker := strings.TrimSpace(src.ReleaseMarkerPath); marker != "" {
markerURL := join(marker, tag)
if mstatus, _, merr := httpGetStatus(markerURL); merr == nil && mstatus == http.StatusOK {
return nil, fmt.Errorf("release %s has %s but no vfxs.yml (%s) — accessory rows would silently vanish; backfill/republish vfxs.yml for %s", tag, marker, manifestURL, tag)
return nil, fmt.Errorf("release %s has %s but no %s (%s) — rows would silently vanish; backfill/republish %s for %s", tag, marker, manifestBase, manifestURL, manifestBase, tag)
}
}
return nil, unavailableAutogenManifest(fmt.Errorf("vfxs.yml 404 %s (no published release for %s)", manifestURL, tag))
return nil, unavailableAutogenManifest(fmt.Errorf("%s 404 %s (no published release for %s)", manifestBase, manifestURL, tag))
default:
return nil, unavailableAutogenManifest(fmt.Errorf("vfxs.yml HTTP %d %s", vstatus, manifestURL))
return nil, unavailableAutogenManifest(fmt.Errorf("%s HTTP %d %s", manifestBase, vstatus, manifestURL))
}
entries, err := filterCDNChannelEntries(vbody, consumer)
if err != nil {
return nil, err // malformed vfxs.yml = HARD fail
return nil, err // malformed manifest = HARD fail
}
progress(fmt.Sprintf("Accessory VFX: resolved %d model entries from %s", len(entries), manifestURL))
progress(fmt.Sprintf("%s: resolved %d model entries from %s", label, len(entries), manifestURL))
return &autogenManifest{ID: consumer.Producer, Ref: tag, Entries: entries}, nil
}
// filterCDNChannelEntries parses vfxs.yml and keeps restype==mdl assets under
// filterCDNChannelEntries dispatches per-tag manifest parsing on the consumer
// mode. parts_rows consumes part.yml (the parts inventory contract); every other
// mode consumes vfxs.yml (the accessory-VFX contract).
func filterCDNChannelEntries(raw []byte, consumer project.AutogenConsumerConfig) ([]autogenManifestEntry, error) {
switch strings.TrimSpace(consumer.Mode) {
case "parts_rows":
return filterPartsCDNChannelEntries(raw)
default:
return filterVFXCDNChannelEntries(raw, consumer)
}
}
// filterPartsCDNChannelEntries parses part.yml and keeps restype==mdl assets
// under part/<supported-category>/.../<stem><digits>.mdl. The asset category is
// the path segment after part/; the row ID is the trailing decimal of the
// filename stem. Body/race/gender/left-right variants dedup by (category,rowID).
// Zero accepted rows is a HARD fail (silent-drop guard).
func filterPartsCDNChannelEntries(raw []byte) ([]autogenManifestEntry, error) {
var manifest struct {
Assets *[]struct {
Path string `yaml:"path"`
Restype string `yaml:"restype"`
} `yaml:"assets"`
}
if err := yaml.Unmarshal(raw, &manifest); err != nil {
return nil, fmt.Errorf("malformed part.yml (cannot parse): %w", err)
}
if manifest.Assets == nil {
return nil, fmt.Errorf("malformed part.yml (no assets array)")
}
seen := map[string]struct{}{}
var entries []autogenManifestEntry
for _, a := range *manifest.Assets {
if a.Restype != "mdl" {
continue
}
path := filepath.ToSlash(strings.TrimSpace(a.Path))
if !strings.HasPrefix(path, "part/") {
continue
}
rel := strings.TrimPrefix(path, "part/")
segs := strings.Split(rel, "/")
if len(segs) < 2 {
continue
}
category := segs[0]
if !isSupportedPartCategory(category) {
continue // ignores _masters, cloak, head, helm, tail, wings
}
stem := strings.TrimSuffix(segs[len(segs)-1], ".mdl")
match := trailingNumberRegex.FindString(stem)
if match == "" {
return nil, fmt.Errorf("part.yml: supported-category model %q has no trailing row number", path)
}
rowID, err := strconv.Atoi(match)
if err != nil || rowID == 0 {
return nil, fmt.Errorf("part.yml: supported-category model %q resolves to invalid row id %q", path, match)
}
key := category + "/" + strconv.Itoa(rowID)
if _, dup := seen[key]; dup {
continue
}
seen[key] = struct{}{}
entries = append(entries, autogenManifestEntry{
Source: rel, Group: category, ModelStem: stem, RowID: rowID,
})
}
if len(entries) == 0 {
return nil, fmt.Errorf("part.yml: zero supported part rows after filtering")
}
slices.SortFunc(entries, func(a, b autogenManifestEntry) int {
return strings.Compare(a.Source, b.Source)
})
return entries, nil
}
// filterVFXCDNChannelEntries parses vfxs.yml and keeps restype==mdl assets under
// vfxs/<group>/ for the consumer's 4 accessory groups, stripping the leading
// vfxs/ from each source path. A present-but-malformed manifest (unparseable, or
// no assets array) is an error; an empty assets array yields zero entries.
func filterCDNChannelEntries(raw []byte, consumer project.AutogenConsumerConfig) ([]autogenManifestEntry, error) {
func filterVFXCDNChannelEntries(raw []byte, consumer project.AutogenConsumerConfig) ([]autogenManifestEntry, error) {
var manifest struct {
Assets *[]struct {
Path string `yaml:"path"`