Phase 5: scaffold Crucible (sow-tools) — dispatcher + builder shims, container, PR-first CI, fail-closed (D11, D19).

This commit is contained in:
2026-06-11 20:36:13 +02:00
parent ff264f0d7c
commit 97f8427c4b
154 changed files with 948 additions and 76902 deletions
-692
View File
@@ -1,692 +0,0 @@
package topdata
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"regexp"
"slices"
"strconv"
"strings"
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
)
// supportedPartCategories defines the part types that should be auto-discovered
// from the sow-assets repository.
var supportedPartCategories = []string{
"belt",
"bicep",
"chest",
"foot",
"forearm",
"leg",
"neck",
"pelvis",
"robe",
"shin",
"shoulder",
}
var partDatasetToAssetCategory = map[string]string{
"belt": "belt",
"bicep": "bicep",
"chest": "chest",
"foot": "foot",
"forearm": "forearm",
"legs": "leg",
"neck": "neck",
"pelvis": "pelvis",
"robe": "robe",
"shin": "shin",
"shoulder": "shoulder",
}
// trailingNumberRegex matches trailing digits at the end of a string
var trailingNumberRegex = regexp.MustCompile(`(\d+)$`)
// isPartsDataset checks if a dataset name corresponds to a parts dataset
func isPartsDataset(datasetName string) bool {
// Check if the dataset is under parts/ directory
if strings.HasPrefix(datasetName, "parts/") {
return true
}
// Check if it's exactly a parts file (e.g., "parts/belt")
parts := strings.Split(datasetName, "/")
if len(parts) == 2 && parts[0] == "parts" {
return true
}
return false
}
func isAutogenEligiblePartsDataset(dataset nativeCollectedDataset) bool {
if !isPartsDataset(dataset.Dataset.Name) {
return false
}
for _, row := range dataset.Rows {
key, ok := row["key"].(string)
if ok && strings.TrimSpace(key) != "" {
return false
}
}
return true
}
// extractPartCategory extracts the part category from a dataset name
// e.g., "parts/belt" -> "belt", "parts/bicep" -> "bicep"
func extractPartCategory(datasetName string) string {
parts := strings.Split(datasetName, "/")
if len(parts) >= 2 {
return parts[len(parts)-1]
}
return ""
}
func partAssetCategoryForDataset(datasetName string) string {
category := extractPartCategory(datasetName)
if mapped, ok := partDatasetToAssetCategory[category]; ok {
return mapped
}
return category
}
// isSupportedPartCategory checks if the given category is in the supported list
func isSupportedPartCategory(category string) bool {
for _, c := range supportedPartCategories {
if c == category {
return true
}
}
return false
}
// extractTrailingNumericSuffix extracts the trailing numeric suffix from a filename stem
// e.g., "pfa0_belt018" -> 18, "pfa0_belt171" -> 171
// Returns 0 and false if no trailing numeric suffix is found
func extractTrailingNumericSuffix(filenameStem string) (int, bool) {
matches := trailingNumberRegex.FindStringSubmatch(filenameStem)
if len(matches) < 2 {
return 0, false
}
num, err := strconv.Atoi(matches[1])
if err != nil {
return 0, false
}
return num, true
}
// DiscoveredPartModel represents a discovered model file with its row ID
type DiscoveredPartModel struct {
RowID int
Filename string
}
// DiscoverPartModels scans the assets/part directory for .mdl files
// and returns a map of rowID -> default row data for the given category.
// The assetsDir should be the root assets directory (containing the part/ subdirectory).
func DiscoverPartModels(assetsDir string, category string) (map[int]*DiscoveredPartModel, error) {
result := make(map[int]*DiscoveredPartModel)
if !isSupportedPartCategory(category) {
return result, nil
}
partRoot, err := resolvePartsRoot(assetsDir)
if err != nil {
return nil, err
}
if partRoot == "" {
return result, nil
}
// Construct the path to the part category directory
partDir := filepath.Join(partRoot, category)
// Check if the directory exists
info, err := os.Stat(partDir)
if err != nil {
if os.IsNotExist(err) {
return result, nil
}
return nil, fmt.Errorf("failed to access part directory %s: %w", partDir, err)
}
if !info.IsDir() {
return nil, fmt.Errorf("part directory %s is not a directory", partDir)
}
// Walk the directory and find .mdl files
err = filepath.Walk(partDir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info.IsDir() {
return nil
}
// Only process .mdl files
if !strings.HasSuffix(strings.ToLower(info.Name()), ".mdl") {
return nil
}
// Extract the filename stem (without extension)
filenameStem := strings.TrimSuffix(info.Name(), filepath.Ext(info.Name()))
// Extract trailing numeric suffix
rowID, hasSuffix := extractTrailingNumericSuffix(filenameStem)
if !hasSuffix {
return nil
}
// Store the discovered model
if existing, ok := result[rowID]; ok {
// If we already have this row ID, keep the first one found (alphabetically)
if path < existing.Filename {
result[rowID] = &DiscoveredPartModel{
RowID: rowID,
Filename: path,
}
}
} else {
result[rowID] = &DiscoveredPartModel{
RowID: rowID,
Filename: path,
}
}
return nil
})
if err != nil {
return nil, fmt.Errorf("failed to scan part directory %s: %w", partDir, err)
}
return result, nil
}
// CreateDefaultPartRow creates a default row for a discovered part model.
// Base defaults are COSTMODIFIER=0 and ACBONUS=0.00. Any robe HIDE* columns
// present in the dataset columns default to 0 as well.
func CreateDefaultPartRow(rowID int) map[string]any {
return CreateDefaultPartRowForColumns(rowID, nil)
}
func CreateDefaultPartRowForColumns(rowID int, columns []string) map[string]any {
return createDefaultPartRowForDataset(rowID, "", columns, project.PartsRowsConfig{})
}
func createDefaultPartRowForDataset(rowID int, datasetName string, columns []string, cfg project.PartsRowsConfig) map[string]any {
row := map[string]any{
"id": rowID,
"COSTMODIFIER": "0",
"ACBONUS": "0.00",
}
for field, value := range cfg.RowDefaults {
if strings.TrimSpace(field) == "" {
continue
}
row[field] = value
}
if value, ok := configuredPartsRowsACBonusValue(rowID, datasetName, cfg); ok {
row["ACBONUS"] = value
}
for _, column := range columns {
if strings.HasPrefix(column, "HIDE") {
row[column] = "0"
}
}
return row
}
func isUnsetPartValue(value any) bool {
switch typed := value.(type) {
case nil:
return true
case string:
return typed == "" || typed == "****"
}
return false
}
func applyDiscoveredPartDefaults(row map[string]any, columns []string) {
applyDiscoveredPartDefaultsWithConfig(row, "", columns, project.PartsRowsConfig{})
}
func applyDiscoveredPartDefaultsWithConfig(row map[string]any, datasetName string, columns []string, cfg project.PartsRowsConfig) {
if isUnsetPartValue(row["COSTMODIFIER"]) {
row["COSTMODIFIER"] = "0"
}
for field, value := range cfg.RowDefaults {
if strings.TrimSpace(field) == "" {
continue
}
if isUnsetPartValue(row[field]) {
row[field] = value
}
}
if isUnsetPartValue(row["ACBONUS"]) {
row["ACBONUS"] = "0.00"
}
if value, ok := configuredPartsRowsACBonusValue(rowIDFromPartRow(row), datasetName, cfg); ok {
row["ACBONUS"] = value
}
for _, column := range columns {
if !strings.HasPrefix(column, "HIDE") {
continue
}
if isUnsetPartValue(row[column]) {
row[column] = "0"
}
}
}
func rowIDFromPartRow(row map[string]any) int {
rowID, _ := row["id"].(int)
return rowID
}
func resolvePartsRoot(root string) (string, error) {
if strings.TrimSpace(root) == "" {
return "", nil
}
candidates := []string{
filepath.Join(root, "assets", "parts"),
filepath.Join(root, "assets", "part"),
filepath.Join(root, "parts"),
filepath.Join(root, "part"),
}
for _, candidate := range candidates {
info, err := os.Stat(candidate)
if err == nil && info.IsDir() {
return candidate, nil
}
if err != nil && !os.IsNotExist(err) {
return "", fmt.Errorf("failed to access parts root %s: %w", candidate, err)
}
}
return "", nil
}
// scanAutogeneratedParts scans the assets/part directory for all supported part categories
// and returns a map of category -> set of row IDs that should be auto-generated.
// The assetsDir parameter should be the path to the assets directory that contains the part/ subdirectory.
func scanAutogeneratedParts(assetsDir string) (map[string]map[int]struct{}, error) {
if assetsDir == "" {
return nil, nil
}
result := make(map[string]map[int]struct{})
for _, category := range supportedPartCategories {
discovered, err := DiscoverPartModels(assetsDir, category)
if err != nil {
return nil, err
}
// Always include the category, even if no IDs were discovered
ids := make(map[int]struct{}, len(discovered))
for rowID := range discovered {
ids[rowID] = struct{}{}
}
result[category] = ids
}
return result, nil
}
func resolveAutogeneratedPartsInventory(p *project.Project, progress func(string)) (map[string]map[int]struct{}, error) {
overrideDir, err := resolvePartsAssetsOverrideDir(p)
if err != nil {
return nil, err
}
if overrideDir != "" {
if progress != nil {
progress(fmt.Sprintf("Scanning part models from local topdata.assets override %s...", overrideDir))
}
return scanAutogeneratedParts(overrideDir)
}
return resolveReleasedPartsManifest(p, progress)
}
func hasCollectedPartsDatasets(collected []nativeCollectedDataset) bool {
for _, dataset := range collected {
if isPartsDataset(dataset.Dataset.Name) {
return true
}
}
return false
}
func loadPartOverrides(path string) ([]map[string]any, error) {
if _, err := os.Stat(path); err != nil {
if os.IsNotExist(err) {
return nil, nil
}
return nil, err
}
obj, err := loadJSONObject(path)
if err != nil {
return nil, err
}
rawOverrides, ok := obj["overrides"]
if !ok {
return nil, nil
}
overrideList, ok := rawOverrides.([]any)
if !ok {
return nil, fmt.Errorf("%s: overrides must be an array", path)
}
overrides := make([]map[string]any, 0, len(overrideList))
for index, rawOverride := range overrideList {
override, ok := rawOverride.(map[string]any)
if !ok {
return nil, fmt.Errorf("%s: override %d must be an object", path, index)
}
overrides = append(overrides, override)
}
return overrides, nil
}
func applyPartOverrides(sourceDir string, collected []nativeCollectedDataset) ([]nativeCollectedDataset, error) {
return applyPartOverridesWithConfig(sourceDir, collected, project.PartsRowsConfig{})
}
func applyPartOverridesWithConfig(sourceDir string, collected []nativeCollectedDataset, cfg project.PartsRowsConfig) ([]nativeCollectedDataset, error) {
result := make([]nativeCollectedDataset, len(collected))
copy(result, collected)
for i, dataset := range result {
if !strings.HasPrefix(dataset.Dataset.Name, "parts/") {
continue
}
category := extractPartCategory(dataset.Dataset.Name)
if category == "" {
continue
}
overrides, err := loadPartOverridesForCategory(sourceDir, category)
if err != nil {
return nil, err
}
if len(overrides) == 0 {
continue
}
rows := make([]map[string]any, len(dataset.Rows))
rowByID := make(map[int]map[string]any, len(dataset.Rows))
for index, row := range dataset.Rows {
cloned := cloneRowMap(row)
rows[index] = cloned
if rowID, ok := cloned["id"].(int); ok {
rowByID[rowID] = cloned
}
}
for index, override := range overrides {
rawID, ok := override["id"]
if !ok {
return nil, fmt.Errorf("parts/%s override %d is missing id", category, index)
}
rowID, err := asInt(rawID)
if err != nil {
return nil, fmt.Errorf("parts/%s override %d id is not numeric", category, index)
}
row, ok := rowByID[rowID]
if !ok {
row = createDefaultPartRowForDataset(rowID, dataset.Dataset.Name, dataset.Columns, cfg)
rows = append(rows, row)
rowByID[rowID] = row
}
for field, value := range override {
if field == "id" {
continue
}
row[field] = normalizePartOverrideValue(value)
}
}
slices.SortFunc(rows, func(a, b map[string]any) int {
idA := a["id"].(int)
idB := b["id"].(int)
return idA - idB
})
result[i].Rows = rows
}
return result, nil
}
func normalizePartsRowsACBonus(collected []nativeCollectedDataset, cfg project.PartsRowsConfig) ([]nativeCollectedDataset, error) {
if !partsRowsACBonusConfigured(cfg) && len(cfg.RowDefaults) == 0 {
return collected, nil
}
result := make([]nativeCollectedDataset, len(collected))
copy(result, collected)
for i, dataset := range result {
if !strings.HasPrefix(dataset.Dataset.Name, "parts/") {
continue
}
rows := make([]map[string]any, len(dataset.Rows))
for index, row := range dataset.Rows {
cloned := cloneRowMap(row)
for field, value := range cfg.RowDefaults {
if strings.TrimSpace(field) == "" {
continue
}
if isUnsetPartValue(cloned[field]) {
cloned[field] = value
}
}
rowID, ok := cloned["id"].(int)
if isUnsetPartValue(cloned["ACBONUS"]) {
cloned["COSTMODIFIER"] = nullValue
} else if ok {
if value, configured := configuredPartsRowsACBonusValue(rowID, dataset.Dataset.Name, cfg); configured {
cloned["ACBONUS"] = value
}
}
rows[index] = cloned
}
result[i].Rows = rows
}
return result, nil
}
func partsRowsACBonusConfigured(cfg project.PartsRowsConfig) bool {
if strings.TrimSpace(cfg.ACBonus.Default.Strategy) != "" {
return true
}
if len(cfg.ACBonus.Datasets) > 0 {
return true
}
for _, dataset := range cfg.Datasets {
if strings.TrimSpace(dataset.ACBonus.Strategy) != "" {
return true
}
}
return false
}
func configuredPartsRowsACBonusValue(rowID int, datasetName string, cfg project.PartsRowsConfig) (string, bool) {
policy, ok := partsRowsACBonusPolicyForDataset(datasetName, cfg)
if !ok {
return "", false
}
switch strings.TrimSpace(policy.Strategy) {
case "ascending_row_id_sort_key":
format := strings.TrimSpace(policy.Format)
if format == "" {
format = "%.2f"
}
return fmt.Sprintf(format, float64(rowID)/float64(policy.Divisor)), true
case "descending_row_id_sort_key":
format := strings.TrimSpace(policy.Format)
if format == "" {
format = "%.2f"
}
return fmt.Sprintf(format, float64(policy.MaxRowID-rowID)/float64(policy.Divisor)), true
case "fixed":
return policy.Value, true
default:
return "", false
}
}
func partsRowsACBonusPolicyForDataset(datasetName string, cfg project.PartsRowsConfig) (project.PartsRowsACBonusPolicy, bool) {
category := extractPartCategory(datasetName)
if policy, ok := cfg.ACBonus.Datasets[category]; ok && strings.TrimSpace(policy.Strategy) != "" {
return policy, true
}
if dataset, ok := cfg.Datasets[category]; ok && strings.TrimSpace(dataset.ACBonus.Strategy) != "" {
return dataset.ACBonus, true
}
if strings.TrimSpace(cfg.ACBonus.Default.Strategy) != "" {
return cfg.ACBonus.Default, true
}
return project.PartsRowsACBonusPolicy{}, false
}
func loadPartOverridesForCategory(sourceDir, category string) ([]map[string]any, error) {
overrides := []map[string]any{}
legacyPath := filepath.Join(sourceDir, "data", "parts", "overrides", category+".json")
legacyOverrides, err := loadPartOverrides(legacyPath)
if err != nil {
return nil, err
}
overrides = append(overrides, legacyOverrides...)
moduleDir := filepath.Join(sourceDir, "data", "parts", "modules")
modulePaths, err := partModuleOverridePaths(moduleDir, category)
if err != nil {
return nil, err
}
for _, path := range modulePaths {
moduleOverrides, err := loadPartOverrides(path)
if err != nil {
return nil, err
}
overrides = append(overrides, moduleOverrides...)
}
return overrides, nil
}
func partModuleOverridePaths(moduleDir, category string) ([]string, error) {
info, err := os.Stat(moduleDir)
if err != nil {
if os.IsNotExist(err) {
return nil, nil
}
return nil, err
}
if !info.IsDir() {
return nil, fmt.Errorf("parts module override path %s is not a directory", moduleDir)
}
var paths []string
err = filepath.WalkDir(moduleDir, func(path string, d os.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() {
return nil
}
if strings.ToLower(filepath.Ext(path)) != ".json" {
return nil
}
if partModuleOverrideMatchesCategory(path, category) {
paths = append(paths, path)
}
return nil
})
if err != nil {
return nil, err
}
slices.Sort(paths)
return paths, nil
}
func partModuleOverrideMatchesCategory(path, category string) bool {
stem := strings.TrimSuffix(strings.ToLower(filepath.Base(path)), strings.ToLower(filepath.Ext(path)))
category = strings.ToLower(strings.TrimSpace(category))
return stem == category ||
strings.HasPrefix(stem, category+"_") ||
strings.HasSuffix(stem, "_"+category) ||
strings.Contains(stem, "_"+category+"_")
}
func normalizePartOverrideValue(value any) any {
switch typed := value.(type) {
case float64:
if typed == float64(int(typed)) {
return strconv.Itoa(int(typed))
}
case int:
return strconv.Itoa(typed)
case json.Number:
return typed.String()
}
return value
}
// augmentWithAutogeneratedParts applies repository-backed discovery defaults.
// Missing discovered IDs are added with default values. Existing discovered rows
// retain authored values unless they still use placeholder values, in which case
// discovery activates them with engine-visible defaults.
func augmentWithAutogeneratedParts(collected []nativeCollectedDataset, autogenerated map[string]map[int]struct{}) []nativeCollectedDataset {
return augmentWithAutogeneratedPartsWithConfig(collected, autogenerated, project.PartsRowsConfig{})
}
func augmentWithAutogeneratedPartsWithConfig(collected []nativeCollectedDataset, autogenerated map[string]map[int]struct{}, cfg project.PartsRowsConfig) []nativeCollectedDataset {
if len(autogenerated) == 0 {
return collected
}
result := make([]nativeCollectedDataset, len(collected))
copy(result, collected)
for i, dataset := range collected {
if !isAutogenEligiblePartsDataset(dataset) {
continue
}
category := partAssetCategoryForDataset(dataset.Dataset.Name)
if !isSupportedPartCategory(category) {
continue
}
ids, ok := autogenerated[category]
if !ok || len(ids) == 0 {
continue
}
// Build a map of existing row IDs.
existingRows := make(map[int]map[string]any, len(dataset.Rows))
for _, row := range dataset.Rows {
if id, ok := row["id"].(int); ok {
existingRows[id] = row
}
}
// Add rows for discovered IDs that don't already exist.
newRows := make([]map[string]any, 0, len(dataset.Rows))
newRows = append(newRows, dataset.Rows...)
for rowID := range ids {
if row, exists := existingRows[rowID]; exists {
applyDiscoveredPartDefaultsWithConfig(row, dataset.Dataset.Name, dataset.Columns, cfg)
continue
}
if _, exists := existingRows[rowID]; !exists {
newRow := createDefaultPartRowForDataset(rowID, dataset.Dataset.Name, dataset.Columns, cfg)
newRows = append(newRows, newRow)
}
}
// Sort rows by ID
slices.SortFunc(newRows, func(a, b map[string]any) int {
idA := a["id"].(int)
idB := b["id"].(int)
return idA - idB
})
result[i].Rows = newRows
}
return result
}