ShadowsOverWestgate/sow-tools
7
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases 38 Activity

Support for global.json files

Browse Source
This commit is contained in:
archvillainette
2026-06-05 10:36:47 +02:00
parent d0e065536c
commit c672a1ff33
5 changed files with 804 additions and 264 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/topdata/native.go
+395 -57
View File
@@ -26,6 +26,7 @@ type nativeDataset struct {
LockPath string
ModulesDir string
GeneratedDir string
GlobalPaths []string
OutputName string
Spec datasetTextSpec
Columns []string
@@ -34,6 +35,7 @@ type nativeDataset struct {
RowGeneration string
RowGenerationMinRow int
CompareReference bool
HasGlobalInjections bool
}
type nativeDatasetKind string
@@ -865,6 +867,10 @@ func discoverNativeDatasets(dataDir string) ([]nativeDataset, error) {
if err != nil {
return err
}
globalPaths, err := collectGlobalJSONPaths(dataDir, rootPath)
if err != nil {
return err
}
lockPath := filepath.Join(rootPath, "lock.json")
if !plainDatasetNeedsLock(lockPath, tableData) {
lockPath = ""
@@ -879,6 +885,7 @@ func discoverNativeDatasets(dataDir string) ([]nativeDataset, error) {
RootPath: rootPath,
BasePath: filePath,
LockPath: lockPath,
GlobalPaths: globalPaths,
OutputName: outputName,
Spec: specForDataset(filepath.ToSlash(relName)),
CompareReference: optionalBool(tableData["compare_reference"], true),
@@ -913,6 +920,9 @@ func discoverNativeDatasets(dataDir string) ([]nativeDataset, error) {
if !strings.HasSuffix(strings.ToLower(name), ".json") || name == "lock.json" || name == "base.json" {
continue
}
if isGlobalJSONName(name) {
continue
}
filePath := filepath.Join(path, name)
if err := appendPlainDataset(path, filePath, strings.TrimSuffix(name, filepath.Ext(name))); err != nil {
return err
@@ -948,6 +958,7 @@ func discoverNativeDatasets(dataDir string) ([]nativeDataset, error) {
LockPath: filepath.Join(path, "lock.json"),
ModulesDir: filepath.Join(path, "modules"),
GeneratedDir: filepath.Join(path, "generated"),
GlobalPaths: collectExistingGlobalJSONPath(path),
OutputName: outputName,
Spec: specForDataset(filepath.ToSlash(rel)),
CompareReference: optionalBool(baseData["compare_reference"], true),
@@ -964,7 +975,7 @@ func discoverNativeDatasets(dataDir string) ([]nativeDataset, error) {
continue
}
name := entry.Name()
if strings.HasSuffix(strings.ToLower(name), ".json") && name != "lock.json" {
if strings.HasSuffix(strings.ToLower(name), ".json") && name != "lock.json" && !isGlobalJSONName(name) {
filePath := filepath.Join(path, name)
if err := appendPlainDataset(path, filePath, filepath.Join(rel, strings.TrimSuffix(name, filepath.Ext(name)))); err != nil {
return err
@@ -983,6 +994,47 @@ func discoverNativeDatasets(dataDir string) ([]nativeDataset, error) {
return datasets, nil
}
func isGlobalJSONName(name string) bool {
return strings.EqualFold(strings.TrimSpace(name), "global.json")
}
func collectExistingGlobalJSONPath(dir string) []string {
path := filepath.Join(dir, "global.json")
if _, err := os.Stat(path); err == nil {
return []string{path}
}
return nil
}
func collectGlobalJSONPaths(dataDir, datasetDir string) ([]string, error) {
rel, err := filepath.Rel(dataDir, datasetDir)
if err != nil {
return nil, err
}
dirs := []string{dataDir}
if rel != "." {
current := dataDir
for _, segment := range strings.Split(filepath.ToSlash(rel), "/") {
if strings.TrimSpace(segment) == "" || segment == "." {
continue
}
current = filepath.Join(current, segment)
dirs = append(dirs, current)
}
}
paths := make([]string, 0, len(dirs))
for _, dir := range dirs {
path := filepath.Join(dir, "global.json")
if _, err := os.Stat(path); err == nil {
paths = append(paths, path)
continue
} else if !errors.Is(err, os.ErrNotExist) {
return nil, err
}
}
return paths, nil
}
func plainDatasetNeedsLock(lockPath string, tableData map[string]any) bool {
if _, err := os.Stat(lockPath); err == nil {
return true
@@ -1076,12 +1128,22 @@ func collectBaseDataset(dataset nativeDataset) (nativeCollectedDataset, error) {
Data: obj,
})
}
globalData, err := loadGlobalModules(dataset.GlobalPaths)
if err != nil {
return nativeCollectedDataset{}, err
}
for _, module := range moduleData {
columns, err = extendColumns(columns, module.Data, dataset.Name, module.Name)
if err != nil {
return nativeCollectedDataset{}, err
}
}
for _, module := range globalData {
columns, err = extendColumns(columns, module.Data, dataset.Name, module.Name)
if err != nil {
return nativeCollectedDataset{}, err
}
}
rawBaseRows, ok := baseData["rows"].([]any)
if !ok {
return nativeCollectedDataset{}, fmt.Errorf("dataset %s: rows must be an array", dataset.Name)
@@ -1116,7 +1178,7 @@ func collectBaseDataset(dataset nativeDataset) (nativeCollectedDataset, error) {
}
explicitModuleIDs := map[string]int{}
for _, module := range moduleData {
for _, module := range append(append([]nativeGeneratedModule(nil), moduleData...), globalData...) {
explicitIDs, err := collectExplicitModuleIDs(dataset.Name, module.Name, module.Data)
if err != nil {
return nativeCollectedDataset{}, err
@@ -1279,7 +1341,61 @@ func collectBaseDataset(dataset nativeDataset) (nativeCollectedDataset, error) {
}
return true
}
applyModuleData := func(sourceLabel string, moduleData map[string]any) error {
applyOverrideFields := func(override map[string]any, row map[string]any) (bool, string, error) {
candidate := map[string]any{}
for field, value := range row {
candidate[field] = value
}
for field, value := range override {
if field == "id" || field == "key" || field == "_tlk" || field == "match" {
continue
}
if isMetadataField(field) {
meta, err := parseRowMetadata(value)
if err != nil {
return false, "", fmt.Errorf("dataset %s: %w", dataset.Name, err)
}
if meta != nil {
existingMeta, _ := parseExistingMetadata(candidate)
candidate["meta"] = mergeMetadataMaps(existingMeta, meta)
}
continue
}
if isDeprecatedAuthoringOnlyField(dataset.Name, field) {
if err := setLegacyWikiMetadata(candidate, field, value); err != nil {
return false, "", fmt.Errorf("dataset %s: %w", dataset.Name, err)
}
continue
}
columnName, ok := canonicalColumn(columns, field)
if !ok {
if isDeprecatedAuthoringOnlyField(dataset.Name, field) {
continue
}
return false, "", fmt.Errorf("dataset %s: override field %q is not a known column", dataset.Name, field)
}
candidate[columnName] = value
}
if rawKey, ok := override["key"]; ok {
candidate["key"] = rawKey
}
expanded, err := expandRowAuthoringSugar(dataset, columns, override, candidate)
if err != nil {
return false, "", err
}
keyChanged, retiredKey, err := updateOverrideRowKey(dataset.Name, row, expanded, rowByKey, lockData)
if err != nil {
return false, "", err
}
for field, value := range expanded {
if field == "id" || field == "_tlk" {
continue
}
row[field] = value
}
return keyChanged, retiredKey, nil
}
applyModuleData := func(sourceLabel string, moduleData map[string]any, globalSource bool) error {
if rawEntries, ok := moduleData["entries"]; ok {
entries, ok := rawEntries.(map[string]any)
if !ok {
@@ -1371,6 +1487,28 @@ func collectBaseDataset(dataset nativeDataset) (nativeCollectedDataset, error) {
if !ok {
return fmt.Errorf("dataset %s: override %d in %s must be an object", dataset.Name, index, sourceLabel)
}
if rawMatch, hasMatch := override["match"]; hasMatch {
match, ok := rawMatch.(string)
if !ok || !strings.EqualFold(strings.TrimSpace(match), "all") {
return fmt.Errorf("dataset %s: override %d in %s has unsupported match value %v", dataset.Name, index, sourceLabel, rawMatch)
}
if !globalSource {
return fmt.Errorf("dataset %s: override %d in %s uses match, which is only supported in global.json", dataset.Name, index, sourceLabel)
}
for _, row := range rows {
keyChanged, retiredKey, err := applyOverrideFields(override, row)
if err != nil {
return err
}
if keyChanged {
lockModified = true
}
if retiredKey != "" {
retiredKeys[retiredKey] = struct{}{}
}
}
continue
}
override, err = stripMismatchedLockedOverrideID(override)
if err != nil {
return err
@@ -1428,48 +1566,7 @@ func collectBaseDataset(dataset nativeDataset) (nativeCollectedDataset, error) {
nullifyOverrideRow(row, columns, rowByKey)
continue
}
candidate := map[string]any{}
for field, value := range row {
candidate[field] = value
}
for field, value := range override {
if field == "id" || field == "key" || field == "_tlk" {
continue
}
if isMetadataField(field) {
meta, err := parseRowMetadata(value)
if err != nil {
return fmt.Errorf("dataset %s: %w", dataset.Name, err)
}
if meta != nil {
existingMeta, _ := parseExistingMetadata(candidate)
candidate["meta"] = mergeMetadataMaps(existingMeta, meta)
}
continue
}
if isDeprecatedAuthoringOnlyField(dataset.Name, field) {
if err := setLegacyWikiMetadata(candidate, field, value); err != nil {
return fmt.Errorf("dataset %s: %w", dataset.Name, err)
}
continue
}
columnName, ok := canonicalColumn(columns, field)
if !ok {
if isDeprecatedAuthoringOnlyField(dataset.Name, field) {
continue
}
return fmt.Errorf("dataset %s: override field %q is not a known column", dataset.Name, field)
}
candidate[columnName] = value
}
if rawKey, ok := override["key"]; ok {
candidate["key"] = rawKey
}
expanded, err := expandRowAuthoringSugar(dataset, columns, override, candidate)
if err != nil {
return err
}
keyChanged, retiredKey, err := updateOverrideRowKey(dataset.Name, row, expanded, rowByKey, lockData)
keyChanged, retiredKey, err := applyOverrideFields(override, row)
if err != nil {
return err
}
@@ -1479,12 +1576,6 @@ func collectBaseDataset(dataset nativeDataset) (nativeCollectedDataset, error) {
if retiredKey != "" {
retiredKeys[retiredKey] = struct{}{}
}
for field, value := range expanded {
if field == "id" || field == "_tlk" {
continue
}
row[field] = value
}
}
}
return nil
@@ -1508,13 +1599,13 @@ func collectBaseDataset(dataset nativeDataset) (nativeCollectedDataset, error) {
if module.ApplyAfterModules {
continue
}
if err := applyModuleData(module.Name, module.Data); err != nil {
if err := applyModuleData(module.Name, module.Data, false); err != nil {
return nativeCollectedDataset{}, err
}
}
for _, module := range moduleData {
if err := applyModuleData(module.Name, module.Data); err != nil {
if err := applyModuleData(module.Name, module.Data, false); err != nil {
return nativeCollectedDataset{}, err
}
}
@@ -1522,7 +1613,12 @@ func collectBaseDataset(dataset nativeDataset) (nativeCollectedDataset, error) {
if !module.ApplyAfterModules {
continue
}
if err := applyModuleData(module.Name, module.Data); err != nil {
if err := applyModuleData(module.Name, module.Data, false); err != nil {
return nativeCollectedDataset{}, err
}
}
for _, module := range globalData {
if err := applyModuleData(module.Name, module.Data, true); err != nil {
return nativeCollectedDataset{}, err
}
}
@@ -3413,11 +3509,21 @@ func collectPlainDataset(dataset nativeDataset) (nativeCollectedDataset, error)
if err != nil {
return nativeCollectedDataset{}, err
}
globalData, err := loadGlobalModules(dataset.GlobalPaths)
if err != nil {
return nativeCollectedDataset{}, err
}
columns, err := parseColumns(tableData, dataset.Name)
if err != nil {
return nativeCollectedDataset{}, err
}
for _, module := range globalData {
columns, err = extendColumns(columns, module.Data, dataset.Name, module.Name)
if err != nil {
return nativeCollectedDataset{}, err
}
}
rawRows, ok := tableData["rows"].([]any)
if !ok {
return nativeCollectedDataset{}, fmt.Errorf("dataset %s: rows must be an array", dataset.Name)
@@ -3455,6 +3561,11 @@ func collectPlainDataset(dataset nativeDataset) (nativeCollectedDataset, error)
}
}
rows, explicitID, dataset.HasGlobalInjections, err = applyPlainDatasetGlobalInjections(dataset, columns, rows, explicitID, globalData)
if err != nil {
return nativeCollectedDataset{}, err
}
nextID := nextAvailableID(usedIDs)
for index, row := range rows {
key, hasKey := row["key"].(string)
@@ -3491,13 +3602,237 @@ func collectPlainDataset(dataset nativeDataset) (nativeCollectedDataset, error)
}, nil
}
func loadGlobalModules(paths []string) ([]nativeGeneratedModule, error) {
modules := make([]nativeGeneratedModule, 0, len(paths))
for _, path := range paths {
obj, err := loadJSONObject(path)
if err != nil {
return nil, err
}
modules = append(modules, nativeGeneratedModule{Name: path, Data: obj})
}
return modules, nil
}
func applyPlainDatasetGlobalInjections(dataset nativeDataset, columns []string, rows []map[string]any, explicitID []bool, modules []nativeGeneratedModule) ([]map[string]any, []bool, bool, error) {
if len(modules) == 0 {
return rows, explicitID, false, nil
}
prefixRows := []map[string]any{}
prefixExplicit := []bool{}
suffixRows := []map[string]any{}
suffixExplicit := []bool{}
hasInjections := false
currentRows := func() []map[string]any {
out := make([]map[string]any, 0, len(prefixRows)+len(rows)+len(suffixRows))
out = append(out, prefixRows...)
out = append(out, rows...)
out = append(out, suffixRows...)
return out
}
for _, module := range modules {
rawInjections, ok := module.Data["injections"]
if !ok {
continue
}
injections, ok := rawInjections.([]any)
if !ok {
return nil, nil, false, fmt.Errorf("dataset %s: injections in %s must be an array", dataset.Name, module.Name)
}
position, err := globalManifestPosition(module)
if err != nil {
return nil, nil, false, fmt.Errorf("dataset %s: %w", dataset.Name, err)
}
for index, rawInjection := range injections {
injection, ok := rawInjection.(map[string]any)
if !ok {
return nil, nil, false, fmt.Errorf("dataset %s: injection %d in %s must be an object", dataset.Name, index, module.Name)
}
rawRow, ok := injection["row"].(map[string]any)
if !ok {
return nil, nil, false, fmt.Errorf("dataset %s: injection %d in %s must contain row object", dataset.Name, index, module.Name)
}
presentRows := currentRows()
matches, err := globalInjectionConditionsMatch(injection, presentRows)
if err != nil {
return nil, nil, false, fmt.Errorf("dataset %s: injection %d in %s: %w", dataset.Name, index, module.Name, err)
}
if !matches {
continue
}
if globalRowIdentityPresent(rawRow, presentRows) {
continue
}
row, err := canonicalizeBaseRow(dataset, columns, rawRow, len(presentRows))
if err != nil {
return nil, nil, false, fmt.Errorf("dataset %s: injection %d in %s: %w", dataset.Name, index, module.Name, err)
}
explicit := false
if _, ok := rawRow["id"]; ok {
explicit = true
}
if position == "prepend" {
prefixRows = append(prefixRows, row)
prefixExplicit = append(prefixExplicit, explicit)
} else {
suffixRows = append(suffixRows, row)
suffixExplicit = append(suffixExplicit, explicit)
}
hasInjections = true
}
}
outRows := make([]map[string]any, 0, len(prefixRows)+len(rows)+len(suffixRows))
outExplicit := make([]bool, 0, len(prefixExplicit)+len(explicitID)+len(suffixExplicit))
outRows = append(outRows, prefixRows...)
outExplicit = append(outExplicit, prefixExplicit...)
outRows = append(outRows, rows...)
outExplicit = append(outExplicit, explicitID...)
outRows = append(outRows, suffixRows...)
outExplicit = append(outExplicit, suffixExplicit...)
return outRows, outExplicit, hasInjections, nil
}
func globalManifestPosition(module nativeGeneratedModule) (string, error) {
raw, ok := module.Data["position"]
if !ok || raw == nil {
return "append", nil
}
position, ok := raw.(string)
if !ok {
return "", fmt.Errorf("position in %s must be a string", module.Name)
}
switch strings.ToLower(strings.TrimSpace(position)) {
case "", "append":
return "append", nil
case "prepend":
return "prepend", nil
default:
return "", fmt.Errorf("position in %s must be append or prepend", module.Name)
}
}
func globalInjectionConditionsMatch(injection map[string]any, rows []map[string]any) (bool, error) {
if rawRequired, ok := injection["require_present"]; ok {
matches, err := globalConditionListMatches("require_present", rawRequired, rows, true)
if err != nil || !matches {
return matches, err
}
}
if rawBlocked, ok := injection["unless_present"]; ok {
matches, err := globalConditionListMatches("unless_present", rawBlocked, rows, false)
if err != nil || !matches {
return matches, err
}
}
return true, nil
}
func globalConditionListMatches(name string, raw any, rows []map[string]any, required bool) (bool, error) {
conditions, ok := raw.([]any)
if !ok {
return false, fmt.Errorf("%s must be an array", name)
}
for index, rawCondition := range conditions {
condition, ok := rawCondition.(map[string]any)
if !ok {
return false, fmt.Errorf("%s[%d] must be an object", name, index)
}
field, _ := condition["field"].(string)
id, _ := condition["id"].(string)
if strings.TrimSpace(field) == "" {
return false, fmt.Errorf("%s[%d].field is required", name, index)
}
if strings.TrimSpace(id) == "" {
return false, fmt.Errorf("%s[%d].id is required", name, index)
}
found := globalReferencePresent(rows, field, id)
if required && !found {
return false, nil
}
if !required && found {
return false, nil
}
}
return true, nil
}
func globalRowIdentityPresent(row map[string]any, rows []map[string]any) bool {
if key, ok := row["key"].(string); ok && strings.TrimSpace(key) != "" {
normalized := normalizeGlobalReferenceID(key)
for _, existing := range rows {
if existingKey, ok := existing["key"].(string); ok && normalizeGlobalReferenceID(existingKey) == normalized {
return true
}
}
return false
}
for field, value := range row {
ref, ok := value.(map[string]any)
if !ok {
continue
}
id, _ := ref["id"].(string)
if strings.TrimSpace(id) == "" {
continue
}
if globalReferencePresent(rows, field, id) {
return true
}
}
return false
}
func globalReferencePresent(rows []map[string]any, field, id string) bool {
field = strings.TrimSpace(field)
id = normalizeGlobalReferenceID(id)
if field == "" || id == "" {
return false
}
for _, row := range rows {
value, ok := row[field]
if !ok {
if canonical, exists := canonicalColumn(sortedStringMapKeys(row), field); exists {
value = row[canonical]
ok = true
}
}
if !ok {
continue
}
ref, ok := value.(map[string]any)
if !ok {
continue
}
existingID, _ := ref["id"].(string)
if normalizeGlobalReferenceID(existingID) == id {
return true
}
}
return false
}
func normalizeGlobalReferenceID(value string) string {
trimmed := strings.ToLower(strings.TrimSpace(value))
if trimmed == "" {
return ""
}
parts := strings.SplitN(trimmed, ":", 2)
if len(parts) != 2 {
return strings.ReplaceAll(trimmed, "_", "")
}
return parts[0] + ":" + strings.ReplaceAll(parts[1], "_", "")
}
func resolveNativeDataset(dataset nativeCollectedDataset, keyToID map[string]int, globalRowByKey map[string]map[string]any, tableRegistry resolvedTableRegistry, compiler *tlkCompiler, classFeatInjections project.TopDataClassFeatInjectionConfig, sidecars *nativeSidecarCollector) (map[string]any, error) {
rows := dataset.Rows
if strings.HasPrefix(filepath.ToSlash(dataset.Dataset.Name), "classes/feats/") {
classKey := "classes:" + dataset.Dataset.Name[strings.LastIndex(dataset.Dataset.Name, "/")+1:]
featSuccessors := buildFeatSuccessorsIndex(globalRowByKey, keyToID)
classSkills := buildClassSkillsIndex(tableRegistry, classKey)
expanded, err := expandClassesFeatRows(rows, keyToID, globalRowByKey, featSuccessors, classSkills, globalRowByKey, classKey, classFeatInjections)
expanded, err := expandClassesFeatRows(rows, keyToID, globalRowByKey, featSuccessors, classSkills, globalRowByKey, classKey, classFeatInjections, !dataset.Dataset.HasGlobalInjections)
if err != nil {
return nil, fmt.Errorf("dataset %s: %w", dataset.Dataset.Name, err)
}
@@ -3546,9 +3881,12 @@ var (
}
)
func expandClassesFeatRows(rows []map[string]any, keyToID map[string]int, rowByKey map[string]map[string]any, featSuccessors map[string]string, classSkills map[string]bool, allRowByKey map[string]map[string]any, classKey string, classFeatInjections project.TopDataClassFeatInjectionConfig) ([]map[string]any, error) {
func expandClassesFeatRows(rows []map[string]any, keyToID map[string]int, rowByKey map[string]map[string]any, featSuccessors map[string]string, classSkills map[string]bool, allRowByKey map[string]map[string]any, classKey string, classFeatInjections project.TopDataClassFeatInjectionConfig, useConfiguredInjections bool) ([]map[string]any, error) {
globalRules, classSkillRules := effectiveClassFeatInjectionRules(classFeatInjections)
globalRules, classSkillRules := []project.TopDataClassFeatGlobalRule{}, []project.TopDataClassFeatMasterfeatRule{}
if useConfiguredInjections {
globalRules, classSkillRules = effectiveClassFeatInjectionRules(classFeatInjections)
}
injected := make([]map[string]any, 0, len(globalRules)+len(classSkillRules))
presentRefIDs := make(map[string]struct{}, len(rows))
for _, row := range rows {