Remove completed legacy-migration cluster from topdata (#45)
The legacy normalize migration is done (confirmed). This deletes the dead migration path: NormalizeProject + 26 *_migrate.go files, the importLegacyItempropsRegistry cluster, loadLegacyTLK, and their tests (~7.9k lines, -33 files worth). mergeLegacyTLK moved into tlk_native.go since the live TLK compiler still uses it for base_dialog.json. make check passes. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Reviewed-on: #45 Reviewed-by: gitea-bot <gitea-bot@noreply.git.westgate.pw> Co-authored-by: vickydotbat <vickydotbat@tutamail.com> Co-committed-by: vickydotbat <vickydotbat@tutamail.com>
This commit was merged in pull request #45.
This commit is contained in:
@@ -601,397 +601,6 @@ func deepCopyValue(value any) any {
|
||||
return out
|
||||
}
|
||||
|
||||
func importLegacyItempropsRegistry(referenceBuilderDir, dataDir string, legacyTLK *legacyTLKData) (int, error) {
|
||||
legacyDataDir := filepath.Join(referenceBuilderDir, "data")
|
||||
registryDir := filepath.Join(dataDir, filepath.FromSlash(itempropsRegistryDirName))
|
||||
if _, err := os.Stat(filepath.Join(registryDir, "properties.json")); err == nil {
|
||||
if refs, err := countLegacyTLKRefsInRegistry(registryDir); err == nil && refs == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(legacyDataDir, "itemprops")); err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return 0, nil
|
||||
}
|
||||
return 0, err
|
||||
}
|
||||
|
||||
defs, err := collectBaseDataset(nativeDataset{
|
||||
Name: "itemprops/defs",
|
||||
BasePath: filepath.Join(legacyDataDir, "itemprops", "defs", "base.json"),
|
||||
LockPath: filepath.Join(legacyDataDir, "itemprops", "defs", "lock.json"),
|
||||
ModulesDir: filepath.Join(legacyDataDir, "itemprops", "defs", "modules"),
|
||||
Spec: specForDataset("itemprops"),
|
||||
})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
sets, err := collectBaseDataset(nativeDataset{
|
||||
Name: "itemprops/sets",
|
||||
BasePath: filepath.Join(legacyDataDir, "itemprops", "sets", "base.json"),
|
||||
LockPath: filepath.Join(legacyDataDir, "itemprops", "sets", "lock.json"),
|
||||
ModulesDir: filepath.Join(legacyDataDir, "itemprops", "sets", "modules"),
|
||||
Spec: specForDataset("itemprops"),
|
||||
})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
costIndex, err := collectBaseDataset(nativeDataset{
|
||||
Name: "itemprops/costtables/index",
|
||||
BasePath: filepath.Join(legacyDataDir, "itemprops", "costtables", "index", "base.json"),
|
||||
LockPath: filepath.Join(legacyDataDir, "itemprops", "costtables", "index", "lock.json"),
|
||||
ModulesDir: filepath.Join(legacyDataDir, "itemprops", "costtables", "index", "modules"),
|
||||
Spec: specForDataset("itemprops"),
|
||||
})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
paramIndex, err := collectBaseDataset(nativeDataset{
|
||||
Name: "itemprops/paramtables/index",
|
||||
BasePath: filepath.Join(legacyDataDir, "itemprops", "paramtables", "index", "base.json"),
|
||||
LockPath: filepath.Join(legacyDataDir, "itemprops", "paramtables", "index", "lock.json"),
|
||||
ModulesDir: filepath.Join(legacyDataDir, "itemprops", "paramtables", "index", "modules"),
|
||||
Spec: specForDataset("itemprops"),
|
||||
})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
ownedCostTables, err := collectLegacyOwnedItempropTables(filepath.Join(legacyDataDir, "itemprops", "costtables"), "index")
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
ownedParamTables, err := collectLegacyOwnedItempropTables(filepath.Join(legacyDataDir, "itemprops", "paramtables"), "index")
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
ownedSubtypeTables, err := collectLegacyOwnedItempropTables(filepath.Join(legacyDataDir, "itemprops", "subtypes"), "")
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
setsByID := map[int]map[string]any{}
|
||||
for _, row := range sets.Rows {
|
||||
setsByID[row["id"].(int)] = row
|
||||
}
|
||||
costByID := map[int]map[string]any{}
|
||||
for _, row := range costIndex.Rows {
|
||||
costByID[row["id"].(int)] = row
|
||||
}
|
||||
paramByID := map[int]map[string]any{}
|
||||
for _, row := range paramIndex.Rows {
|
||||
paramByID[row["id"].(int)] = row
|
||||
}
|
||||
|
||||
registryLock := map[string]int{}
|
||||
legacyKeyByID := map[string]string{}
|
||||
if legacyTLK != nil {
|
||||
for key, id := range legacyTLK.Lock {
|
||||
legacyKeyByID[strconv.Itoa(id)] = key
|
||||
}
|
||||
}
|
||||
properties := make([]map[string]any, 0, len(defs.Rows))
|
||||
subtypeRegistryByKey := map[string]map[string]any{}
|
||||
for _, row := range defs.Rows {
|
||||
rowID := row["id"].(int)
|
||||
key := canonicalItempropPropertyKey(row)
|
||||
registryLock[key] = rowID
|
||||
|
||||
property := map[string]any{
|
||||
"key": key,
|
||||
"label": row["Label"],
|
||||
"name": deepCopyValue(row["Name"]),
|
||||
"property_text": deepCopyValue(row["GameStrRef"]),
|
||||
}
|
||||
if description := nullableRegistryField(row, "Description"); description != nullValue {
|
||||
property["description"] = description
|
||||
}
|
||||
if setRow, ok := setsByID[rowID]; ok {
|
||||
property["availability"] = importLegacyAvailability(setRow)
|
||||
if setName := nullableRegistryField(setRow, "StringRef"); setName != nullValue && setName != stringField(row, "Name") {
|
||||
property["set_name"] = setName
|
||||
}
|
||||
if setLabel := stringField(setRow, "Label"); setLabel != "" && setLabel != stringField(row, "Label") {
|
||||
property["set_label"] = setLabel
|
||||
}
|
||||
} else {
|
||||
property["availability"] = []any{}
|
||||
}
|
||||
|
||||
if subtypeResRef := stringField(row, "SubTypeResRef"); subtypeResRef != "" && subtypeResRef != nullValue {
|
||||
subtypeKey := canonicalModelKey("subtype_models", subtypeResRef)
|
||||
property["subtype"] = map[string]any{"ref": subtypeKey}
|
||||
if _, ok := subtypeRegistryByKey[subtypeKey]; !ok {
|
||||
subtypeRegistryByKey[subtypeKey] = buildLegacySubtypeModel(subtypeKey, subtypeResRef, ownedSubtypeTables)
|
||||
}
|
||||
}
|
||||
costValue := stringField(row, "Cost")
|
||||
costID := stringField(row, "CostTableResRef")
|
||||
if costValue != "" && costValue != nullValue || costID != "" && costID != nullValue {
|
||||
cost := map[string]any{}
|
||||
if costValue != "" && costValue != nullValue {
|
||||
cost["value"] = costValue
|
||||
}
|
||||
if costID != "" && costID != nullValue {
|
||||
id, err := strconv.Atoi(costID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
costRow, ok := costByID[id]
|
||||
if !ok {
|
||||
return 0, fmt.Errorf("%s references missing cost table index %s", key, costID)
|
||||
}
|
||||
cost["ref"] = canonicalModelKey("cost_models", stringField(costRow, "Name"))
|
||||
}
|
||||
property["cost"] = cost
|
||||
}
|
||||
if paramID := stringField(row, "Param1ResRef"); paramID != "" && paramID != nullValue {
|
||||
id, err := strconv.Atoi(paramID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
paramRow, ok := paramByID[id]
|
||||
if !ok {
|
||||
return 0, fmt.Errorf("%s references missing param table index %s", key, paramID)
|
||||
}
|
||||
property["param"] = map[string]any{"ref": canonicalModelKey("param_models", stringField(paramRow, "TableResRef"))}
|
||||
}
|
||||
if legacyTLK != nil {
|
||||
property["name"] = importLegacyStrrefValue(property["name"], key+".name", legacyTLK, legacyKeyByID)
|
||||
property["property_text"] = importLegacyStrrefValue(property["property_text"], key+".property_text", legacyTLK, legacyKeyByID)
|
||||
if description, ok := property["description"]; ok {
|
||||
property["description"] = importLegacyStrrefValue(description, key+".description", legacyTLK, legacyKeyByID)
|
||||
}
|
||||
if updated, _, err := inlineLegacyTLKValue(property, legacyTLK); err != nil {
|
||||
return 0, err
|
||||
} else if updated {
|
||||
// property updated in place
|
||||
}
|
||||
}
|
||||
properties = append(properties, property)
|
||||
}
|
||||
|
||||
costModels := make([]map[string]any, 0, len(costIndex.Rows))
|
||||
for _, row := range costIndex.Rows {
|
||||
key := canonicalModelKey("cost_models", stringField(row, "Name"))
|
||||
registryLock[key] = row["id"].(int)
|
||||
model := map[string]any{
|
||||
"key": key,
|
||||
"index_name": row["Name"],
|
||||
"label": row["Label"],
|
||||
"client_load": row["ClientLoad"],
|
||||
"table": buildLegacyOwnedModelTable(stringField(row, "Name"), ownedCostTables),
|
||||
}
|
||||
costModels = append(costModels, model)
|
||||
}
|
||||
|
||||
paramModels := make([]map[string]any, 0, len(paramIndex.Rows))
|
||||
for _, row := range paramIndex.Rows {
|
||||
key := canonicalModelKey("param_models", stringField(row, "TableResRef"))
|
||||
registryLock[key] = row["id"].(int)
|
||||
model := map[string]any{
|
||||
"key": key,
|
||||
"name": deepCopyValue(row["Name"]),
|
||||
"label": row["Lable"],
|
||||
"table": buildLegacyOwnedModelTable(stringField(row, "TableResRef"), ownedParamTables),
|
||||
}
|
||||
paramModels = append(paramModels, model)
|
||||
}
|
||||
|
||||
subtypeModels := make([]map[string]any, 0, len(subtypeRegistryByKey))
|
||||
for _, key := range sortedKeysAny(subtypeRegistryByKey) {
|
||||
subtypeModels = append(subtypeModels, subtypeRegistryByKey[key])
|
||||
}
|
||||
|
||||
writes := []struct {
|
||||
path string
|
||||
obj map[string]any
|
||||
}{
|
||||
{filepath.Join(registryDir, "properties.json"), map[string]any{"rows": properties}},
|
||||
{filepath.Join(registryDir, "cost_models.json"), map[string]any{"rows": costModels}},
|
||||
{filepath.Join(registryDir, "param_models.json"), map[string]any{"rows": paramModels}},
|
||||
{filepath.Join(registryDir, "subtype_models.json"), map[string]any{"rows": subtypeModels}},
|
||||
{filepath.Join(registryDir, itempropsRegistryLock), anyMapInt(registryLock)},
|
||||
}
|
||||
if err := os.MkdirAll(registryDir, 0o755); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
for _, write := range writes {
|
||||
raw, err := json.MarshalIndent(write.obj, "", " ")
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
raw = append(raw, '\n')
|
||||
if err := os.WriteFile(write.path, raw, 0o644); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
return len(writes), nil
|
||||
}
|
||||
|
||||
func countLegacyTLKRefsInRegistry(registryDir string) (int, error) {
|
||||
paths, err := collectDataJSONPaths(registryDir)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
refs := 0
|
||||
for _, path := range paths {
|
||||
obj, err := loadJSONObject(path)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
refs += countLegacyTLKRefsInValue(obj)
|
||||
}
|
||||
return refs, nil
|
||||
}
|
||||
|
||||
func collectLegacyOwnedItempropTables(root, skipDir string) (map[string]map[string]any, error) {
|
||||
result := map[string]map[string]any{}
|
||||
entries, err := os.ReadDir(root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, entry := range entries {
|
||||
if !entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
if skipDir != "" && entry.Name() == skipDir {
|
||||
continue
|
||||
}
|
||||
dir := filepath.Join(root, entry.Name())
|
||||
basePath := filepath.Join(dir, "base.json")
|
||||
if _, err := os.Stat(basePath); err != nil {
|
||||
continue
|
||||
}
|
||||
collected, err := collectBaseDataset(nativeDataset{
|
||||
Name: filepath.ToSlash(filepath.Join("itemprops", filepath.Base(root), entry.Name())),
|
||||
BasePath: basePath,
|
||||
LockPath: filepath.Join(dir, "lock.json"),
|
||||
ModulesDir: filepath.Join(dir, "modules"),
|
||||
Spec: specForDataset("itemprops"),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result[strings.ToLower(outputStem(collected.Dataset.OutputName))] = map[string]any{
|
||||
"ownership": "owned",
|
||||
"resref": outputStem(collected.Dataset.OutputName),
|
||||
"output": collected.Dataset.OutputName,
|
||||
"columns": stringSliceToAny(collected.Columns),
|
||||
"rows": rowsToAny(collected.Rows),
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func buildLegacySubtypeModel(key, resref string, owned map[string]map[string]any) map[string]any {
|
||||
return map[string]any{
|
||||
"key": key,
|
||||
"table_resref": resref,
|
||||
"table": buildLegacyOwnedModelTable(resref, owned),
|
||||
}
|
||||
}
|
||||
|
||||
func buildLegacyOwnedModelTable(resref string, owned map[string]map[string]any) map[string]any {
|
||||
if table, ok := owned[strings.ToLower(resref)]; ok {
|
||||
return table
|
||||
}
|
||||
return map[string]any{
|
||||
"ownership": "external",
|
||||
"resref": resref,
|
||||
}
|
||||
}
|
||||
|
||||
func canonicalItempropPropertyKey(row map[string]any) string {
|
||||
if key := stringField(row, "key"); key != "" {
|
||||
return "itemprop:" + strings.TrimPrefix(key, "itempropdef:")
|
||||
}
|
||||
return canonicalModelKey("itemprop", stringField(row, "Label"))
|
||||
}
|
||||
|
||||
func canonicalModelKey(prefix, source string) string {
|
||||
normalized := strings.ToLower(strings.TrimSpace(source))
|
||||
normalized = strings.ReplaceAll(normalized, " ", "")
|
||||
normalized = strings.ReplaceAll(normalized, "-", "")
|
||||
normalized = strings.ReplaceAll(normalized, "_", "")
|
||||
return prefix + ":" + normalized
|
||||
}
|
||||
|
||||
func importLegacyStrrefValue(value any, fallbackKey string, legacy *legacyTLKData, keyByID map[string]string) any {
|
||||
if legacy == nil {
|
||||
return value
|
||||
}
|
||||
if _, ok, _ := parseTLKPayload(value, true); ok {
|
||||
return value
|
||||
}
|
||||
_ = keyByID
|
||||
key := fallbackKey
|
||||
entry, ok := legacy.Entries[key]
|
||||
if !ok {
|
||||
return value
|
||||
}
|
||||
payload := map[string]any{
|
||||
"key": key,
|
||||
"text": entry.Text,
|
||||
}
|
||||
if entry.SoundResRef != "" {
|
||||
payload["sound_resref"] = entry.SoundResRef
|
||||
}
|
||||
if entry.SoundLength != 0 {
|
||||
payload["sound_length"] = entry.SoundLength
|
||||
}
|
||||
return map[string]any{"tlk": payload}
|
||||
}
|
||||
|
||||
func importLegacyAvailability(row map[string]any) []any {
|
||||
out := make([]any, 0)
|
||||
for _, column := range itempropsAvailabilityColumns {
|
||||
value, ok := row[column]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
text := formatRegistryScalar(value)
|
||||
if text != "1" {
|
||||
continue
|
||||
}
|
||||
out = append(out, normalizeAvailabilityName(column))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func anyMapInt(values map[string]int) map[string]any {
|
||||
out := make(map[string]any, len(values))
|
||||
for key, value := range values {
|
||||
out[key] = value
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func stringSliceToAny(values []string) []any {
|
||||
out := make([]any, 0, len(values))
|
||||
for _, value := range values {
|
||||
out = append(out, value)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func rowsToAny(rows []map[string]any) []any {
|
||||
out := make([]any, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
out = append(out, deepCopyValue(row))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func sortedKeysAny(values map[string]map[string]any) []string {
|
||||
out := make([]string, 0, len(values))
|
||||
for key := range values {
|
||||
out = append(out, key)
|
||||
}
|
||||
slices.Sort(out)
|
||||
return out
|
||||
}
|
||||
|
||||
func validateItempropsRegistryGraph(dataDir string, report *ValidationReport) {
|
||||
registry, err := loadItempropsRegistry(dataDir)
|
||||
if err != nil {
|
||||
|
||||
Reference in New Issue
Block a user