Files
sow-tools/internal/topdata/tlk_native.go
T
archvillainette cf89c166fe
test / test (push) Has been cancelled
build-image / build-image (push) Has been cancelled
claude: fold in old sow-tools
2026-06-13 09:49:29 +02:00

730 lines
18 KiB
Go

package topdata
import (
"bytes"
"encoding/binary"
"encoding/json"
"fmt"
"math"
"os"
"path/filepath"
"slices"
"strconv"
"strings"
"golang.org/x/text/encoding/charmap"
)
const (
defaultTLKName = "sow_tlk.tlk"
tlkStateFile = ".tlk_state.json"
customTLKBase = 0x01000000
)
type textRole string
const (
textRoleName textRole = "name"
textRoleDescription textRole = "description"
)
type datasetTextSpec struct {
NameFields []string
DescriptionFields []string
}
var datasetTextSpecs = map[string]datasetTextSpec{
"baseitems": {
NameFields: []string{"Name"},
DescriptionFields: []string{"Description"},
},
"classes": {
NameFields: []string{"Short", "Name", "Plural", "Lower", "CLASS"},
DescriptionFields: []string{"Description"},
},
"feat": {
NameFields: []string{"FEAT"},
DescriptionFields: []string{"DESCRIPTION"},
},
"masterfeats": {
NameFields: []string{"STRREF"},
DescriptionFields: []string{"DESCRIPTION"},
},
"itemprops": {
NameFields: []string{"Name", "StringRef"},
DescriptionFields: []string{"Description", "GameStrRef"},
},
"racialtypes": {
NameFields: []string{"Name", "RACIALTYPE"},
DescriptionFields: []string{"Description"},
},
"skills": {
NameFields: []string{"Name", "SKILL"},
DescriptionFields: []string{"Description"},
},
"spells": {
NameFields: []string{"Name", "SPELL"},
DescriptionFields: []string{"SpellDesc", "DESCRIPTION"},
},
}
type tlkEntryData struct {
Text string
SoundResRef string
SoundLength float32
}
type tlkStateDocument struct {
Version int `json:"version"`
Language string `json:"language"`
Entries map[string]tlkStateMapping `json:"entries"`
}
type tlkStateMapping struct {
ID int `json:"id"`
Retired bool `json:"retired,omitempty"`
}
type tlkCompiledValue struct {
Value any
TLKKey string
}
type tlkCompiler struct {
language string
statePath string
state tlkStateDocument
legacy map[string]tlkEntryData
active map[string]tlkEntryData
activeKeys map[string]struct{}
reservedByID map[int]string
nextID int
}
func newTLKCompiler(sourceDir string, legacy *legacyTLKData) (*tlkCompiler, error) {
statePath := filepath.Join(sourceDir, tlkStateFile)
state, err := loadTLKState(statePath)
if err != nil {
return nil, err
}
baseDialog, err := loadBaseDialogData(filepath.Join(sourceDir, "base_dialog.json"))
if err != nil {
return nil, err
}
legacy = mergeLegacyTLK(baseDialog, legacy)
language := state.Language
if language == "" {
language = "en"
}
if legacy != nil && legacy.Language != "" {
if state.Language == "" {
language = legacy.Language
}
for key, id := range legacy.Lock {
if _, ok := state.Entries[key]; !ok {
state.Entries[key] = tlkStateMapping{ID: id}
}
}
}
reserved := map[int]string{}
nextID := 0
for key, entry := range state.Entries {
if owner, ok := reserved[entry.ID]; ok && owner != key {
return nil, fmt.Errorf("tlk state id collision between %q and %q", owner, key)
}
reserved[entry.ID] = key
if entry.ID >= nextID {
nextID = entry.ID + 1
}
}
compiler := &tlkCompiler{
language: language,
statePath: statePath,
state: state,
legacy: map[string]tlkEntryData{},
active: map[string]tlkEntryData{},
activeKeys: map[string]struct{}{},
reservedByID: reserved,
nextID: nextID,
}
if legacy != nil {
for key, entry := range legacy.Entries {
compiler.legacy[key] = entry
}
}
return compiler, nil
}
func loadBaseDialogData(path string) (*legacyTLKData, error) {
if _, err := os.Stat(path); err != nil {
if os.IsNotExist(err) {
return nil, nil
}
return nil, fmt.Errorf("stat %s: %w", path, err)
}
obj, err := loadJSONObject(path)
if err != nil {
return nil, err
}
result := &legacyTLKData{
Language: "en",
Entries: map[string]tlkEntryData{},
Lock: map[string]int{},
}
if language, ok := obj["language"].(string); ok && strings.TrimSpace(language) != "" {
result.Language = language
}
rawEntries, ok := obj["entries"]
if !ok {
return result, nil
}
entries, err := normalizeBaseDialogEntries(rawEntries)
if err != nil {
return nil, err
}
normalized, err := normalizeLegacyTLKEntries(entries)
if err != nil {
return nil, err
}
for key, entry := range normalized {
result.Entries[key] = entry
}
return result, nil
}
func normalizeBaseDialogEntries(raw any) (map[string]any, error) {
switch typed := raw.(type) {
case map[string]any:
return typed, nil
case []any:
entries := make(map[string]any, len(typed))
for index, item := range typed {
obj, ok := item.(map[string]any)
if !ok {
return nil, fmt.Errorf("base_dialog.json entry %d must be an object", index)
}
rawID, ok := obj["id"]
if !ok {
return nil, fmt.Errorf("base_dialog.json entry %d is missing id", index)
}
key, err := legacyDialogIDKey(rawID)
if err != nil {
return nil, fmt.Errorf("base_dialog.json entry %d: %w", index, err)
}
entry := map[string]any{}
if text, ok := obj["text"]; ok {
entry["text"] = text
}
if sound, ok := obj["sound_resref"]; ok {
entry["sound_resref"] = sound
}
if length, ok := obj["sound_length"]; ok {
entry["sound_length"] = length
}
entries[key] = entry
}
return entries, nil
default:
return nil, fmt.Errorf("base_dialog.json entries must be an object or array")
}
}
func legacyDialogIDKey(raw any) (string, error) {
switch typed := raw.(type) {
case string:
if strings.TrimSpace(typed) == "" {
return "", fmt.Errorf("id must not be empty")
}
return typed, nil
case float64:
return strconv.Itoa(int(typed)), nil
case int:
return strconv.Itoa(typed), nil
default:
return "", fmt.Errorf("id must be a string or number")
}
}
func loadTLKState(path string) (tlkStateDocument, error) {
doc := tlkStateDocument{
Version: 1,
Entries: map[string]tlkStateMapping{},
}
raw, err := os.ReadFile(path)
if err != nil {
if os.IsNotExist(err) {
return doc, nil
}
return doc, fmt.Errorf("read %s: %w", path, err)
}
if err := json.Unmarshal(raw, &doc); err != nil {
return doc, fmt.Errorf("parse %s: %w", path, err)
}
if doc.Version == 0 {
doc.Version = 1
}
if doc.Entries == nil {
doc.Entries = map[string]tlkStateMapping{}
}
return doc, nil
}
func saveTLKState(path string, doc tlkStateDocument) error {
doc.Version = 1
if doc.Language == "" {
doc.Language = "en"
}
if doc.Entries == nil {
doc.Entries = map[string]tlkStateMapping{}
}
raw, err := json.MarshalIndent(doc, "", " ")
if err != nil {
return fmt.Errorf("marshal tlk state: %w", err)
}
raw = append(raw, '\n')
current, err := os.ReadFile(path)
if err == nil && bytes.Equal(current, raw) {
return nil
}
if err != nil && !os.IsNotExist(err) {
return fmt.Errorf("read %s: %w", path, err)
}
if err := os.WriteFile(path, raw, 0o644); err != nil {
return fmt.Errorf("write %s: %w", path, err)
}
return nil
}
func (c *tlkCompiler) resolveLegacyKey(key string) (tlkCompiledValue, error) {
if _, ok := c.legacy[key]; !ok {
if active, ok := c.active[key]; ok {
return tlkCompiledValue{
Value: c.customStrrefForKey(key),
TLKKey: key,
}, c.markActive(key, active)
}
if len(c.legacy) == 0 {
return tlkCompiledValue{}, fmt.Errorf("unknown TLK reference: %s (inline TLK text is required for native builds)", key)
}
return tlkCompiledValue{}, fmt.Errorf("unknown TLK reference: %s", key)
}
if err := c.markActive(key, c.legacy[key]); err != nil {
return tlkCompiledValue{}, err
}
return tlkCompiledValue{
Value: c.customStrrefForKey(key),
TLKKey: key,
}, nil
}
func (c *tlkCompiler) registerInline(key string, entry tlkEntryData) (tlkCompiledValue, error) {
if err := c.markActive(key, entry); err != nil {
return tlkCompiledValue{}, err
}
return tlkCompiledValue{
Value: c.customStrrefForKey(key),
TLKKey: key,
}, nil
}
func (c *tlkCompiler) customStrrefForKey(key string) int {
return customTLKBase + c.state.Entries[key].ID
}
func (c *tlkCompiler) markActive(key string, entry tlkEntryData) error {
mapping, ok := c.state.Entries[key]
if !ok {
mapping = tlkStateMapping{ID: c.allocateID(), Retired: false}
c.state.Entries[key] = mapping
}
existing, ok := c.active[key]
if ok && existing != entry {
return fmt.Errorf("conflicting TLK content for key %q", key)
}
c.active[key] = entry
c.activeKeys[key] = struct{}{}
mapping.Retired = false
c.state.Entries[key] = mapping
return nil
}
func (c *tlkCompiler) allocateID() int {
id := c.nextID
c.nextID++
return id
}
func (c *tlkCompiler) finish(outputDir, tlkName string) (int, error) {
if err := os.MkdirAll(outputDir, 0o755); err != nil {
return 0, fmt.Errorf("create tlk output dir: %w", err)
}
for key, mapping := range c.state.Entries {
if _, ok := c.activeKeys[key]; ok {
mapping.Retired = false
} else {
mapping.Retired = true
}
c.state.Entries[key] = mapping
}
pruneRetiredGeneratedFeatTLKEntries(filepath.Dir(c.statePath), c.state.Entries)
if err := saveTLKState(c.statePath, c.state); err != nil {
return 0, err
}
maxID := -1
for key := range c.activeKeys {
id := c.state.Entries[key].ID
if id > maxID {
maxID = id
}
}
entries := make([]tlkEntryData, maxID+1)
for key := range c.activeKeys {
id := c.state.Entries[key].ID
entries[id] = c.active[key]
}
if strings.TrimSpace(tlkName) == "" {
tlkName = defaultTLKName
}
if err := writeTLKBinary(filepath.Join(outputDir, tlkName), entries, c.language); err != nil {
return 0, err
}
if maxID < 0 {
return 1, nil
}
return 1, nil
}
func pruneRetiredGeneratedFeatTLKEntries(sourceDir string, entries map[string]tlkStateMapping) {
prefixes := collectGeneratedFeatTLKPrefixes(sourceDir)
for key, mapping := range entries {
if !mapping.Retired || !isRetiredGeneratedFeatTLKKey(key, prefixes) {
continue
}
delete(entries, key)
}
}
func isRetiredGeneratedFeatTLKKey(key string, prefixes []string) bool {
if !strings.HasPrefix(key, "feat:") {
return false
}
for _, prefix := range prefixes {
if strings.HasPrefix(key, prefix) {
return true
}
}
return false
}
func collectGeneratedFeatTLKPrefixes(sourceDir string) []string {
featGeneratedDir := filepath.Join(sourceDir, "data", "feat", "generated")
paths, err := collectModulePaths(featGeneratedDir)
if err != nil {
return nil
}
prefixes := make([]string, 0, len(paths))
seen := map[string]struct{}{}
for _, path := range paths {
obj, err := loadJSONObject(path)
if err != nil || !isFamilyExpansionObject(obj) {
continue
}
spec, err := parseFamilyExpansionSpec(path, obj)
if err != nil || spec.FamilyKey == "" {
continue
}
prefix := "feat:" + spec.FamilyKey + "_"
if _, ok := seen[prefix]; ok {
continue
}
seen[prefix] = struct{}{}
prefixes = append(prefixes, prefix)
}
slices.Sort(prefixes)
return prefixes
}
func writeTLKBinary(path string, entries []tlkEntryData, language string) error {
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
return fmt.Errorf("create tlk output parent: %w", err)
}
file, err := os.Create(path)
if err != nil {
return fmt.Errorf("create tlk output: %w", err)
}
defer file.Close()
langID, err := languageID(language)
if err != nil {
return err
}
if _, err := file.Write([]byte("TLK V3.0")); err != nil {
return err
}
if err := binary.Write(file, binary.LittleEndian, uint32(langID)); err != nil {
return err
}
if err := binary.Write(file, binary.LittleEndian, uint32(len(entries))); err != nil {
return err
}
if err := binary.Write(file, binary.LittleEndian, uint32(20+len(entries)*40)); err != nil {
return err
}
stringData := make([][]byte, len(entries))
offset := 0
for index, entry := range entries {
flags := uint32(0)
if entry.Text != "" {
flags |= 0x1
}
if entry.SoundResRef != "" {
flags |= 0x2
}
if entry.SoundLength != 0 {
flags |= 0x4
}
encodedText, err := charmap.Windows1252.NewEncoder().Bytes([]byte(entry.Text))
if err != nil {
return fmt.Errorf("encode tlk text at row %d: %w", index, err)
}
stringData[index] = encodedText
soundBytes := make([]byte, 16)
if len(entry.SoundResRef) > 16 {
return fmt.Errorf("sound resref at row %d exceeds 16 characters", index)
}
copy(soundBytes, []byte(entry.SoundResRef))
if err := binary.Write(file, binary.LittleEndian, flags); err != nil {
return err
}
if _, err := file.Write(soundBytes); err != nil {
return err
}
if err := binary.Write(file, binary.LittleEndian, uint32(0)); err != nil {
return err
}
if err := binary.Write(file, binary.LittleEndian, uint32(0)); err != nil {
return err
}
if err := binary.Write(file, binary.LittleEndian, uint32(offset)); err != nil {
return err
}
if err := binary.Write(file, binary.LittleEndian, uint32(len(encodedText))); err != nil {
return err
}
if err := binary.Write(file, binary.LittleEndian, entry.SoundLength); err != nil {
return err
}
offset += len(encodedText)
}
for _, encoded := range stringData {
if _, err := file.Write(encoded); err != nil {
return err
}
}
return nil
}
func languageID(code string) (int, error) {
switch strings.ToLower(strings.TrimSpace(code)) {
case "", "en":
return 0, nil
case "fr":
return 1, nil
case "de":
return 2, nil
case "it":
return 3, nil
case "es":
return 4, nil
case "pl":
return 5, nil
default:
return 0, fmt.Errorf("unknown TLK language code %q", code)
}
}
func specForDataset(name string) datasetTextSpec {
best := datasetTextSpecs[name]
bestLength := 0
for prefix, spec := range datasetTextSpecs {
if name != prefix && !strings.HasPrefix(name, prefix+"/") {
continue
}
if len(prefix) <= bestLength {
continue
}
best = spec
bestLength = len(prefix)
}
return best
}
func roleForField(spec datasetTextSpec, field string) textRole {
for _, candidate := range spec.NameFields {
if strings.EqualFold(candidate, field) {
return textRoleName
}
}
for _, candidate := range spec.DescriptionFields {
if strings.EqualFold(candidate, field) {
return textRoleDescription
}
}
return textRole(strings.ToLower(field))
}
func columnForRole(columns []string, spec datasetTextSpec, role textRole) string {
var candidates []string
switch role {
case textRoleName:
candidates = spec.NameFields
case textRoleDescription:
candidates = spec.DescriptionFields
default:
return ""
}
for _, candidate := range candidates {
if column, ok := canonicalColumn(columns, candidate); ok {
return column
}
}
return ""
}
type parsedTLKPayload struct {
Key string
Text string
Ref string
RefField string
SoundResRef string
SoundLength float32
}
func parseTLKPayload(value any, allowBare bool) (parsedTLKPayload, bool, error) {
switch typed := value.(type) {
case map[string]any:
if rawTLK, ok := typed["tlk"]; ok {
if key, ok := rawTLK.(string); ok {
return parsedTLKPayload{Key: key}, true, nil
}
return parseTLKPayload(rawTLK, true)
}
if allowBare || hasNonRefTLKPayloadKeys(typed) {
if _, ok := typed["text"]; ok {
return buildParsedTLKPayload(typed)
}
if _, ok := typed["ref"]; ok {
return buildParsedTLKPayload(typed)
}
if _, ok := typed["key"]; ok {
return buildParsedTLKPayload(typed)
}
}
return parsedTLKPayload{}, false, nil
case string:
return parsedTLKPayload{}, false, nil
default:
return parsedTLKPayload{}, false, nil
}
}
func hasNonRefTLKPayloadKeys(obj map[string]any) bool {
for _, key := range []string{"text", "key", "sound_resref", "sound_length"} {
if _, ok := obj[key]; ok {
return true
}
}
return false
}
func buildParsedTLKPayload(obj map[string]any) (parsedTLKPayload, bool, error) {
payload := parsedTLKPayload{}
if rawKey, ok := obj["key"]; ok {
key, ok := rawKey.(string)
if !ok {
return payload, false, fmt.Errorf("tlk key must be a string")
}
payload.Key = key
}
if rawText, ok := obj["text"]; ok {
text, ok := rawText.(string)
if !ok {
return payload, false, fmt.Errorf("tlk text must be a string")
}
payload.Text = text
}
if rawRef, ok := obj["ref"]; ok {
ref, ok := rawRef.(string)
if !ok {
return payload, false, fmt.Errorf("tlk ref must be a string")
}
payload.Ref = ref
}
if rawField, ok := obj["field"]; ok {
field, ok := rawField.(string)
if !ok {
return payload, false, fmt.Errorf("tlk field must be a string")
}
payload.RefField = field
}
if rawSound, ok := obj["sound_resref"]; ok {
sound, ok := rawSound.(string)
if !ok {
return payload, false, fmt.Errorf("tlk sound_resref must be a string")
}
payload.SoundResRef = sound
}
if rawLength, ok := obj["sound_length"]; ok {
switch typed := rawLength.(type) {
case float64:
payload.SoundLength = float32(typed)
case int:
payload.SoundLength = float32(typed)
default:
return payload, false, fmt.Errorf("tlk sound_length must be numeric")
}
}
if payload.Ref != "" && payload.RefField == "" {
return payload, false, fmt.Errorf("tlk ref requires field")
}
if payload.Ref != "" && payload.Text != "" {
return payload, false, fmt.Errorf("tlk payload cannot contain both text and ref")
}
if payload.Ref == "" && payload.Text == "" && payload.Key == "" {
return payload, false, fmt.Errorf("tlk payload must contain text, ref, or key")
}
return payload, true, nil
}
func deriveAutoTLKKey(rowIdentity string, field string) string {
fieldText := strings.ToLower(strings.TrimSpace(field))
if fieldText == "" {
fieldText = "text"
}
return rowIdentity + "." + fieldText
}
func sortedKeys[M ~map[string]V, V any](input M) []string {
keys := make([]string, 0, len(input))
for key := range input {
keys = append(keys, key)
}
slices.Sort(keys)
return keys
}
func almostEqualFloat32(a, b float32) bool {
return math.Abs(float64(a-b)) < 0.0001
}