package topdata import ( "encoding/binary" "encoding/json" "fmt" "math" "os" "path/filepath" "slices" "strings" "golang.org/x/text/encoding/charmap" ) const ( defaultTLKName = "sow_tlk.tlk" tlkStateFile = ".tlk_state.json" customTLKBase = 0x01000000 ) type textRole string const ( textRoleName textRole = "name" textRoleDescription textRole = "description" ) type datasetTextSpec struct { NameFields []string DescriptionFields []string } var datasetTextSpecs = map[string]datasetTextSpec{ "baseitems": { NameFields: []string{"Name"}, DescriptionFields: []string{"Description"}, }, "classes": { NameFields: []string{"Short", "Name", "Plural", "Lower", "CLASS"}, DescriptionFields: []string{"Description"}, }, "feat": { NameFields: []string{"FEAT"}, DescriptionFields: []string{"DESCRIPTION"}, }, "masterfeats": { NameFields: []string{"STRREF"}, DescriptionFields: []string{"DESCRIPTION"}, }, "itemprops": { NameFields: []string{"Name", "StringRef"}, DescriptionFields: []string{"Description", "GameStrRef"}, }, "racialtypes": { NameFields: []string{"Name", "RACIALTYPE"}, DescriptionFields: []string{"Description"}, }, "skills": { NameFields: []string{"Name", "SKILL"}, DescriptionFields: []string{"Description"}, }, "spells": { NameFields: []string{"Name", "SPELL"}, DescriptionFields: []string{"SpellDesc", "DESCRIPTION"}, }, } type tlkEntryData struct { Text string SoundResRef string SoundLength float32 } type tlkStateDocument struct { Version int `json:"version"` Language string `json:"language"` Entries map[string]tlkStateMapping `json:"entries"` } type tlkStateMapping struct { ID int `json:"id"` Retired bool `json:"retired,omitempty"` } type tlkCompiledValue struct { Value any TLKKey string } type tlkCompiler struct { language string statePath string state tlkStateDocument legacy map[string]tlkEntryData active map[string]tlkEntryData activeKeys map[string]struct{} reservedByID map[int]string nextID int } func newTLKCompiler(sourceDir string, legacy *legacyTLKData) (*tlkCompiler, error) { statePath := filepath.Join(sourceDir, tlkStateFile) state, err := loadTLKState(statePath) if err != nil { return nil, err } baseDialog, err := loadBaseDialogData(filepath.Join(sourceDir, "base_dialog.json")) if err != nil { return nil, err } legacy = mergeLegacyTLK(baseDialog, legacy) language := state.Language if language == "" { language = "en" } if legacy != nil && legacy.Language != "" { if state.Language == "" { language = legacy.Language } for key, id := range legacy.Lock { if _, ok := state.Entries[key]; !ok { state.Entries[key] = tlkStateMapping{ID: id} } } } reserved := map[int]string{} nextID := 0 for key, entry := range state.Entries { if owner, ok := reserved[entry.ID]; ok && owner != key { return nil, fmt.Errorf("tlk state id collision between %q and %q", owner, key) } reserved[entry.ID] = key if entry.ID >= nextID { nextID = entry.ID + 1 } } compiler := &tlkCompiler{ language: language, statePath: statePath, state: state, legacy: map[string]tlkEntryData{}, active: map[string]tlkEntryData{}, activeKeys: map[string]struct{}{}, reservedByID: reserved, nextID: nextID, } if legacy != nil { for key, entry := range legacy.Entries { compiler.legacy[key] = entry } } return compiler, nil } func loadBaseDialogData(path string) (*legacyTLKData, error) { if _, err := os.Stat(path); err != nil { if os.IsNotExist(err) { return nil, nil } return nil, fmt.Errorf("stat %s: %w", path, err) } obj, err := loadJSONObject(path) if err != nil { return nil, err } result := &legacyTLKData{ Language: "en", Entries: map[string]tlkEntryData{}, Lock: map[string]int{}, } if language, ok := obj["language"].(string); ok && strings.TrimSpace(language) != "" { result.Language = language } rawEntries, ok := obj["entries"] if !ok { return result, nil } entries, ok := rawEntries.(map[string]any) if !ok { return nil, fmt.Errorf("base_dialog.json entries must be an object") } normalized, err := normalizeLegacyTLKEntries(entries) if err != nil { return nil, err } for key, entry := range normalized { result.Entries[key] = entry } return result, nil } func loadTLKState(path string) (tlkStateDocument, error) { doc := tlkStateDocument{ Version: 1, Entries: map[string]tlkStateMapping{}, } raw, err := os.ReadFile(path) if err != nil { if os.IsNotExist(err) { return doc, nil } return doc, fmt.Errorf("read %s: %w", path, err) } if err := json.Unmarshal(raw, &doc); err != nil { return doc, fmt.Errorf("parse %s: %w", path, err) } if doc.Version == 0 { doc.Version = 1 } if doc.Entries == nil { doc.Entries = map[string]tlkStateMapping{} } return doc, nil } func saveTLKState(path string, doc tlkStateDocument) error { doc.Version = 1 if doc.Language == "" { doc.Language = "en" } if doc.Entries == nil { doc.Entries = map[string]tlkStateMapping{} } raw, err := json.MarshalIndent(doc, "", " ") if err != nil { return fmt.Errorf("marshal tlk state: %w", err) } raw = append(raw, '\n') if err := os.WriteFile(path, raw, 0o644); err != nil { return fmt.Errorf("write %s: %w", path, err) } return nil } func (c *tlkCompiler) resolveLegacyKey(key string) (tlkCompiledValue, error) { if _, ok := c.legacy[key]; !ok { if active, ok := c.active[key]; ok { return tlkCompiledValue{ Value: c.customStrrefForKey(key), TLKKey: key, }, c.markActive(key, active) } if len(c.legacy) == 0 { return tlkCompiledValue{}, fmt.Errorf("unknown TLK reference: %s (inline TLK text is required for native builds)", key) } return tlkCompiledValue{}, fmt.Errorf("unknown TLK reference: %s", key) } if err := c.markActive(key, c.legacy[key]); err != nil { return tlkCompiledValue{}, err } return tlkCompiledValue{ Value: c.customStrrefForKey(key), TLKKey: key, }, nil } func (c *tlkCompiler) registerInline(key string, entry tlkEntryData) (tlkCompiledValue, error) { if err := c.markActive(key, entry); err != nil { return tlkCompiledValue{}, err } return tlkCompiledValue{ Value: c.customStrrefForKey(key), TLKKey: key, }, nil } func (c *tlkCompiler) customStrrefForKey(key string) int { return customTLKBase + c.state.Entries[key].ID } func (c *tlkCompiler) markActive(key string, entry tlkEntryData) error { mapping, ok := c.state.Entries[key] if !ok { mapping = tlkStateMapping{ID: c.allocateID(), Retired: false} c.state.Entries[key] = mapping } existing, ok := c.active[key] if ok && existing != entry { return fmt.Errorf("conflicting TLK content for key %q", key) } c.active[key] = entry c.activeKeys[key] = struct{}{} mapping.Retired = false c.state.Entries[key] = mapping return nil } func (c *tlkCompiler) allocateID() int { id := c.nextID c.nextID++ return id } func (c *tlkCompiler) finish(outputDir string) (int, error) { if err := os.MkdirAll(outputDir, 0o755); err != nil { return 0, fmt.Errorf("create tlk output dir: %w", err) } for key, mapping := range c.state.Entries { if _, ok := c.activeKeys[key]; ok { mapping.Retired = false } else { mapping.Retired = true } c.state.Entries[key] = mapping } pruneRetiredGeneratedFeatTLKEntries(c.state.Entries) if err := saveTLKState(c.statePath, c.state); err != nil { return 0, err } maxID := -1 for key := range c.activeKeys { id := c.state.Entries[key].ID if id > maxID { maxID = id } } entries := make([]tlkEntryData, maxID+1) for key := range c.activeKeys { id := c.state.Entries[key].ID entries[id] = c.active[key] } if err := writeTLKBinary(filepath.Join(outputDir, defaultTLKName), entries, c.language); err != nil { return 0, err } if maxID < 0 { return 1, nil } return 1, nil } var generatedFeatTLKPrefixes = []string{ "feat:skillfocus_", "feat:greaterskillfocus_", "feat:weaponfocus_", "feat:weaponspecialization_", "feat:greaterweaponfocus_", "feat:greaterweaponspecialization_", "feat:improvedcritical_", "feat:overwhelmingcritical_", "feat:weaponproficiencycommoner_", } func pruneRetiredGeneratedFeatTLKEntries(entries map[string]tlkStateMapping) { for key, mapping := range entries { if !mapping.Retired || !isRetiredGeneratedFeatTLKKey(key) { continue } delete(entries, key) } } func isRetiredGeneratedFeatTLKKey(key string) bool { if !strings.HasPrefix(key, "feat:") { return false } for _, prefix := range generatedFeatTLKPrefixes { if strings.HasPrefix(key, prefix) { return true } } return false } func writeTLKBinary(path string, entries []tlkEntryData, language string) error { if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { return fmt.Errorf("create tlk output parent: %w", err) } file, err := os.Create(path) if err != nil { return fmt.Errorf("create tlk output: %w", err) } defer file.Close() langID, err := languageID(language) if err != nil { return err } if _, err := file.Write([]byte("TLK V3.0")); err != nil { return err } if err := binary.Write(file, binary.LittleEndian, uint32(langID)); err != nil { return err } if err := binary.Write(file, binary.LittleEndian, uint32(len(entries))); err != nil { return err } if err := binary.Write(file, binary.LittleEndian, uint32(20+len(entries)*40)); err != nil { return err } stringData := make([][]byte, len(entries)) offset := 0 for index, entry := range entries { flags := uint32(0) if entry.Text != "" { flags |= 0x1 } if entry.SoundResRef != "" { flags |= 0x2 } if entry.SoundLength != 0 { flags |= 0x4 } encodedText, err := charmap.Windows1252.NewEncoder().Bytes([]byte(entry.Text)) if err != nil { return fmt.Errorf("encode tlk text at row %d: %w", index, err) } stringData[index] = encodedText soundBytes := make([]byte, 16) if len(entry.SoundResRef) > 16 { return fmt.Errorf("sound resref at row %d exceeds 16 characters", index) } copy(soundBytes, []byte(entry.SoundResRef)) if err := binary.Write(file, binary.LittleEndian, flags); err != nil { return err } if _, err := file.Write(soundBytes); err != nil { return err } if err := binary.Write(file, binary.LittleEndian, uint32(0)); err != nil { return err } if err := binary.Write(file, binary.LittleEndian, uint32(0)); err != nil { return err } if err := binary.Write(file, binary.LittleEndian, uint32(offset)); err != nil { return err } if err := binary.Write(file, binary.LittleEndian, uint32(len(encodedText))); err != nil { return err } if err := binary.Write(file, binary.LittleEndian, entry.SoundLength); err != nil { return err } offset += len(encodedText) } for _, encoded := range stringData { if _, err := file.Write(encoded); err != nil { return err } } return nil } func languageID(code string) (int, error) { switch strings.ToLower(strings.TrimSpace(code)) { case "", "en": return 0, nil case "fr": return 1, nil case "de": return 2, nil case "it": return 3, nil case "es": return 4, nil case "pl": return 5, nil default: return 0, fmt.Errorf("unknown TLK language code %q", code) } } func specForDataset(name string) datasetTextSpec { best := datasetTextSpecs[name] bestLength := 0 for prefix, spec := range datasetTextSpecs { if name != prefix && !strings.HasPrefix(name, prefix+"/") { continue } if len(prefix) <= bestLength { continue } best = spec bestLength = len(prefix) } return best } func roleForField(spec datasetTextSpec, field string) textRole { for _, candidate := range spec.NameFields { if strings.EqualFold(candidate, field) { return textRoleName } } for _, candidate := range spec.DescriptionFields { if strings.EqualFold(candidate, field) { return textRoleDescription } } return textRole(strings.ToLower(field)) } func columnForRole(columns []string, spec datasetTextSpec, role textRole) string { var candidates []string switch role { case textRoleName: candidates = spec.NameFields case textRoleDescription: candidates = spec.DescriptionFields default: return "" } for _, candidate := range candidates { if column, ok := canonicalColumn(columns, candidate); ok { return column } } return "" } type parsedTLKPayload struct { Key string Text string Ref string RefField string SoundResRef string SoundLength float32 } func parseTLKPayload(value any, allowBare bool) (parsedTLKPayload, bool, error) { switch typed := value.(type) { case map[string]any: if rawTLK, ok := typed["tlk"]; ok { if key, ok := rawTLK.(string); ok { return parsedTLKPayload{Key: key}, true, nil } return parseTLKPayload(rawTLK, true) } if allowBare || hasNonRefTLKPayloadKeys(typed) { if _, ok := typed["text"]; ok { return buildParsedTLKPayload(typed) } if _, ok := typed["ref"]; ok { return buildParsedTLKPayload(typed) } if _, ok := typed["key"]; ok { return buildParsedTLKPayload(typed) } } return parsedTLKPayload{}, false, nil case string: return parsedTLKPayload{}, false, nil default: return parsedTLKPayload{}, false, nil } } func hasNonRefTLKPayloadKeys(obj map[string]any) bool { for _, key := range []string{"text", "key", "sound_resref", "sound_length"} { if _, ok := obj[key]; ok { return true } } return false } func buildParsedTLKPayload(obj map[string]any) (parsedTLKPayload, bool, error) { payload := parsedTLKPayload{} if rawKey, ok := obj["key"]; ok { key, ok := rawKey.(string) if !ok { return payload, false, fmt.Errorf("tlk key must be a string") } payload.Key = key } if rawText, ok := obj["text"]; ok { text, ok := rawText.(string) if !ok { return payload, false, fmt.Errorf("tlk text must be a string") } payload.Text = text } if rawRef, ok := obj["ref"]; ok { ref, ok := rawRef.(string) if !ok { return payload, false, fmt.Errorf("tlk ref must be a string") } payload.Ref = ref } if rawField, ok := obj["field"]; ok { field, ok := rawField.(string) if !ok { return payload, false, fmt.Errorf("tlk field must be a string") } payload.RefField = field } if rawSound, ok := obj["sound_resref"]; ok { sound, ok := rawSound.(string) if !ok { return payload, false, fmt.Errorf("tlk sound_resref must be a string") } payload.SoundResRef = sound } if rawLength, ok := obj["sound_length"]; ok { switch typed := rawLength.(type) { case float64: payload.SoundLength = float32(typed) case int: payload.SoundLength = float32(typed) default: return payload, false, fmt.Errorf("tlk sound_length must be numeric") } } if payload.Ref != "" && payload.RefField == "" { return payload, false, fmt.Errorf("tlk ref requires field") } if payload.Ref != "" && payload.Text != "" { return payload, false, fmt.Errorf("tlk payload cannot contain both text and ref") } if payload.Ref == "" && payload.Text == "" && payload.Key == "" { return payload, false, fmt.Errorf("tlk payload must contain text, ref, or key") } return payload, true, nil } func deriveAutoTLKKey(rowIdentity string, field string) string { fieldText := strings.ToLower(strings.TrimSpace(field)) if fieldText == "" { fieldText = "text" } return rowIdentity + "." + fieldText } func sortedKeys[M ~map[string]V, V any](input M) []string { keys := make([]string, 0, len(input)) for key := range input { keys = append(keys, key) } slices.Sort(keys) return keys } func almostEqualFloat32(a, b float32) bool { return math.Abs(float64(a-b)) < 0.0001 }