Fixed plain datasets generating lock ids when not necessary

This commit is contained in:
2026-05-27 23:51:49 +02:00
parent a2d9820f8d
commit c7dc81b80f
2 changed files with 97 additions and 4 deletions
+33 -2
View File
@@ -865,6 +865,10 @@ func discoverNativeDatasets(dataDir string) ([]nativeDataset, error) {
if err != nil {
return err
}
lockPath := filepath.Join(rootPath, "lock.json")
if !plainDatasetNeedsLock(lockPath, tableData) {
lockPath = ""
}
outputName, _ := tableData["output"].(string)
if strings.TrimSpace(outputName) == "" {
outputName = nativeDatasetDefaultOutputName(filePath, nativeDatasetPlain)
@@ -874,7 +878,7 @@ func discoverNativeDatasets(dataDir string) ([]nativeDataset, error) {
Name: filepath.ToSlash(relName),
RootPath: rootPath,
BasePath: filePath,
LockPath: filepath.Join(rootPath, "lock.json"),
LockPath: lockPath,
OutputName: outputName,
Spec: specForDataset(filepath.ToSlash(relName)),
CompareReference: optionalBool(tableData["compare_reference"], true),
@@ -979,6 +983,29 @@ func discoverNativeDatasets(dataDir string) ([]nativeDataset, error) {
return datasets, nil
}
func plainDatasetNeedsLock(lockPath string, tableData map[string]any) bool {
if _, err := os.Stat(lockPath); err == nil {
return true
}
rawRows, ok := tableData["rows"].([]any)
if !ok {
return false
}
for _, raw := range rawRows {
row, ok := raw.(map[string]any)
if !ok {
continue
}
key, hasKey := row["key"].(string)
if hasKey && strings.TrimSpace(key) != "" {
if _, hasID := row["id"]; !hasID {
return true
}
}
}
return false
}
func discoverNativeOutputCatalog(dataDir string) (map[string]string, error) {
datasets, err := discoverNativeDatasets(dataDir)
if err != nil {
@@ -3396,10 +3423,14 @@ func collectPlainDataset(dataset nativeDataset) (nativeCollectedDataset, error)
return nativeCollectedDataset{}, fmt.Errorf("dataset %s: rows must be an array", dataset.Name)
}
lockData, err := loadLockfile(dataset.LockPath)
lockData := map[string]int{}
if dataset.LockPath != "" {
var err error
lockData, err = loadLockfile(dataset.LockPath)
if err != nil {
return nativeCollectedDataset{}, fmt.Errorf("dataset %s: %w", dataset.Name, err)
}
}
usedIDs := map[int]struct{}{}
for _, rowID := range lockData {
usedIDs[rowID] = struct{}{}
+62
View File
@@ -3,6 +3,7 @@ package topdata
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/http/httptest"
@@ -13544,6 +13545,67 @@ func TestBuildNativeSupportsLoosePlainTableAtTopdataDataRoot(t *testing.T) {
}
}
func TestBuildNativeDoesNotCreateRootLockfileForExplicitRootPlainTable(t *testing.T) {
root := testProjectRoot(t)
mkdirAll(t, filepath.Join(root, "topdata", "data"))
writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n")
writeFile(t, filepath.Join(root, "topdata", "data", "categories.json"), `{
"output": "categories.2da",
"columns": ["Category"],
"rows": [
{"id": 1, "key": "categories:harmful", "Category": "Harmful"},
{"id": 2, "key": "categories:beneficial", "Category": "Beneficial"}
]
}`+"\n")
proj := testProject(root)
proj.Config.TopData.ReferenceBuilder = ""
result, err := BuildNativeWithOptions(proj, NativeBuildOptions{BuildWiki: false}, nil)
if err != nil {
t.Fatalf("expected explicit loose root table to build: %v", err)
}
if _, err := os.Stat(filepath.Join(result.Output2DADir, "categories.2da")); err != nil {
t.Fatalf("expected categories.2da output for loose root table: %v", err)
}
if _, err := os.Stat(filepath.Join(root, "topdata", "data", "lock.json")); !errors.Is(err, os.ErrNotExist) {
t.Fatalf("expected no root lock.json for fully explicit loose root table, got err %v", err)
}
}
func TestBuildNativeCreatesRootLockfileForImplicitRootPlainTableKeys(t *testing.T) {
root := testProjectRoot(t)
mkdirAll(t, filepath.Join(root, "topdata", "data"))
writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n")
writeFile(t, filepath.Join(root, "topdata", "data", "categories.json"), `{
"output": "categories.2da",
"columns": ["Category"],
"rows": [
{"key": "categories:harmful", "Category": "Harmful"},
{"id": 5, "key": "categories:beneficial", "Category": "Beneficial"}
]
}`+"\n")
proj := testProject(root)
proj.Config.TopData.ReferenceBuilder = ""
if _, err := BuildNativeWithOptions(proj, NativeBuildOptions{BuildWiki: false}, nil); err != nil {
t.Fatalf("expected implicit loose root table to build: %v", err)
}
lockRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "lock.json"))
if err != nil {
t.Fatalf("expected root lock.json for implicit keyed row: %v", err)
}
var lockData map[string]int
if err := json.Unmarshal(lockRaw, &lockData); err != nil {
t.Fatalf("unmarshal lockfile: %v", err)
}
if lockData["categories:harmful"] != 0 || lockData["categories:beneficial"] != 5 {
t.Fatalf("unexpected lock data: %#v", lockData)
}
}
func TestBuildNativeExtendsConfiguredPlainRowsByRepeatingLastLevel(t *testing.T) {
root := testProjectRoot(t)
mkdirAll(t, filepath.Join(root, "topdata", "data", "classes", "spellsgained"))