ShadowsOverWestgate/sow-tools
7
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases 38 Activity

Build Path Cleanup

Browse Source
This commit is contained in:
archvillainette
2026-04-17 21:09:02 +02:00
parent d77b44c210
commit f32dd224fe
6 changed files with 178 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/topdata/expansion_native.go
+2 -5
View File
@@ -158,6 +158,7 @@ func mergeExpansionData(collected []nativeCollectedDataset) ([]nativeCollectedDa
targetDS.LockData[key] = rowID
lockModified = true
targetDS.LockAdded++
targetDS.LockModified = true
}
}
@@ -198,13 +199,9 @@ func mergeExpansionData(collected []nativeCollectedDataset) ([]nativeCollectedDa
}
if pruned, updated := pruneLockDataToActiveRows(targetDS.LockData, targetDS.Rows, referencedKeys); pruned > 0 || updated > 0 {
targetDS.LockPruned += pruned
targetDS.LockModified = true
collected[targetIndex] = targetDS
}
if targetDS.Dataset.LockPath != "" {
if err := saveLockfile(targetDS.Dataset.LockPath, targetDS.LockData); err != nil {
return nil, fmt.Errorf("dataset %s: %w", targetDS.Dataset.Name, err)
}
}
}
return collected, nil
internal/topdata/native.go
+65 -27
View File
@@ -39,13 +39,14 @@ const (
)
type nativeCollectedDataset struct {
Dataset nativeDataset
Columns []string
Rows []map[string]any
LockData map[string]int
TableKey string
LockAdded int
LockPruned int
Dataset nativeDataset
Columns []string
Rows []map[string]any
LockData map[string]int
TableKey string
LockAdded int
LockPruned int
LockModified bool
}
type resolvedTable struct {
@@ -262,7 +263,10 @@ func buildNativeUnchecked(p *project.Project, progress func(string), prebuiltWik
if err != nil {
return BuildResult{}, err
}
lockAdded, lockPruned := nativeLockChangeStats(collected)
lockAdded, lockPruned, err := saveNativeBaseLockfiles(collected)
if err != nil {
return BuildResult{}, err
}
if lockAdded > 0 || lockPruned > 0 {
progress(fmt.Sprintf("Updated native lockfiles (%d new IDs, %d stale IDs pruned)", lockAdded, lockPruned))
}
@@ -340,9 +344,6 @@ func buildNativeUnchecked(p *project.Project, progress func(string), prebuiltWik
if err != nil {
return BuildResult{}, err
}
if wikiBuild.PageCount > 0 && wikiBuild.Status == wikiSkippedStatus {
wikiBuild.Status = wikiGeneratedStatus
}
}
return BuildResult{
@@ -370,14 +371,49 @@ func nativeCompileGroup(datasetName string) string {
}
}
func nativeLockChangeStats(collected []nativeCollectedDataset) (int, int) {
func saveNativeBaseLockfiles(collected []nativeCollectedDataset) (int, int, error) {
added := 0
pruned := 0
for _, dataset := range collected {
added += dataset.LockAdded
pruned += dataset.LockPruned
if dataset.Dataset.Kind != nativeDatasetBase || dataset.Dataset.LockPath == "" {
continue
}
current, err := loadLockfile(dataset.Dataset.LockPath)
if err != nil {
return 0, 0, fmt.Errorf("dataset %s: %w", dataset.Dataset.Name, err)
}
added += countAddedLockEntries(current, dataset.LockData)
pruned += countAddedLockEntries(dataset.LockData, current)
if lockDataEqual(current, dataset.LockData) {
continue
}
if err := saveLockfile(dataset.Dataset.LockPath, dataset.LockData); err != nil {
return 0, 0, fmt.Errorf("dataset %s: %w", dataset.Dataset.Name, err)
}
}
return added, pruned
return added, pruned, nil
}
func countAddedLockEntries(before, after map[string]int) int {
count := 0
for key, afterID := range after {
if beforeID, ok := before[key]; !ok || beforeID != afterID {
count++
}
}
return count
}
func lockDataEqual(a, b map[string]int) bool {
if len(a) != len(b) {
return false
}
for key, aID := range a {
if bID, ok := b[key]; !ok || bID != aID {
return false
}
}
return true
}
type nativeCompileGroupStat struct {
@@ -993,23 +1029,18 @@ func collectBaseDataset(dataset nativeDataset) (nativeCollectedDataset, error) {
lockPruned += pruned
}
if lockModified {
if err := saveLockfile(dataset.LockPath, lockData); err != nil {
return nativeCollectedDataset{}, fmt.Errorf("dataset %s: %w", dataset.Name, err)
}
}
rows = dedupeCollectedRows(rows)
slices.SortFunc(rows, func(a, b map[string]any) int {
return a["id"].(int) - b["id"].(int)
})
return nativeCollectedDataset{
Dataset: dataset,
Columns: columns,
Rows: rows,
LockData: lockData,
LockAdded: lockAdded,
LockPruned: lockPruned,
Dataset: dataset,
Columns: columns,
Rows: rows,
LockData: lockData,
LockAdded: lockAdded,
LockPruned: lockPruned,
LockModified: lockModified,
}, nil
}
@@ -3919,6 +3950,13 @@ func saveLockfile(path string, lockData map[string]int) error {
if err != nil {
return fmt.Errorf("marshal lockfile: %w", err)
}
current, err := os.ReadFile(path)
if err == nil && bytes.Equal(current, raw) {
return nil
}
if err != nil && !errors.Is(err, os.ErrNotExist) {
return fmt.Errorf("read lockfile %s: %w", path, err)
}
if err := os.WriteFile(path, raw, 0o644); err != nil {
return fmt.Errorf("write lockfile %s: %w", path, err)
}
internal/topdata/parts_manifest.go
+46 -2
View File
@@ -1,6 +1,7 @@
package topdata
import (
"bytes"
"encoding/json"
"fmt"
"io"
@@ -21,6 +22,7 @@ const (
partsManifestReleaseTag = "parts-manifest-current"
partsManifestAssetName = "sow-parts-manifest.json"
partsManifestCacheFileName = "sow-parts-manifest.json"
partsManifestCacheMaxAge = time.Hour
partsManifestRequestTimeout = 30 * time.Second
)
@@ -51,12 +53,21 @@ func resolveReleasedPartsManifest(p *project.Project, progress func(string)) (ma
if err != nil {
return nil, err
}
cachePath := filepath.Join(p.Root, ".cache", "topdata", partsManifestCacheFileName)
if strings.TrimSpace(os.Getenv("SOW_PARTS_MANIFEST_REFRESH")) == "" {
manifest, fresh, err := readFreshPartsManifestCache(cachePath, partsManifestCacheMaxAge)
if err != nil {
progress(fmt.Sprintf("Ignoring cached parts manifest: %v", err))
} else if fresh {
progress(fmt.Sprintf("Using cached released parts manifest from %s...", cachePath))
return inventoryFromPartsManifest(manifest), nil
}
}
progress(fmt.Sprintf("Fetching released parts manifest from %s...", manifestURL))
manifest, err := fetchPartsManifest(manifestURL)
if err != nil {
return nil, err
}
cachePath := filepath.Join(p.Root, ".cache", "topdata", partsManifestCacheFileName)
if err := writePartsManifestCache(cachePath, manifest); err != nil {
return nil, err
}
@@ -144,6 +155,31 @@ func fetchPartsManifest(manifestURL string) (*partsManifest, error) {
return &manifest, nil
}
func readFreshPartsManifestCache(path string, maxAge time.Duration) (*partsManifest, bool, error) {
info, err := os.Stat(path)
if err != nil {
if os.IsNotExist(err) {
return nil, false, nil
}
return nil, false, err
}
if maxAge > 0 && time.Since(info.ModTime()) > maxAge {
return nil, false, nil
}
raw, err := os.ReadFile(path)
if err != nil {
return nil, false, err
}
var manifest partsManifest
if err := json.Unmarshal(raw, &manifest); err != nil {
return nil, false, err
}
if len(manifest.Categories) == 0 {
return nil, false, fmt.Errorf("categories is empty")
}
return &manifest, true, nil
}
func fetchJSON(target string, out any) error {
req, err := http.NewRequest(http.MethodGet, target, nil)
if err != nil {
@@ -181,8 +217,16 @@ func writePartsManifestCache(path string, manifest *partsManifest) error {
if err != nil {
return fmt.Errorf("marshal parts manifest cache: %w", err)
}
payload = append(payload, '\n')
current, err := os.ReadFile(path)
if err == nil && bytes.Equal(current, payload) {
return nil
}
if err != nil && !os.IsNotExist(err) {
return fmt.Errorf("read parts manifest cache: %w", err)
}
tmpPath := path + ".tmp"
if err := os.WriteFile(tmpPath, append(payload, '\n'), 0o644); err != nil {
if err := os.WriteFile(tmpPath, payload, 0o644); err != nil {
return fmt.Errorf("write parts manifest cache: %w", err)
}
if err := os.Rename(tmpPath, path); err != nil {
internal/topdata/tlk_native.go
+8
View File
@@ -1,6 +1,7 @@
package topdata
import (
"bytes"
"encoding/binary"
"encoding/json"
"fmt"
@@ -283,6 +284,13 @@ func saveTLKState(path string, doc tlkStateDocument) error {
return fmt.Errorf("marshal tlk state: %w", err)
}
raw = append(raw, '\n')
current, err := os.ReadFile(path)
if err == nil && bytes.Equal(current, raw) {
return nil
}
if err != nil && !os.IsNotExist(err) {
return fmt.Errorf("read %s: %w", path, err)
}
if err := os.WriteFile(path, raw, 0o644); err != nil {
return fmt.Errorf("write %s: %w", path, err)
}