Build Path Cleanup

build-tool.ps1

@@ -7,4 +7,35 @@ Set-Location $repoRoot
 $env:GOCACHE = Join-Path $repoRoot ".cache/go-build"
 $outputPath = Join-Path $repoRoot "tools/sow-toolkit.exe"
 
+function Test-ToolSourcesAreNewer {
+    if (-not (Test-Path $outputPath)) {
+        return $true
+    }
+
+    $outputTime = (Get-Item -LiteralPath $outputPath).LastWriteTimeUtc
+    $sourcePaths = @("go.mod", "go.sum", "cmd", "internal")
+    foreach ($sourcePath in $sourcePaths) {
+        $fullPath = Join-Path $repoRoot $sourcePath
+        if (-not (Test-Path $fullPath)) {
+            continue
+        }
+        $newerSource = Get-ChildItem -LiteralPath $fullPath -Recurse -File |
+            Where-Object { $_.LastWriteTimeUtc -gt $outputTime } |
+            Select-Object -First 1
+        if ($newerSource) {
+            return $true
+        }
+    }
+
+    return $false
+}
+
+New-Item -ItemType Directory -Force -Path (Join-Path $repoRoot "tools") | Out-Null
+New-Item -ItemType Directory -Force -Path $env:GOCACHE | Out-Null
+if (-not (Test-ToolSourcesAreNewer)) {
+    Write-Host "sow-toolkit is up to date."
+    exit 0
+}
+
+Write-Host "Building sow-toolkit..."
 go build -o $outputPath ./cmd/nwn-tool

build-tool.sh

@@ -2,5 +2,30 @@
 set -eu
 
 cd "$(dirname "$0")"
+output="./tools/sow-toolkit"
+
+tool_sources_are_newer() {
+  if [ ! -x "$output" ]; then
+    return 0
+  fi
+
+  for path in go.mod go.sum cmd internal; do
+    if [ ! -e "$path" ]; then
+      continue
+    fi
+    if [ -n "$(find "$path" -type f -newer "$output" -print -quit)" ]; then
+      return 0
+    fi
+  done
+
+  return 1
+}
+
 mkdir -p tools .cache/go-build
-GOCACHE="${PWD}/.cache/go-build" go build -o ./tools/sow-toolkit ./cmd/nwn-tool
+if ! tool_sources_are_newer; then
+  echo "sow-toolkit is up to date."
+  exit 0
+fi
+
+echo "Building sow-toolkit..."
+GOCACHE="${PWD}/.cache/go-build" go build -o "$output" ./cmd/nwn-tool

internal/topdata/expansion_native.go

@@ -158,6 +158,7 @@ func mergeExpansionData(collected []nativeCollectedDataset) ([]nativeCollectedDa
 						targetDS.LockData[key] = rowID
 						lockModified = true
 						targetDS.LockAdded++
+						targetDS.LockModified = true
 					}
 				}
 
@@ -198,13 +199,9 @@ func mergeExpansionData(collected []nativeCollectedDataset) ([]nativeCollectedDa
 		}
 		if pruned, updated := pruneLockDataToActiveRows(targetDS.LockData, targetDS.Rows, referencedKeys); pruned > 0 || updated > 0 {
 			targetDS.LockPruned += pruned
+			targetDS.LockModified = true
 			collected[targetIndex] = targetDS
 		}
-		if targetDS.Dataset.LockPath != "" {
-			if err := saveLockfile(targetDS.Dataset.LockPath, targetDS.LockData); err != nil {
-				return nil, fmt.Errorf("dataset %s: %w", targetDS.Dataset.Name, err)
-			}
-		}
 	}
 
 	return collected, nil

internal/topdata/native.go

@@ -39,13 +39,14 @@ const (
 )
 
 type nativeCollectedDataset struct {
-	Dataset    nativeDataset
+	Dataset      nativeDataset
-	Columns    []string
+	Columns      []string
-	Rows       []map[string]any
+	Rows         []map[string]any
-	LockData   map[string]int
+	LockData     map[string]int
-	TableKey   string
+	TableKey     string
-	LockAdded  int
+	LockAdded    int
-	LockPruned int
+	LockPruned   int
+	LockModified bool
 }
 
 type resolvedTable struct {
@@ -262,7 +263,10 @@ func buildNativeUnchecked(p *project.Project, progress func(string), prebuiltWik
 	if err != nil {
 		return BuildResult{}, err
 	}
-	lockAdded, lockPruned := nativeLockChangeStats(collected)
+	lockAdded, lockPruned, err := saveNativeBaseLockfiles(collected)
+	if err != nil {
+		return BuildResult{}, err
+	}
 	if lockAdded > 0 || lockPruned > 0 {
 		progress(fmt.Sprintf("Updated native lockfiles (%d new IDs, %d stale IDs pruned)", lockAdded, lockPruned))
 	}
@@ -340,9 +344,6 @@ func buildNativeUnchecked(p *project.Project, progress func(string), prebuiltWik
 		if err != nil {
 			return BuildResult{}, err
 		}
-		if wikiBuild.PageCount > 0 && wikiBuild.Status == wikiSkippedStatus {
-			wikiBuild.Status = wikiGeneratedStatus
-		}
 	}
 
 	return BuildResult{
@@ -370,14 +371,49 @@ func nativeCompileGroup(datasetName string) string {
 	}
 }
 
-func nativeLockChangeStats(collected []nativeCollectedDataset) (int, int) {
+func saveNativeBaseLockfiles(collected []nativeCollectedDataset) (int, int, error) {
 	added := 0
 	pruned := 0
 	for _, dataset := range collected {
-		added += dataset.LockAdded
+		if dataset.Dataset.Kind != nativeDatasetBase || dataset.Dataset.LockPath == "" {
-		pruned += dataset.LockPruned
+			continue
+		}
+		current, err := loadLockfile(dataset.Dataset.LockPath)
+		if err != nil {
+			return 0, 0, fmt.Errorf("dataset %s: %w", dataset.Dataset.Name, err)
+		}
+		added += countAddedLockEntries(current, dataset.LockData)
+		pruned += countAddedLockEntries(dataset.LockData, current)
+		if lockDataEqual(current, dataset.LockData) {
+			continue
+		}
+		if err := saveLockfile(dataset.Dataset.LockPath, dataset.LockData); err != nil {
+			return 0, 0, fmt.Errorf("dataset %s: %w", dataset.Dataset.Name, err)
+		}
 	}
-	return added, pruned
+	return added, pruned, nil
+}
+
+func countAddedLockEntries(before, after map[string]int) int {
+	count := 0
+	for key, afterID := range after {
+		if beforeID, ok := before[key]; !ok || beforeID != afterID {
+			count++
+		}
+	}
+	return count
+}
+
+func lockDataEqual(a, b map[string]int) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for key, aID := range a {
+		if bID, ok := b[key]; !ok || bID != aID {
+			return false
+		}
+	}
+	return true
 }
 
 type nativeCompileGroupStat struct {
@@ -993,23 +1029,18 @@ func collectBaseDataset(dataset nativeDataset) (nativeCollectedDataset, error) {
 		lockPruned += pruned
 	}
 
-	if lockModified {
-		if err := saveLockfile(dataset.LockPath, lockData); err != nil {
-			return nativeCollectedDataset{}, fmt.Errorf("dataset %s: %w", dataset.Name, err)
-		}
-	}
-
 	rows = dedupeCollectedRows(rows)
 	slices.SortFunc(rows, func(a, b map[string]any) int {
 		return a["id"].(int) - b["id"].(int)
 	})
 	return nativeCollectedDataset{
-		Dataset:    dataset,
+		Dataset:      dataset,
-		Columns:    columns,
+		Columns:      columns,
-		Rows:       rows,
+		Rows:         rows,
-		LockData:   lockData,
+		LockData:     lockData,
-		LockAdded:  lockAdded,
+		LockAdded:    lockAdded,
-		LockPruned: lockPruned,
+		LockPruned:   lockPruned,
+		LockModified: lockModified,
 	}, nil
 }
 
@@ -3919,6 +3950,13 @@ func saveLockfile(path string, lockData map[string]int) error {
 	if err != nil {
 		return fmt.Errorf("marshal lockfile: %w", err)
 	}
+	current, err := os.ReadFile(path)
+	if err == nil && bytes.Equal(current, raw) {
+		return nil
+	}
+	if err != nil && !errors.Is(err, os.ErrNotExist) {
+		return fmt.Errorf("read lockfile %s: %w", path, err)
+	}
 	if err := os.WriteFile(path, raw, 0o644); err != nil {
 		return fmt.Errorf("write lockfile %s: %w", path, err)
 	}

internal/topdata/parts_manifest.go

@@ -1,6 +1,7 @@
 package topdata
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -21,6 +22,7 @@ const (
 	partsManifestReleaseTag     = "parts-manifest-current"
 	partsManifestAssetName      = "sow-parts-manifest.json"
 	partsManifestCacheFileName  = "sow-parts-manifest.json"
+	partsManifestCacheMaxAge    = time.Hour
 	partsManifestRequestTimeout = 30 * time.Second
 )
 
@@ -51,12 +53,21 @@ func resolveReleasedPartsManifest(p *project.Project, progress func(string)) (ma
 	if err != nil {
 		return nil, err
 	}
+	cachePath := filepath.Join(p.Root, ".cache", "topdata", partsManifestCacheFileName)
+	if strings.TrimSpace(os.Getenv("SOW_PARTS_MANIFEST_REFRESH")) == "" {
+		manifest, fresh, err := readFreshPartsManifestCache(cachePath, partsManifestCacheMaxAge)
+		if err != nil {
+			progress(fmt.Sprintf("Ignoring cached parts manifest: %v", err))
+		} else if fresh {
+			progress(fmt.Sprintf("Using cached released parts manifest from %s...", cachePath))
+			return inventoryFromPartsManifest(manifest), nil
+		}
+	}
 	progress(fmt.Sprintf("Fetching released parts manifest from %s...", manifestURL))
 	manifest, err := fetchPartsManifest(manifestURL)
 	if err != nil {
 		return nil, err
 	}
-	cachePath := filepath.Join(p.Root, ".cache", "topdata", partsManifestCacheFileName)
 	if err := writePartsManifestCache(cachePath, manifest); err != nil {
 		return nil, err
 	}
@@ -144,6 +155,31 @@ func fetchPartsManifest(manifestURL string) (*partsManifest, error) {
 	return &manifest, nil
 }
 
+func readFreshPartsManifestCache(path string, maxAge time.Duration) (*partsManifest, bool, error) {
+	info, err := os.Stat(path)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, false, nil
+		}
+		return nil, false, err
+	}
+	if maxAge > 0 && time.Since(info.ModTime()) > maxAge {
+		return nil, false, nil
+	}
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return nil, false, err
+	}
+	var manifest partsManifest
+	if err := json.Unmarshal(raw, &manifest); err != nil {
+		return nil, false, err
+	}
+	if len(manifest.Categories) == 0 {
+		return nil, false, fmt.Errorf("categories is empty")
+	}
+	return &manifest, true, nil
+}
+
 func fetchJSON(target string, out any) error {
 	req, err := http.NewRequest(http.MethodGet, target, nil)
 	if err != nil {
@@ -181,8 +217,16 @@ func writePartsManifestCache(path string, manifest *partsManifest) error {
 	if err != nil {
 		return fmt.Errorf("marshal parts manifest cache: %w", err)
 	}
+	payload = append(payload, '\n')
+	current, err := os.ReadFile(path)
+	if err == nil && bytes.Equal(current, payload) {
+		return nil
+	}
+	if err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("read parts manifest cache: %w", err)
+	}
 	tmpPath := path + ".tmp"
-	if err := os.WriteFile(tmpPath, append(payload, '\n'), 0o644); err != nil {
+	if err := os.WriteFile(tmpPath, payload, 0o644); err != nil {
 		return fmt.Errorf("write parts manifest cache: %w", err)
 	}
 	if err := os.Rename(tmpPath, path); err != nil {

internal/topdata/tlk_native.go

@@ -1,6 +1,7 @@
 package topdata
 
 import (
+	"bytes"
 	"encoding/binary"
 	"encoding/json"
 	"fmt"
@@ -283,6 +284,13 @@ func saveTLKState(path string, doc tlkStateDocument) error {
 		return fmt.Errorf("marshal tlk state: %w", err)
 	}
 	raw = append(raw, '\n')
+	current, err := os.ReadFile(path)
+	if err == nil && bytes.Equal(current, raw) {
+		return nil
+	}
+	if err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("read %s: %w", path, err)
+	}
 	if err := os.WriteFile(path, raw, 0o644); err != nil {
 		return fmt.Errorf("write %s: %w", path, err)
 	}