diff --git a/build-tool.ps1 b/build-tool.ps1 index d7b5aaf..e5affb7 100644 --- a/build-tool.ps1 +++ b/build-tool.ps1 @@ -7,4 +7,35 @@ Set-Location $repoRoot $env:GOCACHE = Join-Path $repoRoot ".cache/go-build" $outputPath = Join-Path $repoRoot "tools/sow-toolkit.exe" +function Test-ToolSourcesAreNewer { + if (-not (Test-Path $outputPath)) { + return $true + } + + $outputTime = (Get-Item -LiteralPath $outputPath).LastWriteTimeUtc + $sourcePaths = @("go.mod", "go.sum", "cmd", "internal") + foreach ($sourcePath in $sourcePaths) { + $fullPath = Join-Path $repoRoot $sourcePath + if (-not (Test-Path $fullPath)) { + continue + } + $newerSource = Get-ChildItem -LiteralPath $fullPath -Recurse -File | + Where-Object { $_.LastWriteTimeUtc -gt $outputTime } | + Select-Object -First 1 + if ($newerSource) { + return $true + } + } + + return $false +} + +New-Item -ItemType Directory -Force -Path (Join-Path $repoRoot "tools") | Out-Null +New-Item -ItemType Directory -Force -Path $env:GOCACHE | Out-Null +if (-not (Test-ToolSourcesAreNewer)) { + Write-Host "sow-toolkit is up to date." + exit 0 +} + +Write-Host "Building sow-toolkit..." go build -o $outputPath ./cmd/nwn-tool diff --git a/build-tool.sh b/build-tool.sh index 22abccd..20afa82 100755 --- a/build-tool.sh +++ b/build-tool.sh @@ -2,5 +2,30 @@ set -eu cd "$(dirname "$0")" +output="./tools/sow-toolkit" + +tool_sources_are_newer() { + if [ ! -x "$output" ]; then + return 0 + fi + + for path in go.mod go.sum cmd internal; do + if [ ! -e "$path" ]; then + continue + fi + if [ -n "$(find "$path" -type f -newer "$output" -print -quit)" ]; then + return 0 + fi + done + + return 1 +} + mkdir -p tools .cache/go-build -GOCACHE="${PWD}/.cache/go-build" go build -o ./tools/sow-toolkit ./cmd/nwn-tool +if ! tool_sources_are_newer; then + echo "sow-toolkit is up to date." + exit 0 +fi + +echo "Building sow-toolkit..." +GOCACHE="${PWD}/.cache/go-build" go build -o "$output" ./cmd/nwn-tool diff --git a/internal/topdata/expansion_native.go b/internal/topdata/expansion_native.go index a5aaaad..b4cdcd2 100644 --- a/internal/topdata/expansion_native.go +++ b/internal/topdata/expansion_native.go @@ -158,6 +158,7 @@ func mergeExpansionData(collected []nativeCollectedDataset) ([]nativeCollectedDa targetDS.LockData[key] = rowID lockModified = true targetDS.LockAdded++ + targetDS.LockModified = true } } @@ -198,13 +199,9 @@ func mergeExpansionData(collected []nativeCollectedDataset) ([]nativeCollectedDa } if pruned, updated := pruneLockDataToActiveRows(targetDS.LockData, targetDS.Rows, referencedKeys); pruned > 0 || updated > 0 { targetDS.LockPruned += pruned + targetDS.LockModified = true collected[targetIndex] = targetDS } - if targetDS.Dataset.LockPath != "" { - if err := saveLockfile(targetDS.Dataset.LockPath, targetDS.LockData); err != nil { - return nil, fmt.Errorf("dataset %s: %w", targetDS.Dataset.Name, err) - } - } } return collected, nil diff --git a/internal/topdata/native.go b/internal/topdata/native.go index 0890f91..8e1a8f9 100644 --- a/internal/topdata/native.go +++ b/internal/topdata/native.go @@ -39,13 +39,14 @@ const ( ) type nativeCollectedDataset struct { - Dataset nativeDataset - Columns []string - Rows []map[string]any - LockData map[string]int - TableKey string - LockAdded int - LockPruned int + Dataset nativeDataset + Columns []string + Rows []map[string]any + LockData map[string]int + TableKey string + LockAdded int + LockPruned int + LockModified bool } type resolvedTable struct { @@ -262,7 +263,10 @@ func buildNativeUnchecked(p *project.Project, progress func(string), prebuiltWik if err != nil { return BuildResult{}, err } - lockAdded, lockPruned := nativeLockChangeStats(collected) + lockAdded, lockPruned, err := saveNativeBaseLockfiles(collected) + if err != nil { + return BuildResult{}, err + } if lockAdded > 0 || lockPruned > 0 { progress(fmt.Sprintf("Updated native lockfiles (%d new IDs, %d stale IDs pruned)", lockAdded, lockPruned)) } @@ -340,9 +344,6 @@ func buildNativeUnchecked(p *project.Project, progress func(string), prebuiltWik if err != nil { return BuildResult{}, err } - if wikiBuild.PageCount > 0 && wikiBuild.Status == wikiSkippedStatus { - wikiBuild.Status = wikiGeneratedStatus - } } return BuildResult{ @@ -370,14 +371,49 @@ func nativeCompileGroup(datasetName string) string { } } -func nativeLockChangeStats(collected []nativeCollectedDataset) (int, int) { +func saveNativeBaseLockfiles(collected []nativeCollectedDataset) (int, int, error) { added := 0 pruned := 0 for _, dataset := range collected { - added += dataset.LockAdded - pruned += dataset.LockPruned + if dataset.Dataset.Kind != nativeDatasetBase || dataset.Dataset.LockPath == "" { + continue + } + current, err := loadLockfile(dataset.Dataset.LockPath) + if err != nil { + return 0, 0, fmt.Errorf("dataset %s: %w", dataset.Dataset.Name, err) + } + added += countAddedLockEntries(current, dataset.LockData) + pruned += countAddedLockEntries(dataset.LockData, current) + if lockDataEqual(current, dataset.LockData) { + continue + } + if err := saveLockfile(dataset.Dataset.LockPath, dataset.LockData); err != nil { + return 0, 0, fmt.Errorf("dataset %s: %w", dataset.Dataset.Name, err) + } } - return added, pruned + return added, pruned, nil +} + +func countAddedLockEntries(before, after map[string]int) int { + count := 0 + for key, afterID := range after { + if beforeID, ok := before[key]; !ok || beforeID != afterID { + count++ + } + } + return count +} + +func lockDataEqual(a, b map[string]int) bool { + if len(a) != len(b) { + return false + } + for key, aID := range a { + if bID, ok := b[key]; !ok || bID != aID { + return false + } + } + return true } type nativeCompileGroupStat struct { @@ -993,23 +1029,18 @@ func collectBaseDataset(dataset nativeDataset) (nativeCollectedDataset, error) { lockPruned += pruned } - if lockModified { - if err := saveLockfile(dataset.LockPath, lockData); err != nil { - return nativeCollectedDataset{}, fmt.Errorf("dataset %s: %w", dataset.Name, err) - } - } - rows = dedupeCollectedRows(rows) slices.SortFunc(rows, func(a, b map[string]any) int { return a["id"].(int) - b["id"].(int) }) return nativeCollectedDataset{ - Dataset: dataset, - Columns: columns, - Rows: rows, - LockData: lockData, - LockAdded: lockAdded, - LockPruned: lockPruned, + Dataset: dataset, + Columns: columns, + Rows: rows, + LockData: lockData, + LockAdded: lockAdded, + LockPruned: lockPruned, + LockModified: lockModified, }, nil } @@ -3919,6 +3950,13 @@ func saveLockfile(path string, lockData map[string]int) error { if err != nil { return fmt.Errorf("marshal lockfile: %w", err) } + current, err := os.ReadFile(path) + if err == nil && bytes.Equal(current, raw) { + return nil + } + if err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("read lockfile %s: %w", path, err) + } if err := os.WriteFile(path, raw, 0o644); err != nil { return fmt.Errorf("write lockfile %s: %w", path, err) } diff --git a/internal/topdata/parts_manifest.go b/internal/topdata/parts_manifest.go index 49d885f..37d8895 100644 --- a/internal/topdata/parts_manifest.go +++ b/internal/topdata/parts_manifest.go @@ -1,6 +1,7 @@ package topdata import ( + "bytes" "encoding/json" "fmt" "io" @@ -21,6 +22,7 @@ const ( partsManifestReleaseTag = "parts-manifest-current" partsManifestAssetName = "sow-parts-manifest.json" partsManifestCacheFileName = "sow-parts-manifest.json" + partsManifestCacheMaxAge = time.Hour partsManifestRequestTimeout = 30 * time.Second ) @@ -51,12 +53,21 @@ func resolveReleasedPartsManifest(p *project.Project, progress func(string)) (ma if err != nil { return nil, err } + cachePath := filepath.Join(p.Root, ".cache", "topdata", partsManifestCacheFileName) + if strings.TrimSpace(os.Getenv("SOW_PARTS_MANIFEST_REFRESH")) == "" { + manifest, fresh, err := readFreshPartsManifestCache(cachePath, partsManifestCacheMaxAge) + if err != nil { + progress(fmt.Sprintf("Ignoring cached parts manifest: %v", err)) + } else if fresh { + progress(fmt.Sprintf("Using cached released parts manifest from %s...", cachePath)) + return inventoryFromPartsManifest(manifest), nil + } + } progress(fmt.Sprintf("Fetching released parts manifest from %s...", manifestURL)) manifest, err := fetchPartsManifest(manifestURL) if err != nil { return nil, err } - cachePath := filepath.Join(p.Root, ".cache", "topdata", partsManifestCacheFileName) if err := writePartsManifestCache(cachePath, manifest); err != nil { return nil, err } @@ -144,6 +155,31 @@ func fetchPartsManifest(manifestURL string) (*partsManifest, error) { return &manifest, nil } +func readFreshPartsManifestCache(path string, maxAge time.Duration) (*partsManifest, bool, error) { + info, err := os.Stat(path) + if err != nil { + if os.IsNotExist(err) { + return nil, false, nil + } + return nil, false, err + } + if maxAge > 0 && time.Since(info.ModTime()) > maxAge { + return nil, false, nil + } + raw, err := os.ReadFile(path) + if err != nil { + return nil, false, err + } + var manifest partsManifest + if err := json.Unmarshal(raw, &manifest); err != nil { + return nil, false, err + } + if len(manifest.Categories) == 0 { + return nil, false, fmt.Errorf("categories is empty") + } + return &manifest, true, nil +} + func fetchJSON(target string, out any) error { req, err := http.NewRequest(http.MethodGet, target, nil) if err != nil { @@ -181,8 +217,16 @@ func writePartsManifestCache(path string, manifest *partsManifest) error { if err != nil { return fmt.Errorf("marshal parts manifest cache: %w", err) } + payload = append(payload, '\n') + current, err := os.ReadFile(path) + if err == nil && bytes.Equal(current, payload) { + return nil + } + if err != nil && !os.IsNotExist(err) { + return fmt.Errorf("read parts manifest cache: %w", err) + } tmpPath := path + ".tmp" - if err := os.WriteFile(tmpPath, append(payload, '\n'), 0o644); err != nil { + if err := os.WriteFile(tmpPath, payload, 0o644); err != nil { return fmt.Errorf("write parts manifest cache: %w", err) } if err := os.Rename(tmpPath, path); err != nil { diff --git a/internal/topdata/tlk_native.go b/internal/topdata/tlk_native.go index da331e0..4c1f5df 100644 --- a/internal/topdata/tlk_native.go +++ b/internal/topdata/tlk_native.go @@ -1,6 +1,7 @@ package topdata import ( + "bytes" "encoding/binary" "encoding/json" "fmt" @@ -283,6 +284,13 @@ func saveTLKState(path string, doc tlkStateDocument) error { return fmt.Errorf("marshal tlk state: %w", err) } raw = append(raw, '\n') + current, err := os.ReadFile(path) + if err == nil && bytes.Equal(current, raw) { + return nil + } + if err != nil && !os.IsNotExist(err) { + return fmt.Errorf("read %s: %w", path, err) + } if err := os.WriteFile(path, raw, 0o644); err != nil { return fmt.Errorf("write %s: %w", path, err) }