package topdata import ( "errors" "fmt" "io" "io/fs" "os" "path/filepath" "slices" "strings" "time" "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf" "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" ) const ( PackageHAKResRef = "sow_top" PackageHAKFileName = PackageHAKResRef + ".hak" PackageTLKFileName = defaultTLKName ) func compiled2DAOutputDir(p *project.Project) string { return p.TopDataCompiled2DADir() } func topDataBuildUsesRepoCache(p *project.Project) bool { return p.TopDataUsesRepoCache() } func compiledTLKOutputPath(p *project.Project) string { return p.TopDataCompiledTLKPath() } func compiledTLKOutputDir(p *project.Project) string { return p.TopDataCompiledTLKDir() } func wikiOutputRootDir(p *project.Project) string { return p.TopDataWikiRootDir() } func wikiOutputPagesDir(p *project.Project) string { return p.TopDataWikiPagesDir() } func wikiOutputStatePath(p *project.Project) string { return p.TopDataWikiStatePath() } func legacyCompiled2DAOutputDir(p *project.Project) string { return filepath.Join(p.TopDataSourceDir(), "assets", "2da") } func legacyWikiOutputRootDir(p *project.Project) string { return filepath.Join(p.TopDataSourceDir(), legacyWikiRootDirName) } func BuildPackage(p *project.Project, progress func(string)) (PackageResult, error) { if progress == nil { progress = func(string) {} } nativeResult, err := packagedBuildResult(p) if err != nil { return PackageResult{}, err } return packageBuiltTopData(p, nativeResult, progress) } func BuildAndPackage(p *project.Project, progress func(string)) (PackageResult, error) { return BuildAndPackageWithOptions(p, BuildAndPackageOptions{}, progress) } type BuildAndPackageOptions struct { Force bool BuildWiki bool } func BuildAndPackageWithOptions(p *project.Project, opts BuildAndPackageOptions, progress func(string)) (PackageResult, error) { if progress == nil { progress = func(string) {} } if !opts.Force { incrementalResult, ok, err := currentTopPackageResult(p) if err != nil { return PackageResult{}, err } if ok { progress("Topdata outputs are current; skipping native build and top package refresh.") return incrementalResult, nil } nativeResult, ok, err := currentCompiledBuildResult(p) if err != nil { return PackageResult{}, err } if ok { progress(fmt.Sprintf("Compiled topdata outputs are current; refreshing %s and %s only.", p.TopDataPackageHAKName(), p.TopDataPackageTLKName())) return packageBuiltTopData(p, nativeResult, progress) } } return buildAndPackageFull(p, opts, progress) } func buildAndPackageFull(p *project.Project, opts BuildAndPackageOptions, progress func(string)) (PackageResult, error) { nativeResult, err := BuildNativeWithOptions(p, NativeBuildOptions{BuildWiki: opts.BuildWiki, Force: opts.Force}, progress) if err != nil { return PackageResult{}, err } return packageBuiltTopData(p, nativeResult, progress) } func BuildPackageFromBuild(p *project.Project, nativeResult BuildResult, progress func(string)) (PackageResult, error) { if progress == nil { progress = func(string) {} } return packageBuiltTopData(p, nativeResult, progress) } func packageBuiltTopData(p *project.Project, nativeResult BuildResult, progress func(string)) (PackageResult, error) { if progress == nil { progress = func(string) {} } outputHAK := p.TopDataPackageHAKPath() outputTLK := p.TopDataPackageTLKPath() progress(fmt.Sprintf("Packaging compiled topdata resources into %s...", filepath.Base(outputHAK))) resources, assetFiles, err := collectTopPackageResources(p, nativeResult.Output2DADir) if err != nil { return PackageResult{}, err } if err := writeHAK(outputHAK, resources); err != nil { return PackageResult{}, err } progress(fmt.Sprintf("Preparing compiled TLK release output %s...", filepath.Base(outputTLK))) sourceTLK := compiledTLKOutputPath(p) if nativeResult.OutputTLKDir != "" { sourceTLK, err = resolveCompiledTLKPath(nativeResult.OutputTLKDir) if err != nil { return PackageResult{}, err } } if err := copyFile(outputTLK, sourceTLK); err != nil { return PackageResult{}, err } return PackageResult{ Mode: nativeResult.Mode, Output2DADir: nativeResult.Output2DADir, OutputTLKDir: nativeResult.OutputTLKDir, OutputHAKPath: outputHAK, OutputTLKPath: outputTLK, Files2DA: nativeResult.Files2DA, FilesTLK: nativeResult.FilesTLK, HAKResources: len(resources), AssetFiles: assetFiles, WikiOutputDir: nativeResult.WikiOutputDir, WikiPages: nativeResult.WikiPages, WikiStatus: nativeResult.WikiStatus, }, nil } func collectTopPackageResources(p *project.Project, compiled2DADir string) ([]erf.Resource, int, error) { resourceByKey := map[string]erf.Resource{} resourceSourceByKey := map[string]string{} entries, err := os.ReadDir(compiled2DADir) if err != nil { return nil, 0, fmt.Errorf("read compiled 2da output dir: %w", err) } for _, entry := range entries { if entry.IsDir() || !strings.EqualFold(filepath.Ext(entry.Name()), ".2da") { continue } path := filepath.Join(compiled2DADir, entry.Name()) resource, err := topPackageResourceFromPath(path) if err != nil { return nil, 0, err } key := topPackageResourceKey(resource) resourceByKey[key] = resource resourceSourceByKey[key] = path } assetFiles := 0 assetsDir := filepath.Join(p.TopDataSourceDir(), "assets") skipDirs := map[string]struct{}{ filepath.Clean(compiled2DADir): {}, filepath.Clean(legacyCompiled2DAOutputDir(p)): {}, } info, err := os.Stat(assetsDir) if err == nil && info.IsDir() { err = filepath.WalkDir(assetsDir, func(path string, d os.DirEntry, walkErr error) error { if walkErr != nil { return walkErr } if d.IsDir() { if _, ok := skipDirs[filepath.Clean(path)]; ok { return filepath.SkipDir } } if d.IsDir() { return nil } if strings.HasPrefix(filepath.Base(path), ".") { return nil } resource, err := topPackageResourceFromPath(path) if err != nil { return err } key := topPackageResourceKey(resource) if existing, ok := resourceSourceByKey[key]; ok { return fmt.Errorf("topdata asset %s collides with %s for top package resource %s", path, existing, key) } resourceByKey[key] = resource resourceSourceByKey[key] = path assetFiles++ return nil }) if err != nil { return nil, 0, err } } else if err != nil && !os.IsNotExist(err) { return nil, 0, fmt.Errorf("stat topdata assets dir: %w", err) } keys := make([]string, 0, len(resourceByKey)) for key := range resourceByKey { keys = append(keys, key) } slices.Sort(keys) resources := make([]erf.Resource, 0, len(keys)) for _, key := range keys { resources = append(resources, resourceByKey[key]) } return resources, assetFiles, nil } func shouldSkipTopDataSourceDir(path string, skipDirs map[string]struct{}) bool { _, ok := skipDirs[filepath.Clean(path)] return ok } func topDataSourceSkipDirs(p *project.Project) map[string]struct{} { return map[string]struct{}{ filepath.Clean(filepath.Join(p.TopDataSourceDir(), "templates")): {}, filepath.Clean(legacyWikiOutputRootDir(p)): {}, filepath.Clean(legacyCompiled2DAOutputDir(p)): {}, filepath.Clean(wikiOutputRootDir(p)): {}, } } func newestTopDataSource(p *project.Project) (time.Time, string, error) { newest := time.Time{} newestPath := "" skipDirs := topDataSourceSkipDirs(p) sourceDir := p.TopDataSourceDir() err := filepath.WalkDir(sourceDir, func(path string, d fs.DirEntry, walkErr error) error { if walkErr != nil { return walkErr } if d.IsDir() && shouldSkipTopDataSourceDir(path, skipDirs) { return filepath.SkipDir } if d.IsDir() { return nil } info, err := d.Info() if err != nil { return err } if newest.IsZero() || info.ModTime().After(newest) { newest = info.ModTime() newestPath = path } return nil }) if err != nil { return time.Time{}, "", fmt.Errorf("scan topdata sources: %w", err) } return newest, newestPath, nil } func newestTopDataInput(p *project.Project, now time.Time) (time.Time, string, error) { newest, newestPath, err := newestTopDataSource(p) if err != nil { return time.Time{}, "", err } autogenTime, autogenPath, err := newestAutogenInput(p, now) if err != nil { return time.Time{}, "", err } if autogenTime.After(newest) { return autogenTime, autogenPath, nil } return newest, newestPath, nil } func newestAutogenInput(p *project.Project, now time.Time) (time.Time, string, error) { newest := time.Time{} newestPath := "" for _, consumer := range p.Config.Autogen.Consumers { overrideRoot, err := resolveAutogenLocalOverrideRoot(p, consumer) if err != nil { return time.Time{}, "", err } if overrideRoot != "" { candidateRoot := filepath.Join(overrideRoot, consumer.Root) candidateTime, candidatePath, err := newestMatchingAutogenOverrideInput(candidateRoot, consumer.Include) if err != nil { return time.Time{}, "", err } if candidateTime.After(newest) { newest = candidateTime newestPath = candidatePath } continue } cachePath := filepath.Join(p.Root, ".cache", consumer.Manifest.CacheName) candidateTime, candidatePath, err := newestReleasedAutogenManifestInput(p, consumer, now, cachePath) if err != nil { return time.Time{}, "", err } if candidateTime.After(newest) { newest = candidateTime newestPath = candidatePath } } return newest, newestPath, nil } func newestMatchingAutogenOverrideInput(scanRoot string, include []string) (time.Time, string, error) { info, err := os.Stat(scanRoot) if err != nil { if os.IsNotExist(err) { return time.Time{}, "", nil } return time.Time{}, "", fmt.Errorf("stat autogen override root %s: %w", scanRoot, err) } if !info.IsDir() { return time.Time{}, "", fmt.Errorf("autogen override root %s is not a directory", scanRoot) } newest := time.Time{} newestPath := "" err = filepath.WalkDir(scanRoot, func(path string, d fs.DirEntry, walkErr error) error { if walkErr != nil { return walkErr } if d.IsDir() { return nil } rel, err := filepath.Rel(scanRoot, path) if err != nil { return err } if !matchesAutogenInclude(filepath.ToSlash(rel), include) { return nil } info, err := d.Info() if err != nil { return err } if info.ModTime().After(newest) { newest = info.ModTime() newestPath = path } return nil }) if err != nil { return time.Time{}, "", fmt.Errorf("scan autogen override %s: %w", scanRoot, err) } return newest, newestPath, nil } func topPackageResourceFromPath(path string) (erf.Resource, error) { extension := strings.TrimPrefix(strings.ToLower(filepath.Ext(path)), ".") resourceType, ok := erf.HAKResourceTypeForExtension(extension) if !ok { return erf.Resource{}, fmt.Errorf("unsupported top package HAK resource extension %q", filepath.Ext(path)) } info, err := os.Stat(path) if err != nil { return erf.Resource{}, fmt.Errorf("stat %s: %w", path, err) } return erf.Resource{ Name: strings.ToLower(strings.TrimSuffix(filepath.Base(path), filepath.Ext(path))), Type: resourceType, SourcePath: path, Size: info.Size(), }, nil } func topPackageResourceKey(resource erf.Resource) string { extension, ok := erf.ExtensionForResourceType(resource.Type) if !ok { return strings.ToLower(resource.Name) } return strings.ToLower(resource.Name) + "." + strings.TrimPrefix(strings.ToLower(extension), ".") } func writeHAK(path string, resources []erf.Resource) error { output, err := os.Create(path) if err != nil { return fmt.Errorf("create top package hak: %w", err) } defer output.Close() if err := erf.Write(output, erf.New("HAK ", resources)); err != nil { return fmt.Errorf("write top package hak: %w", err) } return nil } func resolveCompiledTLKPath(outputDir string) (string, error) { entries, err := os.ReadDir(outputDir) if err != nil { return "", fmt.Errorf("read compiled tlk output dir: %w", err) } var tlkFiles []string for _, entry := range entries { if entry.IsDir() || !strings.EqualFold(filepath.Ext(entry.Name()), ".tlk") { continue } tlkFiles = append(tlkFiles, filepath.Join(outputDir, entry.Name())) } if len(tlkFiles) != 1 { return "", fmt.Errorf("expected exactly 1 compiled tlk output, found %d", len(tlkFiles)) } return tlkFiles[0], nil } func copyFile(dst, src string) error { if samePath(dst, src) { return nil } input, err := os.Open(src) if err != nil { return fmt.Errorf("open %s: %w", src, err) } defer input.Close() output, err := os.Create(dst) if err != nil { return fmt.Errorf("create %s: %w", dst, err) } defer output.Close() if _, err := io.Copy(output, input); err != nil { return fmt.Errorf("copy %s to %s: %w", src, dst, err) } return nil } func samePath(left, right string) bool { leftAbs, leftErr := filepath.Abs(left) rightAbs, rightErr := filepath.Abs(right) if leftErr != nil || rightErr != nil { return left == right } return leftAbs == rightAbs } func packagedBuildResult(p *project.Project) (BuildResult, error) { if !p.HasTopData() { return BuildResult{}, errors.New("topdata is not configured for this project") } output2DA := compiled2DAOutputDir(p) outputTLK := compiledTLKOutputDir(p) if _, err := os.Stat(output2DA); err != nil { return BuildResult{}, fmt.Errorf("topdata build output missing: run build-topdata first") } if _, err := os.Stat(compiledTLKOutputPath(p)); err != nil { return BuildResult{}, fmt.Errorf("topdata tlk output missing: run build-topdata first") } files2DA, oldest2DA, err := countCompiledFiles(output2DA, ".2da") if err != nil { return BuildResult{}, err } filesTLK := 1 tlkInfo, err := os.Stat(compiledTLKOutputPath(p)) if err != nil { return BuildResult{}, fmt.Errorf("stat compiled tlk output %s: %w", compiledTLKOutputPath(p), err) } oldestTLK := tlkInfo.ModTime() if files2DA == 0 { return BuildResult{}, fmt.Errorf("topdata build output missing: run build-topdata first") } newestSource, newestPath, err := newestTopDataInput(p, time.Now()) if err != nil { return BuildResult{}, err } oldestBuild := oldest2DA if oldestBuild.IsZero() || (!oldestTLK.IsZero() && oldestTLK.Before(oldestBuild)) { oldestBuild = oldestTLK } if !newestSource.IsZero() && !oldestBuild.IsZero() && newestSource.After(oldestBuild) { return BuildResult{}, fmt.Errorf("topdata build output is older than source file %s; run build-topdata first", newestPath) } return BuildResult{ Mode: "native", Output2DADir: output2DA, OutputTLKDir: outputTLK, Files2DA: files2DA, FilesTLK: filesTLK, }, nil } func currentCompiledBuildResult(p *project.Project) (BuildResult, bool, error) { nativeResult, err := packagedBuildResult(p) if err == nil { return nativeResult, true, nil } if strings.Contains(err.Error(), "run build-topdata first") { return BuildResult{}, false, nil } return BuildResult{}, false, err } func currentTopPackageResult(p *project.Project) (PackageResult, bool, error) { nativeResult, ok, err := currentCompiledBuildResult(p) if err != nil || !ok { return PackageResult{}, ok, err } outputHAK := p.TopDataPackageHAKPath() outputTLK := p.TopDataPackageTLKPath() hakInfo, err := os.Stat(outputHAK) if err != nil { if os.IsNotExist(err) { return PackageResult{}, false, nil } return PackageResult{}, false, fmt.Errorf("stat top package hak %s: %w", outputHAK, err) } tlkInfo, err := os.Stat(outputTLK) if err != nil { if os.IsNotExist(err) { return PackageResult{}, false, nil } return PackageResult{}, false, fmt.Errorf("stat top package tlk %s: %w", outputTLK, err) } newestSource, _, err := newestTopDataInput(p, time.Now()) if err != nil { return PackageResult{}, false, err } newestCompiled, err := newestFileModTime(nativeResult.Output2DADir, ".2da") if err != nil { return PackageResult{}, false, err } compiledTLKInfo, err := os.Stat(compiledTLKOutputPath(p)) if err != nil { return PackageResult{}, false, fmt.Errorf("stat compiled tlk output %s: %w", compiledTLKOutputPath(p), err) } if compiledTLKInfo.ModTime().After(newestCompiled) { newestCompiled = compiledTLKInfo.ModTime() } newestInput := newestSource if newestCompiled.After(newestInput) { newestInput = newestCompiled } oldestPackage := hakInfo.ModTime() if tlkInfo.ModTime().Before(oldestPackage) { oldestPackage = tlkInfo.ModTime() } if !newestInput.IsZero() && newestInput.After(oldestPackage) { return PackageResult{}, false, nil } resources, assetFiles, err := collectTopPackageResources(p, nativeResult.Output2DADir) if err != nil { return PackageResult{}, false, err } return PackageResult{ Mode: "incremental-noop", Output2DADir: nativeResult.Output2DADir, OutputTLKDir: nativeResult.OutputTLKDir, OutputHAKPath: outputHAK, OutputTLKPath: outputTLK, Files2DA: nativeResult.Files2DA, FilesTLK: nativeResult.FilesTLK, HAKResources: len(resources), AssetFiles: assetFiles, WikiOutputDir: nativeResult.WikiOutputDir, WikiPages: nativeResult.WikiPages, WikiStatus: nativeResult.WikiStatus, }, true, nil } func newestFileModTime(dir, extension string) (time.Time, error) { entries, err := os.ReadDir(dir) if err != nil { return time.Time{}, fmt.Errorf("read output dir %s: %w", dir, err) } newest := time.Time{} for _, entry := range entries { if entry.IsDir() || !strings.EqualFold(filepath.Ext(entry.Name()), extension) { continue } info, err := entry.Info() if err != nil { return time.Time{}, fmt.Errorf("stat output %s: %w", filepath.Join(dir, entry.Name()), err) } if newest.IsZero() || info.ModTime().After(newest) { newest = info.ModTime() } } return newest, nil } func countCompiledFiles(dir, extension string) (int, time.Time, error) { entries, err := os.ReadDir(dir) if err != nil { return 0, time.Time{}, fmt.Errorf("read compiled output dir %s: %w", dir, err) } count := 0 oldest := time.Time{} for _, entry := range entries { if entry.IsDir() || !strings.EqualFold(filepath.Ext(entry.Name()), extension) { continue } info, err := entry.Info() if err != nil { return 0, time.Time{}, fmt.Errorf("stat compiled output %s: %w", filepath.Join(dir, entry.Name()), err) } count++ if oldest.IsZero() || info.ModTime().Before(oldest) { oldest = info.ModTime() } } return count, oldest, nil }