package topdata import ( "bytes" "encoding/json" "fmt" "io" "net/http" "net/url" "os" "path/filepath" "strings" "time" "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" ) const ( defaultSowAssetsRepoOwner = "ShadowsOverWestgate" defaultSowAssetsRepoName = "sow-assets" defaultGiteaBaseURL = "https://gitea.westgate.pw" partsManifestReleaseTag = "parts-manifest-current" partsManifestAssetName = "sow-parts-manifest.json" partsManifestCacheFileName = "sow-parts-manifest.json" partsManifestCacheMaxAge = time.Hour partsManifestRequestTimeout = 30 * time.Second ) type partsManifest struct { Repo string `json:"repo"` Ref string `json:"ref"` GeneratedAt string `json:"generated_at"` Categories map[string][]int `json:"categories"` } type giteaRepoSpec struct { BaseURL string Owner string Repo string } var partsManifestHTTPClient = &http.Client{Timeout: partsManifestRequestTimeout} func resolveReleasedPartsManifest(p *project.Project, progress func(string)) (map[string]map[int]struct{}, error) { if progress == nil { progress = func(string) {} } cachePath := filepath.Join(p.Root, ".cache", partsManifestCacheFileName) if strings.TrimSpace(os.Getenv("SOW_PARTS_MANIFEST_REFRESH")) == "" { manifest, fresh, err := readFreshPartsManifestCache(cachePath, partsManifestCacheMaxAge) if err != nil { progress(fmt.Sprintf("Ignoring cached parts manifest: %v", err)) } else if fresh { progress(fmt.Sprintf("Using cached released parts manifest from %s...", cachePath)) return inventoryFromPartsManifest(manifest), nil } } spec, err := deriveSowAssetsRepoSpec(p.Root) if err != nil { return nil, err } manifestURL, err := resolvePartsManifestAssetURL(spec) if err != nil { return nil, err } progress(fmt.Sprintf("Fetching released parts manifest from %s...", manifestURL)) manifest, err := fetchPartsManifest(manifestURL) if err != nil { return nil, err } if err := writePartsManifestCache(cachePath, manifest); err != nil { return nil, err } return inventoryFromPartsManifest(manifest), nil } func deriveSowAssetsRepoSpec(root string) (giteaRepoSpec, error) { serverOverride := strings.TrimSpace(os.Getenv("SOW_ASSETS_SERVER_URL")) repoOverride := strings.TrimSpace(os.Getenv("SOW_ASSETS_REPO")) remote, err := gitOutput(root, "remote", "get-url", "origin") spec := giteaRepoSpec{BaseURL: defaultGiteaBaseURL, Owner: defaultSowAssetsRepoOwner, Repo: defaultSowAssetsRepoName} if err == nil && strings.TrimSpace(remote) != "" { if derived, ok := parseRepoSpec(strings.TrimSpace(remote)); ok { derived.Repo = defaultSowAssetsRepoName spec = derived } } if serverOverride != "" { spec.BaseURL = strings.TrimRight(serverOverride, "/") } if repoOverride != "" { owner, repo, ok := parseOwnerRepo(repoOverride) if !ok { return giteaRepoSpec{}, fmt.Errorf("invalid SOW_ASSETS_REPO %q; expected owner/repo", repoOverride) } spec.Owner = owner spec.Repo = repo } return spec, nil } func parseOwnerRepo(raw string) (string, string, bool) { parts := strings.Split(strings.Trim(strings.TrimSuffix(raw, ".git"), "/"), "/") if len(parts) < 2 { return "", "", false } owner := strings.TrimSpace(parts[len(parts)-2]) repo := strings.TrimSpace(parts[len(parts)-1]) if owner == "" || repo == "" { return "", "", false } return owner, repo, true } func parseRepoSpec(raw string) (giteaRepoSpec, bool) { if strings.HasPrefix(raw, "ssh://") || strings.HasPrefix(raw, "http://") || strings.HasPrefix(raw, "https://") { u, err := url.Parse(raw) if err != nil || u.Host == "" { return giteaRepoSpec{}, false } parts := strings.Split(strings.Trim(strings.TrimSuffix(u.Path, ".git"), "/"), "/") if len(parts) < 2 { return giteaRepoSpec{}, false } base := (&url.URL{Scheme: "https", Host: normalizeGiteaHTTPHost(u.Hostname())}).String() if u.Scheme == "http" || u.Scheme == "https" { base = (&url.URL{Scheme: u.Scheme, Host: u.Host}).String() } return giteaRepoSpec{BaseURL: strings.TrimRight(base, "/"), Owner: parts[len(parts)-2], Repo: parts[len(parts)-1]}, true } if strings.HasPrefix(raw, "git@") { withoutUser := strings.TrimPrefix(raw, "git@") hostAndPath := strings.SplitN(withoutUser, ":", 2) if len(hostAndPath) != 2 { return giteaRepoSpec{}, false } parts := strings.Split(strings.Trim(strings.TrimSuffix(hostAndPath[1], ".git"), "/"), "/") if len(parts) < 2 { return giteaRepoSpec{}, false } base := (&url.URL{Scheme: "https", Host: normalizeGiteaHTTPHost(hostAndPath[0])}).String() return giteaRepoSpec{BaseURL: strings.TrimRight(base, "/"), Owner: parts[len(parts)-2], Repo: parts[len(parts)-1]}, true } return giteaRepoSpec{}, false } func normalizeGiteaHTTPHost(host string) string { host = strings.TrimSpace(host) if strings.HasPrefix(host, "git-ssh.") { return "gitea." + strings.TrimPrefix(host, "git-ssh.") } return host } func resolvePartsManifestAssetURL(spec giteaRepoSpec) (string, error) { apiURL := fmt.Sprintf("%s/api/v1/repos/%s/%s/releases/tags/%s", strings.TrimRight(spec.BaseURL, "/"), url.PathEscape(spec.Owner), url.PathEscape(spec.Repo), url.PathEscape(partsManifestReleaseTag)) var release struct { Assets []struct { Name string `json:"name"` BrowserDownloadURL string `json:"browser_download_url"` DownloadURL string `json:"download_url"` } `json:"assets"` } if err := fetchJSON(apiURL, &release); err != nil { return "", fmt.Errorf("fetch parts manifest release metadata: %w", err) } for _, asset := range release.Assets { if asset.Name != partsManifestAssetName { continue } if asset.DownloadURL != "" { return asset.DownloadURL, nil } if asset.BrowserDownloadURL != "" { return asset.BrowserDownloadURL, nil } } return "", fmt.Errorf("parts manifest asset %s not found in release %s/%s:%s", partsManifestAssetName, spec.Owner, spec.Repo, partsManifestReleaseTag) } func fetchPartsManifest(manifestURL string) (*partsManifest, error) { var manifest partsManifest if err := fetchJSON(manifestURL, &manifest); err != nil { return nil, fmt.Errorf("download parts manifest: %w", err) } if len(manifest.Categories) == 0 { return nil, fmt.Errorf("download parts manifest: categories is empty") } return &manifest, nil } func readFreshPartsManifestCache(path string, maxAge time.Duration) (*partsManifest, bool, error) { info, err := os.Stat(path) if err != nil { if os.IsNotExist(err) { return nil, false, nil } return nil, false, err } if maxAge > 0 && time.Since(info.ModTime()) > maxAge { return nil, false, nil } raw, err := os.ReadFile(path) if err != nil { return nil, false, err } var manifest partsManifest if err := json.Unmarshal(raw, &manifest); err != nil { return nil, false, err } if len(manifest.Categories) == 0 { return nil, false, fmt.Errorf("categories is empty") } return &manifest, true, nil } func fetchJSON(target string, out any) error { req, err := http.NewRequest(http.MethodGet, target, nil) if err != nil { return err } if token := strings.TrimSpace(os.Getenv("SOW_ASSETS_TOKEN")); token != "" { req.Header.Set("Authorization", "token "+token) } else if token := strings.TrimSpace(os.Getenv("GITEA_TOKEN")); token != "" { req.Header.Set("Authorization", "token "+token) } resp, err := partsManifestHTTPClient.Do(req) if err != nil { return err } defer resp.Body.Close() if resp.StatusCode < 200 || resp.StatusCode >= 300 { body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) if resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden { return fmt.Errorf("HTTP %d from %s; set SOW_ASSETS_TOKEN or GITEA_TOKEN if the repo is private", resp.StatusCode, target) } return fmt.Errorf("HTTP %d from %s: %s", resp.StatusCode, target, strings.TrimSpace(string(body))) } decoder := json.NewDecoder(resp.Body) if err := decoder.Decode(out); err != nil { return err } return nil } func writePartsManifestCache(path string, manifest *partsManifest) error { if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { return fmt.Errorf("create parts manifest cache dir: %w", err) } payload, err := json.MarshalIndent(manifest, "", " ") if err != nil { return fmt.Errorf("marshal parts manifest cache: %w", err) } payload = append(payload, '\n') current, err := os.ReadFile(path) if err == nil && bytes.Equal(current, payload) { now := time.Now() if err := os.Chtimes(path, now, now); err != nil { return fmt.Errorf("refresh parts manifest cache timestamp: %w", err) } return nil } if err != nil && !os.IsNotExist(err) { return fmt.Errorf("read parts manifest cache: %w", err) } tmpPath := path + ".tmp" if err := os.WriteFile(tmpPath, payload, 0o644); err != nil { return fmt.Errorf("write parts manifest cache: %w", err) } if err := os.Rename(tmpPath, path); err != nil { return fmt.Errorf("replace parts manifest cache: %w", err) } return nil } func inventoryFromPartsManifest(manifest *partsManifest) map[string]map[int]struct{} { result := make(map[string]map[int]struct{}, len(supportedPartCategories)) for _, category := range supportedPartCategories { ids := make(map[int]struct{}) for _, rowID := range manifest.Categories[category] { ids[rowID] = struct{}{} } result[category] = ids } return result }