301 lines
9.7 KiB
Go
301 lines
9.7 KiB
Go
package topdata
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
|
|
)
|
|
|
|
const (
|
|
defaultSowAssetsRepoOwner = "ShadowsOverWestgate"
|
|
defaultSowAssetsRepoName = "sow-assets"
|
|
defaultGiteaBaseURL = "https://git.westgate.pw"
|
|
partsManifestReleaseTag = "parts-manifest-current"
|
|
partsManifestAssetName = "sow-parts-manifest.json"
|
|
partsManifestCacheFileName = "sow-parts-manifest.json"
|
|
partsManifestCacheMaxAge = time.Hour
|
|
partsManifestRequestTimeout = 30 * time.Second
|
|
)
|
|
|
|
type partsManifest struct {
|
|
Repo string `json:"repo"`
|
|
Ref string `json:"ref"`
|
|
GeneratedAt string `json:"generated_at"`
|
|
Categories map[string][]int `json:"categories"`
|
|
}
|
|
|
|
type giteaRepoSpec struct {
|
|
BaseURL string
|
|
Owner string
|
|
Repo string
|
|
}
|
|
|
|
var partsManifestHTTPClient = &http.Client{Timeout: partsManifestRequestTimeout}
|
|
|
|
func resolveReleasedPartsManifest(p *project.Project, progress func(string)) (map[string]map[int]struct{}, error) {
|
|
if progress == nil {
|
|
progress = func(string) {}
|
|
}
|
|
cachePath := p.AutogenCachePath(partsManifestCacheFileName)
|
|
if strings.TrimSpace(os.Getenv(p.PartsManifestRefreshEnv())) == "" {
|
|
manifest, fresh, err := readFreshPartsManifestCache(cachePath, partsManifestCacheMaxAge)
|
|
if err != nil {
|
|
progress(fmt.Sprintf("Ignoring cached parts manifest: %v", err))
|
|
} else if fresh {
|
|
progress(fmt.Sprintf("Using cached released parts manifest from %s...", cachePath))
|
|
return inventoryFromPartsManifest(manifest), nil
|
|
}
|
|
}
|
|
spec, err := deriveSowAssetsRepoSpec(p)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
manifestURL, err := resolvePartsManifestAssetURL(spec)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
progress(fmt.Sprintf("Fetching released parts manifest from %s...", manifestURL))
|
|
manifest, err := fetchPartsManifest(manifestURL)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if err := writePartsManifestCache(cachePath, manifest); err != nil {
|
|
return nil, err
|
|
}
|
|
return inventoryFromPartsManifest(manifest), nil
|
|
}
|
|
|
|
func deriveSowAssetsRepoSpec(p *project.Project) (giteaRepoSpec, error) {
|
|
serverOverride := strings.TrimSpace(os.Getenv(p.AssetsServerURLEnv()))
|
|
repoOverride := strings.TrimSpace(os.Getenv(p.AssetsRepoEnv()))
|
|
remote, err := gitOutput(p.Root, "remote", "get-url", "origin")
|
|
spec := giteaRepoSpec{BaseURL: defaultGiteaBaseURL, Owner: defaultSowAssetsRepoOwner, Repo: defaultSowAssetsRepoName}
|
|
if err == nil && strings.TrimSpace(remote) != "" {
|
|
if derived, ok := parseRepoSpec(strings.TrimSpace(remote)); ok {
|
|
derived.Repo = defaultSowAssetsRepoName
|
|
spec = derived
|
|
}
|
|
}
|
|
|
|
if serverOverride != "" {
|
|
spec.BaseURL = strings.TrimRight(serverOverride, "/")
|
|
}
|
|
if repoOverride != "" {
|
|
owner, repo, ok := parseOwnerRepo(repoOverride)
|
|
if !ok {
|
|
return giteaRepoSpec{}, fmt.Errorf("invalid SOW_ASSETS_REPO %q; expected owner/repo", repoOverride)
|
|
}
|
|
spec.Owner = owner
|
|
spec.Repo = repo
|
|
}
|
|
return spec, nil
|
|
}
|
|
|
|
func parseOwnerRepo(raw string) (string, string, bool) {
|
|
parts := strings.Split(strings.Trim(strings.TrimSuffix(raw, ".git"), "/"), "/")
|
|
if len(parts) < 2 {
|
|
return "", "", false
|
|
}
|
|
owner := strings.TrimSpace(parts[len(parts)-2])
|
|
repo := strings.TrimSpace(parts[len(parts)-1])
|
|
if owner == "" || repo == "" {
|
|
return "", "", false
|
|
}
|
|
return owner, repo, true
|
|
}
|
|
|
|
func parseRepoSpec(raw string) (giteaRepoSpec, bool) {
|
|
if strings.HasPrefix(raw, "ssh://") || strings.HasPrefix(raw, "http://") || strings.HasPrefix(raw, "https://") {
|
|
u, err := url.Parse(raw)
|
|
if err != nil || u.Host == "" {
|
|
return giteaRepoSpec{}, false
|
|
}
|
|
parts := strings.Split(strings.Trim(strings.TrimSuffix(u.Path, ".git"), "/"), "/")
|
|
if len(parts) < 2 {
|
|
return giteaRepoSpec{}, false
|
|
}
|
|
base := (&url.URL{Scheme: "https", Host: normalizeGiteaHTTPHost(u.Hostname())}).String()
|
|
if u.Scheme == "http" || u.Scheme == "https" {
|
|
base = (&url.URL{Scheme: u.Scheme, Host: u.Host}).String()
|
|
}
|
|
return giteaRepoSpec{BaseURL: strings.TrimRight(base, "/"), Owner: parts[len(parts)-2], Repo: parts[len(parts)-1]}, true
|
|
}
|
|
if strings.HasPrefix(raw, "git@") {
|
|
withoutUser := strings.TrimPrefix(raw, "git@")
|
|
hostAndPath := strings.SplitN(withoutUser, ":", 2)
|
|
if len(hostAndPath) != 2 {
|
|
return giteaRepoSpec{}, false
|
|
}
|
|
parts := strings.Split(strings.Trim(strings.TrimSuffix(hostAndPath[1], ".git"), "/"), "/")
|
|
if len(parts) < 2 {
|
|
return giteaRepoSpec{}, false
|
|
}
|
|
base := (&url.URL{Scheme: "https", Host: normalizeGiteaHTTPHost(hostAndPath[0])}).String()
|
|
return giteaRepoSpec{BaseURL: strings.TrimRight(base, "/"), Owner: parts[len(parts)-2], Repo: parts[len(parts)-1]}, true
|
|
}
|
|
return giteaRepoSpec{}, false
|
|
}
|
|
|
|
func normalizeGiteaHTTPHost(host string) string {
|
|
host = strings.TrimSpace(host)
|
|
if strings.HasPrefix(host, "git-ssh.") {
|
|
return "git." + strings.TrimPrefix(host, "git-ssh.")
|
|
}
|
|
return host
|
|
}
|
|
|
|
func resolvePartsManifestAssetURL(spec giteaRepoSpec) (string, error) {
|
|
apiURL := fmt.Sprintf("%s/api/v1/repos/%s/%s/releases/tags/%s", strings.TrimRight(spec.BaseURL, "/"), url.PathEscape(spec.Owner), url.PathEscape(spec.Repo), url.PathEscape(partsManifestReleaseTag))
|
|
var release struct {
|
|
Assets []struct {
|
|
Name string `json:"name"`
|
|
BrowserDownloadURL string `json:"browser_download_url"`
|
|
DownloadURL string `json:"download_url"`
|
|
} `json:"assets"`
|
|
}
|
|
if err := fetchJSON(apiURL, &release); err != nil {
|
|
return "", fmt.Errorf("fetch parts manifest release metadata: %w", err)
|
|
}
|
|
for _, asset := range release.Assets {
|
|
if asset.Name != partsManifestAssetName {
|
|
continue
|
|
}
|
|
if asset.DownloadURL != "" {
|
|
return asset.DownloadURL, nil
|
|
}
|
|
if asset.BrowserDownloadURL != "" {
|
|
return asset.BrowserDownloadURL, nil
|
|
}
|
|
}
|
|
return "", fmt.Errorf("parts manifest asset %s not found in release %s/%s:%s", partsManifestAssetName, spec.Owner, spec.Repo, partsManifestReleaseTag)
|
|
}
|
|
|
|
func fetchPartsManifest(manifestURL string) (*partsManifest, error) {
|
|
var manifest partsManifest
|
|
if err := fetchJSON(manifestURL, &manifest); err != nil {
|
|
return nil, fmt.Errorf("download parts manifest: %w", err)
|
|
}
|
|
if len(manifest.Categories) == 0 {
|
|
return nil, fmt.Errorf("download parts manifest: categories is empty")
|
|
}
|
|
return &manifest, nil
|
|
}
|
|
|
|
func readFreshPartsManifestCache(path string, maxAge time.Duration) (*partsManifest, bool, error) {
|
|
info, err := os.Stat(path)
|
|
if err != nil {
|
|
if os.IsNotExist(err) {
|
|
return nil, false, nil
|
|
}
|
|
return nil, false, err
|
|
}
|
|
if maxAge > 0 && time.Since(info.ModTime()) > maxAge {
|
|
return nil, false, nil
|
|
}
|
|
raw, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
var manifest partsManifest
|
|
if err := json.Unmarshal(raw, &manifest); err != nil {
|
|
return nil, false, err
|
|
}
|
|
if len(manifest.Categories) == 0 {
|
|
return nil, false, fmt.Errorf("categories is empty")
|
|
}
|
|
return &manifest, true, nil
|
|
}
|
|
|
|
func fetchJSON(target string, out any) error {
|
|
req, err := http.NewRequest(http.MethodGet, target, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if token := strings.TrimSpace(os.Getenv("SOW_ASSETS_TOKEN")); token != "" {
|
|
req.Header.Set("Authorization", "token "+token)
|
|
} else if token := strings.TrimSpace(os.Getenv("GITEA_TOKEN")); token != "" {
|
|
req.Header.Set("Authorization", "token "+token)
|
|
}
|
|
resp, err := partsManifestHTTPClient.Do(req)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer resp.Body.Close()
|
|
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
|
body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
|
|
if resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden {
|
|
return fmt.Errorf("HTTP %d from %s; set SOW_ASSETS_TOKEN or GITEA_TOKEN if the repo is private", resp.StatusCode, target)
|
|
}
|
|
return fmt.Errorf("HTTP %d from %s: %s", resp.StatusCode, target, strings.TrimSpace(string(body)))
|
|
}
|
|
decoder := json.NewDecoder(resp.Body)
|
|
if err := decoder.Decode(out); err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func writePartsManifestCache(path string, manifest *partsManifest) error {
|
|
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
|
return fmt.Errorf("create parts manifest cache dir: %w", err)
|
|
}
|
|
payload, err := json.MarshalIndent(manifest, "", " ")
|
|
if err != nil {
|
|
return fmt.Errorf("marshal parts manifest cache: %w", err)
|
|
}
|
|
payload = append(payload, '\n')
|
|
current, err := os.ReadFile(path)
|
|
if err == nil && bytes.Equal(current, payload) {
|
|
now := time.Now()
|
|
if err := os.Chtimes(path, now, now); err != nil {
|
|
return fmt.Errorf("refresh parts manifest cache timestamp: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
if err != nil && !os.IsNotExist(err) {
|
|
return fmt.Errorf("read parts manifest cache: %w", err)
|
|
}
|
|
tmpPath := path + ".tmp"
|
|
if err := os.WriteFile(tmpPath, payload, 0o644); err != nil {
|
|
return fmt.Errorf("write parts manifest cache: %w", err)
|
|
}
|
|
if err := os.Rename(tmpPath, path); err != nil {
|
|
return fmt.Errorf("replace parts manifest cache: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// httpGetStatus performs an anonymous GET and returns the status code plus the
|
|
// body (capped). No auth header: cdn_channel inputs (channels.json, vfxs.yml,
|
|
// haks.json) are public CDN text — R1.
|
|
func httpGetStatus(target string) (int, []byte, error) {
|
|
resp, err := partsManifestHTTPClient.Get(target)
|
|
if err != nil {
|
|
return 0, nil, err
|
|
}
|
|
defer resp.Body.Close()
|
|
body, _ := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
|
|
return resp.StatusCode, body, nil
|
|
}
|
|
|
|
func inventoryFromPartsManifest(manifest *partsManifest) map[string]map[int]struct{} {
|
|
result := make(map[string]map[int]struct{}, len(supportedPartCategories))
|
|
for _, category := range supportedPartCategories {
|
|
ids := make(map[int]struct{})
|
|
for _, rowID := range manifest.Categories[category] {
|
|
ids[rowID] = struct{}{}
|
|
}
|
|
result[category] = ids
|
|
}
|
|
return result
|
|
}
|