Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e9860c12c6 | ||
|
|
825fff8b67 | ||
|
|
754375fb08 | ||
|
|
8a90713122 | ||
|
|
895f63a81c | ||
|
|
018b0f7686 | ||
|
|
cec3466779 | ||
|
|
9357b30994 | ||
|
|
448ebc74d4 | ||
|
|
bf787a6458 | ||
|
|
dd92379b68 | ||
|
|
06e5893734 | ||
|
|
332a24fd3e | ||
|
|
da6bf17f69 | ||
|
|
396236f242 | ||
|
|
f257672427 | ||
|
|
6afac1a4d9 | ||
|
|
79595e55be | ||
|
|
eefe00ebc5 | ||
|
|
eb5d15061c | ||
|
|
5c46824ebd | ||
|
|
2ea7959693 | ||
|
|
3315f8b7eb | ||
|
|
faee2cde95 | ||
|
|
d2f7c420ba | ||
|
|
97f4c00393 | ||
|
|
4bab09412d | ||
|
|
13c8ced5e8 |
@@ -9,6 +9,11 @@ on:
|
||||
push:
|
||||
branches: [main]
|
||||
tags: ['v*']
|
||||
paths-ignore:
|
||||
- "docs/**"
|
||||
- "README.md"
|
||||
- "AGENTS.md"
|
||||
- "LICENSE"
|
||||
|
||||
jobs:
|
||||
build-binaries:
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
# Auto-PR canonical wrapper updates to consumer repos when wrappers/ changes on
|
||||
# main. Maintenance automation (not artifact publishing), so it is allowed on a
|
||||
# main push under the D7 trigger standard. Requires WRAPPER_SYNC_TOKEN: a bot
|
||||
# user's token with content+PR write to the consumer repos (never committed).
|
||||
# main push under the D7 trigger standard. Requires BOT_TOKEN: the gitea-bot
|
||||
# org token (content+PR write to the consumer repos; never committed).
|
||||
#
|
||||
# Consumer drift checks run only after their sync PRs merge to main. They must
|
||||
# not run on the sync PR itself, which can create recursive cross-repo checks.
|
||||
name: sync-wrappers
|
||||
|
||||
on:
|
||||
@@ -19,12 +22,13 @@ jobs:
|
||||
|
||||
- name: Open sync PRs to consumers
|
||||
env:
|
||||
TOKEN: ${{ secrets.WRAPPER_SYNC_TOKEN }}
|
||||
TOKEN: ${{ secrets.BOT_TOKEN }}
|
||||
SERVER: ${{ github.server_url }}
|
||||
SRC_SHA: ${{ github.sha }}
|
||||
run: |
|
||||
nix develop --command bash -c '
|
||||
set -euo pipefail
|
||||
[ -n "${TOKEN}" ] || { echo "::error::BOT_TOKEN secret is empty — set the gitea-bot org token"; exit 1; }
|
||||
host="$(echo "$SERVER" | sed -E "s#https?://##")"
|
||||
branch="chore/sync-wrappers-$(echo "$SRC_SHA" | cut -c1-12)"
|
||||
grep -vE "^\s*#|^\s*$" wrappers/consumers.txt | while read -r target; do
|
||||
@@ -40,10 +44,19 @@ jobs:
|
||||
git add crucible.sh crucible.ps1
|
||||
git commit -m "chore: sync crucible wrappers from sow-tools@${SRC_SHA}"
|
||||
git push -f origin "$branch"
|
||||
curl -fsS -X POST \
|
||||
# Capture HTTP status: 201=created, 422=PR already open for this
|
||||
# branch (fine, the force-push above refreshed it). Anything else
|
||||
# (401/404/...) is a real failure — fail loud, do not swallow it.
|
||||
resp="$(mktemp)"
|
||||
code="$(curl -sS -o "$resp" -w "%{http_code}" -X POST \
|
||||
-H "Authorization: token ${TOKEN}" -H "Content-Type: application/json" \
|
||||
"${SERVER}/api/v1/repos/${target}/pulls" \
|
||||
-d "{\"head\":\"${branch}\",\"base\":\"main\",\"title\":\"chore: sync crucible wrappers from sow-tools\"}" || true
|
||||
-d "{\"head\":\"${branch}\",\"base\":\"main\",\"title\":\"chore: sync crucible wrappers from sow-tools\"}")"
|
||||
case "$code" in
|
||||
201) echo "opened sync PR for $target" ;;
|
||||
422) echo "sync PR already open for $target; refreshed its branch" ;;
|
||||
*) echo "::error::PR create failed for $target (HTTP $code)"; cat "$resp"; exit 1 ;;
|
||||
esac
|
||||
)
|
||||
rm -rf "$work"
|
||||
done
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
# Test-build the Crucible image on PRs and main. Proves `nix build .#image`
|
||||
# still works (daemonless, Nix-built OCI tarball) but does NOT publish —
|
||||
# release publishing happens in build-image.yml on v* tags.
|
||||
# Test-build the Crucible image on PRs. Proves `nix build .#image` still works
|
||||
# (daemonless, Nix-built OCI tarball) but does NOT publish — release publishing
|
||||
# happens in build-image.yml on v* tags. PR-only: with up-to-date-before-merge
|
||||
# protection, main == the tested PR head, so a throwaway post-merge rebuild that
|
||||
# publishes nothing is pure waste.
|
||||
name: test-image
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
|
||||
jobs:
|
||||
|
||||
@@ -4,6 +4,11 @@ name: test
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths-ignore:
|
||||
- "docs/**"
|
||||
- "README.md"
|
||||
- "AGENTS.md"
|
||||
- "LICENSE"
|
||||
pull_request:
|
||||
|
||||
jobs:
|
||||
|
||||
@@ -18,3 +18,4 @@ result-*
|
||||
*.swp
|
||||
.idea/
|
||||
.vscode/
|
||||
.direnv/
|
||||
|
||||
@@ -9,11 +9,52 @@ This repo owns the **builder logic:** one Go module
|
||||
(`git.westgate.pw/ShadowsOverWestgate/sow-tools`) producing the `crucible`
|
||||
dispatcher and the `crucible-<name>` binaries.
|
||||
|
||||
## What it is / part it serves
|
||||
|
||||
Shadows Over Westgate (SoW) is a Neverwinter Nights: Enhanced Edition
|
||||
persistent world, split into single-purpose repos. This repo is Crucible, the
|
||||
Go build toolkit. The content repos hold game source (module areas, rules
|
||||
data, binary assets) and call Crucible to turn that source into artifacts:
|
||||
the `.mod` file, 2DA/TLK tables, HAKs, wiki pages. Crucible is the only place
|
||||
builder logic lives; content repos only run it through thin wrapper scripts.
|
||||
|
||||
## Guidance map
|
||||
|
||||
| Where | What |
|
||||
|-------|------|
|
||||
| `cmd/crucible/`, `cmd/crucible-<name>/` | dispatcher + per-builder shims (thin `main.go` files) |
|
||||
| `internal/dispatch/` | the command registry — single source of truth for the command surface |
|
||||
| `internal/` (app, pipeline, project, erf, gff, topdata, changelog, validator, depot, menu, buildinfo) | the actual builder logic |
|
||||
| `wrappers/` | canonical bootstrap wrappers (`crucible`, `crucible.ps1`) synced to consumer repos; `wrappers/consumers.txt` lists targets |
|
||||
| `docs/command-surface.md` | every command, old `nwn-tool` name → new home |
|
||||
| `docs/consumer-contract.md` | how consumer repos resolve/pin a Crucible binary |
|
||||
| `docs/migration-from-nwn-tool.md` | migration status, what remains |
|
||||
| `tests/`, `Makefile`, `flake.nix` | checks, targets, dev shell |
|
||||
|
||||
Task routing: adding/changing a command → read `docs/command-surface.md`
|
||||
first, then `internal/dispatch`. Changing how consumers get binaries →
|
||||
`docs/consumer-contract.md` + `wrappers/`. Release/CI questions → README "CI"
|
||||
section and `.gitea/workflows/`.
|
||||
|
||||
## How it is used
|
||||
|
||||
- Dev loop: `nix develop`, then `make check` / `make build` / `make smoke`
|
||||
(see Commands below).
|
||||
- Release: push a `v*` tag. CI uploads cross-built binaries + wrappers to the
|
||||
Gitea release and publishes the `crucible` container image.
|
||||
- Consumers (they download released binaries via the wrapper; they never
|
||||
vendor a toolkit):
|
||||
- sow-module — https://git.westgate.pw/ShadowsOverWestgate/sow-module
|
||||
- sow-topdata — https://git.westgate.pw/ShadowsOverWestgate/sow-topdata
|
||||
- sow-assets-manifest — https://git.westgate.pw/ShadowsOverWestgate/sow-assets-manifest
|
||||
- sow-platform (deploys the released image/pins) —
|
||||
https://git.westgate.pw/ShadowsOverWestgate/sow-platform
|
||||
|
||||
## What this repo owns / does not own
|
||||
|
||||
Owns: build/extract/validate/compare pipeline, ERF/HAK packing, topdata 2da/tlk
|
||||
compilation, wiki rendering/deploy, depot blob verify, music conversion,
|
||||
changelog. Does **not** own authored game content (that is `sow-module` /
|
||||
compilation, wiki rendering/deploy, depot blob verify, changelog. Does **not**
|
||||
own authored game content (that is `sow-module` /
|
||||
`sow-topdata` / `sow-assets-manifest`) or any production deploy authority (that
|
||||
is `sow-platform`).
|
||||
|
||||
|
||||
@@ -0,0 +1,674 @@
|
||||
GNU GENERAL PUBLIC LICENSE
|
||||
Version 3, 29 June 2007
|
||||
|
||||
Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
|
||||
Everyone is permitted to copy and distribute verbatim copies
|
||||
of this license document, but changing it is not allowed.
|
||||
|
||||
Preamble
|
||||
|
||||
The GNU General Public License is a free, copyleft license for
|
||||
software and other kinds of works.
|
||||
|
||||
The licenses for most software and other practical works are designed
|
||||
to take away your freedom to share and change the works. By contrast,
|
||||
the GNU General Public License is intended to guarantee your freedom to
|
||||
share and change all versions of a program--to make sure it remains free
|
||||
software for all its users. We, the Free Software Foundation, use the
|
||||
GNU General Public License for most of our software; it applies also to
|
||||
any other work released this way by its authors. You can apply it to
|
||||
your programs, too.
|
||||
|
||||
When we speak of free software, we are referring to freedom, not
|
||||
price. Our General Public Licenses are designed to make sure that you
|
||||
have the freedom to distribute copies of free software (and charge for
|
||||
them if you wish), that you receive source code or can get it if you
|
||||
want it, that you can change the software or use pieces of it in new
|
||||
free programs, and that you know you can do these things.
|
||||
|
||||
To protect your rights, we need to prevent others from denying you
|
||||
these rights or asking you to surrender the rights. Therefore, you have
|
||||
certain responsibilities if you distribute copies of the software, or if
|
||||
you modify it: responsibilities to respect the freedom of others.
|
||||
|
||||
For example, if you distribute copies of such a program, whether
|
||||
gratis or for a fee, you must pass on to the recipients the same
|
||||
freedoms that you received. You must make sure that they, too, receive
|
||||
or can get the source code. And you must show them these terms so they
|
||||
know their rights.
|
||||
|
||||
Developers that use the GNU GPL protect your rights with two steps:
|
||||
(1) assert copyright on the software, and (2) offer you this License
|
||||
giving you legal permission to copy, distribute and/or modify it.
|
||||
|
||||
For the developers' and authors' protection, the GPL clearly explains
|
||||
that there is no warranty for this free software. For both users' and
|
||||
authors' sake, the GPL requires that modified versions be marked as
|
||||
changed, so that their problems will not be attributed erroneously to
|
||||
authors of previous versions.
|
||||
|
||||
Some devices are designed to deny users access to install or run
|
||||
modified versions of the software inside them, although the manufacturer
|
||||
can do so. This is fundamentally incompatible with the aim of
|
||||
protecting users' freedom to change the software. The systematic
|
||||
pattern of such abuse occurs in the area of products for individuals to
|
||||
use, which is precisely where it is most unacceptable. Therefore, we
|
||||
have designed this version of the GPL to prohibit the practice for those
|
||||
products. If such problems arise substantially in other domains, we
|
||||
stand ready to extend this provision to those domains in future versions
|
||||
of the GPL, as needed to protect the freedom of users.
|
||||
|
||||
Finally, every program is threatened constantly by software patents.
|
||||
States should not allow patents to restrict development and use of
|
||||
software on general-purpose computers, but in those that do, we wish to
|
||||
avoid the special danger that patents applied to a free program could
|
||||
make it effectively proprietary. To prevent this, the GPL assures that
|
||||
patents cannot be used to render the program non-free.
|
||||
|
||||
The precise terms and conditions for copying, distribution and
|
||||
modification follow.
|
||||
|
||||
TERMS AND CONDITIONS
|
||||
|
||||
0. Definitions.
|
||||
|
||||
"This License" refers to version 3 of the GNU General Public License.
|
||||
|
||||
"Copyright" also means copyright-like laws that apply to other kinds of
|
||||
works, such as semiconductor masks.
|
||||
|
||||
"The Program" refers to any copyrightable work licensed under this
|
||||
License. Each licensee is addressed as "you". "Licensees" and
|
||||
"recipients" may be individuals or organizations.
|
||||
|
||||
To "modify" a work means to copy from or adapt all or part of the work
|
||||
in a fashion requiring copyright permission, other than the making of an
|
||||
exact copy. The resulting work is called a "modified version" of the
|
||||
earlier work or a work "based on" the earlier work.
|
||||
|
||||
A "covered work" means either the unmodified Program or a work based
|
||||
on the Program.
|
||||
|
||||
To "propagate" a work means to do anything with it that, without
|
||||
permission, would make you directly or secondarily liable for
|
||||
infringement under applicable copyright law, except executing it on a
|
||||
computer or modifying a private copy. Propagation includes copying,
|
||||
distribution (with or without modification), making available to the
|
||||
public, and in some countries other activities as well.
|
||||
|
||||
To "convey" a work means any kind of propagation that enables other
|
||||
parties to make or receive copies. Mere interaction with a user through
|
||||
a computer network, with no transfer of a copy, is not conveying.
|
||||
|
||||
An interactive user interface displays "Appropriate Legal Notices"
|
||||
to the extent that it includes a convenient and prominently visible
|
||||
feature that (1) displays an appropriate copyright notice, and (2)
|
||||
tells the user that there is no warranty for the work (except to the
|
||||
extent that warranties are provided), that licensees may convey the
|
||||
work under this License, and how to view a copy of this License. If
|
||||
the interface presents a list of user commands or options, such as a
|
||||
menu, a prominent item in the list meets this criterion.
|
||||
|
||||
1. Source Code.
|
||||
|
||||
The "source code" for a work means the preferred form of the work
|
||||
for making modifications to it. "Object code" means any non-source
|
||||
form of a work.
|
||||
|
||||
A "Standard Interface" means an interface that either is an official
|
||||
standard defined by a recognized standards body, or, in the case of
|
||||
interfaces specified for a particular programming language, one that
|
||||
is widely used among developers working in that language.
|
||||
|
||||
The "System Libraries" of an executable work include anything, other
|
||||
than the work as a whole, that (a) is included in the normal form of
|
||||
packaging a Major Component, but which is not part of that Major
|
||||
Component, and (b) serves only to enable use of the work with that
|
||||
Major Component, or to implement a Standard Interface for which an
|
||||
implementation is available to the public in source code form. A
|
||||
"Major Component", in this context, means a major essential component
|
||||
(kernel, window system, and so on) of the specific operating system
|
||||
(if any) on which the executable work runs, or a compiler used to
|
||||
produce the work, or an object code interpreter used to run it.
|
||||
|
||||
The "Corresponding Source" for a work in object code form means all
|
||||
the source code needed to generate, install, and (for an executable
|
||||
work) run the object code and to modify the work, including scripts to
|
||||
control those activities. However, it does not include the work's
|
||||
System Libraries, or general-purpose tools or generally available free
|
||||
programs which are used unmodified in performing those activities but
|
||||
which are not part of the work. For example, Corresponding Source
|
||||
includes interface definition files associated with source files for
|
||||
the work, and the source code for shared libraries and dynamically
|
||||
linked subprograms that the work is specifically designed to require,
|
||||
such as by intimate data communication or control flow between those
|
||||
subprograms and other parts of the work.
|
||||
|
||||
The Corresponding Source need not include anything that users
|
||||
can regenerate automatically from other parts of the Corresponding
|
||||
Source.
|
||||
|
||||
The Corresponding Source for a work in source code form is that
|
||||
same work.
|
||||
|
||||
2. Basic Permissions.
|
||||
|
||||
All rights granted under this License are granted for the term of
|
||||
copyright on the Program, and are irrevocable provided the stated
|
||||
conditions are met. This License explicitly affirms your unlimited
|
||||
permission to run the unmodified Program. The output from running a
|
||||
covered work is covered by this License only if the output, given its
|
||||
content, constitutes a covered work. This License acknowledges your
|
||||
rights of fair use or other equivalent, as provided by copyright law.
|
||||
|
||||
You may make, run and propagate covered works that you do not
|
||||
convey, without conditions so long as your license otherwise remains
|
||||
in force. You may convey covered works to others for the sole purpose
|
||||
of having them make modifications exclusively for you, or provide you
|
||||
with facilities for running those works, provided that you comply with
|
||||
the terms of this License in conveying all material for which you do
|
||||
not control copyright. Those thus making or running the covered works
|
||||
for you must do so exclusively on your behalf, under your direction
|
||||
and control, on terms that prohibit them from making any copies of
|
||||
your copyrighted material outside their relationship with you.
|
||||
|
||||
Conveying under any other circumstances is permitted solely under
|
||||
the conditions stated below. Sublicensing is not allowed; section 10
|
||||
makes it unnecessary.
|
||||
|
||||
3. Protecting Users' Legal Rights From Anti-Circumvention Law.
|
||||
|
||||
No covered work shall be deemed part of an effective technological
|
||||
measure under any applicable law fulfilling obligations under article
|
||||
11 of the WIPO copyright treaty adopted on 20 December 1996, or
|
||||
similar laws prohibiting or restricting circumvention of such
|
||||
measures.
|
||||
|
||||
When you convey a covered work, you waive any legal power to forbid
|
||||
circumvention of technological measures to the extent such circumvention
|
||||
is effected by exercising rights under this License with respect to
|
||||
the covered work, and you disclaim any intention to limit operation or
|
||||
modification of the work as a means of enforcing, against the work's
|
||||
users, your or third parties' legal rights to forbid circumvention of
|
||||
technological measures.
|
||||
|
||||
4. Conveying Verbatim Copies.
|
||||
|
||||
You may convey verbatim copies of the Program's source code as you
|
||||
receive it, in any medium, provided that you conspicuously and
|
||||
appropriately publish on each copy an appropriate copyright notice;
|
||||
keep intact all notices stating that this License and any
|
||||
non-permissive terms added in accord with section 7 apply to the code;
|
||||
keep intact all notices of the absence of any warranty; and give all
|
||||
recipients a copy of this License along with the Program.
|
||||
|
||||
You may charge any price or no price for each copy that you convey,
|
||||
and you may offer support or warranty protection for a fee.
|
||||
|
||||
5. Conveying Modified Source Versions.
|
||||
|
||||
You may convey a work based on the Program, or the modifications to
|
||||
produce it from the Program, in the form of source code under the
|
||||
terms of section 4, provided that you also meet all of these conditions:
|
||||
|
||||
a) The work must carry prominent notices stating that you modified
|
||||
it, and giving a relevant date.
|
||||
|
||||
b) The work must carry prominent notices stating that it is
|
||||
released under this License and any conditions added under section
|
||||
7. This requirement modifies the requirement in section 4 to
|
||||
"keep intact all notices".
|
||||
|
||||
c) You must license the entire work, as a whole, under this
|
||||
License to anyone who comes into possession of a copy. This
|
||||
License will therefore apply, along with any applicable section 7
|
||||
additional terms, to the whole of the work, and all its parts,
|
||||
regardless of how they are packaged. This License gives no
|
||||
permission to license the work in any other way, but it does not
|
||||
invalidate such permission if you have separately received it.
|
||||
|
||||
d) If the work has interactive user interfaces, each must display
|
||||
Appropriate Legal Notices; however, if the Program has interactive
|
||||
interfaces that do not display Appropriate Legal Notices, your
|
||||
work need not make them do so.
|
||||
|
||||
A compilation of a covered work with other separate and independent
|
||||
works, which are not by their nature extensions of the covered work,
|
||||
and which are not combined with it such as to form a larger program,
|
||||
in or on a volume of a storage or distribution medium, is called an
|
||||
"aggregate" if the compilation and its resulting copyright are not
|
||||
used to limit the access or legal rights of the compilation's users
|
||||
beyond what the individual works permit. Inclusion of a covered work
|
||||
in an aggregate does not cause this License to apply to the other
|
||||
parts of the aggregate.
|
||||
|
||||
6. Conveying Non-Source Forms.
|
||||
|
||||
You may convey a covered work in object code form under the terms
|
||||
of sections 4 and 5, provided that you also convey the
|
||||
machine-readable Corresponding Source under the terms of this License,
|
||||
in one of these ways:
|
||||
|
||||
a) Convey the object code in, or embodied in, a physical product
|
||||
(including a physical distribution medium), accompanied by the
|
||||
Corresponding Source fixed on a durable physical medium
|
||||
customarily used for software interchange.
|
||||
|
||||
b) Convey the object code in, or embodied in, a physical product
|
||||
(including a physical distribution medium), accompanied by a
|
||||
written offer, valid for at least three years and valid for as
|
||||
long as you offer spare parts or customer support for that product
|
||||
model, to give anyone who possesses the object code either (1) a
|
||||
copy of the Corresponding Source for all the software in the
|
||||
product that is covered by this License, on a durable physical
|
||||
medium customarily used for software interchange, for a price no
|
||||
more than your reasonable cost of physically performing this
|
||||
conveying of source, or (2) access to copy the
|
||||
Corresponding Source from a network server at no charge.
|
||||
|
||||
c) Convey individual copies of the object code with a copy of the
|
||||
written offer to provide the Corresponding Source. This
|
||||
alternative is allowed only occasionally and noncommercially, and
|
||||
only if you received the object code with such an offer, in accord
|
||||
with subsection 6b.
|
||||
|
||||
d) Convey the object code by offering access from a designated
|
||||
place (gratis or for a charge), and offer equivalent access to the
|
||||
Corresponding Source in the same way through the same place at no
|
||||
further charge. You need not require recipients to copy the
|
||||
Corresponding Source along with the object code. If the place to
|
||||
copy the object code is a network server, the Corresponding Source
|
||||
may be on a different server (operated by you or a third party)
|
||||
that supports equivalent copying facilities, provided you maintain
|
||||
clear directions next to the object code saying where to find the
|
||||
Corresponding Source. Regardless of what server hosts the
|
||||
Corresponding Source, you remain obligated to ensure that it is
|
||||
available for as long as needed to satisfy these requirements.
|
||||
|
||||
e) Convey the object code using peer-to-peer transmission, provided
|
||||
you inform other peers where the object code and Corresponding
|
||||
Source of the work are being offered to the general public at no
|
||||
charge under subsection 6d.
|
||||
|
||||
A separable portion of the object code, whose source code is excluded
|
||||
from the Corresponding Source as a System Library, need not be
|
||||
included in conveying the object code work.
|
||||
|
||||
A "User Product" is either (1) a "consumer product", which means any
|
||||
tangible personal property which is normally used for personal, family,
|
||||
or household purposes, or (2) anything designed or sold for incorporation
|
||||
into a dwelling. In determining whether a product is a consumer product,
|
||||
doubtful cases shall be resolved in favor of coverage. For a particular
|
||||
product received by a particular user, "normally used" refers to a
|
||||
typical or common use of that class of product, regardless of the status
|
||||
of the particular user or of the way in which the particular user
|
||||
actually uses, or expects or is expected to use, the product. A product
|
||||
is a consumer product regardless of whether the product has substantial
|
||||
commercial, industrial or non-consumer uses, unless such uses represent
|
||||
the only significant mode of use of the product.
|
||||
|
||||
"Installation Information" for a User Product means any methods,
|
||||
procedures, authorization keys, or other information required to install
|
||||
and execute modified versions of a covered work in that User Product from
|
||||
a modified version of its Corresponding Source. The information must
|
||||
suffice to ensure that the continued functioning of the modified object
|
||||
code is in no case prevented or interfered with solely because
|
||||
modification has been made.
|
||||
|
||||
If you convey an object code work under this section in, or with, or
|
||||
specifically for use in, a User Product, and the conveying occurs as
|
||||
part of a transaction in which the right of possession and use of the
|
||||
User Product is transferred to the recipient in perpetuity or for a
|
||||
fixed term (regardless of how the transaction is characterized), the
|
||||
Corresponding Source conveyed under this section must be accompanied
|
||||
by the Installation Information. But this requirement does not apply
|
||||
if neither you nor any third party retains the ability to install
|
||||
modified object code on the User Product (for example, the work has
|
||||
been installed in ROM).
|
||||
|
||||
The requirement to provide Installation Information does not include a
|
||||
requirement to continue to provide support service, warranty, or updates
|
||||
for a work that has been modified or installed by the recipient, or for
|
||||
the User Product in which it has been modified or installed. Access to a
|
||||
network may be denied when the modification itself materially and
|
||||
adversely affects the operation of the network or violates the rules and
|
||||
protocols for communication across the network.
|
||||
|
||||
Corresponding Source conveyed, and Installation Information provided,
|
||||
in accord with this section must be in a format that is publicly
|
||||
documented (and with an implementation available to the public in
|
||||
source code form), and must require no special password or key for
|
||||
unpacking, reading or copying.
|
||||
|
||||
7. Additional Terms.
|
||||
|
||||
"Additional permissions" are terms that supplement the terms of this
|
||||
License by making exceptions from one or more of its conditions.
|
||||
Additional permissions that are applicable to the entire Program shall
|
||||
be treated as though they were included in this License, to the extent
|
||||
that they are valid under applicable law. If additional permissions
|
||||
apply only to part of the Program, that part may be used separately
|
||||
under those permissions, but the entire Program remains governed by
|
||||
this License without regard to the additional permissions.
|
||||
|
||||
When you convey a copy of a covered work, you may at your option
|
||||
remove any additional permissions from that copy, or from any part of
|
||||
it. (Additional permissions may be written to require their own
|
||||
removal in certain cases when you modify the work.) You may place
|
||||
additional permissions on material, added by you to a covered work,
|
||||
for which you have or can give appropriate copyright permission.
|
||||
|
||||
Notwithstanding any other provision of this License, for material you
|
||||
add to a covered work, you may (if authorized by the copyright holders of
|
||||
that material) supplement the terms of this License with terms:
|
||||
|
||||
a) Disclaiming warranty or limiting liability differently from the
|
||||
terms of sections 15 and 16 of this License; or
|
||||
|
||||
b) Requiring preservation of specified reasonable legal notices or
|
||||
author attributions in that material or in the Appropriate Legal
|
||||
Notices displayed by works containing it; or
|
||||
|
||||
c) Prohibiting misrepresentation of the origin of that material, or
|
||||
requiring that modified versions of such material be marked in
|
||||
reasonable ways as different from the original version; or
|
||||
|
||||
d) Limiting the use for publicity purposes of names of licensors or
|
||||
authors of the material; or
|
||||
|
||||
e) Declining to grant rights under trademark law for use of some
|
||||
trade names, trademarks, or service marks; or
|
||||
|
||||
f) Requiring indemnification of licensors and authors of that
|
||||
material by anyone who conveys the material (or modified versions of
|
||||
it) with contractual assumptions of liability to the recipient, for
|
||||
any liability that these contractual assumptions directly impose on
|
||||
those licensors and authors.
|
||||
|
||||
All other non-permissive additional terms are considered "further
|
||||
restrictions" within the meaning of section 10. If the Program as you
|
||||
received it, or any part of it, contains a notice stating that it is
|
||||
governed by this License along with a term that is a further
|
||||
restriction, you may remove that term. If a license document contains
|
||||
a further restriction but permits relicensing or conveying under this
|
||||
License, you may add to a covered work material governed by the terms
|
||||
of that license document, provided that the further restriction does
|
||||
not survive such relicensing or conveying.
|
||||
|
||||
If you add terms to a covered work in accord with this section, you
|
||||
must place, in the relevant source files, a statement of the
|
||||
additional terms that apply to those files, or a notice indicating
|
||||
where to find the applicable terms.
|
||||
|
||||
Additional terms, permissive or non-permissive, may be stated in the
|
||||
form of a separately written license, or stated as exceptions;
|
||||
the above requirements apply either way.
|
||||
|
||||
8. Termination.
|
||||
|
||||
You may not propagate or modify a covered work except as expressly
|
||||
provided under this License. Any attempt otherwise to propagate or
|
||||
modify it is void, and will automatically terminate your rights under
|
||||
this License (including any patent licenses granted under the third
|
||||
paragraph of section 11).
|
||||
|
||||
However, if you cease all violation of this License, then your
|
||||
license from a particular copyright holder is reinstated (a)
|
||||
provisionally, unless and until the copyright holder explicitly and
|
||||
finally terminates your license, and (b) permanently, if the copyright
|
||||
holder fails to notify you of the violation by some reasonable means
|
||||
prior to 60 days after the cessation.
|
||||
|
||||
Moreover, your license from a particular copyright holder is
|
||||
reinstated permanently if the copyright holder notifies you of the
|
||||
violation by some reasonable means, this is the first time you have
|
||||
received notice of violation of this License (for any work) from that
|
||||
copyright holder, and you cure the violation prior to 30 days after
|
||||
your receipt of the notice.
|
||||
|
||||
Termination of your rights under this section does not terminate the
|
||||
licenses of parties who have received copies or rights from you under
|
||||
this License. If your rights have been terminated and not permanently
|
||||
reinstated, you do not qualify to receive new licenses for the same
|
||||
material under section 10.
|
||||
|
||||
9. Acceptance Not Required for Having Copies.
|
||||
|
||||
You are not required to accept this License in order to receive or
|
||||
run a copy of the Program. Ancillary propagation of a covered work
|
||||
occurring solely as a consequence of using peer-to-peer transmission
|
||||
to receive a copy likewise does not require acceptance. However,
|
||||
nothing other than this License grants you permission to propagate or
|
||||
modify any covered work. These actions infringe copyright if you do
|
||||
not accept this License. Therefore, by modifying or propagating a
|
||||
covered work, you indicate your acceptance of this License to do so.
|
||||
|
||||
10. Automatic Licensing of Downstream Recipients.
|
||||
|
||||
Each time you convey a covered work, the recipient automatically
|
||||
receives a license from the original licensors, to run, modify and
|
||||
propagate that work, subject to this License. You are not responsible
|
||||
for enforcing compliance by third parties with this License.
|
||||
|
||||
An "entity transaction" is a transaction transferring control of an
|
||||
organization, or substantially all assets of one, or subdividing an
|
||||
organization, or merging organizations. If propagation of a covered
|
||||
work results from an entity transaction, each party to that
|
||||
transaction who receives a copy of the work also receives whatever
|
||||
licenses to the work the party's predecessor in interest had or could
|
||||
give under the previous paragraph, plus a right to possession of the
|
||||
Corresponding Source of the work from the predecessor in interest, if
|
||||
the predecessor has it or can get it with reasonable efforts.
|
||||
|
||||
You may not impose any further restrictions on the exercise of the
|
||||
rights granted or affirmed under this License. For example, you may
|
||||
not impose a license fee, royalty, or other charge for exercise of
|
||||
rights granted under this License, and you may not initiate litigation
|
||||
(including a cross-claim or counterclaim in a lawsuit) alleging that
|
||||
any patent claim is infringed by making, using, selling, offering for
|
||||
sale, or importing the Program or any portion of it.
|
||||
|
||||
11. Patents.
|
||||
|
||||
A "contributor" is a copyright holder who authorizes use under this
|
||||
License of the Program or a work on which the Program is based. The
|
||||
work thus licensed is called the contributor's "contributor version".
|
||||
|
||||
A contributor's "essential patent claims" are all patent claims
|
||||
owned or controlled by the contributor, whether already acquired or
|
||||
hereafter acquired, that would be infringed by some manner, permitted
|
||||
by this License, of making, using, or selling its contributor version,
|
||||
but do not include claims that would be infringed only as a
|
||||
consequence of further modification of the contributor version. For
|
||||
purposes of this definition, "control" includes the right to grant
|
||||
patent sublicenses in a manner consistent with the requirements of
|
||||
this License.
|
||||
|
||||
Each contributor grants you a non-exclusive, worldwide, royalty-free
|
||||
patent license under the contributor's essential patent claims, to
|
||||
make, use, sell, offer for sale, import and otherwise run, modify and
|
||||
propagate the contents of its contributor version.
|
||||
|
||||
In the following three paragraphs, a "patent license" is any express
|
||||
agreement or commitment, however denominated, not to enforce a patent
|
||||
(such as an express permission to practice a patent or covenant not to
|
||||
sue for patent infringement). To "grant" such a patent license to a
|
||||
party means to make such an agreement or commitment not to enforce a
|
||||
patent against the party.
|
||||
|
||||
If you convey a covered work, knowingly relying on a patent license,
|
||||
and the Corresponding Source of the work is not available for anyone
|
||||
to copy, free of charge and under the terms of this License, through a
|
||||
publicly available network server or other readily accessible means,
|
||||
then you must either (1) cause the Corresponding Source to be so
|
||||
available, or (2) arrange to deprive yourself of the benefit of the
|
||||
patent license for this particular work, or (3) arrange, in a manner
|
||||
consistent with the requirements of this License, to extend the patent
|
||||
license to downstream recipients. "Knowingly relying" means you have
|
||||
actual knowledge that, but for the patent license, your conveying the
|
||||
covered work in a country, or your recipient's use of the covered work
|
||||
in a country, would infringe one or more identifiable patents in that
|
||||
country that you have reason to believe are valid.
|
||||
|
||||
If, pursuant to or in connection with a single transaction or
|
||||
arrangement, you convey, or propagate by procuring conveyance of, a
|
||||
covered work, and grant a patent license to some of the parties
|
||||
receiving the covered work authorizing them to use, propagate, modify
|
||||
or convey a specific copy of the covered work, then the patent license
|
||||
you grant is automatically extended to all recipients of the covered
|
||||
work and works based on it.
|
||||
|
||||
A patent license is "discriminatory" if it does not include within
|
||||
the scope of its coverage, prohibits the exercise of, or is
|
||||
conditioned on the non-exercise of one or more of the rights that are
|
||||
specifically granted under this License. You may not convey a covered
|
||||
work if you are a party to an arrangement with a third party that is
|
||||
in the business of distributing software, under which you make payment
|
||||
to the third party based on the extent of your activity of conveying
|
||||
the work, and under which the third party grants, to any of the
|
||||
parties who would receive the covered work from you, a discriminatory
|
||||
patent license (a) in connection with copies of the covered work
|
||||
conveyed by you (or copies made from those copies), or (b) primarily
|
||||
for and in connection with specific products or compilations that
|
||||
contain the covered work, unless you entered into that arrangement,
|
||||
or that patent license was granted, prior to 28 March 2007.
|
||||
|
||||
Nothing in this License shall be construed as excluding or limiting
|
||||
any implied license or other defenses to infringement that may
|
||||
otherwise be available to you under applicable patent law.
|
||||
|
||||
12. No Surrender of Others' Freedom.
|
||||
|
||||
If conditions are imposed on you (whether by court order, agreement or
|
||||
otherwise) that contradict the conditions of this License, they do not
|
||||
excuse you from the conditions of this License. If you cannot convey a
|
||||
covered work so as to satisfy simultaneously your obligations under this
|
||||
License and any other pertinent obligations, then as a consequence you may
|
||||
not convey it at all. For example, if you agree to terms that obligate you
|
||||
to collect a royalty for further conveying from those to whom you convey
|
||||
the Program, the only way you could satisfy both those terms and this
|
||||
License would be to refrain entirely from conveying the Program.
|
||||
|
||||
13. Use with the GNU Affero General Public License.
|
||||
|
||||
Notwithstanding any other provision of this License, you have
|
||||
permission to link or combine any covered work with a work licensed
|
||||
under version 3 of the GNU Affero General Public License into a single
|
||||
combined work, and to convey the resulting work. The terms of this
|
||||
License will continue to apply to the part which is the covered work,
|
||||
but the special requirements of the GNU Affero General Public License,
|
||||
section 13, concerning interaction through a network will apply to the
|
||||
combination as such.
|
||||
|
||||
14. Revised Versions of this License.
|
||||
|
||||
The Free Software Foundation may publish revised and/or new versions of
|
||||
the GNU General Public License from time to time. Such new versions will
|
||||
be similar in spirit to the present version, but may differ in detail to
|
||||
address new problems or concerns.
|
||||
|
||||
Each version is given a distinguishing version number. If the
|
||||
Program specifies that a certain numbered version of the GNU General
|
||||
Public License "or any later version" applies to it, you have the
|
||||
option of following the terms and conditions either of that numbered
|
||||
version or of any later version published by the Free Software
|
||||
Foundation. If the Program does not specify a version number of the
|
||||
GNU General Public License, you may choose any version ever published
|
||||
by the Free Software Foundation.
|
||||
|
||||
If the Program specifies that a proxy can decide which future
|
||||
versions of the GNU General Public License can be used, that proxy's
|
||||
public statement of acceptance of a version permanently authorizes you
|
||||
to choose that version for the Program.
|
||||
|
||||
Later license versions may give you additional or different
|
||||
permissions. However, no additional obligations are imposed on any
|
||||
author or copyright holder as a result of your choosing to follow a
|
||||
later version.
|
||||
|
||||
15. Disclaimer of Warranty.
|
||||
|
||||
THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
|
||||
APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
|
||||
HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
|
||||
OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
|
||||
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
||||
PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
|
||||
IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
|
||||
ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
|
||||
|
||||
16. Limitation of Liability.
|
||||
|
||||
IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
|
||||
WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
|
||||
THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
|
||||
GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
|
||||
USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
|
||||
DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
|
||||
PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
|
||||
EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
|
||||
SUCH DAMAGES.
|
||||
|
||||
17. Interpretation of Sections 15 and 16.
|
||||
|
||||
If the disclaimer of warranty and limitation of liability provided
|
||||
above cannot be given local legal effect according to their terms,
|
||||
reviewing courts shall apply local law that most closely approximates
|
||||
an absolute waiver of all civil liability in connection with the
|
||||
Program, unless a warranty or assumption of liability accompanies a
|
||||
copy of the Program in return for a fee.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
How to Apply These Terms to Your New Programs
|
||||
|
||||
If you develop a new program, and you want it to be of the greatest
|
||||
possible use to the public, the best way to achieve this is to make it
|
||||
free software which everyone can redistribute and change under these terms.
|
||||
|
||||
To do so, attach the following notices to the program. It is safest
|
||||
to attach them to the start of each source file to most effectively
|
||||
state the exclusion of warranty; and each file should have at least
|
||||
the "copyright" line and a pointer to where the full notice is found.
|
||||
|
||||
<one line to give the program's name and a brief idea of what it does.>
|
||||
Copyright (C) <year> <name of author>
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
Also add information on how to contact you by electronic and paper mail.
|
||||
|
||||
If the program does terminal interaction, make it output a short
|
||||
notice like this when it starts in an interactive mode:
|
||||
|
||||
<program> Copyright (C) <year> <name of author>
|
||||
This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
|
||||
This is free software, and you are welcome to redistribute it
|
||||
under certain conditions; type `show c' for details.
|
||||
|
||||
The hypothetical commands `show w' and `show c' should show the appropriate
|
||||
parts of the General Public License. Of course, your program's commands
|
||||
might be different; for a GUI interface, you would use an "about box".
|
||||
|
||||
You should also get your employer (if you work as a programmer) or school,
|
||||
if any, to sign a "copyright disclaimer" for the program, if necessary.
|
||||
For more information on this, and how to apply and follow the GNU GPL, see
|
||||
<https://www.gnu.org/licenses/>.
|
||||
|
||||
The GNU General Public License does not permit incorporating your program
|
||||
into proprietary programs. If your program is a subroutine library, you
|
||||
may consider it more useful to permit linking proprietary applications with
|
||||
the library. If this is what you want to do, use the GNU Lesser General
|
||||
Public License instead of this License. But first, please read
|
||||
<https://www.gnu.org/licenses/why-not-lgpl.html>.
|
||||
@@ -29,7 +29,7 @@ command lands is mapped in [`docs/command-surface.md`](docs/command-surface.md).
|
||||
|
||||
## Status (cutover performed)
|
||||
|
||||
The internal `app`/`pipeline`/`project`/`erf`/`gff`/`topdata`/`music`/`changelog`/
|
||||
The internal `app`/`pipeline`/`project`/`erf`/`gff`/`topdata`/`changelog`/
|
||||
`validator` packages from `gitea/sow-tools` have been migrated into this tree, and
|
||||
the `module`, `topdata`, `hak`, and `wiki` builders now **delegate to the migrated
|
||||
`nwn-tool` command surface** (mapped in
|
||||
@@ -49,7 +49,7 @@ which downloads the latest released `crucible` for your OS and runs it:
|
||||
```bash
|
||||
./crucible # interactive menu (pick a command)
|
||||
./crucible module build
|
||||
./crucible topdata validate-topdata
|
||||
./crucible topdata validate
|
||||
```
|
||||
|
||||
Windows (PowerShell):
|
||||
@@ -62,15 +62,12 @@ The binary is cached under `~/.cache/crucible/<version>/` (`%LOCALAPPDATA%\cruci
|
||||
on Windows); `--repo-local` caches inside the repo instead. Private releases:
|
||||
set `CRUCIBLE_TOKEN` or write the token to `~/.config/crucible/token`.
|
||||
|
||||
Only the **music** builder needs `ffmpeg`; everything else has zero dependencies.
|
||||
If you run a music command without it, Crucible prints a per-OS install hint.
|
||||
|
||||
## Develop
|
||||
|
||||
Self-contained (D8) — a host with only Nix can run everything:
|
||||
|
||||
```bash
|
||||
nix develop # Go + ffmpeg + shellcheck + yamllint + make
|
||||
nix develop # Go + shellcheck + yamllint + make
|
||||
make check # go vet + go test + shellcheck + yamllint
|
||||
make build # build every cmd/* into ./bin (gitignored)
|
||||
make smoke # build + assert the fail-closed contract
|
||||
@@ -83,7 +80,8 @@ This retires the old habit of checking in `nwn-tool` / `sow-toolkit`.
|
||||
## CI
|
||||
|
||||
PR-first (D7): checks run on pull requests and on push to `main`; the only
|
||||
publish event is a `v*` tag (see `sow-docs/runbooks/ci-trigger-standard.md`).
|
||||
publish event is a `v*` tag (see `runbooks/ci-trigger-standard.md` in sow-docs,
|
||||
https://git.westgate.pw/ShadowsOverWestgate/sow-docs).
|
||||
|
||||
- `test.yml` — vet, test, shellcheck, yamllint, binary smoke (PR + main).
|
||||
- `test-image.yml` — build the OCI image to prove it compiles (PR + main, no push).
|
||||
@@ -94,8 +92,10 @@ publish event is a `v*` tag (see `sow-docs/runbooks/ci-trigger-standard.md`).
|
||||
- `publish-image.yml` — manual `workflow_dispatch` break-glass republish.
|
||||
- `sync-wrappers.yml` — on a `main` push that touches `wrappers/`, auto-PR the
|
||||
canonical wrappers to the consumer repos in `wrappers/consumers.txt`.
|
||||
Consumer drift checks run after those PRs merge to `main`, not on the PRs
|
||||
themselves, to avoid recursive cross-repo checks.
|
||||
|
||||
## Consumers
|
||||
|
||||
How the artifact repos resolve a Crucible binary (and the `NWN_ROOT` rule) is
|
||||
How the artifact repos resolve a Crucible binary is
|
||||
documented in [`docs/consumer-contract.md`](docs/consumer-contract.md).
|
||||
|
||||
+1
-6
@@ -6,10 +6,6 @@
|
||||
# and ships them on a static base. The `crucible` dispatcher is the entrypoint;
|
||||
# consumer CI can also call the standalone crucible-<name> binaries by path.
|
||||
#
|
||||
# NOTE: the internal pipeline is migrated and the music conversion path is wired,
|
||||
# so the runtime stage is debian-slim with ffmpeg on PATH (BMU encode/decode).
|
||||
# See docs/migration-from-nwn-tool.md.
|
||||
|
||||
FROM golang:1.26-alpine AS build
|
||||
WORKDIR /src
|
||||
RUN apk add --no-cache git
|
||||
@@ -30,11 +26,10 @@ RUN set -eux; \
|
||||
done
|
||||
|
||||
FROM debian:12-slim
|
||||
# ffmpeg: the migrated music pipeline shells out to it for BMU conversion.
|
||||
# ca-certificates: builders fetch published manifests over HTTPS.
|
||||
RUN set -eux; \
|
||||
apt-get update; \
|
||||
apt-get install -y --no-install-recommends ffmpeg ca-certificates; \
|
||||
apt-get install -y --no-install-recommends ca-certificates; \
|
||||
rm -rf /var/lib/apt/lists/*; \
|
||||
useradd --system --create-home --uid 65532 nonroot
|
||||
COPY --from=build /out/ /usr/local/bin/
|
||||
|
||||
+56
-46
@@ -1,67 +1,77 @@
|
||||
# Crucible command surface
|
||||
|
||||
Crucible re-homes the single `nwn-tool` (a.k.a. `sow-toolkit`) command surface
|
||||
into five builders plus a dispatcher (D11). This table maps every legacy command
|
||||
to its Crucible home. It is the migration contract — keep it in sync with
|
||||
`internal/dispatch.Registry`.
|
||||
`internal/dispatch.Registry` is the single source of truth for Crucible's
|
||||
builders, visible commands, descriptions, help, and hidden compatibility
|
||||
aliases.
|
||||
|
||||
## Builders
|
||||
## Visible commands
|
||||
|
||||
| Builder | Legacy `nwn-tool` commands subsumed |
|
||||
| ------------------ | ------------------------------------------------------------------------------------- |
|
||||
| `crucible-module` | `build`, `build-module`, `extract`, `validate`, `compare`, `apply-hak-manifest`† |
|
||||
| `crucible-topdata` | `validate-topdata`, `build-topdata`, `build-top-package`, `compare-topdata`, `convert-topdata` |
|
||||
| `crucible-hak` | `build-haks`, `apply-hak-manifest` |
|
||||
| `crucible-wiki` | `build-wiki`, `deploy-wiki` |
|
||||
| `crucible-depot` | (new) depot blob verify/move — was shell `mc`/S3 in `sow-assets-manifest` |
|
||||
| Builder | Command | Purpose |
|
||||
| --- | --- | --- |
|
||||
| `hak` | `build` | Build configured HAK archives and their manifest. |
|
||||
| `hak` | `manifest` | Apply a generated HAK list to module source. |
|
||||
| `module` | `build` | Build the module and configured project outputs. |
|
||||
| `module` | `extract` | Extract built archives into configured source trees. |
|
||||
| `module` | `validate` | Validate project layout, config, and source inventory. |
|
||||
| `module` | `compare` | Compare source content with built archives. |
|
||||
| `module` | `manifest` | Apply a generated HAK list to module source. |
|
||||
| `topdata` | `validate` | Validate topdata source and canonical JSON compatibility. |
|
||||
| `topdata` | `build` | Compile topdata and refresh the configured package. |
|
||||
| `topdata` | `package` | Package cached outputs into HAK and TLK files. |
|
||||
| `topdata` | `compare` | Compare current output with a fresh native build. |
|
||||
| `topdata` | `convert` | Convert between 2DA and native JSON/module formats. |
|
||||
| `wiki` | `build` | Render wiki page drafts from compiled topdata. |
|
||||
| `wiki` | `deploy` | Deploy generated wiki pages to NodeBB. |
|
||||
|
||||
† `apply-hak-manifest` is HAK-list maintenance on a module; it is reachable from
|
||||
both `crucible-module` and `crucible-hak`. Canonical home is `crucible-hak`.
|
||||
`crucible-depot` remains registered but unwired. It fails closed with exit `70`
|
||||
and never emits placeholder artifacts.
|
||||
|
||||
## Folded / global commands
|
||||
## Hidden compatibility aliases
|
||||
|
||||
These legacy commands are **not** top-level builders:
|
||||
Existing scripts may continue using these names indefinitely. They are accepted
|
||||
but omitted from routine help and the interactive menu:
|
||||
|
||||
- `music *` (`list-datasets`, `scan`, `build`, `credits`, `validate`, `manifest`,
|
||||
`normalize`) → folds into the **module/hak build pipeline** (music BMUs are
|
||||
packed into HAKs at build time). Exposed as `crucible-module music ...` /
|
||||
`crucible-hak music ...`. Needs `ffmpeg` at runtime (in the image).
|
||||
- `config *` (`validate`, `effective`, `inspect`, `explain`, `sources`) → a
|
||||
**global** concern shared by every builder; exposed as the global
|
||||
`crucible config ...`, not its own binary.
|
||||
- `build-changelog` → release tooling; exposed as the global `crucible changelog`
|
||||
rather than a content builder.
|
||||
| Builder | Hidden alias | Implementation |
|
||||
| --- | --- | --- |
|
||||
| `hak` | `build-haks` | `build-haks` |
|
||||
| `hak` | `apply-hak-manifest` | `apply-hak-manifest` |
|
||||
| `module` | `build-module` | module-only `build-module` |
|
||||
| `module` | `apply-hak-manifest` | `apply-hak-manifest` |
|
||||
| `topdata` | `validate-topdata` | `validate-topdata` |
|
||||
| `topdata` | `build-topdata` | `build-topdata` |
|
||||
| `topdata` | `build-top-package` | `build-top-package` |
|
||||
| `topdata` | `compare-topdata` | `compare-topdata` |
|
||||
| `topdata` | `convert-topdata` | `convert-topdata` |
|
||||
| `wiki` | `build-wiki` | `build-wiki` |
|
||||
| `wiki` | `deploy-wiki` | `deploy-wiki` |
|
||||
|
||||
## Global commands (implemented)
|
||||
`module build-module` deliberately retains its narrower module-only behavior;
|
||||
it is not redirected to the broader `module build`.
|
||||
|
||||
## Global commands
|
||||
|
||||
```text
|
||||
crucible help | -h usage + builder list
|
||||
crucible version | -V build version (git sha via -ldflags)
|
||||
crucible list builders, one per line: name<TAB>bin<TAB>summary
|
||||
crucible config [args] -> legacy `config` command group
|
||||
crucible changelog [args] -> legacy `build-changelog`
|
||||
crucible version | -V build version
|
||||
crucible list name<TAB>binary<TAB>summary
|
||||
crucible config [args] inspect and validate effective configuration
|
||||
crucible changelog [args] generate a release changelog
|
||||
```
|
||||
|
||||
`crucible list` is machine-readable so CI can enumerate builders without parsing
|
||||
help text.
|
||||
|
||||
## HAK builder source modes
|
||||
## HAK build options
|
||||
|
||||
```text
|
||||
build-haks [--hak <hak-name> ...] [--archive <archive-name> ...]
|
||||
[--source-manifest <path>] [--content-addressed-root <path>]
|
||||
[--plan-only] [--skip-music] [--music-dataset <id> ...]
|
||||
crucible hak build
|
||||
[--hak <hak-name> ...]
|
||||
[--archive <archive-name> ...]
|
||||
[--source-manifest <path>]
|
||||
[--content-addressed-root <path>]
|
||||
[--plan-only]
|
||||
[--quiet|--verbose|--debug]
|
||||
```
|
||||
|
||||
When a source manifest contains `asset_sources`, pass
|
||||
`--content-addressed-root <directory>` (or
|
||||
`--content-addressed-root=<directory>`). Crucible resolves each declared SHA-256
|
||||
`--content-addressed-root <directory>`. Crucible resolves each declared SHA-256
|
||||
under `sha256/<first-two>/<next-two>/<full-sha256>` and verifies streamed bytes
|
||||
while writing the selected HAK archives. Source manifests without
|
||||
`asset_sources` continue to use the configured assets tree.
|
||||
|
||||
## Global flags (planned, at wiring time)
|
||||
|
||||
`--quiet`, `--verbose`, `--debug` (the legacy verbosity model), passed after the
|
||||
builder name, e.g. `crucible topdata build-topdata --verbose`.
|
||||
while writing selected HAK archives. Authored `.bmu` files are ordinary assets
|
||||
and require no conversion pipeline.
|
||||
|
||||
+24
-17
@@ -22,30 +22,37 @@ If none resolve, the wrapper exits non-zero with a Phase 5 message. The
|
||||
wrappers prefer the single-token `crucible-<name>` shim so `"$builder" args`
|
||||
quoting stays correct.
|
||||
|
||||
## In CI (preferred)
|
||||
## Parity contract — wrappers are not authoritative for build inputs
|
||||
|
||||
**OPERATOR NOTE: Stop. Do not do it this way.**
|
||||
**Use the pinned `prod.yml` version. That's what it's there for.**
|
||||
A consumer build wrapper may only:
|
||||
|
||||
~~Run the consumer job inside the pinned image and the binaries are on `PATH`:~~
|
||||
1. **validate** — a pre-build gate that does not change output;
|
||||
2. **publish** — a post-build step on a separate artifact;
|
||||
3. **resolve the crucible binary** — the bootstrap above.
|
||||
|
||||
```yaml
|
||||
container:
|
||||
image: registry.westgate.pw/deployment/crucible:<sha> # pinned, immutable
|
||||
```
|
||||
A wrapper MUST NOT fetch, resolve, generate, or stage any input Crucible reads to
|
||||
produce the artifact. If Crucible needs an input, Crucible acquires it. This makes
|
||||
the "no local-machine assumptions" + "same inputs → identical output" rules
|
||||
explicit: local `crucible <build>` and CI `crucible <build>` against the same
|
||||
committed config must produce identical bytes, with no CI-only pre-steps.
|
||||
|
||||
~~No host install, no `$HOME` layout, no developer machine assumptions.~~
|
||||
Builders read every input from the project tree. No NWN install is required. A
|
||||
builder that one day needs NWN game data auto-detects the install (the NWN home
|
||||
and Steam path are reliably discoverable), with an optional explicit override for
|
||||
non-standard layouts — never a required hand-set env.
|
||||
|
||||
## `NWN_ROOT` rule
|
||||
## In CI
|
||||
|
||||
Crucible **never guesses `NWN_ROOT` from `$HOME`** (this was a deploy-notes
|
||||
pain point). The NWN install/data root is passed explicitly:
|
||||
Builds run the Crucible **binary** — from the `crucible.sh` / `crucible.ps1`
|
||||
bootstrap (anonymous download) or, for Nix users, from the flake devshell pinned
|
||||
by `flake.lock`. CI runs the _same_ `crucible <build>` as local dev, inside the
|
||||
nix devshell (binary-cache fast). No build container, no token, no CI-only
|
||||
pre-steps.
|
||||
|
||||
- env `NWN_ROOT=/path/to/nwn`, or
|
||||
- flag `--nwn-root /path/to/nwn`.
|
||||
|
||||
A builder that needs `NWN_ROOT` and receives neither must fail closed with a
|
||||
clear message, not fall back to a home-directory default.
|
||||
The `registry.westgate.pw/deployment/crucible:<sha>` image is **deployment-only**:
|
||||
`prod.yml` pins it so tools travel with the runtime host (`nwn.enable`), a
|
||||
disabled placeholder until the NWN stack lands. It is **not** a build tool and no
|
||||
build/CI job consumes it.
|
||||
|
||||
## Determinism
|
||||
|
||||
|
||||
@@ -27,7 +27,7 @@ checklist is kept as the record of what was done.
|
||||
## Cutover steps
|
||||
|
||||
1. **Move the internal packages.** Bring `internal/{app,pipeline,project,erf,
|
||||
gff,topdata,music,changelog,validator}` from `gitea/sow-tools` into this repo.
|
||||
gff,topdata,changelog,validator}` from `gitea/sow-tools` into this repo.
|
||||
Re-home `internal/app` command wiring behind `internal/dispatch` instead of
|
||||
the old flag parser in `cmd/nwn-tool`.
|
||||
2. **Restore dependencies.** Add `golang.org/x/text` and `gopkg.in/yaml.v3` back
|
||||
@@ -43,10 +43,7 @@ checklist is kept as the record of what was done.
|
||||
artifacts and image layers only.
|
||||
6. **Carry the determinism test.** Port
|
||||
`TestBuildNativeKeepsGeneratedIDsStableAcrossRepeatedBuilds` and keep it green.
|
||||
7. **Runtime base for music.** If music conversion is wired, switch the final
|
||||
Docker stage from `distroless/static` to a debian-slim base that includes
|
||||
`ffmpeg`.
|
||||
8. **Update `make smoke`.** Once a builder is wired, change the smoke
|
||||
7. **Update `make smoke`.** Once a builder is wired, change the smoke
|
||||
expectation for it from exit `70` to a real success path.
|
||||
|
||||
## Wrapper contract carried over
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,346 @@
|
||||
# Crucible Command Surface Cleanup Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use
|
||||
> `superpowers:subagent-driven-development` or
|
||||
> `superpowers:executing-plans` to implement this plan task-by-task. Steps use
|
||||
> checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Replace repetitive Crucible command names with a descriptive,
|
||||
registry-driven public surface, retain long names as hidden aliases, and remove
|
||||
the unused music conversion pipeline without affecting authored `.bmu` assets.
|
||||
|
||||
**Architecture:** `internal/dispatch.Registry` gains explicit command records
|
||||
that drive lookup, help, and the interactive menu while translating to unchanged
|
||||
`internal/app` implementation names. The unused music subsystem is removed
|
||||
through its app, pipeline, project-config, validator, packaging, and consumer
|
||||
boundaries. The two active `sow-assets-manifest` call sites are decoupled in the
|
||||
same worktree session.
|
||||
|
||||
**Tech Stack:** Go 1.26, Go tests, Nix flakes, Dockerfile, Bash/Bats.
|
||||
|
||||
## Global Constraints
|
||||
|
||||
- Never commit or push.
|
||||
- Do not touch secrets or `.sops` data.
|
||||
- Keep `internal/dispatch.Registry` synchronized with
|
||||
`docs/command-surface.md`.
|
||||
- Preserve deterministic artifact behavior.
|
||||
- Preserve every existing long command as a hidden alias except all music
|
||||
commands, which are removed.
|
||||
- Preserve `.bmu` as an ordinary HAK asset.
|
||||
- Do not retain generated binaries or build output.
|
||||
|
||||
---
|
||||
|
||||
### Task 1: Registry-driven canonical commands and hidden aliases
|
||||
|
||||
**Files:**
|
||||
|
||||
- Modify: `internal/dispatch/dispatch.go`
|
||||
- Modify: `internal/dispatch/dispatch_test.go`
|
||||
|
||||
**Interfaces:**
|
||||
|
||||
- Produces a `Command` record owned by each `Builder`.
|
||||
- Produces lookup that returns the command record and exact app target.
|
||||
- Preserves `Builder.Legacy` only if required by migration tests; otherwise
|
||||
replaces it with command-derived compatibility metadata.
|
||||
|
||||
- [ ] Add failing tests covering the visible command table:
|
||||
`hak build/manifest`, `module build/extract/validate/compare/manifest`,
|
||||
`topdata validate/build/package/compare/convert`, and `wiki build/deploy`.
|
||||
- [ ] Add failing tests proving long aliases resolve to the same app target and
|
||||
that `module build-module` resolves to `build-module`, not `build`.
|
||||
- [ ] Add a failing test proving all music commands are rejected.
|
||||
- [ ] Run:
|
||||
|
||||
```bash
|
||||
nix develop --command go test ./internal/dispatch -run 'TestCanonical|TestHidden|TestMusic' -v
|
||||
```
|
||||
|
||||
Expected: failures caused by the old registry shape and exposed music
|
||||
commands.
|
||||
|
||||
- [ ] Introduce command records with fields equivalent to:
|
||||
|
||||
```go
|
||||
type Command struct {
|
||||
Name string
|
||||
Summary string
|
||||
AppCommand string
|
||||
Usage string
|
||||
Options []string
|
||||
HiddenAlias []string
|
||||
}
|
||||
```
|
||||
|
||||
Use the minimum final field names that keep lookup, menu, and help readable.
|
||||
|
||||
- [ ] Replace builder-level subcommand acceptance with command lookup. Translate
|
||||
canonical and hidden aliases to `AppCommand` before calling `app.Run`.
|
||||
- [ ] Define the approved command table and remove dispatcher music entries.
|
||||
- [ ] Re-run the focused dispatch tests and the full dispatch package:
|
||||
|
||||
```bash
|
||||
nix develop --command go test ./internal/dispatch -v
|
||||
```
|
||||
|
||||
Expected: pass.
|
||||
|
||||
### Task 2: Descriptive menu and command-specific help
|
||||
|
||||
**Files:**
|
||||
|
||||
- Modify: `internal/menu/menu.go`
|
||||
- Modify: `internal/menu/menu_test.go`
|
||||
- Modify: `internal/dispatch/dispatch.go`
|
||||
- Modify: `internal/dispatch/dispatch_test.go`
|
||||
|
||||
**Interfaces:**
|
||||
|
||||
- `menu.Item` carries display label, summary, dispatcher args, usage, and option
|
||||
lines.
|
||||
- `menu.Select` returns selected dispatcher args plus entered arguments.
|
||||
- Dispatcher command help uses the same registry guidance as the menu.
|
||||
|
||||
- [ ] Add failing menu tests proving descriptions start at one aligned column
|
||||
for labels of different lengths and hidden aliases never enter `menuItems()`.
|
||||
- [ ] Add a failing test selecting a command and asserting output contains its
|
||||
usage, option guidance, and
|
||||
`arguments (press Enter to use defaults):`.
|
||||
- [ ] Add failing dispatcher tests proving
|
||||
`crucible topdata build --help` returns `0`, prints canonical usage/options,
|
||||
and does not delegate into project loading.
|
||||
- [ ] Run:
|
||||
|
||||
```bash
|
||||
nix develop --command go test ./internal/menu ./internal/dispatch -run 'Test.*Menu|Test.*Help' -v
|
||||
```
|
||||
|
||||
Expected: fail against the current fixed-width menu and builder-only help.
|
||||
|
||||
- [ ] Compute menu label width from visible items and render descriptions with
|
||||
that width.
|
||||
- [ ] Print command guidance immediately after a selection and before reading
|
||||
optional arguments.
|
||||
- [ ] Add command-level help handling before app delegation.
|
||||
- [ ] Ensure builder help lists only canonical names and their summaries.
|
||||
- [ ] Run:
|
||||
|
||||
```bash
|
||||
nix develop --command go test ./internal/menu ./internal/dispatch -v
|
||||
```
|
||||
|
||||
Expected: pass.
|
||||
|
||||
### Task 3: Remove music from HAK app and pipeline
|
||||
|
||||
**Files:**
|
||||
|
||||
- Delete: `internal/pipeline/music.go`
|
||||
- Modify: `internal/pipeline/build.go`
|
||||
- Modify: `internal/pipeline/pipeline_test.go`
|
||||
- Modify: `internal/app/app.go`
|
||||
- Modify: `internal/app/app_test.go`
|
||||
- Delete: `internal/music/config.go`
|
||||
- Delete: `internal/music/credits.go`
|
||||
- Delete: `internal/music/metadata.go`
|
||||
- Delete: `internal/music/music.go`
|
||||
- Delete: `internal/music/music_test.go`
|
||||
- Delete: `internal/music/naming.go`
|
||||
|
||||
**Interfaces:**
|
||||
|
||||
- `pipeline.BuildHAKOptions` retains only HAK selection, source-manifest,
|
||||
content-addressed-root, and progress controls.
|
||||
- `pipeline.BuildResult` retains artifact fields unrelated to generated music
|
||||
credits.
|
||||
- `build-haks` accepts no music-specific flags.
|
||||
|
||||
- [ ] Replace music pipeline tests with a failing contract test that builds a
|
||||
HAK containing a pre-authored `.bmu` and proves the file appears in the
|
||||
manifest/archive without conversion setup.
|
||||
- [ ] Add failing parser tests proving `--skip-music` and `--music-dataset`
|
||||
return unknown-argument errors and help does not mention them.
|
||||
- [ ] Run:
|
||||
|
||||
```bash
|
||||
nix develop --command go test ./internal/app ./internal/pipeline -run 'Test.*Music|Test.*BMU|TestParseBuildHAK' -v
|
||||
```
|
||||
|
||||
Expected: at least the removed-flag assertions fail.
|
||||
|
||||
- [ ] Remove the app `music` command, argument parser, emitters, music HAK
|
||||
options, and credits console output.
|
||||
- [ ] Remove music fields from `BuildHAKOptions` and `BuildResult`.
|
||||
- [ ] Remove preparation/cleanup calls from HAK builds so asset collection reads
|
||||
authored resources directly.
|
||||
- [ ] Delete the music implementation packages and obsolete conversion tests.
|
||||
- [ ] Keep `.bmu` handling in ordinary ERF/resource and asset collection paths.
|
||||
- [ ] Run:
|
||||
|
||||
```bash
|
||||
nix develop --command go test ./internal/app ./internal/pipeline -v
|
||||
```
|
||||
|
||||
Expected: pass.
|
||||
|
||||
### Task 4: Remove music configuration and validation
|
||||
|
||||
**Files:**
|
||||
|
||||
- Modify: `internal/project/project.go`
|
||||
- Modify: `internal/project/effective.go`
|
||||
- Modify: `internal/project/project_test.go`
|
||||
- Modify: `internal/validator/validator.go`
|
||||
- Modify: `internal/validator/validator_test.go`
|
||||
|
||||
**Interfaces:**
|
||||
|
||||
- `project.Config` and `project.EffectiveConfig` contain no music section.
|
||||
- Project scanning uses only configured inventory extensions.
|
||||
- `.bmu` and `.wav` remain default asset extensions; `.mp3` and `.ogg` do not.
|
||||
|
||||
- [ ] Add or adjust failing project tests asserting default extensions include
|
||||
`.bmu` and `.wav` but exclude `.mp3` and `.ogg`.
|
||||
- [ ] Add a failing strict-decoder test proving a top-level `music:` field is
|
||||
rejected as unknown.
|
||||
- [ ] Run:
|
||||
|
||||
```bash
|
||||
nix develop --command go test ./internal/project ./internal/validator -run 'Test.*Music|Test.*AssetExtension|Test.*BMU' -v
|
||||
```
|
||||
|
||||
Expected: fail while music config is still accepted.
|
||||
|
||||
- [ ] Remove music config types, defaults, normalization, validation, accessors,
|
||||
effective config, provenance defaults, and environment overrides.
|
||||
- [ ] Simplify `Project.Scan` to accept assets solely from
|
||||
`effective.Inventory.AssetExtensions`.
|
||||
- [ ] Remove validator logic that exempts conversion source files.
|
||||
- [ ] Remove conversion-only `.mp3` and `.ogg` from default asset extensions;
|
||||
retain `.bmu` and `.wav`.
|
||||
- [ ] Remove obsolete music-focused project and validator tests.
|
||||
- [ ] Run:
|
||||
|
||||
```bash
|
||||
nix develop --command go test ./internal/project ./internal/validator -v
|
||||
```
|
||||
|
||||
Expected: pass.
|
||||
|
||||
### Task 5: Remove ffmpeg packaging and stale documentation
|
||||
|
||||
**Files:**
|
||||
|
||||
- Modify: `flake.nix`
|
||||
- Modify: `docker/Dockerfile`
|
||||
- Modify: `wrappers/crucible.sh`
|
||||
- Modify: `README.md`
|
||||
- Modify: `AGENTS.md`
|
||||
- Modify: `docs/command-surface.md`
|
||||
- Modify: `docs/migration-from-nwn-tool.md`
|
||||
|
||||
**Interfaces:**
|
||||
|
||||
- Nix development shell and image no longer contain ffmpeg.
|
||||
- Docker runtime contains only certificates and the non-root runtime support
|
||||
needed by current builders.
|
||||
- User docs advertise canonical short commands.
|
||||
|
||||
- [ ] Add a shell verification that fails while active packaging still mentions
|
||||
ffmpeg or removed command names:
|
||||
|
||||
```bash
|
||||
! rg -n 'ffmpeg|ffprobe|skip-music|music-dataset|module music|hak music' \
|
||||
flake.nix docker wrappers README.md AGENTS.md docs/command-surface.md
|
||||
```
|
||||
|
||||
- [ ] Remove `pkgs.ffmpeg-headless` and `ffmpeg` from Nix image/dev-shell
|
||||
contents and update comments.
|
||||
- [ ] Return the Docker runtime to a minimal base without ffmpeg while retaining
|
||||
CA certificates and non-root execution.
|
||||
- [ ] Remove the wrapper's `ffmpeg-free` wording.
|
||||
- [ ] Rewrite command-surface documentation around canonical names and a
|
||||
concise hidden-alias compatibility section.
|
||||
- [ ] Update README examples and ownership/status prose.
|
||||
- [ ] Re-run the search. Expected: no matches in active files.
|
||||
|
||||
### Task 6: Decouple active consumer scripts
|
||||
|
||||
**Files in `../sow-assets-manifest`:**
|
||||
|
||||
- Modify: `scripts/pack-haks.sh`
|
||||
- Modify: `scripts/build-local-haks.sh`
|
||||
- Modify: `tests/pack-haks.bats`
|
||||
- Modify only directly related active comments/tests if a focused test proves
|
||||
they require adjustment.
|
||||
|
||||
**Interfaces:**
|
||||
|
||||
- Both scripts call the hidden-compatible `hak build-haks` or canonical
|
||||
`hak build` without `--skip-music`.
|
||||
|
||||
- [ ] Read `../sow-assets-manifest/AGENTS.md` before editing.
|
||||
- [ ] Add or adjust a focused test to reject `--skip-music` in generated
|
||||
Crucible invocations.
|
||||
- [ ] Run the focused test and confirm it fails while the flag is present.
|
||||
- [ ] Remove `--skip-music` and its lockstep comments from both scripts.
|
||||
- [ ] Update the HAK parser test invocation to omit the removed flag.
|
||||
- [ ] Run:
|
||||
|
||||
```bash
|
||||
cd ../sow-assets-manifest
|
||||
nix develop --command bats tests/pack-haks.bats
|
||||
```
|
||||
|
||||
Expected: pass.
|
||||
|
||||
### Task 7: Full verification and scratchpad cleanup
|
||||
|
||||
**Files:**
|
||||
|
||||
- Modify: `../SCRATCHPAD.md`
|
||||
- Do not retain: `bin/`, `.cache/go-build/`, result symlinks, image tarballs, or
|
||||
other generated outputs.
|
||||
|
||||
- [ ] Run formatting:
|
||||
|
||||
```bash
|
||||
nix develop --command gofmt -w internal/app internal/dispatch internal/menu internal/pipeline internal/project internal/validator
|
||||
```
|
||||
|
||||
- [ ] Run full repository checks:
|
||||
|
||||
```bash
|
||||
nix develop --command make check
|
||||
make build
|
||||
make smoke
|
||||
```
|
||||
|
||||
- [ ] Run focused consumer verification from Task 6.
|
||||
- [ ] Verify deleted pipeline identifiers are absent from active code:
|
||||
|
||||
```bash
|
||||
rg -n 'BuildMusic|CreditsSummary|MusicDataset|skip-music|music-dataset|ffmpeg|ffprobe' \
|
||||
--glob '!docs/superpowers/**' .
|
||||
```
|
||||
|
||||
Expected: no matches.
|
||||
|
||||
- [ ] Verify active command docs/examples do not advertise long names:
|
||||
|
||||
```bash
|
||||
rg -n 'topdata (validate-topdata|build-topdata|build-top-package|compare-topdata|convert-topdata)|wiki (build-wiki|deploy-wiki)|hak (build-haks|apply-hak-manifest)' \
|
||||
README.md docs/command-surface.md
|
||||
```
|
||||
|
||||
Expected: matches only inside the explicitly labeled hidden-alias
|
||||
compatibility section.
|
||||
|
||||
- [ ] Inspect `git diff --check`, `git status --short`, and both repository
|
||||
diffs. Remove only generated files created by this work.
|
||||
- [ ] Remove the completed “Simplify crucible commands” to-do and the full
|
||||
“Crucible commands issue” section from `../SCRATCHPAD.md`; do not archive it.
|
||||
- [ ] Record exact verification commands and outcomes in the final handoff.
|
||||
|
||||
@@ -0,0 +1,858 @@
|
||||
# Topdata JSON Validation Tightening Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use
|
||||
> `superpowers:subagent-driven-development` (recommended) or
|
||||
> `superpowers:executing-plans` to implement this plan task-by-task. Steps use
|
||||
> checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Reject unsupported properties in established canonical topdata JSON
|
||||
containers, add the non-empty `any_present` global-injection condition, and keep
|
||||
standalone validation and direct native builds fail-closed and consistent.
|
||||
|
||||
**Architecture:** Add one focused `internal/topdata/json_contract.go` unit that
|
||||
owns deterministic closed-key checks plus parsing and evaluation of global
|
||||
injection condition groups. Existing validation in `topdata.go` uses that unit
|
||||
to collect actionable diagnostics, while `native.go` uses the same parsed
|
||||
conditions during ordered injection evaluation. Dataset payload rows continue
|
||||
through the existing column-aware canonicalization paths and are not assigned a
|
||||
global schema.
|
||||
|
||||
**Tech Stack:** Go 1.26, standard-library `encoding/json`, Go tests, Nix flakes,
|
||||
Make, authored JSON in the sibling `sow-topdata` checkout.
|
||||
|
||||
## Global Constraints
|
||||
|
||||
- Work on a fresh non-`main` branch and open all changes as a pull request.
|
||||
- Never commit or push secrets, `.sops` data, generated binaries, `.cache/`,
|
||||
`generated/`, HAK/TLK artifacts, or result symlinks.
|
||||
- Add no JSON Schema framework and no new validation dependency.
|
||||
- Close only the canonical base/plain rows root, canonical module root,
|
||||
`global.json` root, global injection, global condition, global default rule,
|
||||
global default match object, and global default format object.
|
||||
- Preserve specialized parser-owned JSON dialects outside those canonical
|
||||
containers.
|
||||
- Preserve dataset-specific columns and documented row controls inside `rows`,
|
||||
`entries`, `overrides`, injection `row`, and default `values`.
|
||||
- Preserve current-row, ordered-injection behavior: an earlier injection may
|
||||
satisfy or block a later injection.
|
||||
- Preserve empty-list behavior for `require_present` and `unless_present`;
|
||||
reject an empty `any_present`.
|
||||
- Do not add a compatibility alias for `when_present`.
|
||||
- Use deterministic property ordering in diagnostics.
|
||||
- Inventory evidence requires `compare_reference` to remain valid on
|
||||
base/plain rows roots: `internal/topdata/native.go` reads it,
|
||||
`internal/topdata/feat_migrate.go` emits it, and existing tests exercise it.
|
||||
This is an established canonical property covered by the design's moderate
|
||||
strictness rule even though the initial property table omitted it.
|
||||
- The active `sow-topdata` branch
|
||||
`fix-always-present-metamagic` already replaces the known `when_present` with
|
||||
`require_present` in commit `ed139f5`; preserve that change and use the
|
||||
checkout for integration validation rather than duplicating or rewriting it.
|
||||
|
||||
---
|
||||
|
||||
### Task 1: Introduce the shared closed-object and condition contract
|
||||
|
||||
**Files:**
|
||||
|
||||
- Create: `internal/topdata/json_contract.go`
|
||||
- Create: `internal/topdata/json_contract_test.go`
|
||||
|
||||
**Interfaces:**
|
||||
|
||||
- Produces:
|
||||
|
||||
```go
|
||||
type globalCondition struct {
|
||||
Field string
|
||||
ID string
|
||||
}
|
||||
|
||||
type globalConditionGroups struct {
|
||||
RequirePresent []globalCondition
|
||||
AnyPresent []globalCondition
|
||||
UnlessPresent []globalCondition
|
||||
}
|
||||
|
||||
func unsupportedObjectKeys(obj map[string]any, supported ...string) []string
|
||||
func unsupportedObjectKeysError(label string, obj map[string]any, supported ...string) error
|
||||
func parseGlobalConditionGroups(injection map[string]any) (globalConditionGroups, error)
|
||||
func globalConditionGroupsMatch(groups globalConditionGroups, rows []map[string]any) bool
|
||||
```
|
||||
|
||||
- `unsupportedObjectKeys` returns unknown keys in lexical order.
|
||||
- `parseGlobalConditionGroups` accepts absent groups, permits empty
|
||||
`require_present` and `unless_present`, rejects empty `any_present`, validates
|
||||
condition keys as exactly `field` and `id`, and returns the first deterministic
|
||||
error.
|
||||
- `globalConditionGroupsMatch` implements all/any/none semantics without
|
||||
reparsing JSON.
|
||||
|
||||
- [ ] **Step 1: Create the feature branch after checking it was not previously
|
||||
merged and deleted**
|
||||
|
||||
```bash
|
||||
git fetch origin
|
||||
git branch --show-current
|
||||
git ls-remote --heads origin feat/topdata-json-validation
|
||||
git log --all --oneline --decorate --grep='topdata json validation'
|
||||
```
|
||||
|
||||
Expected: current branch is not `main`. If
|
||||
`feat/topdata-json-validation` has never been used, create it from the approved
|
||||
design commit:
|
||||
|
||||
```bash
|
||||
git switch -c feat/topdata-json-validation
|
||||
```
|
||||
|
||||
If that exact branch name has prior remote history, choose a new descriptive
|
||||
name and record it in the PR; do not reuse the old branch.
|
||||
|
||||
- [ ] **Step 2: Re-run the canonical-format inventory before writing the
|
||||
allowlists**
|
||||
|
||||
```bash
|
||||
for f in $(find ../sow-topdata/data -type f -name 'base.json' | sort); do
|
||||
jq -r --arg f "${f#../sow-topdata/}" \
|
||||
'[$f, (keys | sort | join(","))] | @tsv' "$f"
|
||||
done
|
||||
|
||||
for f in $(find ../sow-topdata/data -type f -name 'global.json' | sort); do
|
||||
jq -r --arg f "${f#../sow-topdata/}" \
|
||||
'[$f, (keys | sort | join(","))] | @tsv' "$f"
|
||||
done
|
||||
|
||||
find ../sow-topdata/data -type f -path '*/modules/*.json' -print0 |
|
||||
while IFS= read -r -d '' f; do
|
||||
jq -r --arg f "${f#../sow-topdata/}" \
|
||||
'[$f, (keys | sort | join(","))] | @tsv' "$f"
|
||||
done | sort
|
||||
|
||||
rg -n 'compare_reference' internal/topdata ../sow-topdata/data
|
||||
```
|
||||
|
||||
Expected: active base roots use `columns`, `key`, `output`, and `rows`;
|
||||
active modules use `columns`, `entries`, `overrides`, or the legitimate
|
||||
`entries`+`overrides` combination; active globals use the approved global
|
||||
keys. `sow-tools` additionally confirms established `compare_reference`
|
||||
support.
|
||||
|
||||
- [ ] **Step 3: Write failing unit tests for deterministic unsupported-key
|
||||
diagnostics**
|
||||
|
||||
Add table-driven tests equivalent to:
|
||||
|
||||
```go
|
||||
func TestUnsupportedObjectKeysAreSorted(t *testing.T) {
|
||||
obj := map[string]any{
|
||||
"row": map[string]any{},
|
||||
"when_present": []any{},
|
||||
"aaa": true,
|
||||
}
|
||||
|
||||
got := unsupportedObjectKeys(
|
||||
obj,
|
||||
"row",
|
||||
"require_present",
|
||||
"any_present",
|
||||
"unless_present",
|
||||
)
|
||||
want := []string{"aaa", "when_present"}
|
||||
if !slices.Equal(got, want) {
|
||||
t.Fatalf("unsupportedObjectKeys() = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnsupportedObjectKeysErrorNamesObjectAndContract(t *testing.T) {
|
||||
err := unsupportedObjectKeysError(
|
||||
"global injection 10",
|
||||
map[string]any{"row": map[string]any{}, "when_present": []any{}},
|
||||
"row",
|
||||
"require_present",
|
||||
"any_present",
|
||||
"unless_present",
|
||||
)
|
||||
if err == nil {
|
||||
t.Fatal("expected unsupported-property error")
|
||||
}
|
||||
for _, want := range []string{
|
||||
"global injection 10",
|
||||
`"when_present"`,
|
||||
"row",
|
||||
"require_present",
|
||||
"any_present",
|
||||
"unless_present",
|
||||
} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Fatalf("error %q does not contain %q", err, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Write failing parser tests for all condition-group contracts**
|
||||
|
||||
Cover these exact cases in `json_contract_test.go`:
|
||||
|
||||
```go
|
||||
tests := []struct {
|
||||
name string
|
||||
injection map[string]any
|
||||
want globalConditionGroups
|
||||
wantError string
|
||||
}{
|
||||
{
|
||||
name: "all groups",
|
||||
injection: map[string]any{
|
||||
"require_present": []any{
|
||||
map[string]any{"field": "FeatIndex", "id": "feat:required"},
|
||||
},
|
||||
"any_present": []any{
|
||||
map[string]any{"field": "FeatIndex", "id": "feat:first"},
|
||||
map[string]any{"field": "FeatIndex", "id": "feat:second"},
|
||||
},
|
||||
"unless_present": []any{
|
||||
map[string]any{"field": "FeatIndex", "id": "feat:blocked"},
|
||||
},
|
||||
},
|
||||
want: globalConditionGroups{
|
||||
RequirePresent: []globalCondition{
|
||||
{Field: "FeatIndex", ID: "feat:required"},
|
||||
},
|
||||
AnyPresent: []globalCondition{
|
||||
{Field: "FeatIndex", ID: "feat:first"},
|
||||
{Field: "FeatIndex", ID: "feat:second"},
|
||||
},
|
||||
UnlessPresent: []globalCondition{
|
||||
{Field: "FeatIndex", ID: "feat:blocked"},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "empty existing groups remain valid",
|
||||
injection: map[string]any{
|
||||
"require_present": []any{},
|
||||
"unless_present": []any{},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "any present must be array",
|
||||
injection: map[string]any{
|
||||
"any_present": "feat:first",
|
||||
},
|
||||
wantError: "any_present must be an array",
|
||||
},
|
||||
{
|
||||
name: "any present must not be empty",
|
||||
injection: map[string]any{
|
||||
"any_present": []any{},
|
||||
},
|
||||
wantError: "any_present must contain at least one condition",
|
||||
},
|
||||
{
|
||||
name: "condition is closed",
|
||||
injection: map[string]any{
|
||||
"require_present": []any{
|
||||
map[string]any{
|
||||
"field": "FeatIndex",
|
||||
"id": "feat:required",
|
||||
"typo": true,
|
||||
},
|
||||
},
|
||||
},
|
||||
wantError: `require_present[0] contains unsupported key "typo"`,
|
||||
},
|
||||
{
|
||||
name: "field is required",
|
||||
injection: map[string]any{
|
||||
"require_present": []any{
|
||||
map[string]any{"id": "feat:required"},
|
||||
},
|
||||
},
|
||||
wantError: "require_present[0].field is required",
|
||||
},
|
||||
{
|
||||
name: "id is required",
|
||||
injection: map[string]any{
|
||||
"require_present": []any{
|
||||
map[string]any{"field": "FeatIndex"},
|
||||
},
|
||||
},
|
||||
wantError: "require_present[0].id is required",
|
||||
},
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 5: Write failing evaluator tests for all/any/none and group
|
||||
conjunction**
|
||||
|
||||
Build rows with `FeatIndex` references for `feat:required`,
|
||||
`feat:second`, and `feat:blocked`, then assert:
|
||||
|
||||
```go
|
||||
if !globalConditionGroupsMatch(globalConditionGroups{
|
||||
RequirePresent: []globalCondition{
|
||||
{Field: "FeatIndex", ID: "feat:required"},
|
||||
},
|
||||
AnyPresent: []globalCondition{
|
||||
{Field: "FeatIndex", ID: "feat:first"},
|
||||
{Field: "FeatIndex", ID: "feat:second"},
|
||||
},
|
||||
}, rowsWithoutBlocked) {
|
||||
t.Fatal("expected required + later any_present alternative to match")
|
||||
}
|
||||
|
||||
if globalConditionGroupsMatch(globalConditionGroups{
|
||||
AnyPresent: []globalCondition{
|
||||
{Field: "FeatIndex", ID: "feat:missing"},
|
||||
},
|
||||
}, rowsWithoutBlocked) {
|
||||
t.Fatal("expected missing any_present alternatives to reject")
|
||||
}
|
||||
|
||||
if globalConditionGroupsMatch(globalConditionGroups{
|
||||
UnlessPresent: []globalCondition{
|
||||
{Field: "FeatIndex", ID: "feat:blocked"},
|
||||
},
|
||||
}, rowsWithBlocked) {
|
||||
t.Fatal("expected unless_present match to reject")
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 6: Run the focused tests and confirm they fail**
|
||||
|
||||
```bash
|
||||
nix develop --command go test ./internal/topdata \
|
||||
-run 'TestUnsupportedObjectKeys|TestParseGlobalConditionGroups|TestGlobalConditionGroupsMatch' \
|
||||
-v
|
||||
```
|
||||
|
||||
Expected: build failure because the shared contract types and functions do
|
||||
not exist.
|
||||
|
||||
- [ ] **Step 7: Implement the minimal shared contract**
|
||||
|
||||
In `json_contract.go`, define fixed key sets and implement:
|
||||
|
||||
```go
|
||||
var (
|
||||
baseOrPlainRootKeys = []string{
|
||||
"output", "key", "columns", "rows", "compare_reference",
|
||||
}
|
||||
canonicalModuleRootKeys = []string{
|
||||
"output", "key", "columns", "entries", "overrides", "rows",
|
||||
}
|
||||
globalRootKeys = []string{
|
||||
"columns", "entries", "overrides", "defaults", "position", "injections",
|
||||
}
|
||||
globalInjectionKeys = []string{
|
||||
"row", "require_present", "any_present", "unless_present",
|
||||
}
|
||||
globalConditionKeys = []string{"field", "id"}
|
||||
globalDefaultRuleKeys = []string{"match", "values"}
|
||||
globalDefaultMatchKeys = []string{"source"}
|
||||
globalDefaultFormatKeys = []string{"format"}
|
||||
)
|
||||
```
|
||||
|
||||
`unsupportedObjectKeysError` must report one deterministic error per call,
|
||||
using the first lexically sorted unsupported key and a supported-key list in
|
||||
declaration order. Implement condition parsing through one private helper:
|
||||
|
||||
```go
|
||||
func parseGlobalConditionList(
|
||||
name string,
|
||||
raw any,
|
||||
requireNonEmpty bool,
|
||||
) ([]globalCondition, error)
|
||||
```
|
||||
|
||||
It must reject non-arrays, reject non-object elements, run the closed-key
|
||||
check before required-field checks, trim `field` and `id`, and enforce the
|
||||
`any_present` non-empty rule.
|
||||
|
||||
`globalConditionGroupsMatch` must call the existing
|
||||
`globalReferencePresent` helper and implement:
|
||||
|
||||
```text
|
||||
require_present: every condition is found
|
||||
any_present: at least one condition is found
|
||||
unless_present: no condition is found
|
||||
all groups: every present group passes
|
||||
```
|
||||
|
||||
- [ ] **Step 8: Run the focused and package tests**
|
||||
|
||||
```bash
|
||||
nix develop --command go test ./internal/topdata \
|
||||
-run 'TestUnsupportedObjectKeys|TestParseGlobalConditionGroups|TestGlobalConditionGroupsMatch' \
|
||||
-v
|
||||
nix develop --command go test ./internal/topdata
|
||||
```
|
||||
|
||||
Expected: pass.
|
||||
|
||||
- [ ] **Step 9: Commit the shared contract**
|
||||
|
||||
```bash
|
||||
git add internal/topdata/json_contract.go internal/topdata/json_contract_test.go
|
||||
git commit -m "feat(topdata): define canonical json contracts"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 2: Close canonical structural objects during project validation
|
||||
|
||||
**Files:**
|
||||
|
||||
- Modify: `internal/topdata/topdata.go`
|
||||
- Modify: `internal/topdata/json_contract_test.go`
|
||||
|
||||
**Interfaces:**
|
||||
|
||||
- Consumes the key sets and condition parser from Task 1.
|
||||
- `validateDataObject` selects exactly one root contract from file context:
|
||||
base/plain rows root, canonical module root, or `global.json` root.
|
||||
- A module validates every present supported container; it does not return
|
||||
after the first of `entries`, `overrides`, `rows`, or `columns`.
|
||||
- Specialized paths continue to return to their existing validators before
|
||||
canonical root closure.
|
||||
|
||||
- [ ] **Step 1: Add failing validation tests for unsupported canonical root
|
||||
properties**
|
||||
|
||||
Add table-driven temporary-project cases covering:
|
||||
- base root with `"typo": true`;
|
||||
- loose plain rows root with `"typo": true`;
|
||||
- module root with `"typo": true`;
|
||||
- `global.json` root with `"typo": true`.
|
||||
|
||||
For every case, call `ValidateProject`, require errors, and assert only stable
|
||||
semantic fragments: the file/object label, `typo`, and `unsupported`.
|
||||
|
||||
- [ ] **Step 2: Add failing validation tests for closed nested global objects**
|
||||
|
||||
Add cases for:
|
||||
|
||||
```json
|
||||
{"injections":[{"row":{},"when_present":[]}]}
|
||||
{"injections":[{"row":{},"require_present":[{"field":"FeatIndex","id":"feat:x","typo":true}]}]}
|
||||
{"defaults":[{"match":"all","values":{},"typo":true}]}
|
||||
{"defaults":[{"match":{"source":"entries","typo":true},"values":{}}]}
|
||||
{"defaults":[{"match":"all","values":{"ImpactScript":{"format":"ss_{id}","typo":true}}}]}
|
||||
```
|
||||
|
||||
Expected diagnostics name `global injection 0`, the condition label,
|
||||
`global default 0`, `global default 0 match`, or
|
||||
`global default 0 values.ImpactScript` respectively.
|
||||
|
||||
- [ ] **Step 3: Add failing validation tests for `any_present` shape**
|
||||
|
||||
Test one accepted non-empty list and rejected string/empty-list cases. The
|
||||
accepted case must use a declared dataset column in the injection row so this
|
||||
also protects payload flexibility.
|
||||
|
||||
- [ ] **Step 4: Add regression tests for canonical combinations and payload
|
||||
flexibility**
|
||||
|
||||
Build one table-driven test that validates without errors:
|
||||
- a columns-only module;
|
||||
- an entries-only module;
|
||||
- an overrides-only module;
|
||||
- a module containing both entries and overrides;
|
||||
- a rows module;
|
||||
- a current base root;
|
||||
- a loose plain rows root;
|
||||
- a base/plain root with `compare_reference: false`;
|
||||
- dataset columns plus `id`, `key`, `inherit`, `_tlk`, and `meta` in row-like
|
||||
payloads where their existing contracts allow them;
|
||||
- an injection row using a declared dataset column.
|
||||
|
||||
The combined entries+overrides case must contain an independently invalid
|
||||
override value first, prove validation catches it, then fix only that value
|
||||
and prove the module passes. This specifically prevents the current
|
||||
first-container early return.
|
||||
|
||||
- [ ] **Step 5: Run the focused tests and confirm failures**
|
||||
|
||||
```bash
|
||||
nix develop --command go test ./internal/topdata \
|
||||
-run 'TestValidateProject(RejectsUnsupportedCanonicalJSON|RejectsUnsupportedGlobalJSON|ValidatesEveryCanonicalModuleContainer|AcceptsCanonicalJSONContracts|AcceptsAnyPresent|RejectsInvalidAnyPresent)' \
|
||||
-v
|
||||
```
|
||||
|
||||
Expected: unknown keys are currently ignored, `any_present` is not validated,
|
||||
and the combined module does not validate all containers.
|
||||
|
||||
- [ ] **Step 6: Add a validation adapter for shared contract errors**
|
||||
|
||||
In `topdata.go`, add:
|
||||
|
||||
```go
|
||||
func reportTopdataContractError(path string, err error, report *ValidationReport) {
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
report.Diagnostics = append(report.Diagnostics, Diagnostic{
|
||||
Severity: SeverityError,
|
||||
Path: path,
|
||||
Message: err.Error(),
|
||||
})
|
||||
}
|
||||
```
|
||||
|
||||
Use `unsupportedObjectKeysError` for canonical roots and nested structural
|
||||
objects. Keep row payload validation in existing `validateRowCollection`,
|
||||
`validateEntriesFile`, and `validateOverridesFile` paths.
|
||||
|
||||
- [ ] **Step 7: Refactor canonical module dispatch to validate all containers**
|
||||
|
||||
After specialized-path checks, classify module files by directory context.
|
||||
For recognized canonical modules:
|
||||
1. check `canonicalModuleRootKeys`;
|
||||
2. call `validateColumnsFile` when `columns` is present;
|
||||
3. call `validateEntriesFile` when `entries` is present;
|
||||
4. call `validateOverridesFile` when `overrides` is present;
|
||||
5. call `validateRowsFile(path, obj, report, false)` when `rows` is present;
|
||||
6. emit the existing warning only when none of those containers is present.
|
||||
|
||||
Avoid duplicate `validateColumnsFile` calls by removing the nested column
|
||||
calls from this module dispatch path or by adding a caller-controlled flag;
|
||||
retain current behavior for callers outside canonical module dispatch.
|
||||
|
||||
- [ ] **Step 8: Close global roots, injections, defaults, matches, and format
|
||||
objects**
|
||||
- Run the global-root key check before validating present containers.
|
||||
- Run the injection key check before validating `row` and condition groups.
|
||||
- Replace calls to `validateGlobalConditionList` with
|
||||
`parseGlobalConditionGroups`; report its error through
|
||||
`reportTopdataContractError`.
|
||||
- Run default-rule and match-object key checks before existing type/value
|
||||
checks.
|
||||
- Treat an object containing `format` as a format object, reject every key
|
||||
other than `format`, and preserve non-format object literals as payload
|
||||
values.
|
||||
|
||||
- [ ] **Step 9: Run focused and full package tests**
|
||||
|
||||
```bash
|
||||
nix develop --command go test ./internal/topdata \
|
||||
-run 'TestValidateProject(RejectsUnsupportedCanonicalJSON|RejectsUnsupportedGlobalJSON|ValidatesEveryCanonicalModuleContainer|AcceptsCanonicalJSONContracts|AcceptsAnyPresent|RejectsInvalidAnyPresent)' \
|
||||
-v
|
||||
nix develop --command go test ./internal/topdata
|
||||
```
|
||||
|
||||
Expected: pass.
|
||||
|
||||
- [ ] **Step 10: Commit validation tightening**
|
||||
|
||||
```bash
|
||||
git add internal/topdata/topdata.go internal/topdata/json_contract_test.go
|
||||
git commit -m "feat(topdata): reject unsupported canonical json properties"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 3: Use shared condition parsing and `any_present` in native builds
|
||||
|
||||
**Files:**
|
||||
|
||||
- Modify: `internal/topdata/native.go`
|
||||
- Modify: `internal/topdata/json_contract_test.go`
|
||||
|
||||
**Interfaces:**
|
||||
|
||||
- Consumes `parseGlobalConditionGroups` and
|
||||
`globalConditionGroupsMatch` from Task 1.
|
||||
- Removes the old boolean-mode
|
||||
`globalConditionListMatches(name, raw, rows, required)` path.
|
||||
- Direct `BuildNativeWithOptions` calls fail before output mutation when
|
||||
canonical control syntax is invalid because the existing build entry point
|
||||
runs `ValidateProject` first and native evaluation uses the same parser.
|
||||
|
||||
- [ ] **Step 1: Add failing native-build tests for `any_present` alternatives**
|
||||
|
||||
Use one compact plain-dataset fixture with:
|
||||
- an authored row satisfying the first alternative;
|
||||
- a second subtest satisfying only the later alternative;
|
||||
- a third subtest satisfying neither alternative.
|
||||
|
||||
Assert the injected row exists in the first two outputs and is absent in the
|
||||
third. Inspect the built 2DA content rather than internal slices.
|
||||
|
||||
- [ ] **Step 2: Add failing native-build tests for condition-group
|
||||
conjunction**
|
||||
|
||||
Add subtests proving:
|
||||
- `require_present` + `any_present` injects only when both pass;
|
||||
- `any_present` + `unless_present` injects only when both pass;
|
||||
- existing `require_present` all semantics remain unchanged;
|
||||
- existing `unless_present` none semantics remain unchanged.
|
||||
|
||||
- [ ] **Step 3: Add a failing ordered-injection test**
|
||||
|
||||
Author injection 0 so it adds the row referenced by injection 1's
|
||||
`any_present`. Assert injection 1 applies. This protects evaluation against
|
||||
the same current-row view and ordered mutation used today.
|
||||
|
||||
- [ ] **Step 4: Add direct-build fail-closed tests**
|
||||
|
||||
Call `BuildNativeWithOptions` without a prior explicit `ValidateProject` call
|
||||
for fixtures containing:
|
||||
- `when_present` on an injection;
|
||||
- an unknown condition key;
|
||||
- an empty `any_present`;
|
||||
- an unknown canonical root key.
|
||||
|
||||
Require an error containing the stable object/property fragments and assert
|
||||
the expected output 2DA does not exist.
|
||||
|
||||
- [ ] **Step 5: Run focused tests and confirm failures**
|
||||
|
||||
```bash
|
||||
nix develop --command go test ./internal/topdata \
|
||||
-run 'TestBuildNative(AnyPresent|CombinesGlobalConditionGroups|UsesCurrentRowsForAnyPresent|RejectsUnsupportedGlobalControlSyntax|RejectsUnsupportedCanonicalRoot)' \
|
||||
-v
|
||||
```
|
||||
|
||||
Expected: `any_present` is ignored by current native evaluation and the new
|
||||
parser/evaluator is not wired.
|
||||
|
||||
- [ ] **Step 6: Replace native condition evaluation with the shared contract**
|
||||
|
||||
Replace `globalInjectionConditionsMatch` and
|
||||
`globalConditionListMatches` with:
|
||||
|
||||
```go
|
||||
func globalInjectionConditionsMatch(
|
||||
injection map[string]any,
|
||||
rows []map[string]any,
|
||||
) (bool, error) {
|
||||
groups, err := parseGlobalConditionGroups(injection)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return globalConditionGroupsMatch(groups, rows), nil
|
||||
}
|
||||
```
|
||||
|
||||
Keep `applyPlainDatasetGlobalInjections` calling this function immediately
|
||||
before deduplication/canonicalization, using `currentRows()` for each
|
||||
injection.
|
||||
|
||||
- [ ] **Step 7: Run focused, package, and repository tests**
|
||||
|
||||
```bash
|
||||
nix develop --command go test ./internal/topdata \
|
||||
-run 'TestBuildNative(AnyPresent|CombinesGlobalConditionGroups|UsesCurrentRowsForAnyPresent|RejectsUnsupportedGlobalControlSyntax|RejectsUnsupportedCanonicalRoot)' \
|
||||
-v
|
||||
nix develop --command go test ./internal/topdata
|
||||
nix develop --command go test ./...
|
||||
```
|
||||
|
||||
Expected: pass.
|
||||
|
||||
- [ ] **Step 8: Commit native `any_present` support**
|
||||
|
||||
```bash
|
||||
git add internal/topdata/native.go internal/topdata/json_contract_test.go
|
||||
git commit -m "feat(topdata): add any-present injection conditions"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 4: Document the global injection grammar
|
||||
|
||||
**Files:**
|
||||
|
||||
- Modify: `internal/topdata/CLASS_FEAT_GLOBAL_INJECTS_CONTRACT.md`
|
||||
|
||||
**Interfaces:**
|
||||
|
||||
- Documents the exact injection keys and all/any/none condition semantics.
|
||||
- States that all present groups must pass and that conditions observe current
|
||||
rows in injection order.
|
||||
- States that `any_present` must be a non-empty array and `when_present` is
|
||||
invalid.
|
||||
|
||||
- [ ] **Step 1: Add a documentation contract check**
|
||||
|
||||
Run this before editing:
|
||||
|
||||
```bash
|
||||
rg -n 'any_present|when_present|all.*any.*none|require_present.*unless_present' \
|
||||
internal/topdata/CLASS_FEAT_GLOBAL_INJECTS_CONTRACT.md
|
||||
```
|
||||
|
||||
Expected: no complete description of `any_present` and group conjunction.
|
||||
|
||||
- [ ] **Step 2: Update the manifest shape and semantics**
|
||||
|
||||
Add an example containing all three groups:
|
||||
|
||||
```json
|
||||
{
|
||||
"row": {
|
||||
"FeatIndex": { "id": "feat:example" }
|
||||
},
|
||||
"require_present": [{ "field": "FeatIndex", "id": "feat:required" }],
|
||||
"any_present": [
|
||||
{ "field": "FeatIndex", "id": "masterfeats:metamagic" },
|
||||
{ "field": "FeatIndex", "id": "masterfeats:combat" }
|
||||
],
|
||||
"unless_present": [{ "field": "FeatIndex", "id": "feat:example" }]
|
||||
}
|
||||
```
|
||||
|
||||
Document:
|
||||
- `require_present`: all listed references must exist;
|
||||
- `any_present`: at least one listed reference must exist and the list cannot
|
||||
be empty;
|
||||
- `unless_present`: none of the listed references may exist;
|
||||
- every present group must pass;
|
||||
- earlier injections affect later conditions;
|
||||
- supported injection keys are `row`, `require_present`, `any_present`, and
|
||||
`unless_present`;
|
||||
- `when_present` is rejected rather than aliased.
|
||||
|
||||
- [ ] **Step 3: Verify the documentation contains the complete contract**
|
||||
|
||||
```bash
|
||||
rg -n 'require_present|any_present|unless_present|when_present|earlier injection|every present group' \
|
||||
internal/topdata/CLASS_FEAT_GLOBAL_INJECTS_CONTRACT.md
|
||||
```
|
||||
|
||||
Expected: each semantic point appears in the contract.
|
||||
|
||||
- [ ] **Step 4: Commit documentation**
|
||||
|
||||
```bash
|
||||
git add internal/topdata/CLASS_FEAT_GLOBAL_INJECTS_CONTRACT.md
|
||||
git commit -m "docs(topdata): define global injection conditions"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 5: Validate the active topdata checkout and complete repository checks
|
||||
|
||||
**Files:**
|
||||
|
||||
- No authored files should change in `sow-tools`.
|
||||
- Preserve existing sibling change:
|
||||
`../sow-topdata/data/classes/feats/global.json`.
|
||||
- Do not retain: `bin/`, `.cache/`, `generated/`, `result`, HAK/TLK files, or
|
||||
temporary Crucible binaries.
|
||||
|
||||
**Interfaces:**
|
||||
|
||||
- The built Crucible validates the active `sow-topdata` checkout.
|
||||
- The active checkout's single-condition correction uses `require_present`;
|
||||
`any_present` remains available for future multi-alternative authoring.
|
||||
|
||||
- [ ] **Step 1: Format and run the full `sow-tools` checks**
|
||||
|
||||
```bash
|
||||
nix develop --command gofmt -w \
|
||||
internal/topdata/json_contract.go \
|
||||
internal/topdata/json_contract_test.go \
|
||||
internal/topdata/topdata.go \
|
||||
internal/topdata/native.go
|
||||
nix develop --command make check
|
||||
```
|
||||
|
||||
Expected: vet, Go tests, shellcheck, and yamllint pass.
|
||||
|
||||
- [ ] **Step 2: Build a temporary Crucible binary without tracking it**
|
||||
|
||||
```bash
|
||||
nix develop --command go build \
|
||||
-o /tmp/crucible-topdata-json-validation \
|
||||
./cmd/crucible
|
||||
```
|
||||
|
||||
Expected: `/tmp/crucible-topdata-json-validation` exists outside the
|
||||
repository.
|
||||
|
||||
- [ ] **Step 3: Confirm the active consumer syntax and repository state**
|
||||
|
||||
```bash
|
||||
git -C ../sow-topdata status --short --branch
|
||||
rg -n 'when_present|require_present|any_present|unless_present' \
|
||||
../sow-topdata/data/classes/feats/global.json
|
||||
```
|
||||
|
||||
Expected: the existing consumer branch contains `require_present` and no
|
||||
`when_present`. Do not alter unrelated consumer work.
|
||||
|
||||
- [ ] **Step 4: Validate and build active `sow-topdata` with the changed
|
||||
Crucible**
|
||||
|
||||
```bash
|
||||
(
|
||||
cd ../sow-topdata
|
||||
nix develop --command /tmp/crucible-topdata-json-validation topdata validate
|
||||
nix develop --command /tmp/crucible-topdata-json-validation topdata build
|
||||
)
|
||||
```
|
||||
|
||||
Expected: validation and build pass, global injections execute, and generated
|
||||
outputs remain ignored.
|
||||
|
||||
- [ ] **Step 5: Verify no generated files became tracked**
|
||||
|
||||
```bash
|
||||
git status --short
|
||||
git -C ../sow-topdata status --short
|
||||
git ls-files --error-unmatch bin generated .cache result 2>/dev/null
|
||||
git -C ../sow-topdata ls-files --error-unmatch generated .cache result 2>/dev/null
|
||||
```
|
||||
|
||||
Expected: the first two commands show only intentional source/doc changes;
|
||||
the `git ls-files --error-unmatch` commands find no newly tracked generated
|
||||
output. Remove only untracked artifacts created by this plan.
|
||||
|
||||
- [ ] **Step 6: Run final semantic searches and diff checks**
|
||||
|
||||
```bash
|
||||
rg -n 'when_present' internal/topdata --glob '!docs/superpowers/**'
|
||||
rg -n 'any_present' \
|
||||
internal/topdata/json_contract.go \
|
||||
internal/topdata/json_contract_test.go \
|
||||
internal/topdata/CLASS_FEAT_GLOBAL_INJECTS_CONTRACT.md
|
||||
git diff --check
|
||||
git status --short --branch
|
||||
git log --oneline origin/main..HEAD
|
||||
```
|
||||
|
||||
Expected: `when_present` appears only in rejection tests/documentation;
|
||||
implementation, tests, and contract document `any_present`; diff check is
|
||||
clean.
|
||||
|
||||
- [ ] **Step 7: Remove the temporary binary**
|
||||
|
||||
```bash
|
||||
rm -f /tmp/crucible-topdata-json-validation
|
||||
```
|
||||
|
||||
- [ ] **Step 8: Push the feature branch and open the required pull request**
|
||||
|
||||
```bash
|
||||
git push -u origin HEAD
|
||||
tea pr create \
|
||||
--title "feat: tighten topdata JSON validation" \
|
||||
--description $'## Summary\n- reject unsupported canonical topdata JSON properties\n- add non-empty any_present global injection conditions\n- keep validation and direct builds on one shared contract\n\n## Verification\n- nix develop --command make check\n- active sow-topdata validate and build with the changed Crucible'
|
||||
```
|
||||
|
||||
Expected: a PR targeting `main`; no production workflow or release tag is
|
||||
triggered.
|
||||
|
||||
- [ ] **Step 9: Record the handoff**
|
||||
|
||||
Report:
|
||||
- the `sow-tools` branch and PR URL;
|
||||
- the existing `sow-topdata` consumer branch/commit used for integration;
|
||||
- exact focused and full verification commands with outcomes;
|
||||
- confirmation that no generated files were tracked;
|
||||
- any diagnostics wording intentionally asserted as public semantic
|
||||
contract.
|
||||
@@ -0,0 +1,330 @@
|
||||
# Crucible Depot Core (Increment 1) Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Implement `crucible depot {status,push,verify,get,pull}` with local/cdn/bunny backends in sow-tools, then cut sow-assets-manifest over to it and delete the bash it replaces.
|
||||
|
||||
**Architecture:** New stdlib-only `internal/depot` package with its own `Run(args) int` entrypoint (exit contract 0/1/2/70 is richer than `internal/app.Run`'s 0/1, so dispatch special-cases depot instead of routing through `app.Run`). Backends implement a small interface; a sweep engine does parallel range-GET probes with serial re-confirm. Cutover repo work happens in sow-assets-manifest on its own branch.
|
||||
|
||||
**Tech Stack:** Go 1.26, stdlib `net/http`/`crypto/sha256`/`sync`, `gopkg.in/yaml.v3` (already a dep — add nothing to go.mod, vendorHash must not change).
|
||||
|
||||
**Spec:** `docs/superpowers/specs/2026-07-04-crucible-depot-core-design.md` — read it before any task; it is authoritative.
|
||||
|
||||
## Global Constraints
|
||||
|
||||
- **No new Go dependencies.** stdlib + existing `gopkg.in/yaml.v3` only.
|
||||
- **HEAD is banned** for existence checks. All probes: IPv4, `GET` with `Range: bytes=0-0`, expect `206` (accept any 2xx). A test must fail if HEAD is introduced.
|
||||
- **Never prompt, never read stdin** for credentials — read path included. Missing key → clear error on stderr, exit `70`.
|
||||
- **No cache of "what I uploaded"** — presence is always probed against the real target backend.
|
||||
- `unconfirmed` is a distinct state from `missing`; exit `2` when unconfirmed-only.
|
||||
- Exit codes: `0` clean, `1` drift (referenced-but-absent), `2` unconfirmed-only, `64` usage, `70` internal/backend error.
|
||||
- Blob key: `sha256/<sha[0:2]>/<sha[2:4]>/<sha>`; sha must match `^[0-9a-f]{64}$`.
|
||||
- Env names (unchanged from bash): `DEPOT_CDN_BASE`/`BUNNY_CDN_BASE` (cdn read base, default `https://cdn-a7f3k9.westgate.pw`), `BUNNY_STORAGE_HOST` (**required for bunny, no default** — spec overrides bash), `BUNNY_STORAGE_ZONE` (default `sow-assets-depot`), `BUNNY_STORAGE_READ_PASSWORD` (falls back to `BUNNY_STORAGE_PASSWORD`) for reads/probes, `BUNNY_STORAGE_PASSWORD` for PUT, `DEPOT_PROBE_JOBS` (default 16), `DEPOT_JOBS` (default 16), `DEPOT_CONNECT_TIMEOUT` (default 10s), `DEPOT_PROBE_MAX_TIME` (default 30s), `DEPOT_CONFIRM_RETRIES` (default 3), `DEPOT_CONFIRM_MAX` (default 200, 0 = unlimited).
|
||||
- Upload: `PUT` with `AccessKey: <write key>` and `Checksum: <UPPERCASE sha>` headers; each sha uploaded at most once per run; probe-before-put.
|
||||
- Every downloaded blob is re-hashed; deleted on mismatch.
|
||||
- `cdn` backend writes fail closed (error, never fake success).
|
||||
- Verify with `make vet test` in sow-tools (run inside `nix develop` if go missing from PATH).
|
||||
- Commit after each task. **Never commit to main.** sow-tools branch: `depot-core-increment-1`. sow-assets-manifest branch: `crucible-depot-cutover`. sow-tools has an unrelated modified spec file on main — leave it out of commits unless the task says otherwise.
|
||||
|
||||
---
|
||||
|
||||
### Task 1: Package skeleton — config + manifest parsing
|
||||
|
||||
**Files:**
|
||||
- Create: `internal/depot/config.go`, `internal/depot/config_test.go`
|
||||
- Create: `internal/depot/manifest.go`, `internal/depot/manifest_test.go`
|
||||
|
||||
**Interfaces (Produces):**
|
||||
|
||||
```go
|
||||
package depot
|
||||
|
||||
// config.go
|
||||
type Config struct {
|
||||
CDNBase string // DEPOT_CDN_BASE || BUNNY_CDN_BASE || default
|
||||
StorageHost string // BUNNY_STORAGE_HOST, no default
|
||||
StorageZone string // BUNNY_STORAGE_ZONE, default "sow-assets-depot"
|
||||
ReadKey string // BUNNY_STORAGE_READ_PASSWORD || BUNNY_STORAGE_PASSWORD
|
||||
WriteKey string // BUNNY_STORAGE_PASSWORD
|
||||
ProbeJobs int // DEPOT_PROBE_JOBS, default 16
|
||||
Jobs int // DEPOT_JOBS, default 16
|
||||
ConnectTimeout time.Duration // DEPOT_CONNECT_TIMEOUT seconds, default 10s
|
||||
ProbeMaxTime time.Duration // DEPOT_PROBE_MAX_TIME seconds, default 30s
|
||||
ConfirmRetries int // DEPOT_CONFIRM_RETRIES, default 3
|
||||
ConfirmMax int // DEPOT_CONFIRM_MAX, default 200 (0 = unlimited)
|
||||
}
|
||||
func LoadConfig(getenv func(string) string) Config
|
||||
|
||||
// manifest.go
|
||||
// ReferencedSHAs parses every *.yml in dir (schema: assets[].{path,sha256,size,restype,hak})
|
||||
// and returns the deduplicated sha set plus per-sha size (for pull).
|
||||
func ReferencedSHAs(dir string) (map[string]int64, error)
|
||||
func ValidSHA(s string) bool // ^[0-9a-f]{64}$
|
||||
func BlobKey(sha string) string // "sha256/ab/cd/<sha>"
|
||||
```
|
||||
|
||||
`LoadConfig` takes `getenv` so tests inject env without `t.Setenv` races. Invalid ints/durations in env → fall back to default (do not error). `ReferencedSHAs` errors on: unreadable dir, no `*.yml` files, YAML parse failure, or an entry whose `sha256` fails `ValidSHA` (report file + path). Manifest struct:
|
||||
|
||||
```go
|
||||
type manifestFile struct {
|
||||
Assets []struct {
|
||||
Path string `yaml:"path"`
|
||||
SHA256 string `yaml:"sha256"`
|
||||
Size int64 `yaml:"size"`
|
||||
} `yaml:"assets"`
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 1:** `git -C sow-tools switch -c depot-core-increment-1` (from main).
|
||||
- [ ] **Step 2:** Write failing tests: `TestLoadConfigDefaults`, `TestLoadConfigReadKeyFallback` (READ unset → WriteKey value; READ set → its own), `TestLoadConfigBadIntFallsBack`, `TestBlobKey` (`BlobKey("0b1d…")=="sha256/0b/1d/0b1d…"`), `TestValidSHA` (rejects uppercase, short, non-hex), `TestReferencedSHAs` (temp dir with two yml files sharing one sha → dedup; bad sha → error naming the file; empty dir → error).
|
||||
- [ ] **Step 3:** `go test ./internal/depot/` — verify FAIL (compile error is fine).
|
||||
- [ ] **Step 4:** Implement `config.go` + `manifest.go` minimally.
|
||||
- [ ] **Step 5:** `go test ./internal/depot/` PASS; `gofmt -l -w internal/depot`.
|
||||
- [ ] **Step 6:** Commit: `feat(depot): config resolution and manifest sha parsing`.
|
||||
|
||||
### Task 2: Local backend
|
||||
|
||||
**Files:**
|
||||
- Create: `internal/depot/backend.go`, `internal/depot/local.go`, `internal/depot/local_test.go`
|
||||
|
||||
**Interfaces (Produces):**
|
||||
|
||||
```go
|
||||
// backend.go
|
||||
type ProbeState int
|
||||
const (
|
||||
Present ProbeState = iota
|
||||
Absent
|
||||
Unconfirmed
|
||||
)
|
||||
type Backend interface {
|
||||
Name() string
|
||||
// Probe returns the existence state of one sha. transient=true means a
|
||||
// retry might change the answer (feeds the serial confirm loop).
|
||||
Probe(ctx context.Context, sha string) (state ProbeState, transient bool, err error)
|
||||
// Get fetches sha into dest (temp file + rename), re-hashes, deletes on mismatch.
|
||||
Get(ctx context.Context, sha, dest string) error
|
||||
// Put uploads bytes from src for sha. Read-only backends return an error.
|
||||
Put(ctx context.Context, sha, src string) error
|
||||
}
|
||||
|
||||
// local.go
|
||||
type LocalBackend struct{ Root string } // implements Backend; Name()=="local"
|
||||
```
|
||||
|
||||
Local semantics: `Probe` = `os.Stat(Root/BlobKey(sha))`, never transient. `Put` = copy to temp file in dest dir then `os.Rename` (MkdirAll parents). `Get` = copy out + re-hash (hash the source bytes while copying via `io.TeeReader` into `sha256.New()`), delete dest and error on mismatch. `err` from Probe only for real I/O errors (permission), not absence.
|
||||
|
||||
- [ ] **Step 1:** Failing tests: `TestLocalRoundTrip` (Put→Probe Present→Get→bytes equal), `TestLocalProbeAbsent`, `TestLocalGetHashMismatch` (hand-write a corrupt blob file at the keyed path, Get returns error and dest does not exist), `TestLocalPutIdempotent`.
|
||||
- [ ] **Step 2:** Verify FAIL. **Step 3:** Implement. **Step 4:** PASS + gofmt.
|
||||
- [ ] **Step 5:** Commit: `feat(depot): backend interface and local backend`.
|
||||
|
||||
### Task 3: Remote backends (cdn, bunny) with fake-Bunny tests
|
||||
|
||||
**Files:**
|
||||
- Create: `internal/depot/remote.go`, `internal/depot/remote_test.go`
|
||||
|
||||
**Interfaces (Consumes):** `Backend`, `ProbeState`, `Config`, `BlobKey`.
|
||||
**Produces:**
|
||||
|
||||
```go
|
||||
// NewBackend returns the backend for name ("local"|"cdn"|"bunny").
|
||||
// localRoot is used only for "local". Fails closed: "bunny" with empty
|
||||
// StorageHost or empty ReadKey returns an error (never prompts);
|
||||
// unknown name returns an error.
|
||||
func NewBackend(name, localRoot string, cfg Config) (Backend, error)
|
||||
|
||||
type httpBackend struct { ... } // shared by cdn and bunny
|
||||
```
|
||||
|
||||
Implementation requirements:
|
||||
|
||||
```go
|
||||
// IPv4-only transport — REQUIRED, spec bans IPv6/HEAD probe hazards.
|
||||
transport := &http.Transport{
|
||||
DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
d := net.Dialer{Timeout: cfg.ConnectTimeout}
|
||||
return d.DialContext(ctx, "tcp4", addr)
|
||||
},
|
||||
MaxIdleConnsPerHost: cfg.ProbeJobs,
|
||||
}
|
||||
client := &http.Client{Transport: transport, Timeout: cfg.ProbeMaxTime}
|
||||
```
|
||||
|
||||
- **Probe** (both remotes): `GET <base>/<BlobKey(sha)>` with header `Range: bytes=0-0`; drain/close body. 2xx → `Present`; 404/410 → `Absent`; anything else (incl. transport error) → `Unconfirmed, transient=true`. **Never http.Head / MethodHead.**
|
||||
- cdn: base = `cfg.CDNBase`, no auth headers, `Put` returns `errors.New("cdn backend is read-only")`.
|
||||
- bunny: probe/read against storage `https://<StorageHost>/<StorageZone>/<BlobKey(sha)>` with `AccessKey: <ReadKey>`. `Get`: try CDN base first, on non-2xx fall back to storage URL with ReadKey (mirrors bash `_depot_bunny_get`); re-hash, delete on mismatch. `Put`: probe first, skip if Present; else `PUT` storage URL, headers `AccessKey: <WriteKey>`, `Checksum: <strings.ToUpper(sha)>`, body streamed from src file; non-2xx → error. `Put` with empty WriteKey → error before any I/O.
|
||||
- Both `Get`s write via temp file + rename and re-hash exactly like local.
|
||||
|
||||
httptest note: `httptest.NewServer` listens on `127.0.0.1`, which the tcp4 dialer reaches — no test shim needed.
|
||||
|
||||
- [ ] **Step 1:** Failing tests against an `httptest` fake Bunny (a handler recording method+path+headers, serving a blob map):
|
||||
- `TestProbeUsesRangeGetNotHead` — probe a present sha; assert recorded request has `Method=="GET"` and `Range=="bytes=0-0"`; **also** assert the depot source contains no HEAD probe: the handler returns 500 for any HEAD request, and the test asserts zero HEAD requests were received across the whole test run (this is the anti-HEAD regression tripwire from field note 3).
|
||||
- `TestProbeStates` — 200/206→Present, 404→Absent, 428→Unconfirmed+transient, 503→Unconfirmed+transient.
|
||||
- `TestBunnyPutSequence` — Put of an absent sha: recorded sequence is [GET range probe, PUT]; PUT has `Checksum` = uppercase sha and `AccessKey` = write key; Put of a present sha: probe only, no PUT.
|
||||
- `TestReadWriteKeySplit` — probe uses ReadKey, PUT uses WriteKey (distinct values asserted).
|
||||
- `TestBunnyNoReadKeyFailsClosed` — `NewBackend("bunny",…)` with empty read key returns error **without reading stdin**: replace `os.Stdin` with the read end of an `os.Pipe()` for the test; after the call, write+close and confirm the pipe is undrained (or simply assert error occurs with no stdin interaction because the code path has no stdin reference — grep-based assertion acceptable: test fails if `internal/depot` sources mention `os.Stdin` or `/dev/tty`).
|
||||
- `TestBunnyNoStorageHostFailsClosed`.
|
||||
- `TestGetHashMismatchDeletes` — server returns wrong bytes; Get errors, dest removed.
|
||||
- `TestCDNPutReadOnly`.
|
||||
- [ ] **Step 2:** Verify FAIL. **Step 3:** Implement `remote.go`. **Step 4:** PASS + gofmt + `go vet ./...`.
|
||||
- [ ] **Step 5:** Commit: `feat(depot): cdn and bunny HTTP backends with fake-Bunny tests`.
|
||||
|
||||
### Task 4: Sweep engine — parallel probe + serial confirm
|
||||
|
||||
**Files:**
|
||||
- Create: `internal/depot/sweep.go`, `internal/depot/sweep_test.go`
|
||||
|
||||
**Interfaces (Consumes):** `Backend`, `Config`, `ProbeState`.
|
||||
**Produces:**
|
||||
|
||||
```go
|
||||
type SweepResult struct {
|
||||
Present []string
|
||||
Missing []string // confirmed absent (404/410 after retries)
|
||||
Unconfirmed []string // probe budget exhausted, state unknown
|
||||
}
|
||||
// Sweep probes every sha against b: one parallel pass (cfg.ProbeJobs workers,
|
||||
// sync.WaitGroup + buffered-channel work queue), then any non-Present result
|
||||
// is re-probed serially with linear backoff (attempt t sleeps t seconds,
|
||||
// cfg.ConfirmRetries attempts) — unless the non-Present count exceeds
|
||||
// cfg.ConfirmMax (>0), in which case the remainder is reported Unconfirmed
|
||||
// without serial re-probe (never collapsed into Missing).
|
||||
// A transient result that stays non-2xx after retries is Unconfirmed;
|
||||
// a 404/410 is Missing. Backend errors (non-transient) abort with error.
|
||||
func Sweep(ctx context.Context, b Backend, shas []string, cfg Config, progress io.Writer) (SweepResult, error)
|
||||
```
|
||||
|
||||
Sort `shas` before sweeping (deterministic output). Progress: print `probed X/N` roughly every 1000 blobs to `progress` (may be `io.Discard`). For tests, make the backoff sleeper injectable: package-level `var confirmSleep = time.Sleep`, overridden in tests.
|
||||
|
||||
- [ ] **Step 1:** Failing tests using a stub Backend (function fields):
|
||||
- `TestSweepAllPresent`, `TestSweepMissing` (404 → Missing, no retries beyond first serial confirm).
|
||||
- `TestSweepThrottledUnconfirmed` — stub always returns Unconfirmed/transient → after retries lands in `Unconfirmed`, never `Missing`; assert `confirmSleep` called with 1s then 2s.
|
||||
- `TestSweepTransientRecovers` — first probe 428, serial re-probe 206 → Present.
|
||||
- `TestSweepConfirmMaxCap` — 5 non-present, ConfirmMax=2 → all 5 Unconfirmed, zero serial re-probes.
|
||||
- `TestSweepParallelBounded` — 100 shas, ProbeJobs=4; stub counts concurrent in-flight probes (atomic), assert max ≤ 4.
|
||||
- [ ] **Step 2:** FAIL. **Step 3:** Implement. **Step 4:** PASS + gofmt.
|
||||
- [ ] **Step 5:** Commit: `feat(depot): parallel probe sweep with serial confirm and unconfirmed state`.
|
||||
|
||||
### Task 5: Commands + Run entrypoint + integration test
|
||||
|
||||
**Files:**
|
||||
- Create: `internal/depot/run.go`, `internal/depot/run_test.go`, `internal/depot/integration_test.go`
|
||||
|
||||
**Interfaces (Consumes):** everything above.
|
||||
**Produces:**
|
||||
|
||||
```go
|
||||
// Run executes a depot subcommand. args[0] is the subcommand
|
||||
// (status|push|verify|get|pull); returns the process exit code.
|
||||
// Exit: 0 clean, 1 drift, 2 unconfirmed-only, 64 usage, 70 internal.
|
||||
func Run(args []string, stdout, stderr io.Writer, getenv func(string) string) int
|
||||
```
|
||||
|
||||
Flag parsing: one `flag.FlagSet` per subcommand (stdlib; `internal/depot` is a fresh package, the hand-rolled loop in `internal/app` is not a constraint here). Flags per spec:
|
||||
|
||||
```
|
||||
status [--manifests DIR] [--source LOCAL_DEPOT] --target bunny|cdn|local
|
||||
push [--manifests DIR] --source LOCAL_DEPOT --target bunny
|
||||
verify [--manifests DIR] --target bunny|cdn [--sample N]
|
||||
get <sha> <dest> --target cdn|bunny|local
|
||||
pull [--manifests DIR] --dest DIR --target cdn|bunny
|
||||
```
|
||||
|
||||
`--manifests` default `assets`. `--target local` uses `--source` as root for status, and `--target`-side root comes from `DEPOT_DIR` env when target is local (matches bash `DEPOT_DIR`); document in usage string. Missing required flag / unknown subcommand / no args → usage on stderr, exit 64.
|
||||
|
||||
Command semantics (all build shas via `ReferencedSHAs`, backend via `NewBackend`, sweep via `Sweep`):
|
||||
|
||||
- **status**: sweep; print `referenced=<n> present=<n> missing=<n> unconfirmed=<n>` to stdout, then each missing sha on its own line prefixed `missing ` and each unconfirmed prefixed `unconfirmed `. Exit 1 if missing>0; else 2 if unconfirmed>0; else 0.
|
||||
- **push**: target must be `bunny` (enforce). Sweep; treat missing∪unconfirmed as to-upload (unconfirmed→re-upload is safe/idempotent per spec). For each, read bytes from `--source` local root; a sha absent from source → report and count as failed. Upload with `cfg.Jobs` parallel workers, each sha at most once. Exit 70 on upload error, 1 if any sha was missing from source, else 0. Print `uploaded=<n> failed=<n>`.
|
||||
- **verify**: sweep (exit 1/2 as status) **plus**, when clean, download `--sample` N (default 3, 0=skip) random blobs (`math/rand` seeded — pick via deterministic-enough `rand.New(rand.NewSource(time.Now().UnixNano()))`) to temp files via `Get` (which re-hashes); any failure → exit 1.
|
||||
- **get**: `Get(sha, dest)`; validate sha; exit 0/70.
|
||||
- **pull**: for every referenced sha, dest path `--dest/BlobKey(sha)`. Incremental: if dest file exists, re-hash it; matching → skip, mismatch → re-download. Download absent/mismatched with `cfg.Jobs` workers via `Get`. Print `pulled=<n> present=<n>`; any failed download → exit 70 after finishing the batch (report each failure). Non-zero missing at the source (404) → exit 1 listing them.
|
||||
|
||||
Wire the binary: **Modify** `internal/dispatch/dispatch.go` — in `runBuilder`, before the `Wired` check/`delegateLegacy` routing, special-case:
|
||||
|
||||
```go
|
||||
if b.Name == "depot" && b.Wired {
|
||||
return depot.Run(args[1:], os.Stdout, errw, os.Getenv)
|
||||
}
|
||||
```
|
||||
|
||||
(Exact insertion point: the implementer must read `runBuilder` and place this after subcommand resolution is bypassed — depot does its own subcommand parsing, so pass the raw args through, i.e. `args[1:]` where `args[0]=="depot"` at the dispatcher level, or the full arg slice in `RunBuilder("depot", …)` context. Verify both `crucible depot status …` and `crucible-depot status …` paths hit `depot.Run`.)
|
||||
|
||||
- [ ] **Step 1:** Failing unit tests (`run_test.go`): `TestRunUsage` (no args→64, unknown cmd→64, push --target cdn→64), `TestStatusExitCodes` (drift→1, unconfirmed-only→2 using a bunny backend pointed at an httptest server that 428s), `TestGetInvalidSHA` (→64 or 70, pick 64), `TestNoKeyStatusBunnyFailsClosedNoStdin` (getenv returns empty keys → exit 70, stderr mentions the env var name).
|
||||
- [ ] **Step 2:** Failing integration test (`integration_test.go`), pure local→local in temp dirs: build manifests yml referencing 3 blobs; source depot has all 3; target depot has 1. Then: `status --target local` → exit 1, missing=2 → `push` (local→local: for the test, allow `push --target local` **only** via an unexported hook? No — keep the spec surface: instead run the integration through bunny backend backed by httptest fake-Bunny with an in-memory blob map, which exercises the real push path) → `status` → 0 → `push` again → 0 uploads (idempotent, assert fake recorded no second PUT). Also `pull` into a pre-populated dest: present-and-hash-ok skipped (fake records no GET for it), corrupt pre-existing file re-downloaded.
|
||||
- [ ] **Step 3:** FAIL. **Step 4:** Implement `run.go`. **Step 5:** `make vet test` PASS + gofmt.
|
||||
- [ ] **Step 6:** Commit: `feat(depot): status/push/verify/get/pull commands with exit-code contract`.
|
||||
|
||||
### Task 6: Registry wiring + docs
|
||||
|
||||
**Files:**
|
||||
- Modify: `internal/dispatch/dispatch.go` (depot Registry entry)
|
||||
- Modify: `internal/dispatch/dispatch_test.go` (if it asserts wired/unwired sets)
|
||||
|
||||
Flip the depot entry:
|
||||
|
||||
```go
|
||||
{
|
||||
Name: "depot",
|
||||
Bin: "crucible-depot",
|
||||
Summary: "content-addressed asset depot (local/cdn/bunny)",
|
||||
Commands: []Command{
|
||||
{Name: "status", Summary: "report referenced-vs-present drift against a backend", Usage: "crucible depot status [--manifests DIR] [--source DIR] --target bunny|cdn|local"},
|
||||
{Name: "push", Summary: "upload referenced-but-absent blobs from a local depot", Usage: "crucible depot push [--manifests DIR] --source DIR --target bunny"},
|
||||
{Name: "verify", Summary: "existence sweep plus sampled download re-hash", Usage: "crucible depot verify [--manifests DIR] --target bunny|cdn [--sample N]"},
|
||||
{Name: "get", Summary: "fetch one blob with sha re-verify", Usage: "crucible depot get <sha> <dest> --target cdn|bunny|local"},
|
||||
{Name: "pull", Summary: "incremental verified pull of every referenced blob", Usage: "crucible depot pull [--manifests DIR] --dest DIR --target cdn|bunny"},
|
||||
},
|
||||
Wired: true,
|
||||
},
|
||||
```
|
||||
|
||||
The stale "(SeaweedFS)" wording must not survive anywhere: `grep -ri seaweedfs` across the repo and fix hits.
|
||||
|
||||
- [ ] **Step 1:** Update registry; run `go test ./internal/dispatch/` — fix any test that pinned depot as unwired; add/extend a test asserting `crucible depot` routes to `depot.Run` (e.g. `RunBuilder("depot", []string{"status"})` with no manifests dir returns 70/64, not the unwired 70 message — assert on stderr text).
|
||||
- [ ] **Step 2:** `make check` (vet+test+shellcheck+yamllint) PASS. Also `make build && ./bin/crucible depot` prints depot usage; `./bin/crucible` interactive menu shows depot entries (manual eyeball via `printf 'q\n' | ./bin/crucible` or equivalent).
|
||||
- [ ] **Step 3:** Commit: `feat(depot): wire depot into dispatch registry and menu`.
|
||||
|
||||
### Task 7: sow-assets-manifest cutover
|
||||
|
||||
**Repo:** `/home/vicky/Projects/westgate/repositories/migration/sow-assets-manifest`, new branch `crucible-depot-cutover` from main.
|
||||
|
||||
**Files:**
|
||||
- Delete: `scripts/mirror.sh`
|
||||
- Modify: `scripts/build-haks.sh` (the `--full` depot-verify block, ~L138-192), `Makefile` (`mirror` target, `haks` target), `.gitea/workflows/release.yml`, `.gitea/workflows/publish-haks.yml`, `.gitea/workflows/build-haks.yml`, `scripts/lib.sh` (header comments only), `AGENTS.md`/`README.md` (author-facing docs)
|
||||
|
||||
**Consumes:** the `crucible depot` CLI from Tasks 1–6. NOTE: CI resolves crucible via the flake input tracking sow-tools' default branch — the sow-tools branch must be merged before this repo's CI can pass. Local verification uses a locally built binary (`sow-tools/bin/crucible` on PATH). Do not run `nix flake update` here; note in the PR description that a `nix flake update sow-tools` (or lock bump) must land with/after the sow-tools merge.
|
||||
|
||||
Changes:
|
||||
|
||||
1. **`release.yml`** "Warm depot cache (verified full pull)" step body (keep name, job env unchanged; drop the ticker/compgen bash):
|
||||
```yaml
|
||||
- name: Warm depot cache (verified full pull)
|
||||
env:
|
||||
BUNNY_STORAGE_READ_PASSWORD: ${{ secrets.BUNNY_STORAGE_READ_PASSWORD }}
|
||||
BUNNY_STORAGE_HOST: storage.bunnycdn.com
|
||||
BUNNY_STORAGE_ZONE: sow-assets-depot
|
||||
run: |
|
||||
nix develop --command bash -c '
|
||||
set -euo pipefail
|
||||
crucible depot status --manifests assets --target bunny
|
||||
crucible depot pull --manifests assets --dest "$DEPOT_CACHE_DIR" --target cdn
|
||||
'
|
||||
```
|
||||
(If the `BUNNY_STORAGE_READ_PASSWORD` secret does not exist in the repo, use `secrets.BUNNY_STORAGE_PASSWORD` and say so in the PR body. Check what other steps use.)
|
||||
2. **`publish-haks.yml`** — same swap of the identical step.
|
||||
3. **`build-haks.yml`** — the PR-check step keeps `build-haks.sh --dry-run --no-verify --base "$base"` (structural check) and **adds** `crucible depot verify --manifests assets --target cdn --sample 3` as the cdn-backend verification.
|
||||
4. **`scripts/build-haks.sh`** — delete the depot existence-sweep + sample-download block inside `--dry-run` (the `depot_has_many`/`depot_get` verify at ~L138-185); replace with a comment pointing at `crucible depot verify`. Remove now-unused locals. `shellcheck scripts/build-haks.sh` must pass.
|
||||
5. **`Makefile`** — `mirror:` target now runs `crucible depot pull --manifests assets --dest "$(DEPOT_DIR)" --target cdn`. `haks:` gains a trailing non-fatal drift report: `-crucible depot status --manifests assets --target cdn` (leading `-` so drift never fails a local build) and an echo pointing at `crucible depot push --source workspace/depot --target bunny` as the standard post-edit step.
|
||||
6. **`scripts/lib.sh`** — no function deletions (caller analysis: every depot function used by mirror.sh/build-haks.sh retains Increment 2–3 callers). Add a header comment block at the top of the depot section: `# NOTE: this bash depot layer is being retired. Increment 1 moved mirror/verify to 'crucible depot' (sow-tools). import/sync go in Increment 2; export/pack/publish/artifact/cdn_purge in Increment 3. Do not add new callers.` **Do not touch the backend-keyed stat-cache in import.sh** (spec forbids it).
|
||||
7. **Docs** — in README.md (or AGENTS.md where depot workflow is documented): document `crucible depot status --manifests assets --target cdn` as the canonical credential-free "is my content live?" author check, and `crucible depot push --source workspace/depot --target bunny` as the standard post-edit publish step.
|
||||
|
||||
- [ ] **Step 1:** Branch; make all edits above.
|
||||
- [ ] **Step 2:** Verify: `shellcheck scripts/*.sh`; `yamllint .gitea`; `grep -rn "mirror.sh" . --exclude-dir=.git` returns nothing; with sow-tools' `bin` on PATH run `crucible depot status --manifests assets --target cdn` for real (credential-free) and confirm it produces a sane report (exit 0 or 2 acceptable; exit 1 means real drift — report it, don't hide it).
|
||||
- [ ] **Step 3:** Commit: `feat: cut depot mirror/verify over to crucible depot, retire mirror.sh`.
|
||||
|
||||
## Self-Review Notes
|
||||
|
||||
- Spec coverage: command surface (T5), drift model + unconfirmed/exit 2 (T4/T5), backends + env + no-prompt (T1/T3), IPv4 range probe + HEAD ban test (T3), Checksum PUT (T3), pull incremental (T5), cutover of all three workflows + Makefile + lib.sh headers + author docs (T7), registry/SeaweedFS fix + menu (T6), testing section requirements all mapped (T1–T5), interim contract `make haks` drift report (T7.5).
|
||||
- Deliberate deviations from spec, justified: `status --target local` root via `DEPOT_DIR`; push treats source-missing blobs as exit 1; pull download failure exits 70 (backend error) but source-404 exits 1 (drift).
|
||||
- Out of scope, per spec non-goals: import/sync, prune/gc/export, mdl checks, rich TUI, cdn_purge.
|
||||
@@ -0,0 +1,357 @@
|
||||
# Crucible build parity — local == CI, zero-config reproducible
|
||||
|
||||
Date: 2026-06-21
|
||||
Status: approved (design); implementation pending
|
||||
Repos touched: `sow-tools` (Crucible + contract doc + image notes),
|
||||
`sow-topdata` (config, flake, wrapper, CI), `sow-assets-manifest` (flake pin),
|
||||
and the consumer wrapper/flake pattern shared by every Crucible repo.
|
||||
|
||||
Supersedes the resolver-script approach in
|
||||
`2026-06-21-topdata-vfx-autogen-channel-design.md` (the channel→tag→`vfxs.yml`
|
||||
resolution moves out of a bash wrapper and into Crucible).
|
||||
|
||||
## Problem
|
||||
|
||||
`crucible topdata build-topdata` run by hand does **not** reproduce the CI
|
||||
build. CI runs `scripts/build-topdata.sh`, which resolves the accessory-VFX
|
||||
model list with `scripts/resolve-accessory-vfx.sh` and writes
|
||||
`.cache/sow-accessory-vfx-manifest.json` **before** Crucible runs; Crucible only
|
||||
reads that pre-staged file (`nwn-tool.yaml: manifest_file:`). Run Crucible
|
||||
directly and the file never exists, the `optional` consumer skips, and
|
||||
`visualeffects.2da` ships **zero accessory rows** — exactly the local build the
|
||||
operator observed (637 rows in CI, 0 locally).
|
||||
|
||||
Root cause: a **build-output-affecting input is resolved outside Crucible**, in
|
||||
a wrapper script. The wrapper, not Crucible, is authoritative for that input.
|
||||
|
||||
## Goal
|
||||
|
||||
Crucible is the single authority for everything that lands in a build artifact.
|
||||
A clone of any consumer repo, on any system, builds the same bytes CI builds —
|
||||
with no secret beyond what cloning already required and no hand-written
|
||||
configuration.
|
||||
|
||||
### Zero-config reproducibility requirements
|
||||
|
||||
- **R1 — No extra secrets.** All build-input reads are anonymous: the asset
|
||||
channel pointer (`releases/haks/channels.json`) and `vfxs.yml` are public CDN
|
||||
text. The crucible binary is fetchable anonymously — **verified**: the
|
||||
sow-tools release assets return HTTP 200 without auth, and the bootstrap tries
|
||||
anon before any token, so `CRUCIBLE_TOKEN` in consumer CI is **vestigial and is
|
||||
removed**. Git-LFS asset bytes use the **same** credentials the clone already
|
||||
used — never a separate key.
|
||||
- **R2 — No hand-written config.** Every default needed to build is baked into
|
||||
committed `nwn-tool.yaml` and Crucible defaults: CDN base, channel selection,
|
||||
source paths. The user types `crucible topdata build-topdata` and nothing else.
|
||||
- **R3 — No host data assumptions, no `NWN_ROOT`.** Builders read every input
|
||||
from the project tree (`nwn-tool.yaml` + committed `data/**/base.json`); no NWN
|
||||
install is required. `NWN_ROOT` is **dead** today — nothing in Crucible reads it
|
||||
(see *NWN_ROOT cleanup* below). The accessory build reads
|
||||
`data/visualeffects/base.json`. (Enforced by the CI parity guard, which runs in
|
||||
a bare checkout with no `NWN_ROOT`.)
|
||||
- **R4 — Nix is convenience, not a requirement.** The flake devshell provides
|
||||
`crucible`, `git-lfs`, `jq`, `yq`, etc. on `PATH` for Nix users. Non-Nix users
|
||||
(incl. Windows) get the same Crucible via the `crucible.sh`/`crucible.ps1`
|
||||
bootstrap and a host `git-lfs`; the build path through Crucible is identical.
|
||||
- **R5 — Same Crucible, both worlds.** Local dev and CI invoke the same Crucible
|
||||
subcommand against the same committed config. No CI-only pre-steps that change
|
||||
output.
|
||||
|
||||
## Parity contract (prevents recurrence)
|
||||
|
||||
Added to `sow-tools/docs/consumer-contract.md`. A consumer build wrapper may only:
|
||||
|
||||
1. **validate** — a pre-build gate that does not change output;
|
||||
2. **publish** — post-build, on a separate artifact;
|
||||
3. **resolve the crucible binary** — the bootstrap in `consumer-contract.md`.
|
||||
|
||||
A wrapper MUST NOT fetch, resolve, generate, or stage any input Crucible reads
|
||||
to produce the artifact. If Crucible needs an input, Crucible acquires it. This
|
||||
is the existing "no local-machine assumptions" + "same inputs → identical
|
||||
output" rule, made explicit: **the wrapper is not allowed to be authoritative
|
||||
for build inputs.**
|
||||
|
||||
## Design
|
||||
|
||||
### A. Accessory-VFX resolution moves into Crucible
|
||||
|
||||
**Config (`sow-topdata/nwn-tool.yaml`).** Drop `manifest_file:`; declare the
|
||||
source on the consumer. The 4-group + `mdl` filter is **not** re-declared — it
|
||||
is derived from the keys already present under `accessory_visualeffects.groups`:
|
||||
|
||||
```yaml
|
||||
autogen:
|
||||
consumers:
|
||||
- id: accessory_visualeffects
|
||||
producer: accessory_visualeffects
|
||||
dataset: visualeffects
|
||||
mode: accessory_visualeffects
|
||||
optional: true
|
||||
source:
|
||||
kind: cdn_channel
|
||||
cdn_base_env: BUNNY_CDN_BASE # default https://depot.westgate.pw
|
||||
channels_path: releases/haks/channels.json
|
||||
manifest_path: releases/haks/{tag}/vfxs.yml
|
||||
release_marker_path: releases/haks/{tag}/haks.json # the broken-release guard probe
|
||||
channel_env: SOW_TOPDATA_ASSET_CHANNEL # else derive from git tag
|
||||
offline_override_env: SOW_VFXS_MANIFEST # dev/air-gapped vfxs.yml path
|
||||
accessory_visualeffects:
|
||||
groups: { chest_accessories: {...}, head_accessories: {...},
|
||||
head_decorations: {...}, head_features: {...} }
|
||||
# ... rest of the policy block unchanged
|
||||
```
|
||||
|
||||
New struct field on `AutogenConsumerConfig` (sow-tools
|
||||
`internal/project/project.go`): `Source AutogenSourceConfig` with the keys
|
||||
above. Validation: when `source.kind == cdn_channel`, `channels_path` and
|
||||
`manifest_path` are required and `manifest_path` must contain `{tag}`.
|
||||
|
||||
**Resolution (`internal/topdata/autogen.go`,
|
||||
`resolveAutogenConsumerManifest`).** A new branch for `source.kind ==
|
||||
cdn_channel`, reusing the existing `fetchJSON` HTTP client and the project YAML
|
||||
parser:
|
||||
|
||||
1. **Offline override.** If `offline_override_env` names a readable file (or a
|
||||
manifest-repo checkout root containing `assets/vfxs.yml`), parse it and skip
|
||||
the network — the dev/air-gapped path.
|
||||
2. **Channel.** `channel_env` if set; else from the git tag / `GITHUB_REF_NAME`
|
||||
(`v*-*` → `testing`, `v*` → `current`, otherwise `current`).
|
||||
3. **CDN base.** `cdn_base_env` value, else the baked default.
|
||||
4. `GET {cdn}/{channels_path}` → JSON → `tag = channels[channel]`.
|
||||
5. `GET {cdn}/{manifest_path with {tag}}` → parse YAML `assets[]`.
|
||||
6. **Filter** to `restype == mdl` whose `path` is under `vfxs/<g>/` for `<g>` in
|
||||
the `accessory_visualeffects.groups` keys. Derive each entry's `source`
|
||||
(leading `vfxs/` stripped), `group`, `subgroup`, `model_stem`. Sort by
|
||||
`source`. Hand the entries to the existing accessory augmentor unchanged.
|
||||
|
||||
**Fail policy (ported verbatim from the resolver — this is the v0.1.4-bug
|
||||
guard).** Distinguish *unreachable* (fail open) from *broken* (hard fail):
|
||||
|
||||
- channels.json unreachable / not JSON / channel absent → **fail open**: the
|
||||
`optional` consumer contributes no rows and `data/visualeffects/lock.json` IDs
|
||||
are preserved.
|
||||
- vfxs.yml network error, or 404 **and** the `release_marker_path` (haks.json)
|
||||
is **absent** for that tag (no published release) → **fail open**.
|
||||
- vfxs.yml 404 **and** haks.json **present** → **HARD fail** ("release `<tag>`
|
||||
has haks.json but no vfxs.yml — accessory rows would silently vanish").
|
||||
- vfxs.yml present but malformed / no `assets` array → **HARD fail**.
|
||||
|
||||
Fail-open writes nothing and lets the optional consumer skip; hard-fail aborts
|
||||
the build with the message above.
|
||||
|
||||
**Deleted:** `sow-topdata/scripts/resolve-accessory-vfx.sh`,
|
||||
`tests/resolve-accessory-vfx-contract.sh`, the `manifest_file:` key, and the
|
||||
resolver pre-step at `build-topdata.sh:19`. The four bash contract cases
|
||||
(offline override, unreachable fail-open, broken-release hard-fail, no-release
|
||||
fail-open) are re-expressed as Go tests in `internal/topdata`.
|
||||
|
||||
### B. Git-LFS materialization moves into Crucible
|
||||
|
||||
So a bare clone + `crucible` packs real bytes, not pointer stubs (today only
|
||||
`build-topdata.sh` pulls LFS, so direct Crucible would ship stubs in CI's
|
||||
non-smudging checkout).
|
||||
|
||||
Before packing assets into a hak, Crucible: detects git-LFS pointer stubs under
|
||||
the asset tree; if any, runs `git lfs install --local` then `git lfs pull` in
|
||||
the repo; re-checks and **hard-fails** if stubs remain ("refusing to build from
|
||||
pointer stubs"). Uses the clone's own credentials (R1). Requires `git` +
|
||||
`git-lfs` on `PATH` — provided by the Nix devshell, a standard host install
|
||||
otherwise (a clone without git-lfs already yields stubs, so this is the existing
|
||||
host expectation, now enforced loudly by Crucible instead of a wrapper).
|
||||
|
||||
A skip control replaces `TOPDATA_SKIP_LFS` for the maintain-tree flow, which
|
||||
regenerates the `data/` tree and **discards** the package: a Crucible flag/env
|
||||
(e.g. `--skip-lfs` / `CRUCIBLE_SKIP_LFS`) so that one job can opt out. Default
|
||||
is fail-closed (pull).
|
||||
|
||||
### C. Wrappers thin out
|
||||
|
||||
`scripts/build-topdata.sh` (and the shared pattern in every Crucible repo)
|
||||
reduces to: validate (gate) → resolve crucible → `crucible <build>`. No input
|
||||
resolution, no LFS pull. `sow-module/scripts/package-module.sh` already matches
|
||||
this shape (validate → resolve → build); it stays the reference.
|
||||
|
||||
### D. CI parity guard (catches the next regression)
|
||||
|
||||
A `sow-tools` integration test builds the topdata package via Crucible **alone**
|
||||
in a bare checkout (no wrapper, clean `.cache`, no `NWN_ROOT`, anonymous CDN)
|
||||
and asserts `visualeffects.2da` contains accessory rows. This is the test that
|
||||
would have caught today's bug: it fails if any build-output-affecting step ever
|
||||
drifts back out of Crucible into a wrapper, and it enforces R2/R3/R5 mechanically.
|
||||
|
||||
The contract rule in `consumer-contract.md` is the written half; this test is
|
||||
the enforced half.
|
||||
|
||||
### E. NWN_ROOT cleanup (stale, contradicts the goal)
|
||||
|
||||
`NWN_ROOT` is unused — no `os.Getenv`, no `--nwn-root` flag, no consumer. The
|
||||
only traces are documentation/help-text describing a rule for a thing that does
|
||||
not exist: the `## NWN_ROOT rule` section in `consumer-contract.md`, the help
|
||||
text in `internal/dispatch/dispatch.go` ("if a builder ever needs an NWN
|
||||
root… crucible never guesses from $HOME"), and the `README.md` pointer. That
|
||||
old rule — *never guess, require an explicit env* — directly contradicts the
|
||||
zero-config goal (R2): it would make a future builder demand a hand-set path.
|
||||
|
||||
- **Remove** the `NWN_ROOT` references: the `consumer-contract.md` section, the
|
||||
dispatch help-text lines, the README pointer. Leave the
|
||||
`migration-from-nwn-tool.md` bullet as historical record.
|
||||
- **Replace the forward rule.** Should a future builder ever need NWN game data,
|
||||
it **auto-detects** it (the NWN home — containing `modules/`, `hak/`, `tlk/` —
|
||||
and the Steam install path are reliably discoverable), with an optional
|
||||
explicit override for non-standard layouts. Auto-detect first, never a required
|
||||
hand-set env. This keeps "clone → run → Just Work" intact even for a builder
|
||||
that one day reads game data. None do today.
|
||||
|
||||
### F. Toolchain convergence — one Crucible, three modes
|
||||
|
||||
Parity needs everyone to run the *same* Crucible. Today three mechanisms
|
||||
diverge: non-Nix uses the `crucible.sh` bootstrap download; sow-assets-manifest's
|
||||
flake pins `crucible v0.2.5` via `fetchurl`+hash; sow-topdata's flake ships no
|
||||
crucible at all (falls back to the bootstrap, with a needless `CRUCIBLE_TOKEN`).
|
||||
The pin is scattered (a hardcoded flake version, none in another flake, an image
|
||||
sha in `prod.yml`, a HEAD-floating CI binary build).
|
||||
|
||||
Converge on **one binary, one pin, three modes — all anonymous, all identical**:
|
||||
|
||||
1. **Non-Nix (local + CI fallback).** `crucible.sh` / `crucible.ps1` bootstrap
|
||||
downloads the release binary anonymously. `CRUCIBLE_TOKEN` is dropped from
|
||||
consumer CI (R1).
|
||||
2. **Nix (local + CI optimized).** Every consumer flake provides `crucible` in
|
||||
its devshell from a **sow-tools flake input**, pinned by that repo's
|
||||
`flake.lock`. Idiomatic, reproducible, no manual hashes; bump with
|
||||
`nix flake update`. This replaces sow-assets-manifest's manual `fetchurl`
|
||||
pin and **adds crucible to sow-topdata's devshell** (today it has none).
|
||||
3. **CI** runs the *same* `crucible <build>` as local, inside the nix devshell
|
||||
(binary-cache fast). No build container, no token, no CI-only pre-steps.
|
||||
|
||||
**Pin source of truth = `flake.lock` per repo**, bumped deliberately in a PR. We
|
||||
**reject floating "latest"**: it would let local drift from CI between releases
|
||||
and break reproducibility. The non-Nix bootstrap, which resolves "latest" by
|
||||
default, is the convenience path; reproducible/parity-critical builds pin (nix
|
||||
devshell, or `CRUCIBLE_*` pin env for the bootstrap).
|
||||
|
||||
### G. The crucible container is deployment-only
|
||||
|
||||
`sow-tools` builds and publishes `registry.westgate.pw/deployment/crucible:<sha>`
|
||||
on `v*` tags (`nix build .#image` → `skopeo`). It exists for **deployment** —
|
||||
`prod.yml` pins it so tools travel with the runtime host (`nwn.enable`), and it
|
||||
is a **disabled placeholder** until the NWN stack lands. No build/CI job consumes
|
||||
it, and the host-mode runners have no container runtime to run it. It is **not a
|
||||
build tool.**
|
||||
|
||||
- Keep the image build (cheap, `v*`-gated) for its deployment purpose.
|
||||
- **Fix `consumer-contract.md`:** remove the crossed-out "run CI inside the
|
||||
pinned image" guidance and its OPERATOR NOTE; state plainly that builds use the
|
||||
binary (bootstrap or nix devshell) and the image is deployment-only. This kills
|
||||
the "why do we build a container we don't use?" confusion.
|
||||
|
||||
### H. Autogen lock identity — model-anchored, immutable
|
||||
|
||||
**General lock contract (hard rule, all datasets).** A dataset lock maps a stable
|
||||
key to a row id. The build MUST: reuse the existing id for a key that still
|
||||
exists; allocate the first free id (lexicographic 0→1→2…) for a new key; and free
|
||||
an id only when its key is gone everywhere. An id, once assigned to a live key,
|
||||
never changes. (Already implemented for the accessory consumer via
|
||||
`historicalLockData` reuse + `nextAvailableAutogenID`; this section makes it the
|
||||
written rule and fixes the autogen-specific anchor below.)
|
||||
|
||||
**The autogen problem.** Autogen rows describe models that do **not** exist in
|
||||
`sow-topdata/data/` — they live in the asset `vfxs.yml` manifest. Today the lock
|
||||
key is the *config-derived presentation key*:
|
||||
`KeyFormat = {dataset}:{group}{delimiter}{category_segment}{stem}`. Group token,
|
||||
category, delimiter, case, and prefix-stripping all come from `nwn-tool.yaml`, so
|
||||
a **config-only** change (`key_format`, `delimiter`, `case`, a group rename,
|
||||
`category_from`, `strip_model_prefixes`) changes the key → the lock no longer
|
||||
matches → a new id is allocated → **every row shifts**. That is a config edit
|
||||
silently reshuffling ids for models that never changed.
|
||||
|
||||
**Fix — anchor the lock identity on the dataset + model, not the presentation.**
|
||||
The lock key for an autogen row becomes `{dataset}:{model_source_path}` — the
|
||||
stable `dataset` namespace prefix (kept as the 2da/namespace linker developers use
|
||||
instead of raw ids) plus the **model source path** from `vfxs.yml`, e.g.
|
||||
`visualeffects:head_accessories/hat/hfx_bandana.mdl` (leading `vfxs/` stripped —
|
||||
the path is the value already on `entry.Source`). Dropped from the key are only
|
||||
the config-derived parts (group token, `category_segment`, `delimiter`, case,
|
||||
prefix-stripping). The row's `key`/`label` *columns* still derive from config via
|
||||
`KeyFormat`/`LabelFormat` — only the **lock identity** is decoupled and
|
||||
model-anchored. Consequences:
|
||||
|
||||
- A config-only change keeps every id (the path is unchanged).
|
||||
- A model re-export (same path, new `sha256`) keeps its id — a content change is
|
||||
not an identity change. (Audit trail comes free from `git diff` on the committed
|
||||
`vfxs.yml`, which carries `path` + `sha256`; the lock stays a flat `string→int`
|
||||
map — no schema change, no fork of the shared lock format. A future `id`+`sha`
|
||||
object form is a clean later add if observability ever needs it, without
|
||||
changing id semantics.)
|
||||
- A renamed/moved/removed model is a genuinely different (or absent) identity.
|
||||
|
||||
**One-time cutover remap.** On the first build under `{dataset}:{path}` keys, for
|
||||
each current model compute its *old* config-key (with the current policy) and its
|
||||
new `{dataset}:{path}` key; if the new key is absent but the old key carries an id
|
||||
in the lock, adopt that id under the new key. So ids carry over with zero shift on
|
||||
cutover; the now-unreferenced old keys fall stale and are pruned (below).
|
||||
|
||||
**Stale pruning.** A lock key absent from the resolved entry set is dropped and
|
||||
its id freed for first-free reuse — but **only on a successful, non-empty
|
||||
resolution**. The fail-open paths (section A) contribute no entries and skip the
|
||||
augmentor entirely, so an unreachable/missing manifest can never mass-prune the
|
||||
lock. This realizes the general "free a fully-stale id" rule without risking the
|
||||
lock on a transient source outage.
|
||||
|
||||
**Testing.** Go tests in `internal/topdata`: config-only change (rename a group
|
||||
token, change `key_format`/`delimiter`/`case`) → ids unchanged; model re-export
|
||||
(same path, different sha) → id unchanged; model rename/remove → old id freed and
|
||||
reusable; cutover remap → existing ids preserved across the key-scheme switch;
|
||||
fail-open → no pruning, lock untouched.
|
||||
|
||||
## Determinism vs. channel mutability (decision)
|
||||
|
||||
The committed topdata commit pins everything **except** the asset channel
|
||||
pointer. `current` resolves through `channels.json`, which is intentionally
|
||||
mutable, so "reproducible" here means **same commit + same channel state →
|
||||
identical bytes**, not "this commit is frozen forever." This is deliberate: the
|
||||
accessory rows must track the asset HAKs shipped beside `sow_top.hak`, which is
|
||||
exactly what the channel pointer expresses. A given asset release **tag** is
|
||||
immutable; the pointer moving is a real input change, and `lock.json` preserves
|
||||
accessory IDs across it (no reshuffle). Builds against an explicit tag
|
||||
(`SOW_TOPDATA_ASSET_CHANNEL` set, or a `v*` ref) are fully pinned.
|
||||
|
||||
This satisfies R1–R5: no secret, no config, deterministic for a fixed input
|
||||
set, on any system.
|
||||
|
||||
## Testing
|
||||
|
||||
- **Go unit tests (`sow-tools/internal/topdata`)** for the `cdn_channel`
|
||||
resolution: offline-override path; unreachable channels.json → fail open;
|
||||
channel absent → fail open; vfxs.yml 404 + no haks.json → fail open; vfxs.yml
|
||||
404 + haks.json present → hard fail (asserts message); malformed vfxs.yml →
|
||||
hard fail; happy path → correct filtered/sorted entries (mdl-only, 4-groups-
|
||||
only, `vfxs/` stripped, group/subgroup/stem). Mirrors the deleted bash
|
||||
contract cases.
|
||||
- **Go config validation test** for the new `source` block.
|
||||
- **CI parity guard** (section D) in the topdata build workflow.
|
||||
- Existing accessory augmentor / lock-preserve tests unchanged.
|
||||
|
||||
## Rollout
|
||||
|
||||
1. sow-tools: add `AutogenSourceConfig`, the `cdn_channel` resolution branch,
|
||||
LFS materialization, the model-anchored lock identity + cutover remap + stale
|
||||
pruning (section H), the parity-guard test; update `consumer-contract.md`
|
||||
(add the parity contract, remove the `NWN_ROOT` rule per section E, replace the
|
||||
image guidance with deployment-only per section G).
|
||||
2. Release Crucible (assets already anon-downloadable — R1).
|
||||
3. Toolchain convergence (section F): expose `crucible` from a sow-tools flake
|
||||
input in each consumer flake — add it to sow-topdata's devshell, switch
|
||||
sow-assets-manifest off its manual `fetchurl` pin; drop `CRUCIBLE_TOKEN` from
|
||||
consumer CI; CI builds inside the nix devshell.
|
||||
4. sow-topdata: swap `manifest_file:` → `source:` in `nwn-tool.yaml`; delete the
|
||||
resolver script + bash contract test; thin `build-topdata.sh` (validate →
|
||||
resolve crucible → build); pin Crucible via `flake.lock`.
|
||||
5. Verify: bare clone → `crucible topdata build-topdata` → 637 accessory rows,
|
||||
no env, no key; and the same command inside `nix develop`.
|
||||
|
||||
## Follow-ups (not this work)
|
||||
|
||||
- Parts `2da` / `cachedmodels` reinstatement (separate spec) can reuse the same
|
||||
`cdn_channel` source type.
|
||||
- Roll the thinned-wrapper pattern review across `sow-module` / `sow-codebase`.
|
||||
@@ -0,0 +1,147 @@
|
||||
# Crucible Command Surface Cleanup Design
|
||||
|
||||
**Date:** 2026-06-25
|
||||
**Status:** Approved
|
||||
**Scope:** `sow-tools`, with the two active `sow-assets-manifest` call sites
|
||||
updated in lockstep
|
||||
|
||||
## Problem
|
||||
|
||||
Crucible exposes migrated `nwn-tool` implementation names directly. Several
|
||||
public commands repeat their builder name (`topdata build-topdata`,
|
||||
`wiki build-wiki`), the interactive menu repeats builder-level descriptions
|
||||
instead of explaining each action, and selected commands only prompt for
|
||||
unexplained "extra args".
|
||||
|
||||
The registry also exposes a migrated music-conversion subsystem that is not part
|
||||
of the current project workflow. Live HAK builds explicitly bypass it with
|
||||
`--skip-music`; authored `.bmu` files are already packed as ordinary assets.
|
||||
|
||||
## Decisions
|
||||
|
||||
### Canonical command names
|
||||
|
||||
The visible command surface is:
|
||||
|
||||
| Builder | Visible commands |
|
||||
| --- | --- |
|
||||
| `hak` | `build`, `manifest` |
|
||||
| `module` | `build`, `extract`, `validate`, `compare`, `manifest` |
|
||||
| `topdata` | `validate`, `build`, `package`, `compare`, `convert` |
|
||||
| `wiki` | `build`, `deploy` |
|
||||
|
||||
The dispatcher translates these names to the existing `internal/app`
|
||||
implementation commands. The app implementation names do not need to be
|
||||
renamed.
|
||||
|
||||
### Hidden compatibility aliases
|
||||
|
||||
Existing long names remain callable indefinitely but are omitted from the
|
||||
interactive menu and routine builder help:
|
||||
|
||||
- `hak build-haks`
|
||||
- `hak apply-hak-manifest`
|
||||
- `module build-module`
|
||||
- `module apply-hak-manifest`
|
||||
- `topdata validate-topdata`
|
||||
- `topdata build-topdata`
|
||||
- `topdata build-top-package`
|
||||
- `topdata compare-topdata`
|
||||
- `topdata convert-topdata`
|
||||
- `wiki build-wiki`
|
||||
- `wiki deploy-wiki`
|
||||
|
||||
Aliases preserve their existing implementation target. In particular,
|
||||
`module build-module` continues to run the module-only implementation; it must
|
||||
not silently become the broader `module build`.
|
||||
|
||||
### Registry model
|
||||
|
||||
`internal/dispatch.Registry` remains the single source of truth. Each wired
|
||||
builder owns command records containing:
|
||||
|
||||
- visible command name;
|
||||
- action-oriented summary;
|
||||
- underlying `internal/app` command;
|
||||
- command usage and option guidance;
|
||||
- hidden compatibility aliases, where applicable.
|
||||
|
||||
Execution, menu items, and builder help are generated from these records.
|
||||
Unknown commands continue to fail with exit `64`.
|
||||
|
||||
### Interactive behavior
|
||||
|
||||
The main menu:
|
||||
|
||||
- lists only visible commands;
|
||||
- aligns the description column from the longest visible label;
|
||||
- shows a distinct description for every command.
|
||||
|
||||
After selection, it prints the selected command's usage and available options,
|
||||
then prompts:
|
||||
|
||||
```text
|
||||
arguments (press Enter to use defaults):
|
||||
```
|
||||
|
||||
Arguments continue to use shell-like whitespace splitting, matching current
|
||||
behavior. Quoted argument parsing is out of scope.
|
||||
|
||||
`crucible <builder> <command> --help` prints the same command-specific guidance,
|
||||
returns success, and performs no project loading or build work.
|
||||
|
||||
### Remove the unused music pipeline
|
||||
|
||||
The music conversion subsystem is deleted rather than hidden:
|
||||
|
||||
- remove the `music` app and dispatcher commands;
|
||||
- remove `internal/music` and `internal/pipeline/music.go`;
|
||||
- remove music config, effective config, validation, environment overrides, and
|
||||
project accessors;
|
||||
- remove HAK music preparation, conversion, credits, generated-manifest result
|
||||
fields, `--skip-music`, and `--music-dataset`;
|
||||
- remove ffmpeg/ffprobe runtime and development dependencies;
|
||||
- remove music-pipeline documentation and wrapper wording.
|
||||
|
||||
`.bmu` remains a supported ordinary asset extension and continues through the
|
||||
same content-addressed HAK packing path as other authored files. Actual game
|
||||
content, category names, or prose that happens to use the word "music" is not
|
||||
part of this deletion.
|
||||
|
||||
The default asset-extension list drops conversion-only `.mp3` and `.ogg`.
|
||||
`.wav` remains because it is an NWN resource type independent of the deleted
|
||||
conversion pipeline.
|
||||
|
||||
Because `sow-assets-manifest` currently passes `--skip-music`, its
|
||||
`scripts/pack-haks.sh` and `scripts/build-local-haks.sh` call sites are updated
|
||||
in the same change before verification.
|
||||
|
||||
## Error handling
|
||||
|
||||
- Canonical and hidden alias lookup is deterministic within one builder.
|
||||
- Registry tests reject duplicate visible names or aliases.
|
||||
- Command help never delegates to `internal/app`.
|
||||
- Removed music commands and flags fail as unknown usage.
|
||||
- Existing project files containing a top-level `music:` configuration become
|
||||
invalid under strict YAML decoding. No active consumer configuration contains
|
||||
that field.
|
||||
|
||||
## Testing
|
||||
|
||||
Tests assert public contracts rather than exact incidental formatting:
|
||||
|
||||
- visible command names and hidden alias translation;
|
||||
- preservation of the distinct `module build-module` target;
|
||||
- hidden aliases absent from menu and normal help;
|
||||
- command-specific help exits successfully without project setup;
|
||||
- menu columns align and selected-command guidance is displayed;
|
||||
- `.bmu` remains accepted as an asset;
|
||||
- `.mp3`/`.ogg` are no longer default packable assets;
|
||||
- HAK builds no longer expose or execute music preparation;
|
||||
- ffmpeg is absent from package/image definitions;
|
||||
- both consumer scripts no longer pass `--skip-music`.
|
||||
|
||||
Final verification is `nix develop --command make check`, `make build`,
|
||||
`make smoke`, focused consumer tests, and repository-wide searches for deleted
|
||||
pipeline identifiers.
|
||||
|
||||
@@ -0,0 +1,269 @@
|
||||
# Topdata JSON Validation Tightening Design
|
||||
|
||||
**Date:** 2026-06-25
|
||||
**Status:** Implemented and reviewed
|
||||
**Scope:** `sow-tools`
|
||||
|
||||
## Problem
|
||||
|
||||
Topdata JSON validation checks many required properties and value types, but it
|
||||
often ignores unknown structural properties. This allows plausible typos to
|
||||
pass validation and then be ignored by the builder.
|
||||
|
||||
The immediate failure occurred in
|
||||
`topdata/data/classes/feats/global.json`: an injection used
|
||||
`when_present` even though the implemented condition is `require_present`.
|
||||
Validation succeeded, the builder ignored `when_present`, and the injection was
|
||||
therefore applied without the intended condition.
|
||||
|
||||
This is a fail-open authoring contract. A misspelled build-control property must
|
||||
not silently change generated game data.
|
||||
|
||||
## Goals
|
||||
|
||||
- Reject unsupported structural properties in canonical topdata JSON files.
|
||||
- Add an `any_present` global injection condition.
|
||||
- Keep standalone validation and direct native builds consistent.
|
||||
- Preserve dataset-specific row flexibility.
|
||||
- Tighten only established canonical formats; do not turn this work into a
|
||||
repository-wide schema rewrite.
|
||||
- Produce diagnostics that identify the file, object, and unsupported property.
|
||||
|
||||
## Non-goals
|
||||
|
||||
- No schema migration or formatting change to authored topdata.
|
||||
- No generic JSON Schema framework or new validation dependency.
|
||||
- No blanket closure of every specialized JSON dialect.
|
||||
- No change to generated feat family, registry, spellbook, TLK, or other
|
||||
specialized formats unless they use the canonical containers covered here.
|
||||
- No restriction of row values beyond existing column, metadata, inheritance,
|
||||
reference, TLK, and authoring-sugar contracts.
|
||||
- No compatibility alias for `when_present`; it is invalid syntax.
|
||||
|
||||
## Design
|
||||
|
||||
### Validation boundary
|
||||
|
||||
Structural objects use closed property sets. Dataset payload objects remain
|
||||
dataset-aware.
|
||||
|
||||
Closed structural objects include:
|
||||
|
||||
| Object | Supported properties |
|
||||
| --- | --- |
|
||||
| Base or plain rows-file root | `output`, `key`, `columns`, `rows` |
|
||||
| Canonical module root | `output`, `key`, `columns`, `entries`, `overrides`, `rows` |
|
||||
| `global.json` root | `columns`, `entries`, `overrides`, `defaults`, `position`, `injections` |
|
||||
| Global injection | `row`, `require_present`, `any_present`, `unless_present` |
|
||||
| Global condition | `field`, `id` |
|
||||
| Global default rule | `match`, `values` |
|
||||
| Global default match object | `source` |
|
||||
| Global default format object | `format` |
|
||||
|
||||
The allowed root set is selected from the file's existing canonical context;
|
||||
properties from unrelated formats are not accepted merely because another
|
||||
topdata file type supports them.
|
||||
|
||||
When a canonical root contains more than one supported container, each present
|
||||
container must be validated. For example, a module containing both `entries`
|
||||
and `overrides` must not stop validation after recognizing `entries`.
|
||||
|
||||
The following remain payload objects rather than global allowlists:
|
||||
|
||||
- objects inside `rows`;
|
||||
- values inside `entries`;
|
||||
- row-like objects inside `overrides`;
|
||||
- an injection's `row`;
|
||||
- a default rule's `values`.
|
||||
|
||||
Those objects continue to accept declared dataset columns plus documented row
|
||||
control and metadata fields. The builder already rejects unknown row and entry
|
||||
columns when it canonicalizes a dataset. This work must reuse or align with
|
||||
that knowledge where practical, but must not invent a single fixed row schema
|
||||
across all datasets.
|
||||
|
||||
### Moderate strictness
|
||||
|
||||
This change closes only canonical containers whose grammar is already
|
||||
established by the parser and current authored data.
|
||||
|
||||
Before adding a closed root check, implementation must inventory active
|
||||
`sow-topdata` files and existing `sow-tools` fixtures for legitimate property
|
||||
combinations. A property used by an established canonical format must be added
|
||||
to that format's explicit set rather than removed from authored data merely to
|
||||
satisfy the validator.
|
||||
|
||||
Specialized parsers remain responsible for their own object shapes. Extending
|
||||
closed-property checks to those formats is separate work.
|
||||
|
||||
### Global injection conditions
|
||||
|
||||
Global injections support three condition groups:
|
||||
|
||||
- `require_present`: every listed condition must match a current row.
|
||||
- `any_present`: at least one listed condition must match a current row.
|
||||
- `unless_present`: no listed condition may match a current row.
|
||||
|
||||
When more than one group is present, every group must pass. Conditions continue
|
||||
to use the existing shape:
|
||||
|
||||
```json
|
||||
{
|
||||
"field": "FeatIndex",
|
||||
"id": "feat:example"
|
||||
}
|
||||
```
|
||||
|
||||
Example:
|
||||
|
||||
```json
|
||||
{
|
||||
"row": {
|
||||
"FeatIndex": {
|
||||
"id": "feat:example"
|
||||
}
|
||||
},
|
||||
"require_present": [
|
||||
{
|
||||
"field": "FeatIndex",
|
||||
"id": "feat:base_requirement"
|
||||
}
|
||||
],
|
||||
"any_present": [
|
||||
{
|
||||
"field": "FeatIndex",
|
||||
"id": "masterfeats:metamagic"
|
||||
},
|
||||
{
|
||||
"field": "FeatIndex",
|
||||
"id": "masterfeats:combat"
|
||||
}
|
||||
],
|
||||
"unless_present": [
|
||||
{
|
||||
"field": "FeatIndex",
|
||||
"id": "feat:example"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
This injection applies only when the required feat exists, either listed
|
||||
masterfeat exists, and the injected feat does not already exist.
|
||||
|
||||
`any_present` must contain at least one condition. An empty list is an
|
||||
authoring error because it cannot express a useful successful condition.
|
||||
Existing empty-list behavior for `require_present` and `unless_present` is not
|
||||
changed by this work.
|
||||
|
||||
Conditions are evaluated against the same current-row view and in the same
|
||||
injection order used today. This preserves the existing behavior where an
|
||||
earlier injection can affect a later injection's conditions.
|
||||
|
||||
### Validation and build consistency
|
||||
|
||||
`ValidateProject` must report unsupported structural properties before a build.
|
||||
The native builder must also fail closed when invoked directly with the same
|
||||
invalid control syntax.
|
||||
|
||||
The implementation should centralize:
|
||||
|
||||
- deterministic unsupported-property checks;
|
||||
- global condition parsing and validation;
|
||||
- condition group evaluation.
|
||||
|
||||
Validation may collect multiple diagnostics while building returns the first
|
||||
blocking error. Both paths must accept and reject the same property names and
|
||||
condition shapes.
|
||||
|
||||
### Diagnostics
|
||||
|
||||
Diagnostics must name the containing object and unsupported property. Property
|
||||
lists must be deterministic.
|
||||
|
||||
For the original error, an acceptable diagnostic is:
|
||||
|
||||
```text
|
||||
global injection 10 contains unsupported key "when_present";
|
||||
supported keys are row, require_present, any_present, and unless_present
|
||||
```
|
||||
|
||||
Equivalent concise wording is acceptable if tests assert the stable semantic
|
||||
parts rather than the entire sentence.
|
||||
|
||||
## Compatibility
|
||||
|
||||
Current canonical `sow-topdata` roots fit the property sets above. The
|
||||
implementation must rerun the inventory immediately before tightening checks
|
||||
and run validation against the active `sow-topdata` checkout.
|
||||
|
||||
Unknown properties are not retained as compatibility behavior. They currently
|
||||
have no defined effect, and silently accepting them is the bug being fixed.
|
||||
Existing documented properties and dataset-specific row fields remain valid.
|
||||
|
||||
`require_present` and `unless_present` retain their current semantics.
|
||||
`any_present` is additive.
|
||||
|
||||
## Testing
|
||||
|
||||
Tests must follow red-green development and cover validation and direct-build
|
||||
paths.
|
||||
|
||||
### Unsupported properties
|
||||
|
||||
- Reject an unknown `global.json` root property.
|
||||
- Reject `when_present` and other unknown injection properties.
|
||||
- Reject an unknown condition property.
|
||||
- Reject unknown properties in default rules, match objects, and format
|
||||
objects.
|
||||
- Reject unknown canonical base/plain rows-file root properties.
|
||||
- Reject unknown canonical module root properties.
|
||||
- Continue accepting legitimate combinations such as:
|
||||
- a columns-only module;
|
||||
- entries-only and overrides-only modules;
|
||||
- a module containing both entries and overrides;
|
||||
- current base and plain rows-file roots.
|
||||
- Continue accepting dataset columns and documented row control fields inside
|
||||
rows, entries, overrides, and injection rows.
|
||||
|
||||
### `any_present`
|
||||
|
||||
- Validation accepts a non-empty `any_present` condition list.
|
||||
- Validation rejects a non-array or empty `any_present`.
|
||||
- A build injects when the first alternative is present.
|
||||
- A build injects when a later alternative is present.
|
||||
- A build skips the injection when no alternative is present.
|
||||
- `any_present` combines correctly with `require_present`.
|
||||
- `any_present` combines correctly with `unless_present`.
|
||||
- Existing `require_present` and `unless_present` behavior remains unchanged.
|
||||
- Direct building rejects unsupported condition syntax even when project
|
||||
validation was not called first.
|
||||
|
||||
### Integration
|
||||
|
||||
- Run focused `internal/topdata` tests.
|
||||
- Run `nix develop --command make check`.
|
||||
- Validate the active `sow-topdata` checkout with the changed Crucible.
|
||||
- Build topdata far enough to exercise global injections and confirm no
|
||||
generated files are tracked.
|
||||
|
||||
## Documentation
|
||||
|
||||
Update the class feat global injection contract to document `require_present`,
|
||||
`any_present`, and `unless_present`, including their all/any/none semantics and
|
||||
how multiple groups combine.
|
||||
|
||||
No broad validation guide is added unless implementation reveals an existing
|
||||
operator document that would otherwise become stale.
|
||||
|
||||
## Acceptance criteria
|
||||
|
||||
- `when_present` in a global injection fails validation and direct building.
|
||||
- `any_present` is valid only as a non-empty list of valid conditions.
|
||||
- `any_present` applies an injection when one or more alternatives exist and
|
||||
skips it when none exist.
|
||||
- Unknown canonical structural properties fail with actionable diagnostics.
|
||||
- Dataset-specific row fields remain valid when declared by the dataset.
|
||||
- Existing active `sow-topdata` validates and builds after replacing invalid
|
||||
syntax with the documented grammar.
|
||||
- No specialized topdata format is tightened outside the stated scope.
|
||||
@@ -0,0 +1,375 @@
|
||||
# Crucible Depot Core Design (Increment 1)
|
||||
|
||||
## Goal
|
||||
|
||||
Give Crucible real ownership of the content-addressed asset depot. Today
|
||||
`crucible-depot` is a registered-but-unwired stub (fails closed, exit `70`), and
|
||||
the actual depot logic lives as ~2000 lines of bash embedded in
|
||||
`sow-assets-manifest/scripts` — a direct violation of the Crucible principle that
|
||||
*artifact repos invoke Crucible through wrappers and never embed a toolkit*.
|
||||
|
||||
Increment 1 wires the depot **core** — blob backends plus `status` / `push` /
|
||||
`verify` / `get` — into `crucible depot`, and cuts the manifest repo over to call
|
||||
it directly. The bash depot layer is **retired, not wrapped**: one tool owns
|
||||
depot byte-motion for both local workflows and CI/CD, with complete parity.
|
||||
|
||||
## Why now (root cause that triggered this)
|
||||
|
||||
Release run 222 (`v0.1.5`) failed in 45s at the "Warm depot cache" step. PR #27's
|
||||
mdl-name normalization recompiled the corpus, producing **5903 net-new sha256s**
|
||||
in `assets/*.yml`. Sampling proved **15/15 net-new blobs return HTTP 404 on the
|
||||
CDN** while **10/10 old blobs return 200** — the CDN works; the new content was
|
||||
never uploaded. The blobs exist in the local depot (`workspace/depot`, 69,473
|
||||
blobs) but never reached Bunny. `mirror.sh` then 404s and dies — the release is
|
||||
correctly failing closed on a depot missing data.
|
||||
|
||||
Root cause of the *missing upload*: `import.sh`'s stat-cache
|
||||
(`.cache/import/<cat>.tsv`) records `(path,size,mtime,sha)` after **any** import
|
||||
and excludes "unchanged" files from the only upload call (`depot_put_many`) —
|
||||
**without tracking which backend those bytes actually landed in**. A prior
|
||||
`local` import poisons the cache so a later `bunny` import silently skips the
|
||||
upload. Combined with `depot_require_write` only normalizing `cdn`/unset→`bunny`
|
||||
(never `local`→anything), it is very easy to leave the manifests referencing
|
||||
blobs that exist only in the local depot.
|
||||
|
||||
The design principle that kills this bug class: **presence is always checked
|
||||
against the real target backend, never a cache.**
|
||||
|
||||
### Update 2026-07-04 — interim bash fix landed + corrected drift measurement
|
||||
|
||||
The bug above was fixed in bash ahead of this rewrite (this spec's Increment 2
|
||||
retires the stat-cache concept entirely, making the fix structural):
|
||||
[`sow-assets-manifest#29`](https://git.westgate.pw/ShadowsOverWestgate/sow-assets-manifest/pulls/29)
|
||||
keys the stat-cache dir by write backend + target
|
||||
(`.cache/import/<target>/`, `scripts/import.sh`), so a `local` import
|
||||
(`make haks`) can no longer mark a blob "done" in a cache the next `bunny` sync
|
||||
trusts. A cross-backend regression test guards it. When Increment 1/2 delete the
|
||||
bash depot layer, this becomes moot — but until then, **do not "simplify" the
|
||||
stat-cache back to a single shared dir**; that reintroduces the exact orphaning
|
||||
that broke run 222.
|
||||
|
||||
**Corrected drift numbers.** The "15/15 sampled net-new blobs 404" reading in the
|
||||
section above is a *sampling artifact*, not the real drift. The CDN edge
|
||||
(`cdn-a7f3k9.westgate.pw`) is IPv6-reachable but resets HTTP/2 from some hosts,
|
||||
and it throttles concurrent HEADs with `428`/`000` — which produce **both**
|
||||
false-404s and false-200s. A reliable sweep (force `curl -4`, 1-byte range GET
|
||||
`-r 0-0` → expect `206`, low concurrency, then serial re-confirm any non-2xx)
|
||||
shows the true missing set was **exactly 2 net-new blobs**, not ~5903:
|
||||
|
||||
- `item/_shared_aenea_weaponvfx/wblpl_fxshblk.mdl` (`0b1d3f…`)
|
||||
- `item/_shared_aenea_weaponvfx/wxdbsc_fxshpur.mdl` (`4e8d4f…`)
|
||||
|
||||
A 600-blob random sample of the pre-existing corpus was 100% present. `mirror.sh`
|
||||
still correctly failed the release — it dies on the first missing blob — but the
|
||||
gap was two files a local `make haks` had primed in the shared cache, not a
|
||||
wholesale upload failure. Both blobs were re-uploaded to Bunny from
|
||||
`workspace/depot` (present in the local depot, sha-verified), unblocking the
|
||||
`v0.1.5` re-run.
|
||||
|
||||
Takeaway for `crucible depot status`/`verify`: probe over IPv4, treat a single
|
||||
concurrent-probe non-2xx as *unconfirmed* (re-check serially) before reporting a
|
||||
blob missing, so the tool doesn't over-report drift the way an ad-hoc HEAD sweep
|
||||
did here.
|
||||
|
||||
### Field notes 2026-07-04 (content-side session) — issues the migration must fix
|
||||
|
||||
Surfaced while verifying a content edit (`over/ravenloft_potm_blood`, 18 blobs) had
|
||||
actually reached the CDN. All corroborate the design above; #1 is a new gap to close in
|
||||
Increment 1.
|
||||
|
||||
1. **`push` must be non-interactive — no tty prompt (new gap).** Today the only write
|
||||
path, `depot_require_write` (`lib.sh`), *prompts* for `BUNNY_STORAGE_PASSWORD` on a tty
|
||||
when it is unset. Headless agents and CI-without-a-pre-exported-secret cannot answer
|
||||
that prompt, so there is no way to push except with a human at a terminal — which is
|
||||
exactly why the current operational workaround is "an agent/human manually pushes each
|
||||
fixed asset." `crucible depot push` must take the write key **from env only**, fail
|
||||
closed with a clear message when it is absent, and **never prompt**. The Backends
|
||||
section lists the env names but does not state the no-prompt requirement — make it
|
||||
explicit and add a test that a missing write key exits non-zero without reading stdin.
|
||||
|
||||
2. **`status --target cdn` is the everyday author check, not only the release gate.** The
|
||||
spec frames `status` as the release pre-flight (correct), but the same command,
|
||||
credential-free against `cdn`, is the "did my edit actually reach the CDN?" check
|
||||
content authors need routinely. This session answered that question by hand-probing 18
|
||||
sha256s with `curl` — precisely the toil `crucible depot status --manifests assets
|
||||
--target cdn` should replace. Document it as the canonical author-facing "is my content
|
||||
live?" command so people stop ad-hoc probing.
|
||||
|
||||
3. **Probe reliability confirmed again — bake it into the tests.** Reproduced the HEAD
|
||||
hazard directly: a `curl -sI` HEAD sweep of the 18 blobs returned all-`200`; the
|
||||
prescribed reliable method (`curl -4`, 1-byte range GET `-r 0-0` → `206`, serial) also
|
||||
returned all-`206` with the fake-sha control at `404`. HEAD did not false-negative this
|
||||
time, but it stays the wrong tool. Make the IPv4 range-GET probe a hard requirement and
|
||||
add a test that fails if a HEAD-based existence check is introduced.
|
||||
|
||||
4. **Drift is recurring, not the single v0.1.5 incident.** Operationally, local→CDN sync is
|
||||
treated as "basically broken": every `make haks` / local import can re-orphan blobs from
|
||||
Bunny, so authors push by hand after edits. That raises Increment 1's priority and argues
|
||||
for making `crucible depot push` a standard post-edit step (or an on-edit CI hook), not
|
||||
only a release-time gate.
|
||||
|
||||
## Ownership principle (from the maintainer)
|
||||
|
||||
- Crucible **owns** the depot tools. No wrapper scripts preserved for their own
|
||||
sake.
|
||||
- Every consumer repo already has `crucible` in its flake; that is the only
|
||||
dependency. Consumers call `crucible depot …` directly (Makefile targets and
|
||||
CI workflow steps alike).
|
||||
- **Complete parity**: the same `crucible depot` commands serve local workflows
|
||||
and CI/CD. No divergent code paths, no "CI-only" or "local-only" logic.
|
||||
- Total clean-out over legacy preservation: bash depot code is **deleted** as its
|
||||
Go replacement lands, not kept as a fallback.
|
||||
|
||||
## Scope
|
||||
|
||||
This spec covers **Increment 1** only. Roadmap for context:
|
||||
|
||||
- **Increment 1 (this spec):** `crucible depot {status, push, verify, get, pull}`
|
||||
+ blob backends (`local` / `cdn` / `bunny`). Cut over `mirror`/verify/release
|
||||
gate. Delete the `lib.sh` depot functions that lose their last caller (see
|
||||
Cutover for what actually remains until Increments 2–3).
|
||||
- **Increment 2:** `crucible depot import` / `sync` (edit-tree → manifest+depot),
|
||||
replacing `import.sh` / `sync-assets.sh`. Retire the stat-cache concept.
|
||||
- **Increment 3:** mdl integrity checks, `mirror`/`pull` ergonomics, `prune`,
|
||||
`gc`, `export`. Delete remaining bash.
|
||||
|
||||
Each increment ships independently and deletes the bash it replaces.
|
||||
|
||||
## Command surface (Increment 1)
|
||||
|
||||
```
|
||||
crucible depot status [--manifests DIR] [--source LOCAL_DEPOT] --target bunny|cdn|local
|
||||
crucible depot push [--manifests DIR] --source LOCAL_DEPOT --target bunny
|
||||
crucible depot verify [--manifests DIR] --target bunny|cdn [--sample N]
|
||||
crucible depot get <sha> <dest> --target cdn|bunny|local
|
||||
crucible depot pull [--manifests DIR] --dest DIR --target cdn|bunny
|
||||
```
|
||||
|
||||
- `pull` is the bulk fetch that replaces `mirror.sh`: every referenced sha into
|
||||
`--dest/sha256/ab/cd/<sha>`, **incremental** (skip blobs already present with a
|
||||
matching hash — the release warm step re-runs against a persistent
|
||||
`/var/cache` dir holding ~69k blobs; a non-incremental pull would re-download
|
||||
the corpus every release), sha re-verified on download, parallel with the same
|
||||
probe/transfer bounds as `status`. Without `pull`, Increment 1 cannot delete
|
||||
`mirror.sh` (single-blob `get` is not a replacement).
|
||||
|
||||
- `--manifests` defaults to `assets/` (the repo's manifest dir). Confirmed
|
||||
decision: `crucible depot` parses `assets/*.yml` directly (same coupling model
|
||||
as `crucible-hak` reading hak manifests); the caller does not pre-extract sha
|
||||
lists.
|
||||
- `--source` is the local content-addressed store used as the byte source for
|
||||
`push` (e.g. `workspace/depot`).
|
||||
- Config/auth is env-first with flags as local aliases (see Backends).
|
||||
|
||||
## Drift model (the fix)
|
||||
|
||||
The manifest is the source of truth for **what** must exist; the target backend
|
||||
is the source of truth for **what does** exist; `push` reconciles them. There is
|
||||
no cache of "what I uploaded."
|
||||
|
||||
- **`status`** — collect every `sha256` referenced by `--manifests/*.yml`, run
|
||||
one parallel existence sweep against `--target`, report
|
||||
`{referenced, present, missing, unconfirmed}`. Non-zero exit if any referenced
|
||||
blob is missing. This is the release pre-flight gate.
|
||||
|
||||
**`unconfirmed` is a distinct state, not `missing`.** Any non-2xx from the
|
||||
parallel sweep is re-probed serially with backoff (the bash
|
||||
`depot_confirm_absent` behavior). If the serial re-probe budget is exhausted
|
||||
(bash caps at 200; keep a cap, `DEPOT_CONFIRM_MAX`), the remainder is reported
|
||||
as `unconfirmed` — **not** collapsed into `missing`. For `push` the
|
||||
distinction is harmless (unconfirmed → treat as absent → re-upload; wasteful
|
||||
but idempotent and safe). For `status` as the release gate it is critical: a
|
||||
throttled CDN edge must produce "N unconfirmed, retry" (distinct exit code,
|
||||
proposed `2`), never a false "thousands missing" hard-fail — exactly the
|
||||
over-reporting the corrected v0.1.5 drift measurement documented above.
|
||||
|
||||
**Target authority split:** `bunny` probes storage — the origin, authoritative,
|
||||
needs the read key; this is what the release gate uses. `cdn` probes the edge —
|
||||
credential-free, what players actually fetch, the author-facing "is my content
|
||||
live?" check; the edge cache can briefly lag storage (bash carries `cdn_purge`
|
||||
for this reason). A blob `present` on bunny but `missing` on cdn is
|
||||
propagation lag, not drift.
|
||||
- **`push`** — run `status`, then for each missing sha read the bytes from
|
||||
`--source` and upload to `--target`. Stateless and idempotent; re-running does
|
||||
nothing once clean. **Cannot silently skip an upload** — "should I upload this?"
|
||||
is answered by probing the target, not by trusting a local record.
|
||||
- **`verify`** — existence sweep plus a bounded random sample downloaded and
|
||||
re-hashed (the current `--full` tag-build check), decoupled from packing.
|
||||
- **`get`** — single-blob fetch with sha re-verify (backfill / debugging).
|
||||
|
||||
## Backends & config
|
||||
|
||||
Faithful port of `lib.sh` semantics — this is a re-home, not a redesign of the
|
||||
wire protocol.
|
||||
|
||||
| Backend | Read | Write | Role |
|
||||
|---|---|---|---|
|
||||
| `local` | filesystem `sha256/ab/cd/<sha>` | filesystem | dev/CI; byte source for `push` |
|
||||
| `cdn` | CDN GET + sha re-verify | ✗ read-only, fails closed | credential-free public reads |
|
||||
| `bunny` | CDN GET, storage fallback | Storage `PUT` | the only writable remote |
|
||||
|
||||
Layout: `blob_key(sha) = sha256/<sha[0:2]>/<sha[2:4]>/<sha>`.
|
||||
|
||||
Config (env names unchanged so existing CI secrets and `release.yml` keep working
|
||||
untouched; flags are local aliases):
|
||||
|
||||
- `DEPOT_CDN_BASE` / `BUNNY_CDN_BASE` — CDN read base.
|
||||
- `BUNNY_STORAGE_HOST`, `BUNNY_STORAGE_ZONE` — Bunny storage endpoint.
|
||||
- **Read/write key split (preserved):** `BUNNY_STORAGE_READ_PASSWORD` (falls back
|
||||
to `BUNNY_STORAGE_PASSWORD`) for probes/reads; `BUNNY_STORAGE_PASSWORD` for
|
||||
`PUT`. Note for the `release.yml` cutover: `status --target bunny` as the gate
|
||||
step needs at least the read key exported in that step's env.
|
||||
- **No prompting, read path included.** The no-prompt rule from field note 1
|
||||
applies to every credentialed operation, not just `push`: `status`/`verify`/
|
||||
`get` against `bunny` with no read key fail closed with a clear message —
|
||||
never a hang, never a tty read.
|
||||
- `BUNNY_STORAGE_HOST` stays **env-required, no baked default** — the bash never
|
||||
defaulted it (`depot_require_write` defaults only the zone
|
||||
`sow-assets-depot` and the CDN base); resolving the earlier open question.
|
||||
- Concurrency: `DEPOT_PROBE_JOBS` (read probes), `DEPOT_JOBS` (uploads);
|
||||
`DEPOT_CONNECT_TIMEOUT`, `DEPOT_PROBE_MAX_TIME` per-transfer bounds.
|
||||
|
||||
Invariants carried over exactly:
|
||||
|
||||
- **Existence probe** is an IPv4 1-byte range request (no body download) — for
|
||||
**all** backends. This is a deliberate deviation from `lib.sh`, whose `cdn`
|
||||
backend probes with HEAD (`_depot_cdn_probe_code`); the "faithful port"
|
||||
clause above does not extend to the probe method. HEAD is banned (see field
|
||||
note 3 and the test requirement there).
|
||||
- **Upload** is `PUT` with `Checksum: <UPPER-sha>` so Bunny rejects corrupt
|
||||
writes server-side; each sha uploaded at most once per run.
|
||||
- **Fail-safe:** a blob with no confirmed-present reply is treated as absent and
|
||||
(re)uploaded — never silently skipped. This is the invariant the stat-cache
|
||||
violated; here it is structural, not cached.
|
||||
- **`get`** re-hashes every downloaded blob and deletes on mismatch.
|
||||
- `cdn` write operations fail closed (never fake a write).
|
||||
|
||||
Stale-wording note to reconcile in code/docs: the dispatch registry summarizes
|
||||
depot as *"(SeaweedFS)"*, but the real depot is Bunny-CDN + local
|
||||
(`sow-assets-manifest/AGENTS.md`: "No S3, no SeaweedFS"). The Go implementation
|
||||
and the registry summary adopt the Bunny/local/cdn reality.
|
||||
|
||||
## Data flow
|
||||
|
||||
```
|
||||
assets/*.yml ──parse──▶ referenced sha set ─┐
|
||||
├─▶ target.HasMany() ─▶ missing set
|
||||
--target (bunny) ───┘ │
|
||||
▼
|
||||
--source (workspace/depot) ──read bytes──▶ target.PutMany()
|
||||
```
|
||||
|
||||
## Cutover / clean-out
|
||||
|
||||
Increment 1 deletes the bash it replaces (no wrappers left behind):
|
||||
|
||||
- `scripts/mirror.sh` → `crucible depot pull`; script removed.
|
||||
- `build-haks.sh`'s `--full` depot-verify block → `crucible depot verify`.
|
||||
- **All three depot-touching workflows switch in the same change** (not just
|
||||
`release.yml`):
|
||||
- `release.yml` "Warm depot cache (verified full pull)" →
|
||||
`crucible depot status --target bunny` (gate) then
|
||||
`crucible depot pull --dest $DEPOT_CACHE_DIR`; the inline bash
|
||||
ticker/compgen logic goes away.
|
||||
- `publish-haks.yml` mirrors the same warm+verify+pack flow — same swap.
|
||||
- `build-haks.yml` (PR check, cdn-backend verification) →
|
||||
`crucible depot verify --target cdn`.
|
||||
- Registry wiring: the `depot` entry in `internal/dispatch` flips to
|
||||
`Wired: true` with its `Commands` list populated (status/push/verify/get/pull)
|
||||
and the stale "(SeaweedFS)" summary corrected. This automatically puts depot
|
||||
in the interactive `crucible` menu alongside the other wired tools — no
|
||||
menu-specific code needed.
|
||||
|
||||
**What `lib.sh` deletion actually looks like.** The depot layer cannot be fully
|
||||
removed in Increment 1 — it is still referenced by scripts outside this
|
||||
increment's scope:
|
||||
|
||||
- `import.sh` / `sync-assets.sh` (`make haks`, `make import`, `make sync`) —
|
||||
Increment 2.
|
||||
- `export.sh` (`make pull`), `pack-haks.sh` (release-time `DEPOT_BACKEND=bunny`
|
||||
packing), `publish-release.sh` + `release_put`, and the path-addressed
|
||||
`artifact_has/get/put` + `cdn_purge` helpers — Increment 3 (these were
|
||||
previously unlisted; they are now explicitly Increment 3 scope).
|
||||
|
||||
Increment 1 deletes the functions whose last caller it removes (the
|
||||
mirror/verify probe-and-get paths) and leaves the rest with a header comment
|
||||
marking them scheduled for deletion in Increments 2–3. No compatibility shims,
|
||||
no new wrappers — but no pretending `lib.sh` dies before its last caller does.
|
||||
|
||||
Consumers call `crucible depot …` directly. Nothing new is wrapped.
|
||||
|
||||
### Interim contract for local builds (until Increment 2)
|
||||
|
||||
`make haks` keeps running bash `sync-assets.sh` with `DEPOT_BACKEND=local` until
|
||||
Increment 2 replaces import/sync. The gap this leaves — local imports creating
|
||||
blobs Bunny never sees — is held closed by two things:
|
||||
|
||||
- The backend-keyed stat-cache
|
||||
([`sow-assets-manifest#29`](https://git.westgate.pw/ShadowsOverWestgate/sow-assets-manifest/pulls/29))
|
||||
stays exactly as it is; do not "simplify" it (see the 2026-07-04 update above).
|
||||
- `make haks` gains a trailing `crucible depot status --manifests assets
|
||||
--target cdn` (credential-free, report-only, non-fatal) so drift is visible
|
||||
the moment it is created instead of at release time; and
|
||||
`crucible depot push --source workspace/depot --target bunny` is documented
|
||||
as the standard post-edit step (field note 4's ask, landed here). Upload when
|
||||
you want the content live; build local-only when you don't — but either way
|
||||
the drift is *reported*, never masked.
|
||||
|
||||
## Exit-code contract
|
||||
|
||||
Matches Crucible's fail-closed convention:
|
||||
|
||||
- `0` — clean / success.
|
||||
- `1` — **drift found** (`status`/`push` saw referenced-but-absent blobs), so
|
||||
`release.yml` can gate on `status` before packing.
|
||||
- `2` — **unconfirmed only**: no confirmed-missing blobs, but the probe budget
|
||||
ran out before every blob got a confirmed reply (throttled edge). The gate
|
||||
should retry, not report drift.
|
||||
- `70` — internal/backend error; never a faked result.
|
||||
|
||||
## Testing
|
||||
|
||||
- **Go unit tests** (`internal/depot`): `local` backend round-trip against a temp
|
||||
dir; manifest parsing; drift computation (referenced − present = missing) as
|
||||
table tests; backend selection + config resolution (incl. read/write key
|
||||
fallback).
|
||||
- **httptest fake-Bunny**: assert the range-probe-then-PUT sequence, the
|
||||
`Checksum: <UPPER-sha>` header, and the read/write key split — no real Bunny.
|
||||
- **Integration (local→local)**: `status` shows drift → `push` clears it →
|
||||
`status` clean → `push` again is a no-op (idempotency).
|
||||
- **New-surface tests**: fake-Bunny throttling (`428`) → blobs land in
|
||||
`unconfirmed`, exit `2`, never `missing`; `pull` into a pre-populated dest
|
||||
skips present-and-hash-ok blobs (incremental); credentialed op with no key
|
||||
exits non-zero without reading stdin (read path and write path both).
|
||||
|
||||
## Non-goals (Increment 1)
|
||||
|
||||
- `import` / `sync` / edit-tree ingestion (Increment 2).
|
||||
- mdl integrity checks, `prune`, `gc`, `export`, `mirror` ergonomics
|
||||
(Increment 3).
|
||||
- Any change to the depot wire protocol, blob layout, or Bunny account config.
|
||||
- A depot-specific rich TUI. **Direction note:** Crucible is headed toward being
|
||||
a TUI overall — the existing interactive dispatcher menu is the first step,
|
||||
and depot joins it for free via the registry wiring above. What Increment 1
|
||||
does not build is anything richer than that menu (progress UIs, pickers,
|
||||
dashboards); those come with the Crucible-wide TUI work, not per-tool.
|
||||
|
||||
## Immediate operational payoff
|
||||
|
||||
`crucible depot push --source workspace/depot --target bunny` is exactly the
|
||||
command that unblocks the stuck `v0.1.5` release: it probes the target and
|
||||
uploads whatever is actually 404 while it re-hydrates from the local depot.
|
||||
(The `5903` figure below is superseded — see the 2026-07-04 update above: the
|
||||
real drift was **2** blobs, already re-uploaded; `push` would have been the
|
||||
clean way to do it and is idempotent once the depot is whole.) Building
|
||||
Increment 1 first both fixes the tooling and clears the current outage.
|
||||
|
||||
## Risks / open questions
|
||||
|
||||
- ~~`BUNNY_STORAGE_HOST` default~~ — resolved: env-required, no baked default
|
||||
(see Backends & config).
|
||||
- Manifest schema coupling: `crucible depot` now depends on the `assets/*.yml`
|
||||
shape (`assets[].{path,sha256,size,restype,hak}`). Acceptable and intentional,
|
||||
but a schema change now touches Crucible.
|
||||
- Cutover ordering: `release.yml` and Makefile must switch to `crucible depot` in
|
||||
the same change that removes the bash, to avoid a window where both exist.
|
||||
@@ -10,12 +10,11 @@
|
||||
in
|
||||
{
|
||||
# Self-contained (D8): a host with ONLY nix can enter this shell and run
|
||||
# `make check`. ffmpeg is present because the migrated music pipeline needs
|
||||
# it; the scaffold itself is pure stdlib Go.
|
||||
# `make check`.
|
||||
# Daemonless image build (D8): `nix build .#image` produces an OCI tarball
|
||||
# with no docker/podman daemon. CI loads/pushes it with skopeo. This is the
|
||||
# Nix-native replacement for `docker build` on the host-mode runner, which
|
||||
# has no container runtime. ffmpeg-headless ships the BMU codec path.
|
||||
# has no container runtime.
|
||||
packages = forAllSystems (pkgs:
|
||||
let
|
||||
version = self.shortRev or self.dirtyShortRev or "unknown";
|
||||
@@ -49,7 +48,7 @@
|
||||
image = pkgs.dockerTools.buildLayeredImage {
|
||||
name = "registry.westgate.pw/deployment/crucible";
|
||||
tag = version;
|
||||
contents = [ crucible pkgs.cacert pkgs.ffmpeg-headless pkgs.fakeNss ];
|
||||
contents = [ crucible pkgs.cacert pkgs.fakeNss ];
|
||||
config = {
|
||||
Entrypoint = [ "/bin/crucible" ];
|
||||
Cmd = [ "help" ];
|
||||
@@ -69,8 +68,8 @@
|
||||
go
|
||||
gopls
|
||||
gotools
|
||||
ffmpeg
|
||||
git
|
||||
tea
|
||||
gnumake
|
||||
bashInteractive
|
||||
shellcheck
|
||||
|
||||
+29
-218
@@ -9,7 +9,6 @@ import (
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -184,11 +183,6 @@ var commands = []command{
|
||||
description: "Inspect, explain, and validate the resolved project configuration.",
|
||||
run: runConfig,
|
||||
},
|
||||
{
|
||||
name: "music",
|
||||
description: "List, scan, build, and validate configured music datasets.",
|
||||
run: runMusic,
|
||||
},
|
||||
{
|
||||
name: "compare",
|
||||
description: "Compare current source content against the built archives.",
|
||||
@@ -470,8 +464,6 @@ type buildHAKOptions struct {
|
||||
filteredArchives []string
|
||||
sourceManifest string
|
||||
contentAddressedRoot string
|
||||
musicDatasets []string
|
||||
skipMusic bool
|
||||
planOnly bool
|
||||
logLevel logLevel
|
||||
}
|
||||
@@ -721,46 +713,11 @@ func (c *buildHAKConsole) emitResult(result pipeline.BuildResult) {
|
||||
spin.linebreak()
|
||||
fmt.Fprintln(c.stdout, "Build HAKs ----------")
|
||||
if c.level != logLevelQuiet {
|
||||
c.emitCreditsSummary(result)
|
||||
c.emitCoreSteps(result)
|
||||
}
|
||||
c.emitFinalSummary(result)
|
||||
}
|
||||
|
||||
func (c *buildHAKConsole) emitCreditsSummary(result pipeline.BuildResult) {
|
||||
summary := result.CreditsSummary
|
||||
if len(summary.ArtifactPaths) == 0 {
|
||||
return
|
||||
}
|
||||
fmt.Fprintln(c.stdout, "Refreshing credits cache")
|
||||
if len(summary.ScannedDirs) > 0 {
|
||||
fmt.Fprintf(c.stdout, " scanned: %s\n", strings.Join(summary.ScannedDirs, ", "))
|
||||
}
|
||||
fmt.Fprintf(c.stdout, " tracks: %d\n", summary.TrackCount)
|
||||
if len(summary.GeneratedPaths) > 0 {
|
||||
for _, path := range summary.GeneratedPaths {
|
||||
fmt.Fprintf(c.stdout, " output: %s\n", c.relPath(path))
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case summary.ChangedFiles == 0:
|
||||
fmt.Fprintln(c.stdout, " status: unchanged")
|
||||
default:
|
||||
fmt.Fprintf(c.stdout, " updated: %d artifact(s)\n", summary.ChangedFiles)
|
||||
}
|
||||
if summary.TrackCount > 0 {
|
||||
if c.level >= logLevelVerbose {
|
||||
fmt.Fprintln(c.stdout, " mappings:")
|
||||
for _, mapping := range summary.Mappings {
|
||||
fmt.Fprintf(c.stdout, " %s -> %s\n", mapping.Source, mapping.Output)
|
||||
}
|
||||
} else {
|
||||
fmt.Fprintf(c.stdout, " mapped: %d music file(s); use --verbose to list mappings\n", summary.TrackCount)
|
||||
}
|
||||
}
|
||||
fmt.Fprintln(c.stdout)
|
||||
}
|
||||
|
||||
func (c *buildHAKConsole) emitCoreSteps(result pipeline.BuildResult) {
|
||||
fmt.Fprintln(c.stdout, "Validating project")
|
||||
fmt.Fprintln(c.stdout, "Collecting asset resources")
|
||||
@@ -808,9 +765,6 @@ func (c *buildHAKConsole) emitFinalSummary(result pipeline.BuildResult) {
|
||||
if result.Manifest != "" {
|
||||
fmt.Fprintf(c.stdout, "manifest: %s\n", c.relPath(result.Manifest))
|
||||
}
|
||||
if len(result.CreditsArtifactPaths) > 0 {
|
||||
fmt.Fprintf(c.stdout, "credits artifacts: %d\n", len(result.CreditsArtifactPaths))
|
||||
}
|
||||
if len(result.AutogenManifestPaths) > 0 {
|
||||
fmt.Fprintf(c.stdout, "autogen manifests: %d\n", len(result.AutogenManifestPaths))
|
||||
}
|
||||
@@ -1050,8 +1004,6 @@ func runBuildHAKs(ctx context) error {
|
||||
ArchiveNames: opts.filteredArchives,
|
||||
SourceManifestPath: opts.sourceManifest,
|
||||
ContentAddressedRoot: opts.contentAddressedRoot,
|
||||
SkipMusic: opts.skipMusic,
|
||||
MusicDatasetIDs: opts.musicDatasets,
|
||||
}
|
||||
if opts.planOnly {
|
||||
result, err = pipeline.PlanHAKsWithOptions(p, pipelineOpts)
|
||||
@@ -1076,7 +1028,7 @@ func parseBuildHAKArgs(args []string) (buildHAKOptions, error) {
|
||||
arg := args[index]
|
||||
switch arg {
|
||||
case "-h", "--help":
|
||||
return opts, errors.New("usage: build-haks [--hak <hak-name> ...] [--archive <archive-name> ...] [--source-manifest <path>] [--content-addressed-root <path>] [--plan-only] [--skip-music] [--music-dataset <id> ...] [--quiet|--verbose|--debug]")
|
||||
return opts, errors.New("usage: build-haks [--hak <hak-name> ...] [--archive <archive-name> ...] [--source-manifest <path>] [--content-addressed-root <path>] [--plan-only] [--quiet|--verbose|--debug]")
|
||||
case "--hak":
|
||||
index++
|
||||
if index >= len(args) {
|
||||
@@ -1091,14 +1043,6 @@ func parseBuildHAKArgs(args []string) (buildHAKOptions, error) {
|
||||
opts.filteredArchives = append(opts.filteredArchives, args[index])
|
||||
case "--plan-only":
|
||||
opts.planOnly = true
|
||||
case "--skip-music":
|
||||
opts.skipMusic = true
|
||||
case "--music-dataset":
|
||||
index++
|
||||
if index >= len(args) {
|
||||
return opts, errors.New("--music-dataset requires a value")
|
||||
}
|
||||
opts.musicDatasets = append(opts.musicDatasets, args[index])
|
||||
case "--quiet":
|
||||
opts.logLevel = logLevelQuiet
|
||||
case "--verbose":
|
||||
@@ -1146,13 +1090,6 @@ func parseBuildHAKArgs(args []string) (buildHAKOptions, error) {
|
||||
opts.contentAddressedRoot = value
|
||||
continue
|
||||
}
|
||||
if value, ok, err := requireInlineFlagValue(arg, "--music-dataset"); ok || err != nil {
|
||||
if err != nil {
|
||||
return opts, err
|
||||
}
|
||||
opts.musicDatasets = append(opts.musicDatasets, value)
|
||||
continue
|
||||
}
|
||||
if arg == "--quiet=true" {
|
||||
opts.logLevel = logLevelQuiet
|
||||
continue
|
||||
@@ -1182,149 +1119,6 @@ func requireInlineFlagValue(arg, name string) (string, bool, error) {
|
||||
return value, true, nil
|
||||
}
|
||||
|
||||
type musicCommandOptions struct {
|
||||
datasets []string
|
||||
json bool
|
||||
dryRun bool
|
||||
check bool
|
||||
force bool
|
||||
}
|
||||
|
||||
func runMusic(ctx context) error {
|
||||
if len(ctx.args) < 2 {
|
||||
return errors.New("usage: music <list-datasets|scan|build|credits|validate|manifest|normalize> [--dataset <id>] [--json] [--dry-run] [--check] [--force]")
|
||||
}
|
||||
p, err := loadProject(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := p.Scan(); err != nil {
|
||||
return err
|
||||
}
|
||||
subcommand := ctx.args[1]
|
||||
opts, err := parseMusicCommandArgs(ctx.args[2:])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
switch subcommand {
|
||||
case "list-datasets":
|
||||
return emitMusicDatasets(ctx, p, opts)
|
||||
case "scan":
|
||||
result, err := pipeline.BuildMusic(p, pipeline.MusicBuildOptions{DatasetIDs: opts.datasets, Write: false})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return emitMusicResult(ctx, "scan", result, opts)
|
||||
case "build", "credits":
|
||||
write := true
|
||||
if subcommand == "build" && opts.dryRun {
|
||||
write = false
|
||||
}
|
||||
if subcommand == "credits" && opts.check {
|
||||
write = false
|
||||
}
|
||||
result, err := pipeline.BuildMusic(p, pipeline.MusicBuildOptions{DatasetIDs: opts.datasets, Write: write})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return emitMusicResult(ctx, subcommand, result, opts)
|
||||
case "manifest", "normalize":
|
||||
result, err := pipeline.BuildMusic(p, pipeline.MusicBuildOptions{DatasetIDs: opts.datasets, Write: false})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return emitMusicResult(ctx, subcommand, result, opts)
|
||||
case "validate":
|
||||
if err := p.ValidateLayout(); err != nil {
|
||||
return err
|
||||
}
|
||||
result, err := pipeline.BuildMusic(p, pipeline.MusicBuildOptions{DatasetIDs: opts.datasets, Write: false})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return emitMusicResult(ctx, "validate", result, opts)
|
||||
default:
|
||||
return fmt.Errorf("unknown music subcommand %q", subcommand)
|
||||
}
|
||||
}
|
||||
|
||||
func parseMusicCommandArgs(args []string) (musicCommandOptions, error) {
|
||||
opts := musicCommandOptions{}
|
||||
for index := 0; index < len(args); index++ {
|
||||
arg := args[index]
|
||||
switch arg {
|
||||
case "--dataset":
|
||||
index++
|
||||
if index >= len(args) || strings.TrimSpace(args[index]) == "" {
|
||||
return opts, errors.New("--dataset requires a value")
|
||||
}
|
||||
opts.datasets = append(opts.datasets, args[index])
|
||||
case "--json":
|
||||
opts.json = true
|
||||
case "--dry-run":
|
||||
opts.dryRun = true
|
||||
case "--check":
|
||||
opts.check = true
|
||||
case "--force":
|
||||
opts.force = true
|
||||
default:
|
||||
if value, ok, err := requireInlineFlagValue(arg, "--dataset"); ok || err != nil {
|
||||
if err != nil {
|
||||
return opts, err
|
||||
}
|
||||
opts.datasets = append(opts.datasets, value)
|
||||
continue
|
||||
}
|
||||
return opts, fmt.Errorf("unknown music argument %q", arg)
|
||||
}
|
||||
}
|
||||
return opts, nil
|
||||
}
|
||||
|
||||
func emitMusicDatasets(ctx context, p *project.Project, opts musicCommandOptions) error {
|
||||
effective := p.EffectiveConfig()
|
||||
if opts.json {
|
||||
return emitConfigValue(ctx.stdout, effective.Music.Datasets, "json")
|
||||
}
|
||||
if len(effective.Music.Datasets) == 0 {
|
||||
fmt.Fprintln(ctx.stdout, "music datasets: none")
|
||||
return nil
|
||||
}
|
||||
fmt.Fprintln(ctx.stdout, "music datasets:")
|
||||
ids := make([]string, 0, len(effective.Music.Datasets))
|
||||
for id := range effective.Music.Datasets {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
sort.Strings(ids)
|
||||
for _, id := range ids {
|
||||
ds := effective.Music.Datasets[id]
|
||||
fmt.Fprintf(ctx.stdout, " %s: source=%s output=%s prefix=%s\n", id, ds.Source, ds.Output, ds.Prefix)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func emitMusicResult(ctx context, action string, result pipeline.BuildResult, opts musicCommandOptions) error {
|
||||
if opts.json {
|
||||
return emitConfigValue(ctx.stdout, result.CreditsSummary, "json")
|
||||
}
|
||||
fmt.Fprintf(ctx.stdout, "music %s\n", action)
|
||||
if len(result.CreditsSummary.ScannedDirs) > 0 {
|
||||
fmt.Fprintf(ctx.stdout, "scanned: %s\n", strings.Join(result.CreditsSummary.ScannedDirs, ", "))
|
||||
}
|
||||
fmt.Fprintf(ctx.stdout, "tracks: %d\n", result.CreditsSummary.TrackCount)
|
||||
if action != "scan" && len(result.CreditsArtifactPaths) > 0 {
|
||||
fmt.Fprintf(ctx.stdout, "credits artifacts: %d\n", len(result.CreditsArtifactPaths))
|
||||
}
|
||||
if len(result.CreditsSummary.Mappings) > 0 && ctx.logLevel >= logLevelVerbose {
|
||||
fmt.Fprintln(ctx.stdout, "mappings:")
|
||||
for _, mapping := range result.CreditsSummary.Mappings {
|
||||
fmt.Fprintf(ctx.stdout, " %s -> %s\n", mapping.Source, mapping.Output)
|
||||
}
|
||||
}
|
||||
fmt.Fprintln(ctx.stdout, "status: ok")
|
||||
return nil
|
||||
}
|
||||
|
||||
func runExtract(ctx context) error {
|
||||
p, err := loadProject(ctx)
|
||||
if err != nil {
|
||||
@@ -1620,16 +1414,20 @@ func runBuildTopData(ctx context) error {
|
||||
return err
|
||||
}
|
||||
|
||||
opts, err := parseBuildTopDataArgs("build-topdata", ctx.args[1:])
|
||||
parsed, err := parseBuildTopDataArgs("build-topdata", ctx.args[1:])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if parsed.skipLFS {
|
||||
os.Setenv("CRUCIBLE_SKIP_LFS", "1") //nolint:errcheck
|
||||
}
|
||||
|
||||
console := newTopdataConsole(ctx, p, "build-topdata")
|
||||
spin.configure(ctx.stderr, console.spinnerEnabled)
|
||||
spin.start("Build Topdata: starting")
|
||||
defer spin.stop()
|
||||
result, err := topdata.BuildAndPackageWithOptions(p, opts, console.progress)
|
||||
result, err := topdata.BuildAndPackageWithOptions(p, parsed.opts, console.progress)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -1644,15 +1442,19 @@ func runBuildTopPackage(ctx context) error {
|
||||
return err
|
||||
}
|
||||
|
||||
opts, err := parseBuildTopDataArgs("build-top-package", ctx.args[1:])
|
||||
parsed, err := parseBuildTopDataArgs("build-top-package", ctx.args[1:])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if opts.BuildWiki {
|
||||
if parsed.opts.BuildWiki {
|
||||
return fmt.Errorf("--wiki is not supported with build-top-package; use build-topdata when wiki generation is required")
|
||||
}
|
||||
|
||||
if parsed.skipLFS {
|
||||
os.Setenv("CRUCIBLE_SKIP_LFS", "1") //nolint:errcheck
|
||||
}
|
||||
|
||||
console := newTopdataConsole(ctx, p, "build-top-package")
|
||||
spin.configure(ctx.stderr, console.spinnerEnabled)
|
||||
spin.start("Build Top Package: starting")
|
||||
@@ -1666,21 +1468,30 @@ func runBuildTopPackage(ctx context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func parseBuildTopDataArgs(commandName string, args []string) (topdata.BuildAndPackageOptions, error) {
|
||||
opts := topdata.BuildAndPackageOptions{}
|
||||
type buildTopDataArgs struct {
|
||||
opts topdata.BuildAndPackageOptions
|
||||
skipLFS bool
|
||||
}
|
||||
|
||||
func parseBuildTopDataArgs(commandName string, args []string) (buildTopDataArgs, error) {
|
||||
var parsed buildTopDataArgs
|
||||
for _, arg := range args {
|
||||
switch arg {
|
||||
case "--force":
|
||||
opts.Force = true
|
||||
parsed.opts.Force = true
|
||||
case "--wiki":
|
||||
opts.BuildWiki = true
|
||||
parsed.opts.BuildWiki = true
|
||||
case "--skip-lfs":
|
||||
// ponytail: CRUCIBLE_SKIP_LFS env is the single skip mechanism; set it at
|
||||
// the CLI entry point rather than carrying a dead field through BuildAndPackageOptions.
|
||||
parsed.skipLFS = true
|
||||
case "-h", "--help":
|
||||
return opts, fmt.Errorf("usage: %s [--force] [--wiki]", commandName)
|
||||
return parsed, fmt.Errorf("usage: %s [--force] [--wiki] [--skip-lfs]", commandName)
|
||||
default:
|
||||
return opts, fmt.Errorf("unknown %s argument %q", commandName, arg)
|
||||
return parsed, fmt.Errorf("unknown %s argument %q", commandName, arg)
|
||||
}
|
||||
}
|
||||
return opts, nil
|
||||
return parsed, nil
|
||||
}
|
||||
|
||||
func runCompareTopData(ctx context) error {
|
||||
|
||||
+53
-210
@@ -2,6 +2,7 @@ package app
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@@ -47,9 +48,31 @@ func TestParseBuildHAKArgsHelpListsContentAddressedRoot(t *testing.T) {
|
||||
if err == nil {
|
||||
t.Fatal("expected help usage error")
|
||||
}
|
||||
const want = "usage: build-haks [--hak <hak-name> ...] [--archive <archive-name> ...] [--source-manifest <path>] [--content-addressed-root <path>] [--plan-only] [--skip-music] [--music-dataset <id> ...] [--quiet|--verbose|--debug]"
|
||||
if err.Error() != want {
|
||||
t.Fatalf("help usage = %q, want %q", err.Error(), want)
|
||||
for _, flag := range []string{
|
||||
"--hak",
|
||||
"--archive",
|
||||
"--source-manifest",
|
||||
"--content-addressed-root",
|
||||
"--plan-only",
|
||||
"--quiet",
|
||||
"--verbose",
|
||||
"--debug",
|
||||
} {
|
||||
if !strings.Contains(err.Error(), flag) {
|
||||
t.Errorf("help usage missing documented flag %q: %v", flag, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseBuildHAKArgsRejectsRemovedMusicFlags(t *testing.T) {
|
||||
for _, args := range [][]string{
|
||||
{"--skip-music"},
|
||||
{"--music-dataset", "westgate"},
|
||||
{"--music-dataset=westgate"},
|
||||
} {
|
||||
if _, err := parseBuildHAKArgs(args); err == nil {
|
||||
t.Errorf("expected removed music arguments %v to fail", args)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -85,9 +108,8 @@ topdata:
|
||||
setFileTime(t, filepath.Join(root, ".cache", "2da", "repadjust.2da"), outputTime)
|
||||
setFileTime(t, filepath.Join(root, "build", "sow_tlk.tlk"), outputTime)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
ctx := context{
|
||||
stdout: &stdout,
|
||||
stdout: &bytes.Buffer{},
|
||||
stderr: &bytes.Buffer{},
|
||||
cwd: root,
|
||||
args: []string{"build-top-package"},
|
||||
@@ -96,19 +118,12 @@ topdata:
|
||||
if err := runBuildTopPackage(ctx); err != nil {
|
||||
t.Fatalf("runBuildTopPackage failed: %v", err)
|
||||
}
|
||||
output := stdout.String()
|
||||
if !strings.Contains(output, "top package hak: build/sow_top.hak") {
|
||||
t.Fatalf("expected build-top-package output, got %q", output)
|
||||
}
|
||||
if strings.Contains(output, "[build-top-package]") {
|
||||
t.Fatalf("did not expect raw progress lines in normal output, got %q", output)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(root, "build", "sow_top.hak")); err != nil {
|
||||
t.Fatalf("expected packaged hak output: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunBuildHAKsEmitsCompactSummary(t *testing.T) {
|
||||
func TestRunBuildHAKsPacksAuthoredBMU(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
mkdirAll(t, filepath.Join(root, "assets", "envi", "music", "westgate"))
|
||||
mkdirAll(t, filepath.Join(root, "build"))
|
||||
@@ -130,23 +145,7 @@ haks:
|
||||
include:
|
||||
- envi/**
|
||||
`)
|
||||
writeFile(t, filepath.Join(root, "assets", "envi", "music", "westgate", "AleandAnecdotes.mp3"), "source-mp3")
|
||||
writeFile(t, filepath.Join(root, "assets", "envi", "music", "westgate", "CREDITS.md"), "# NWN Music Pack Credits\n\n| Artist | Title | Output File | Original File | Album | Date | License / Copyright | Notes |\n|---|---|---|---|---|---|---|---|\n| Darren Curtis | Ale and Anecdotes | `mus_wg_andnc.bmu` | `AleandAnecdotes.mp3` | Darren's Commercially Free Music | | Darren Curtis Music | |\n")
|
||||
|
||||
ffprobePath := filepath.Join(root, "ffprobe")
|
||||
writeFile(t, ffprobePath, "#!/bin/sh\nprintf '%s\\n' '{\"format\":{\"tags\":{\"artist\":\"Unknown\",\"title\":\"Broken Metadata\"}}}'\n")
|
||||
if err := os.Chmod(ffprobePath, 0o755); err != nil {
|
||||
t.Fatalf("chmod ffprobe: %v", err)
|
||||
}
|
||||
|
||||
ffmpegPath := filepath.Join(root, "ffmpeg")
|
||||
writeFile(t, ffmpegPath, "#!/bin/sh\nfor last; do :; done\nprintf 'bmu-data' > \"$last\"\n")
|
||||
if err := os.Chmod(ffmpegPath, 0o755); err != nil {
|
||||
t.Fatalf("chmod ffmpeg: %v", err)
|
||||
}
|
||||
|
||||
t.Setenv("SOW_FFMPEG", ffmpegPath)
|
||||
t.Setenv("SOW_FFPROBE", ffprobePath)
|
||||
writeFile(t, filepath.Join(root, "assets", "envi", "music", "westgate", "westgate_theme.bmu"), "authored-bmu")
|
||||
|
||||
var stdout bytes.Buffer
|
||||
ctx := context{
|
||||
@@ -160,18 +159,12 @@ haks:
|
||||
t.Fatalf("runBuildHAKs failed: %v", err)
|
||||
}
|
||||
|
||||
output := stdout.String()
|
||||
if !strings.Contains(output, "Build HAKs ----------") {
|
||||
t.Fatalf("expected build header, got %q", output)
|
||||
manifest, err := os.ReadFile(filepath.Join(root, "build", "haks.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("read HAK manifest: %v", err)
|
||||
}
|
||||
if !strings.Contains(output, "mapped: 1 music file(s); use --verbose to list mappings") {
|
||||
t.Fatalf("expected compact mapping summary, got %q", output)
|
||||
}
|
||||
if strings.Contains(output, "AleandAnecdotes.mp3 -> mus_wg_andnc.bmu") {
|
||||
t.Fatalf("did not expect verbose mapping in normal mode, got %q", output)
|
||||
}
|
||||
if !strings.Contains(output, "manifest: build/haks.json") {
|
||||
t.Fatalf("expected relative manifest path, got %q", output)
|
||||
if !strings.Contains(string(manifest), "envi/music/westgate/westgate_theme.bmu") {
|
||||
t.Fatalf("authored BMU missing from HAK manifest:\n%s", manifest)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -188,70 +181,6 @@ func setTreeTime(t *testing.T, root string, modTime time.Time) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunBuildHAKsVerboseListsMappings(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
mkdirAll(t, filepath.Join(root, "assets", "envi", "music", "westgate"))
|
||||
mkdirAll(t, filepath.Join(root, "build"))
|
||||
mkdirAll(t, filepath.Join(root, "src"))
|
||||
|
||||
writeFile(t, filepath.Join(root, "nwn-tool.yaml"), `
|
||||
module:
|
||||
name: Test Module
|
||||
resref: testmod
|
||||
paths:
|
||||
source: src
|
||||
assets: assets
|
||||
build: build
|
||||
haks:
|
||||
- name: envi
|
||||
priority: 1
|
||||
max_bytes: 1048576
|
||||
split: false
|
||||
include:
|
||||
- envi/**
|
||||
`)
|
||||
writeFile(t, filepath.Join(root, "assets", "envi", "music", "westgate", "AleandAnecdotes.mp3"), "source-mp3")
|
||||
writeFile(t, filepath.Join(root, "assets", "envi", "music", "westgate", "CREDITS.md"), "# NWN Music Pack Credits\n\n| Artist | Title | Output File | Original File | Album | Date | License / Copyright | Notes |\n|---|---|---|---|---|---|---|---|\n| Darren Curtis | Ale and Anecdotes | `mus_wg_andnc.bmu` | `AleandAnecdotes.mp3` | Darren's Commercially Free Music | | Darren Curtis Music | |\n")
|
||||
|
||||
ffprobePath := filepath.Join(root, "ffprobe")
|
||||
writeFile(t, ffprobePath, "#!/bin/sh\nprintf '%s\\n' '{\"format\":{\"tags\":{\"artist\":\"Unknown\",\"title\":\"Broken Metadata\"}}}'\n")
|
||||
if err := os.Chmod(ffprobePath, 0o755); err != nil {
|
||||
t.Fatalf("chmod ffprobe: %v", err)
|
||||
}
|
||||
|
||||
ffmpegPath := filepath.Join(root, "ffmpeg")
|
||||
writeFile(t, ffmpegPath, "#!/bin/sh\nfor last; do :; done\nprintf 'bmu-data' > \"$last\"\n")
|
||||
if err := os.Chmod(ffmpegPath, 0o755); err != nil {
|
||||
t.Fatalf("chmod ffmpeg: %v", err)
|
||||
}
|
||||
|
||||
t.Setenv("SOW_FFMPEG", ffmpegPath)
|
||||
t.Setenv("SOW_FFPROBE", ffprobePath)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
ctx := context{
|
||||
stdout: &stdout,
|
||||
stderr: &bytes.Buffer{},
|
||||
cwd: root,
|
||||
args: []string{"build-haks", "--verbose"},
|
||||
}
|
||||
|
||||
if err := runBuildHAKs(ctx); err != nil {
|
||||
t.Fatalf("runBuildHAKs failed: %v", err)
|
||||
}
|
||||
|
||||
output := stdout.String()
|
||||
if !strings.Contains(output, "mappings:") {
|
||||
t.Fatalf("expected verbose mappings header, got %q", output)
|
||||
}
|
||||
if !strings.Contains(output, "AleandAnecdotes.mp3 -> mus_wg_andnc.bmu") {
|
||||
t.Fatalf("expected verbose mapping output, got %q", output)
|
||||
}
|
||||
if !strings.Contains(output, "wrote: envi (1 assets)") {
|
||||
t.Fatalf("expected verbose archive action, got %q", output)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTopdataConsoleSuppressesProgressInNormalMode(t *testing.T) {
|
||||
var stdout bytes.Buffer
|
||||
console := &topdataConsole{
|
||||
@@ -292,16 +221,10 @@ func TestTopdataConsoleDebugProgressAndRelativePaths(t *testing.T) {
|
||||
console.emitWikiDeployResult(10, 1, 2, 3, 4, 5, 6, 0, "/workspace/project/build/wiki/deploy-manifest.json")
|
||||
|
||||
output := stdout.String()
|
||||
if !strings.Contains(output, "[debug] NodeBB wiki plan: create 1, update 2, skip 3, stale 4, archive 5, purge 6, drift 0") {
|
||||
if !strings.Contains(output, "NodeBB wiki plan") {
|
||||
t.Fatalf("expected debug progress line, got %q", output)
|
||||
}
|
||||
if !strings.Contains(output, "archived: 5") {
|
||||
t.Fatalf("expected archived deploy count, got %q", output)
|
||||
}
|
||||
if !strings.Contains(output, "purged: 6") {
|
||||
t.Fatalf("expected purged deploy count, got %q", output)
|
||||
}
|
||||
if !strings.Contains(output, "manifest: build/wiki/deploy-manifest.json") {
|
||||
if !strings.Contains(output, "build/wiki/deploy-manifest.json") || strings.Contains(output, "/workspace/project/") {
|
||||
t.Fatalf("expected relative deploy manifest path, got %q", output)
|
||||
}
|
||||
}
|
||||
@@ -359,85 +282,11 @@ func TestProjectConsoleEmitsRelativePaths(t *testing.T) {
|
||||
})
|
||||
|
||||
output := stdout.String()
|
||||
if !strings.Contains(output, "module: build/test.mod") {
|
||||
if !strings.Contains(output, "build/test.mod") || strings.Contains(output, "/workspace/project/") {
|
||||
t.Fatalf("expected relative module path, got %q", output)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunMusicScanUsesConfiguredDatasetsWithoutWriting(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
mkdirAll(t, filepath.Join(root, "assets", "audio", "westgate"))
|
||||
mkdirAll(t, filepath.Join(root, "build"))
|
||||
writeFile(t, filepath.Join(root, "nwn-tool.yaml"), `
|
||||
module:
|
||||
name: Test Module
|
||||
resref: testmod
|
||||
paths:
|
||||
assets: assets
|
||||
build: build
|
||||
music:
|
||||
datasets:
|
||||
westgate_audio:
|
||||
source: audio/westgate
|
||||
output: generated/music
|
||||
prefix: wg_
|
||||
`)
|
||||
writeFile(t, filepath.Join(root, "assets", "audio", "westgate", "Theme Song.mp3"), "source-mp3")
|
||||
|
||||
var stdout bytes.Buffer
|
||||
ctx := context{
|
||||
stdout: &stdout,
|
||||
stderr: &bytes.Buffer{},
|
||||
cwd: root,
|
||||
args: []string{"music", "scan", "--dataset", "westgate_audio"},
|
||||
}
|
||||
|
||||
if err := runMusic(ctx); err != nil {
|
||||
t.Fatalf("runMusic failed: %v", err)
|
||||
}
|
||||
output := stdout.String()
|
||||
if !strings.Contains(output, "music scan") || !strings.Contains(output, "tracks: 1") {
|
||||
t.Fatalf("unexpected music scan output: %q", output)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(root, ".cache", "credits")); !os.IsNotExist(err) {
|
||||
t.Fatalf("music scan should not write credits artifacts, stat err=%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunMusicListDatasets(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
mkdirAll(t, filepath.Join(root, "assets"))
|
||||
mkdirAll(t, filepath.Join(root, "build"))
|
||||
writeFile(t, filepath.Join(root, "nwn-tool.yaml"), `
|
||||
module:
|
||||
name: Test Module
|
||||
resref: testmod
|
||||
paths:
|
||||
assets: assets
|
||||
music:
|
||||
datasets:
|
||||
westgate_audio:
|
||||
source: audio/westgate
|
||||
output: generated/music
|
||||
prefix: wg_
|
||||
`)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
ctx := context{
|
||||
stdout: &stdout,
|
||||
stderr: &bytes.Buffer{},
|
||||
cwd: root,
|
||||
args: []string{"music", "list-datasets"},
|
||||
}
|
||||
|
||||
if err := runMusic(ctx); err != nil {
|
||||
t.Fatalf("runMusic failed: %v", err)
|
||||
}
|
||||
if !strings.Contains(stdout.String(), "westgate_audio: source=audio/westgate output=generated/music prefix=wg_") {
|
||||
t.Fatalf("unexpected dataset list: %q", stdout.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunConfigEffectiveJSONShowsDefaultsAndProvenance(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
writeFile(t, filepath.Join(root, "nwn-tool.yaml"), `
|
||||
@@ -459,12 +308,16 @@ paths:
|
||||
if err := runConfig(ctx); err != nil {
|
||||
t.Fatalf("runConfig failed: %v", err)
|
||||
}
|
||||
output := stdout.String()
|
||||
if !strings.Contains(output, `"hak_manifest": "haks.json"`) {
|
||||
t.Fatalf("expected HAK manifest default in effective config, got %q", output)
|
||||
var effective map[string]any
|
||||
if err := json.Unmarshal(stdout.Bytes(), &effective); err != nil {
|
||||
t.Fatalf("effective config is not JSON: %v\n%s", err, stdout.String())
|
||||
}
|
||||
if !strings.Contains(output, `"paths.build":`) || !strings.Contains(output, `"toolkit default"`) {
|
||||
t.Fatalf("expected default provenance in effective config, got %q", output)
|
||||
provenance, ok := effective["provenance"].(map[string]any)
|
||||
if !ok || len(provenance) == 0 {
|
||||
t.Fatalf("effective config missing provenance: %#v", effective["provenance"])
|
||||
}
|
||||
if _, ok := provenance["paths.build"]; !ok {
|
||||
t.Fatalf("effective config missing provenance for omitted paths.build: %#v", provenance)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -490,10 +343,10 @@ paths:
|
||||
t.Fatalf("runConfig failed: %v", err)
|
||||
}
|
||||
output := stdout.String()
|
||||
if !strings.Contains(output, "value: \"output\"") {
|
||||
if !strings.Contains(output, "output") {
|
||||
t.Fatalf("expected configured build value, got %q", output)
|
||||
}
|
||||
if !strings.Contains(output, "source: yaml") {
|
||||
if !strings.Contains(strings.ToLower(output), "yaml") {
|
||||
t.Fatalf("expected YAML source, got %q", output)
|
||||
}
|
||||
}
|
||||
@@ -510,9 +363,8 @@ paths:
|
||||
build: build
|
||||
`)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
ctx := context{
|
||||
stdout: &stdout,
|
||||
stdout: &bytes.Buffer{},
|
||||
stderr: &bytes.Buffer{},
|
||||
cwd: root,
|
||||
args: []string{"config", "validate"},
|
||||
@@ -521,9 +373,6 @@ paths:
|
||||
if err := runConfig(ctx); err != nil {
|
||||
t.Fatalf("runConfig failed: %v", err)
|
||||
}
|
||||
if !strings.Contains(stdout.String(), "config: ok") {
|
||||
t.Fatalf("expected config validation output, got %q", stdout.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunConfigSourcesListsActiveOverrides(t *testing.T) {
|
||||
@@ -549,25 +398,19 @@ paths:
|
||||
t.Fatalf("runConfig failed: %v", err)
|
||||
}
|
||||
output := stdout.String()
|
||||
if !strings.Contains(output, "active overrides:") {
|
||||
t.Fatalf("expected active overrides section, got %q", output)
|
||||
}
|
||||
if !strings.Contains(output, "build.keep_existing_haks=1") {
|
||||
if !strings.Contains(output, "build.keep_existing_haks") {
|
||||
t.Fatalf("expected keep existing override, got %q", output)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInlineFlagParsersRejectEmptyValues(t *testing.T) {
|
||||
if _, err := parseBuildHAKArgs([]string{"--hak="}); err == nil || !strings.Contains(err.Error(), "--hak requires a value") {
|
||||
if _, err := parseBuildHAKArgs([]string{"--hak="}); err == nil {
|
||||
t.Fatalf("expected empty --hak inline value error, got %v", err)
|
||||
}
|
||||
if _, err := parseMusicCommandArgs([]string{"--dataset="}); err == nil || !strings.Contains(err.Error(), "--dataset requires a value") {
|
||||
t.Fatalf("expected empty --dataset inline value error, got %v", err)
|
||||
}
|
||||
if _, err := parseDeployWikiArgs("deploy-wiki", []string{"--endpoint="}); err == nil || !strings.Contains(err.Error(), "--endpoint requires a value") {
|
||||
if _, err := parseDeployWikiArgs("deploy-wiki", []string{"--endpoint="}); err == nil {
|
||||
t.Fatalf("expected empty --endpoint inline value error, got %v", err)
|
||||
}
|
||||
if _, err := parseBuildChangelogArgs("build-changelog", []string{"--output="}); err == nil || !strings.Contains(err.Error(), "--output requires a value") {
|
||||
if _, err := parseBuildChangelogArgs("build-changelog", []string{"--output="}); err == nil {
|
||||
t.Fatalf("expected empty --output inline value error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -178,11 +178,17 @@ func TestGenerateIncludesPullRequestsAndDirectPushes(t *testing.T) {
|
||||
}
|
||||
|
||||
rendered := stdout.String()
|
||||
if !strings.Contains(rendered, "- Add release summary ([#12]("+repoURL+"/pulls/12)) - From Patch Author") {
|
||||
t.Fatalf("rendered changelog missing pull entry:\n%s", rendered)
|
||||
}
|
||||
if !strings.Contains(rendered, "- Fix direct push handling (["+shortCommitHash(directHash)+"]("+repoURL+"/commit/"+directHash+")) - From Test User") {
|
||||
t.Fatalf("rendered changelog missing direct-push entry:\n%s", rendered)
|
||||
for _, want := range []string{
|
||||
"Add release summary",
|
||||
repoURL + "/pulls/12",
|
||||
"Patch Author",
|
||||
"Fix direct push handling",
|
||||
repoURL + "/commit/" + directHash,
|
||||
"Test User",
|
||||
} {
|
||||
if !strings.Contains(rendered, want) {
|
||||
t.Errorf("rendered changelog missing %q:\n%s", want, rendered)
|
||||
}
|
||||
}
|
||||
if pullRequests != 1 {
|
||||
t.Fatalf("expected 1 pull lookup, got %d", pullRequests)
|
||||
@@ -192,7 +198,7 @@ func TestGenerateIncludesPullRequestsAndDirectPushes(t *testing.T) {
|
||||
func runGit(t *testing.T, repoRoot string, args ...string) string {
|
||||
t.Helper()
|
||||
|
||||
cmd := exec.Command("git", args...)
|
||||
cmd := exec.Command("git", append([]string{"-c", "commit.gpgsign=false", "-c", "tag.gpgsign=false"}, args...)...)
|
||||
cmd.Dir = repoRoot
|
||||
output, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
package depot
|
||||
|
||||
import "context"
|
||||
|
||||
type ProbeState int
|
||||
|
||||
const (
|
||||
Present ProbeState = iota
|
||||
Absent
|
||||
Unconfirmed
|
||||
)
|
||||
|
||||
// Backend defines the interface for blob storage backends.
|
||||
type Backend interface {
|
||||
// Name returns the backend name.
|
||||
Name() string
|
||||
// Probe returns the existence state of one sha. transient=true means a
|
||||
// retry might change the answer (feeds the serial confirm loop).
|
||||
Probe(ctx context.Context, sha string) (state ProbeState, transient bool, err error)
|
||||
// Get fetches sha into dest (temp file + rename), re-hashes, deletes on mismatch.
|
||||
Get(ctx context.Context, sha, dest string) error
|
||||
// Put uploads bytes from src for sha. Read-only backends return an error.
|
||||
Put(ctx context.Context, sha, src string) error
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
package depot
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
CDNBase string
|
||||
StorageHost string
|
||||
StorageZone string
|
||||
ReadKey string
|
||||
WriteKey string
|
||||
ProbeJobs int
|
||||
Jobs int
|
||||
ConnectTimeout time.Duration
|
||||
ProbeMaxTime time.Duration
|
||||
ConfirmRetries int
|
||||
ConfirmMax int
|
||||
}
|
||||
|
||||
func LoadConfig(getenv func(string) string) Config {
|
||||
cfg := Config{
|
||||
CDNBase: "https://cdn-a7f3k9.westgate.pw",
|
||||
StorageZone: "sow-assets-depot",
|
||||
ProbeJobs: 16,
|
||||
Jobs: 16,
|
||||
ConnectTimeout: 10 * time.Second,
|
||||
ProbeMaxTime: 30 * time.Second,
|
||||
ConfirmRetries: 3,
|
||||
ConfirmMax: 200,
|
||||
}
|
||||
|
||||
// DEPOT_CDN_BASE || BUNNY_CDN_BASE
|
||||
if v := getenv("DEPOT_CDN_BASE"); v != "" {
|
||||
cfg.CDNBase = v
|
||||
} else if v := getenv("BUNNY_CDN_BASE"); v != "" {
|
||||
cfg.CDNBase = v
|
||||
}
|
||||
|
||||
// BUNNY_STORAGE_HOST (no default)
|
||||
cfg.StorageHost = getenv("BUNNY_STORAGE_HOST")
|
||||
|
||||
// BUNNY_STORAGE_ZONE (default "sow-assets-depot")
|
||||
if v := getenv("BUNNY_STORAGE_ZONE"); v != "" {
|
||||
cfg.StorageZone = v
|
||||
}
|
||||
|
||||
// BUNNY_STORAGE_PASSWORD
|
||||
cfg.WriteKey = getenv("BUNNY_STORAGE_PASSWORD")
|
||||
|
||||
// BUNNY_STORAGE_READ_PASSWORD || BUNNY_STORAGE_PASSWORD
|
||||
if v := getenv("BUNNY_STORAGE_READ_PASSWORD"); v != "" {
|
||||
cfg.ReadKey = v
|
||||
} else {
|
||||
cfg.ReadKey = cfg.WriteKey
|
||||
}
|
||||
|
||||
// DEPOT_PROBE_JOBS (default 16)
|
||||
if v := getenv("DEPOT_PROBE_JOBS"); v != "" {
|
||||
if i, err := strconv.Atoi(v); err == nil {
|
||||
cfg.ProbeJobs = i
|
||||
}
|
||||
}
|
||||
|
||||
// DEPOT_JOBS (default 16)
|
||||
if v := getenv("DEPOT_JOBS"); v != "" {
|
||||
if i, err := strconv.Atoi(v); err == nil {
|
||||
cfg.Jobs = i
|
||||
}
|
||||
}
|
||||
|
||||
// DEPOT_CONNECT_TIMEOUT (default 10s)
|
||||
if v := getenv("DEPOT_CONNECT_TIMEOUT"); v != "" {
|
||||
if d, err := time.ParseDuration(v + "s"); err == nil {
|
||||
cfg.ConnectTimeout = d
|
||||
}
|
||||
}
|
||||
|
||||
// DEPOT_PROBE_MAX_TIME (default 30s)
|
||||
if v := getenv("DEPOT_PROBE_MAX_TIME"); v != "" {
|
||||
if d, err := time.ParseDuration(v + "s"); err == nil {
|
||||
cfg.ProbeMaxTime = d
|
||||
}
|
||||
}
|
||||
|
||||
// DEPOT_CONFIRM_RETRIES (default 3)
|
||||
if v := getenv("DEPOT_CONFIRM_RETRIES"); v != "" {
|
||||
if i, err := strconv.Atoi(v); err == nil {
|
||||
cfg.ConfirmRetries = i
|
||||
}
|
||||
}
|
||||
|
||||
// DEPOT_CONFIRM_MAX (default 200, 0 = unlimited)
|
||||
if v := getenv("DEPOT_CONFIRM_MAX"); v != "" {
|
||||
if i, err := strconv.Atoi(v); err == nil {
|
||||
cfg.ConfirmMax = i
|
||||
}
|
||||
}
|
||||
|
||||
return cfg
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
package depot
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestLoadConfigDefaults(t *testing.T) {
|
||||
getenv := func(key string) string {
|
||||
return ""
|
||||
}
|
||||
cfg := LoadConfig(getenv)
|
||||
if cfg.CDNBase != "https://cdn-a7f3k9.westgate.pw" {
|
||||
t.Errorf("CDNBase: got %q, want %q", cfg.CDNBase, "https://cdn-a7f3k9.westgate.pw")
|
||||
}
|
||||
if cfg.StorageZone != "sow-assets-depot" {
|
||||
t.Errorf("StorageZone: got %q, want %q", cfg.StorageZone, "sow-assets-depot")
|
||||
}
|
||||
if cfg.ProbeJobs != 16 {
|
||||
t.Errorf("ProbeJobs: got %d, want 16", cfg.ProbeJobs)
|
||||
}
|
||||
if cfg.Jobs != 16 {
|
||||
t.Errorf("Jobs: got %d, want 16", cfg.Jobs)
|
||||
}
|
||||
if cfg.ConnectTimeout != 10*time.Second {
|
||||
t.Errorf("ConnectTimeout: got %v, want 10s", cfg.ConnectTimeout)
|
||||
}
|
||||
if cfg.ProbeMaxTime != 30*time.Second {
|
||||
t.Errorf("ProbeMaxTime: got %v, want 30s", cfg.ProbeMaxTime)
|
||||
}
|
||||
if cfg.ConfirmRetries != 3 {
|
||||
t.Errorf("ConfirmRetries: got %d, want 3", cfg.ConfirmRetries)
|
||||
}
|
||||
if cfg.ConfirmMax != 200 {
|
||||
t.Errorf("ConfirmMax: got %d, want 200", cfg.ConfirmMax)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadConfigReadKeyFallback(t *testing.T) {
|
||||
// When READ unset, falls back to WriteKey
|
||||
getenv := func(key string) string {
|
||||
if key == "BUNNY_STORAGE_PASSWORD" {
|
||||
return "write-key"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
cfg := LoadConfig(getenv)
|
||||
if cfg.ReadKey != "write-key" {
|
||||
t.Errorf("ReadKey fallback: got %q, want %q", cfg.ReadKey, "write-key")
|
||||
}
|
||||
if cfg.WriteKey != "write-key" {
|
||||
t.Errorf("WriteKey: got %q, want %q", cfg.WriteKey, "write-key")
|
||||
}
|
||||
|
||||
// When READ is set, uses its own value
|
||||
getenv = func(key string) string {
|
||||
if key == "BUNNY_STORAGE_READ_PASSWORD" {
|
||||
return "read-key"
|
||||
}
|
||||
if key == "BUNNY_STORAGE_PASSWORD" {
|
||||
return "write-key"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
cfg = LoadConfig(getenv)
|
||||
if cfg.ReadKey != "read-key" {
|
||||
t.Errorf("ReadKey explicit: got %q, want %q", cfg.ReadKey, "read-key")
|
||||
}
|
||||
if cfg.WriteKey != "write-key" {
|
||||
t.Errorf("WriteKey: got %q, want %q", cfg.WriteKey, "write-key")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadConfigBadIntFallsBack(t *testing.T) {
|
||||
getenv := func(key string) string {
|
||||
if key == "DEPOT_PROBE_JOBS" {
|
||||
return "not-a-number"
|
||||
}
|
||||
if key == "DEPOT_JOBS" {
|
||||
return "invalid"
|
||||
}
|
||||
if key == "DEPOT_CONNECT_TIMEOUT" {
|
||||
return "bad"
|
||||
}
|
||||
if key == "DEPOT_PROBE_MAX_TIME" {
|
||||
return "wrong"
|
||||
}
|
||||
if key == "DEPOT_CONFIRM_RETRIES" {
|
||||
return "nope"
|
||||
}
|
||||
if key == "DEPOT_CONFIRM_MAX" {
|
||||
return "nah"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
cfg := LoadConfig(getenv)
|
||||
if cfg.ProbeJobs != 16 {
|
||||
t.Errorf("ProbeJobs fallback: got %d, want 16", cfg.ProbeJobs)
|
||||
}
|
||||
if cfg.Jobs != 16 {
|
||||
t.Errorf("Jobs fallback: got %d, want 16", cfg.Jobs)
|
||||
}
|
||||
if cfg.ConnectTimeout != 10*time.Second {
|
||||
t.Errorf("ConnectTimeout fallback: got %v, want 10s", cfg.ConnectTimeout)
|
||||
}
|
||||
if cfg.ProbeMaxTime != 30*time.Second {
|
||||
t.Errorf("ProbeMaxTime fallback: got %v, want 30s", cfg.ProbeMaxTime)
|
||||
}
|
||||
if cfg.ConfirmRetries != 3 {
|
||||
t.Errorf("ConfirmRetries fallback: got %d, want 3", cfg.ConfirmRetries)
|
||||
}
|
||||
if cfg.ConfirmMax != 200 {
|
||||
t.Errorf("ConfirmMax fallback: got %d, want 200", cfg.ConfirmMax)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,183 @@
|
||||
package depot
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestIntegrationPushStatusPull drives status/push/pull end-to-end against a
|
||||
// fake Bunny backend (real bunny HTTP path, in-memory blob store) since the
|
||||
// command surface has no local->local push (push --target must be bunny).
|
||||
func TestIntegrationPushStatusPull(t *testing.T) {
|
||||
manifestsDir := t.TempDir()
|
||||
sourceDir := t.TempDir()
|
||||
|
||||
blobs := map[string]string{
|
||||
shaOf("blob-one"): "blob-one",
|
||||
shaOf("blob-two"): "blob-two",
|
||||
shaOf("blob-three"): "blob-three",
|
||||
}
|
||||
var manifest strings.Builder
|
||||
manifest.WriteString("assets:\n")
|
||||
for sha, content := range blobs {
|
||||
manifest.WriteString(fmt.Sprintf(" - path: %s\n sha256: %s\n size: %d\n", sha, sha, len(content)))
|
||||
blobPath := filepath.Join(sourceDir, BlobKey(sha))
|
||||
if err := os.MkdirAll(filepath.Dir(blobPath), 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(blobPath, []byte(content), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(manifestsDir, "manifest.yml"), []byte(manifest.String()), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
f := newFakeBunny()
|
||||
// Pre-seed one of the three blobs on the target so status starts with drift=2.
|
||||
var oneSHA string
|
||||
for sha := range blobs {
|
||||
oneSHA = sha
|
||||
break
|
||||
}
|
||||
f.blobs[oneSHA] = []byte(blobs[oneSHA])
|
||||
srv := httptest.NewServer(f.handler())
|
||||
defer srv.Close()
|
||||
|
||||
env := map[string]string{
|
||||
"BUNNY_STORAGE_HOST": hostPort(srv),
|
||||
"BUNNY_STORAGE_READ_PASSWORD": "readkey",
|
||||
"BUNNY_STORAGE_PASSWORD": "writekey",
|
||||
}
|
||||
getenv := testGetenv(env)
|
||||
|
||||
// status: expect exit 1, missing=2
|
||||
var out, errb bytes.Buffer
|
||||
code := Run([]string{"status", "--manifests", manifestsDir, "--target", "bunny"}, &out, &errb, getenv)
|
||||
if code != 1 {
|
||||
t.Fatalf("status: expected 1, got %d (stdout=%s stderr=%s)", code, out.String(), errb.String())
|
||||
}
|
||||
if !bytesContains(out.String(), "missing=2") {
|
||||
t.Fatalf("status: expected missing=2, got %s", out.String())
|
||||
}
|
||||
|
||||
// push: uploads the 2 missing blobs
|
||||
out.Reset()
|
||||
errb.Reset()
|
||||
code = Run([]string{"push", "--manifests", manifestsDir, "--source", sourceDir, "--target", "bunny"}, &out, &errb, getenv)
|
||||
if code != 0 {
|
||||
t.Fatalf("push: expected 0, got %d (stdout=%s stderr=%s)", code, out.String(), errb.String())
|
||||
}
|
||||
if !bytesContains(out.String(), "uploaded=2 failed=0") {
|
||||
t.Fatalf("push: expected uploaded=2 failed=0, got %s", out.String())
|
||||
}
|
||||
|
||||
putCountAfterFirstPush := 0
|
||||
for _, r := range f.requestsSnapshot() {
|
||||
if r.Method == "PUT" {
|
||||
putCountAfterFirstPush++
|
||||
}
|
||||
}
|
||||
if putCountAfterFirstPush != 2 {
|
||||
t.Fatalf("expected 2 PUTs after first push, got %d", putCountAfterFirstPush)
|
||||
}
|
||||
|
||||
// status: now clean
|
||||
out.Reset()
|
||||
errb.Reset()
|
||||
code = Run([]string{"status", "--manifests", manifestsDir, "--target", "bunny"}, &out, &errb, getenv)
|
||||
if code != 0 {
|
||||
t.Fatalf("status after push: expected 0, got %d (stdout=%s stderr=%s)", code, out.String(), errb.String())
|
||||
}
|
||||
|
||||
// push again: idempotent, no new PUTs, uploaded=0
|
||||
out.Reset()
|
||||
errb.Reset()
|
||||
code = Run([]string{"push", "--manifests", manifestsDir, "--source", sourceDir, "--target", "bunny"}, &out, &errb, getenv)
|
||||
if code != 0 {
|
||||
t.Fatalf("second push: expected 0, got %d (stdout=%s stderr=%s)", code, out.String(), errb.String())
|
||||
}
|
||||
if !bytesContains(out.String(), "uploaded=0 failed=0") {
|
||||
t.Fatalf("second push: expected uploaded=0 failed=0, got %s", out.String())
|
||||
}
|
||||
putCountAfterSecondPush := 0
|
||||
for _, r := range f.requestsSnapshot() {
|
||||
if r.Method == "PUT" {
|
||||
putCountAfterSecondPush++
|
||||
}
|
||||
}
|
||||
if putCountAfterSecondPush != putCountAfterFirstPush {
|
||||
t.Fatalf("expected no additional PUTs on idempotent push, before=%d after=%d", putCountAfterFirstPush, putCountAfterSecondPush)
|
||||
}
|
||||
|
||||
// pull into a dest dir: one pre-populated correct file (skipped, no GET),
|
||||
// one pre-populated corrupt file (re-downloaded), one absent (downloaded).
|
||||
destDir := t.TempDir()
|
||||
shas := make([]string, 0, len(blobs))
|
||||
for sha := range blobs {
|
||||
shas = append(shas, sha)
|
||||
}
|
||||
correctSHA := shas[0]
|
||||
corruptSHA := shas[1]
|
||||
// absentSHA := shas[2] // left absent on purpose
|
||||
|
||||
correctPath := filepath.Join(destDir, BlobKey(correctSHA))
|
||||
if err := os.MkdirAll(filepath.Dir(correctPath), 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(correctPath, []byte(blobs[correctSHA]), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
corruptPath := filepath.Join(destDir, BlobKey(corruptSHA))
|
||||
if err := os.MkdirAll(filepath.Dir(corruptPath), 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(corruptPath, []byte("corrupted-content"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
getCountBeforePull := 0
|
||||
for _, r := range f.requestsSnapshot() {
|
||||
if r.Method == "GET" && !strings.Contains(r.Range, "0-0") {
|
||||
getCountBeforePull++
|
||||
}
|
||||
}
|
||||
|
||||
out.Reset()
|
||||
errb.Reset()
|
||||
code = Run([]string{"pull", "--manifests", manifestsDir, "--dest", destDir, "--target", "bunny"}, &out, &errb, getenv)
|
||||
if code != 0 {
|
||||
t.Fatalf("pull: expected 0, got %d (stdout=%s stderr=%s)", code, out.String(), errb.String())
|
||||
}
|
||||
if !bytesContains(out.String(), "pulled=2 present=1") {
|
||||
t.Fatalf("pull: expected pulled=2 present=1, got %s", out.String())
|
||||
}
|
||||
|
||||
// verify the correct pre-existing file was never re-fetched with a full GET.
|
||||
fullGETsForCorrect := 0
|
||||
for _, r := range f.requestsSnapshot() {
|
||||
if r.Method == "GET" && strings.HasSuffix(r.Path, correctSHA) && r.Range != "bytes=0-0" {
|
||||
fullGETsForCorrect++
|
||||
}
|
||||
}
|
||||
if fullGETsForCorrect != 0 {
|
||||
t.Fatalf("expected no full GET for already-correct blob, got %d", fullGETsForCorrect)
|
||||
}
|
||||
|
||||
// all three blobs should now be present and correct in destDir.
|
||||
for sha, content := range blobs {
|
||||
got, err := os.ReadFile(filepath.Join(destDir, BlobKey(sha)))
|
||||
if err != nil {
|
||||
t.Fatalf("dest blob %s: %v", sha, err)
|
||||
}
|
||||
if string(got) != content {
|
||||
t.Fatalf("dest blob %s: expected %q, got %q", sha, content, string(got))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
package depot
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
// LocalBackend implements Backend for local filesystem storage.
|
||||
type LocalBackend struct {
|
||||
Root string
|
||||
}
|
||||
|
||||
// Name returns the backend name.
|
||||
func (b *LocalBackend) Name() string {
|
||||
return "local"
|
||||
}
|
||||
|
||||
// Probe checks if a blob exists.
|
||||
func (b *LocalBackend) Probe(ctx context.Context, sha string) (ProbeState, bool, error) {
|
||||
blobPath := filepath.Join(b.Root, BlobKey(sha))
|
||||
_, err := os.Stat(blobPath)
|
||||
if err == nil {
|
||||
return Present, false, nil
|
||||
}
|
||||
if os.IsNotExist(err) {
|
||||
return Absent, false, nil
|
||||
}
|
||||
// Real I/O error (permission, etc.)
|
||||
return Absent, false, err
|
||||
}
|
||||
|
||||
// Put copies the file from src to the blob storage.
|
||||
func (b *LocalBackend) Put(ctx context.Context, sha, src string) error {
|
||||
blobPath := filepath.Join(b.Root, BlobKey(sha))
|
||||
blobDir := filepath.Dir(blobPath)
|
||||
|
||||
// Create parent directories
|
||||
if err := os.MkdirAll(blobDir, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Create temp file in the same directory for atomic rename
|
||||
tmpFile, err := os.CreateTemp(blobDir, ".tmp-")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.Remove(tmpFile.Name())
|
||||
|
||||
// Copy source to temp file
|
||||
srcFile, err := os.Open(src)
|
||||
if err != nil {
|
||||
tmpFile.Close()
|
||||
return err
|
||||
}
|
||||
defer srcFile.Close()
|
||||
|
||||
_, err = io.Copy(tmpFile, srcFile)
|
||||
tmpFile.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Atomic rename
|
||||
return os.Rename(tmpFile.Name(), blobPath)
|
||||
}
|
||||
|
||||
// Get fetches the blob, re-hashes it, and deletes it if the hash doesn't match.
|
||||
func (b *LocalBackend) Get(ctx context.Context, sha, dest string) error {
|
||||
blobPath := filepath.Join(b.Root, BlobKey(sha))
|
||||
|
||||
// Create temp file in dest directory for atomic rename
|
||||
destDir := filepath.Dir(dest)
|
||||
if err := os.MkdirAll(destDir, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
tmpFile, err := os.CreateTemp(destDir, ".tmp-")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.Remove(tmpFile.Name())
|
||||
|
||||
// Copy and hash simultaneously
|
||||
srcFile, err := os.Open(blobPath)
|
||||
if err != nil {
|
||||
tmpFile.Close()
|
||||
return err
|
||||
}
|
||||
defer srcFile.Close()
|
||||
|
||||
hasher := sha256.New()
|
||||
teeReader := io.TeeReader(srcFile, hasher)
|
||||
|
||||
_, err = io.Copy(tmpFile, teeReader)
|
||||
tmpFile.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Check hash
|
||||
gotHash := fmt.Sprintf("%x", hasher.Sum(nil))
|
||||
if gotHash != sha {
|
||||
os.Remove(tmpFile.Name())
|
||||
return fmt.Errorf("hash mismatch: expected %s, got %s", sha, gotHash)
|
||||
}
|
||||
|
||||
// Atomic rename
|
||||
return os.Rename(tmpFile.Name(), dest)
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
package depot
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestLocalRoundTrip(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
backend := &LocalBackend{Root: tmpDir}
|
||||
|
||||
// Generate test data
|
||||
testData := []byte("hello world")
|
||||
testSHA := fmt.Sprintf("%x", sha256.Sum256(testData))
|
||||
|
||||
// Create source file
|
||||
srcFile := filepath.Join(tmpDir, "source.txt")
|
||||
if err := os.WriteFile(srcFile, testData, 0644); err != nil {
|
||||
t.Fatalf("failed to create source file: %v", err)
|
||||
}
|
||||
|
||||
// Put
|
||||
ctx := context.Background()
|
||||
if err := backend.Put(ctx, testSHA, srcFile); err != nil {
|
||||
t.Fatalf("Put failed: %v", err)
|
||||
}
|
||||
|
||||
// Probe should be Present
|
||||
state, transient, err := backend.Probe(ctx, testSHA)
|
||||
if err != nil {
|
||||
t.Fatalf("Probe failed: %v", err)
|
||||
}
|
||||
if state != Present {
|
||||
t.Fatalf("expected Present, got %v", state)
|
||||
}
|
||||
if transient {
|
||||
t.Fatalf("local backend should never be transient")
|
||||
}
|
||||
|
||||
// Get
|
||||
destFile := filepath.Join(tmpDir, "dest.txt")
|
||||
if err := backend.Get(ctx, testSHA, destFile); err != nil {
|
||||
t.Fatalf("Get failed: %v", err)
|
||||
}
|
||||
|
||||
// Verify bytes match
|
||||
gotData, err := os.ReadFile(destFile)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read dest file: %v", err)
|
||||
}
|
||||
if !bytes.Equal(gotData, testData) {
|
||||
t.Fatalf("data mismatch: want %s, got %s", testData, gotData)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalProbeAbsent(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
backend := &LocalBackend{Root: tmpDir}
|
||||
|
||||
ctx := context.Background()
|
||||
state, transient, err := backend.Probe(ctx, "0b1d0b1d0b1d0b1d0b1d0b1d0b1d0b1d0b1d0b1d0b1d0b1d0b1d0b1d0b1d0b1d")
|
||||
if err != nil {
|
||||
t.Fatalf("Probe should not error on missing blob: %v", err)
|
||||
}
|
||||
if state != Absent {
|
||||
t.Fatalf("expected Absent, got %v", state)
|
||||
}
|
||||
if transient {
|
||||
t.Fatalf("local backend should never be transient")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalGetHashMismatch(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
backend := &LocalBackend{Root: tmpDir}
|
||||
|
||||
// Create a corrupt blob at the expected path
|
||||
sha := "0b1d0b1d0b1d0b1d0b1d0b1d0b1d0b1d0b1d0b1d0b1d0b1d0b1d0b1d0b1d0b1d"
|
||||
blobKey := BlobKey(sha)
|
||||
blobPath := filepath.Join(tmpDir, blobKey)
|
||||
|
||||
if err := os.MkdirAll(filepath.Dir(blobPath), 0755); err != nil {
|
||||
t.Fatalf("failed to create blob dir: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(blobPath, []byte("wrong data"), 0644); err != nil {
|
||||
t.Fatalf("failed to create blob file: %v", err)
|
||||
}
|
||||
|
||||
// Get should error
|
||||
ctx := context.Background()
|
||||
destFile := filepath.Join(tmpDir, "dest.txt")
|
||||
err := backend.Get(ctx, sha, destFile)
|
||||
if err == nil {
|
||||
t.Fatalf("Get should error on hash mismatch")
|
||||
}
|
||||
|
||||
// Destination file should not exist
|
||||
_, err = os.Stat(destFile)
|
||||
if !os.IsNotExist(err) {
|
||||
t.Fatalf("dest file should not exist after Get error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalPutIdempotent(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
backend := &LocalBackend{Root: tmpDir}
|
||||
|
||||
testData := []byte("hello world")
|
||||
testSHA := fmt.Sprintf("%x", sha256.Sum256(testData))
|
||||
|
||||
srcFile := filepath.Join(tmpDir, "source.txt")
|
||||
if err := os.WriteFile(srcFile, testData, 0644); err != nil {
|
||||
t.Fatalf("failed to create source file: %v", err)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Put twice
|
||||
if err := backend.Put(ctx, testSHA, srcFile); err != nil {
|
||||
t.Fatalf("first Put failed: %v", err)
|
||||
}
|
||||
if err := backend.Put(ctx, testSHA, srcFile); err != nil {
|
||||
t.Fatalf("second Put failed: %v", err)
|
||||
}
|
||||
|
||||
// Verify blob exists
|
||||
state, _, err := backend.Probe(ctx, testSHA)
|
||||
if err != nil {
|
||||
t.Fatalf("Probe failed: %v", err)
|
||||
}
|
||||
if state != Present {
|
||||
t.Fatalf("expected Present after Put, got %v", state)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
package depot
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
type manifestFile struct {
|
||||
Assets []struct {
|
||||
Path string `yaml:"path"`
|
||||
SHA256 string `yaml:"sha256"`
|
||||
Size int64 `yaml:"size"`
|
||||
} `yaml:"assets"`
|
||||
}
|
||||
|
||||
var sha256Pattern = regexp.MustCompile(`^[0-9a-f]{64}$`)
|
||||
|
||||
func ValidSHA(s string) bool {
|
||||
return sha256Pattern.MatchString(s)
|
||||
}
|
||||
|
||||
func BlobKey(sha string) string {
|
||||
return fmt.Sprintf("sha256/%s/%s/%s", sha[0:2], sha[2:4], sha)
|
||||
}
|
||||
|
||||
func ReferencedSHAs(dir string) (map[string]int64, error) {
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
result := make(map[string]int64)
|
||||
foundAny := false
|
||||
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
if filepath.Ext(entry.Name()) != ".yml" {
|
||||
continue
|
||||
}
|
||||
|
||||
foundAny = true
|
||||
path := filepath.Join(dir, entry.Name())
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var manifest manifestFile
|
||||
if err := yaml.Unmarshal(data, &manifest); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, asset := range manifest.Assets {
|
||||
if !ValidSHA(asset.SHA256) {
|
||||
return nil, fmt.Errorf("%s: asset %q: invalid sha256 %q", entry.Name(), asset.Path, asset.SHA256)
|
||||
}
|
||||
|
||||
// Keep the largest size for each sha
|
||||
if asset.Size > result[asset.SHA256] {
|
||||
result[asset.SHA256] = asset.Size
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !foundAny {
|
||||
return nil, fmt.Errorf("no *.yml files found in %s", dir)
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
package depot
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestValidSHA(t *testing.T) {
|
||||
tests := []struct {
|
||||
input string
|
||||
want bool
|
||||
}{
|
||||
{"0b1d1234567890abcdef1234567890abcdef1234567890abcdef1234567890ab", true},
|
||||
{"0000000000000000000000000000000000000000000000000000000000000000", true},
|
||||
{"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", true},
|
||||
{"0b1d1234567890ABCDEF1234567890abcdef1234567890abcdef1234567890ab", false}, // uppercase
|
||||
{"0b1d1234567890abcde", false}, // too short
|
||||
{"0b1d1234567890abcdef1234567890abcdef1234567890abcdef1234567890abff", false}, // too long
|
||||
{"0b1d1234567890abcdef1234567890abcdef1234567890abcdef1234567890ag", false}, // invalid hex
|
||||
{"", false},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
got := ValidSHA(tc.input)
|
||||
if got != tc.want {
|
||||
t.Errorf("ValidSHA(%q): got %v, want %v", tc.input, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBlobKey(t *testing.T) {
|
||||
tests := []struct {
|
||||
sha string
|
||||
want string
|
||||
}{
|
||||
{"0b1d1234567890abcdef1234567890abcdef1234567890abcdef1234567890ab", "sha256/0b/1d/0b1d1234567890abcdef1234567890abcdef1234567890abcdef1234567890ab"},
|
||||
{"abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789", "sha256/ab/cd/abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
got := BlobKey(tc.sha)
|
||||
if got != tc.want {
|
||||
t.Errorf("BlobKey(%q): got %q, want %q", tc.sha, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestReferencedSHAs(t *testing.T) {
|
||||
// Test: temp dir with two yml files sharing one sha → dedup
|
||||
tmpdir := t.TempDir()
|
||||
|
||||
// First manifest with sha1 and sha2
|
||||
manifest1 := `assets:
|
||||
- path: file1.txt
|
||||
sha256: 0b1d1234567890abcdef1234567890abcdef1234567890abcdef1234567890ab
|
||||
size: 100
|
||||
- path: file2.txt
|
||||
sha256: abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789
|
||||
size: 200
|
||||
`
|
||||
if err := os.WriteFile(filepath.Join(tmpdir, "manifest1.yml"), []byte(manifest1), 0644); err != nil {
|
||||
t.Fatalf("write manifest1: %v", err)
|
||||
}
|
||||
|
||||
// Second manifest with sha2 and sha3
|
||||
manifest2 := `assets:
|
||||
- path: file3.txt
|
||||
sha256: abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789
|
||||
size: 300
|
||||
- path: file4.txt
|
||||
sha256: fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210
|
||||
size: 400
|
||||
`
|
||||
if err := os.WriteFile(filepath.Join(tmpdir, "manifest2.yml"), []byte(manifest2), 0644); err != nil {
|
||||
t.Fatalf("write manifest2: %v", err)
|
||||
}
|
||||
|
||||
result, err := ReferencedSHAs(tmpdir)
|
||||
if err != nil {
|
||||
t.Fatalf("ReferencedSHAs: %v", err)
|
||||
}
|
||||
|
||||
// Should have 3 unique SHAs
|
||||
if len(result) != 3 {
|
||||
t.Errorf("ReferencedSHAs: got %d unique SHAs, want 3", len(result))
|
||||
}
|
||||
|
||||
expectedSizes := map[string]int64{
|
||||
"0b1d1234567890abcdef1234567890abcdef1234567890abcdef1234567890ab": 100,
|
||||
"abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789": 300, // Largest size for duplicated sha
|
||||
"fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210": 400,
|
||||
}
|
||||
for sha, expectedSize := range expectedSizes {
|
||||
size, exists := result[sha]
|
||||
if !exists {
|
||||
t.Errorf("ReferencedSHAs: sha %q missing", sha)
|
||||
}
|
||||
if size != expectedSize {
|
||||
t.Errorf("ReferencedSHAs: sha %q size: got %d, want %d", sha, size, expectedSize)
|
||||
}
|
||||
}
|
||||
|
||||
// Test: bad sha → error naming the file
|
||||
badManifest := `assets:
|
||||
- path: file5.txt
|
||||
sha256: not_a_valid_sha
|
||||
size: 500
|
||||
`
|
||||
if err := os.WriteFile(filepath.Join(tmpdir, "badsha.yml"), []byte(badManifest), 0644); err != nil {
|
||||
t.Fatalf("write badsha: %v", err)
|
||||
}
|
||||
|
||||
_, err = ReferencedSHAs(tmpdir)
|
||||
if err == nil {
|
||||
t.Fatalf("ReferencedSHAs with bad sha: got nil error, want error naming file and asset path")
|
||||
}
|
||||
want := `badsha.yml: asset "file5.txt": invalid sha256 "not_a_valid_sha"`
|
||||
if err.Error() != want {
|
||||
t.Errorf("ReferencedSHAs error message: got %q, want %q", err.Error(), want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReferencedSHAsEmpty(t *testing.T) {
|
||||
// Test: empty dir → error
|
||||
tmpdir := t.TempDir()
|
||||
_, err := ReferencedSHAs(tmpdir)
|
||||
if err == nil {
|
||||
t.Errorf("ReferencedSHAs on empty dir: got nil error, want error")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,269 @@
|
||||
package depot
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// NewBackend returns the backend for name ("local"|"cdn"|"bunny").
|
||||
// localRoot is used only for "local". Fails closed: "bunny" with empty
|
||||
// StorageHost or empty ReadKey returns an error (never prompts);
|
||||
// unknown name returns an error.
|
||||
func NewBackend(name, localRoot string, cfg Config) (Backend, error) {
|
||||
switch name {
|
||||
case "local":
|
||||
if localRoot == "" {
|
||||
return nil, errors.New("local backend requires a non-empty root")
|
||||
}
|
||||
return &LocalBackend{Root: localRoot}, nil
|
||||
case "cdn":
|
||||
return &httpBackend{
|
||||
name: "cdn",
|
||||
client: newHTTPClient(cfg),
|
||||
cfg: cfg,
|
||||
}, nil
|
||||
case "bunny":
|
||||
if cfg.StorageHost == "" {
|
||||
return nil, errors.New("bunny backend requires BUNNY_STORAGE_HOST")
|
||||
}
|
||||
if cfg.ReadKey == "" {
|
||||
return nil, errors.New("bunny backend requires BUNNY_STORAGE_READ_PASSWORD or BUNNY_STORAGE_PASSWORD")
|
||||
}
|
||||
return &httpBackend{
|
||||
name: "bunny",
|
||||
client: newHTTPClient(cfg),
|
||||
cfg: cfg,
|
||||
}, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unknown backend %q", name)
|
||||
}
|
||||
}
|
||||
|
||||
// newHTTPClient builds an IPv4-only client per spec (HEAD probes are banned;
|
||||
// IPv6 dial hazards are out of scope for this depot).
|
||||
func newHTTPClient(cfg Config) *http.Client {
|
||||
transport := &http.Transport{
|
||||
DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
d := net.Dialer{Timeout: cfg.ConnectTimeout}
|
||||
return d.DialContext(ctx, "tcp4", addr)
|
||||
},
|
||||
MaxIdleConnsPerHost: cfg.ProbeJobs,
|
||||
}
|
||||
return &http.Client{Transport: transport, Timeout: cfg.ProbeMaxTime}
|
||||
}
|
||||
|
||||
// httpBackend implements Backend for both cdn (read-only) and bunny
|
||||
// (read/write) over HTTP, sharing probe/get/put logic.
|
||||
type httpBackend struct {
|
||||
name string
|
||||
client *http.Client
|
||||
cfg Config
|
||||
}
|
||||
|
||||
func (b *httpBackend) Name() string { return b.name }
|
||||
|
||||
func (b *httpBackend) storageURL(sha string) string {
|
||||
host := b.cfg.StorageHost
|
||||
// StorageHost is normally a bare host ("storage.bunnycdn.com"); allow a
|
||||
// full scheme (used by tests against httptest.NewServer) to pass through
|
||||
// unchanged.
|
||||
if strings.Contains(host, "://") {
|
||||
return fmt.Sprintf("%s/%s/%s", strings.TrimSuffix(host, "/"), b.cfg.StorageZone, BlobKey(sha))
|
||||
}
|
||||
return fmt.Sprintf("https://%s/%s/%s", host, b.cfg.StorageZone, BlobKey(sha))
|
||||
}
|
||||
|
||||
func (b *httpBackend) cdnURL(sha string) string {
|
||||
return fmt.Sprintf("%s/%s", b.cfg.CDNBase, BlobKey(sha))
|
||||
}
|
||||
|
||||
// probeRetrySleep is the backoff sleeper for transient probe retries;
|
||||
// tests stub it (same pattern as sweep.go's confirmSleep).
|
||||
var probeRetrySleep = time.Sleep
|
||||
|
||||
// rangeProbe issues GET <url> with Range: bytes=0-0 and classifies the
|
||||
// response. Never HEAD (banned by spec). Transient outcomes (transport
|
||||
// error, or a status that is neither 2xx nor 404/410) retry up to 2 more
|
||||
// times with linear backoff (1s, 2s) — the CDN edge throttles sustained
|
||||
// sweeps; the bash this ports absorbed that with curl --retry 2.
|
||||
func (b *httpBackend) rangeProbe(ctx context.Context, url string, headers map[string]string) (ProbeState, bool, error) {
|
||||
for attempt := 0; ; attempt++ {
|
||||
state, transient, err := b.rangeProbeOnce(ctx, url, headers)
|
||||
if !transient || attempt >= 2 {
|
||||
return state, transient, err
|
||||
}
|
||||
probeRetrySleep(time.Duration(attempt+1) * time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
func (b *httpBackend) rangeProbeOnce(ctx context.Context, url string, headers map[string]string) (ProbeState, bool, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return Unconfirmed, true, err
|
||||
}
|
||||
req.Header.Set("Range", "bytes=0-0")
|
||||
for k, v := range headers {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
|
||||
resp, err := b.client.Do(req)
|
||||
if err != nil {
|
||||
return Unconfirmed, true, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
|
||||
switch {
|
||||
case resp.StatusCode >= 200 && resp.StatusCode < 300:
|
||||
return Present, false, nil
|
||||
case resp.StatusCode == 404 || resp.StatusCode == 410:
|
||||
return Absent, false, nil
|
||||
default:
|
||||
return Unconfirmed, true, nil
|
||||
}
|
||||
}
|
||||
|
||||
// Probe returns the existence state of sha at this backend.
|
||||
func (b *httpBackend) Probe(ctx context.Context, sha string) (ProbeState, bool, error) {
|
||||
if b.name == "cdn" {
|
||||
return b.rangeProbe(ctx, b.cdnURL(sha), nil)
|
||||
}
|
||||
return b.rangeProbe(ctx, b.storageURL(sha), map[string]string{"AccessKey": b.cfg.ReadKey})
|
||||
}
|
||||
|
||||
// Put uploads src for sha. cdn is read-only.
|
||||
func (b *httpBackend) Put(ctx context.Context, sha, src string) error {
|
||||
if b.name == "cdn" {
|
||||
return errors.New("cdn backend is read-only")
|
||||
}
|
||||
if b.cfg.WriteKey == "" {
|
||||
return errors.New("bunny backend requires BUNNY_STORAGE_PASSWORD to write")
|
||||
}
|
||||
|
||||
state, _, err := b.Probe(ctx, sha)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if state == Present {
|
||||
return nil
|
||||
}
|
||||
|
||||
f, err := os.Open(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPut, b.storageURL(sha), f)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("AccessKey", b.cfg.WriteKey)
|
||||
req.Header.Set("Checksum", strings.ToUpper(sha))
|
||||
|
||||
resp, err := b.client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
return fmt.Errorf("bunny put %s: unexpected status %d", sha, resp.StatusCode)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get fetches sha into dest via temp file + rename, re-hashing and deleting
|
||||
// on mismatch. Mirrors bash _depot_bunny_get: try CDN first, fall back to
|
||||
// storage with ReadKey.
|
||||
func (b *httpBackend) Get(ctx context.Context, sha, dest string) error {
|
||||
destDir := filepath.Dir(dest)
|
||||
if err := os.MkdirAll(destDir, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
tmpFile, err := os.CreateTemp(destDir, ".tmp-")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tmpName := tmpFile.Name()
|
||||
defer os.Remove(tmpName)
|
||||
|
||||
resp, err := b.fetch(ctx, sha)
|
||||
if err != nil {
|
||||
tmpFile.Close()
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
hasher := sha256.New()
|
||||
tee := io.TeeReader(resp.Body, hasher)
|
||||
_, err = io.Copy(tmpFile, tee)
|
||||
tmpFile.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
gotHash := fmt.Sprintf("%x", hasher.Sum(nil))
|
||||
if gotHash != sha {
|
||||
os.Remove(tmpName)
|
||||
return fmt.Errorf("hash mismatch: expected %s, got %s", sha, gotHash)
|
||||
}
|
||||
|
||||
return os.Rename(tmpName, dest)
|
||||
}
|
||||
|
||||
// fetch tries the CDN URL first (no auth), falling back to the storage URL
|
||||
// with ReadKey on any non-2xx response or transport error.
|
||||
func (b *httpBackend) fetch(ctx context.Context, sha string) (*http.Response, error) {
|
||||
if b.name != "cdn" {
|
||||
if resp, err := b.get(ctx, b.cdnURL(sha), nil); err == nil && resp.StatusCode >= 200 && resp.StatusCode < 300 {
|
||||
return resp, nil
|
||||
} else if err == nil {
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
resp.Body.Close()
|
||||
}
|
||||
resp, err := b.get(ctx, b.storageURL(sha), map[string]string{"AccessKey": b.cfg.ReadKey})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
resp.Body.Close()
|
||||
return nil, fmt.Errorf("bunny get %s: unexpected status %d", sha, resp.StatusCode)
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
resp, err := b.get(ctx, b.cdnURL(sha), nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
resp.Body.Close()
|
||||
return nil, fmt.Errorf("cdn get %s: unexpected status %d", sha, resp.StatusCode)
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (b *httpBackend) get(ctx context.Context, url string, headers map[string]string) (*http.Response, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for k, v := range headers {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
return b.client.Do(req)
|
||||
}
|
||||
@@ -0,0 +1,549 @@
|
||||
package depot
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func shaOf(s string) string {
|
||||
return fmt.Sprintf("%x", sha256.Sum256([]byte(s)))
|
||||
}
|
||||
|
||||
type recordedReq struct {
|
||||
Method string
|
||||
Path string
|
||||
Range string
|
||||
Headers http.Header
|
||||
}
|
||||
|
||||
// fakeBunny is a minimal recording fake for Bunny storage/CDN endpoints.
|
||||
type fakeBunny struct {
|
||||
mu sync.Mutex
|
||||
requests []recordedReq
|
||||
headCount int
|
||||
blobs map[string][]byte
|
||||
statusFor map[string]int // sha -> status code override for probe/get
|
||||
}
|
||||
|
||||
func newFakeBunny() *fakeBunny {
|
||||
return &fakeBunny{blobs: map[string][]byte{}, statusFor: map[string]int{}}
|
||||
}
|
||||
|
||||
func (f *fakeBunny) handler() http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
f.mu.Lock()
|
||||
f.requests = append(f.requests, recordedReq{
|
||||
Method: r.Method,
|
||||
Path: r.URL.Path,
|
||||
Range: r.Header.Get("Range"),
|
||||
Headers: r.Header.Clone(),
|
||||
})
|
||||
if r.Method == http.MethodHead {
|
||||
f.headCount++
|
||||
f.mu.Unlock()
|
||||
w.WriteHeader(500)
|
||||
return
|
||||
}
|
||||
f.mu.Unlock()
|
||||
|
||||
sha := strings.TrimPrefix(r.URL.Path, "/")
|
||||
// last path element is the sha
|
||||
parts := strings.Split(sha, "/")
|
||||
sha = parts[len(parts)-1]
|
||||
|
||||
if r.Method == http.MethodPut {
|
||||
buf, _ := io.ReadAll(r.Body)
|
||||
f.mu.Lock()
|
||||
f.blobs[sha] = buf
|
||||
f.mu.Unlock()
|
||||
w.WriteHeader(201)
|
||||
return
|
||||
}
|
||||
|
||||
// GET (range probe / actual get)
|
||||
f.mu.Lock()
|
||||
status, hasStatus := f.statusFor[sha]
|
||||
data, ok := f.blobs[sha]
|
||||
f.mu.Unlock()
|
||||
|
||||
if hasStatus {
|
||||
w.WriteHeader(status)
|
||||
return
|
||||
}
|
||||
if !ok {
|
||||
w.WriteHeader(404)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(206)
|
||||
_, _ = w.Write(data)
|
||||
}
|
||||
}
|
||||
|
||||
func (f *fakeBunny) requestsSnapshot() []recordedReq {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
out := make([]recordedReq, len(f.requests))
|
||||
copy(out, f.requests)
|
||||
return out
|
||||
}
|
||||
|
||||
func testCfg(storageHost, readKey, writeKey string) Config {
|
||||
return Config{
|
||||
CDNBase: "http://127.0.0.1:1", // unreachable by default; overridden per test
|
||||
StorageHost: storageHost,
|
||||
StorageZone: "sow-assets-depot",
|
||||
ReadKey: readKey,
|
||||
WriteKey: writeKey,
|
||||
ProbeJobs: 4,
|
||||
Jobs: 4,
|
||||
ConnectTimeout: 2 * time.Second,
|
||||
ProbeMaxTime: 2 * time.Second,
|
||||
ConfirmRetries: 3,
|
||||
ConfirmMax: 200,
|
||||
}
|
||||
}
|
||||
|
||||
// hostPort returns a StorageHost value for tests: a full "http://host:port"
|
||||
// base URL, which storageURL() passes through unchanged (bypassing the
|
||||
// production https:// default so httptest.NewServer works without TLS).
|
||||
func hostPort(srv *httptest.Server) string {
|
||||
return srv.URL
|
||||
}
|
||||
|
||||
// stubProbeSleep replaces probeRetrySleep for the test, recording durations.
|
||||
func stubProbeSleep(t *testing.T) *[]time.Duration {
|
||||
t.Helper()
|
||||
var slept []time.Duration
|
||||
old := probeRetrySleep
|
||||
probeRetrySleep = func(d time.Duration) { slept = append(slept, d) }
|
||||
t.Cleanup(func() { probeRetrySleep = old })
|
||||
return &slept
|
||||
}
|
||||
|
||||
func TestProbeRetriesTransientThenPresent(t *testing.T) {
|
||||
slept := stubProbeSleep(t)
|
||||
sha := shaOf("throttled-blob")
|
||||
|
||||
var calls int
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
calls++
|
||||
if calls == 1 {
|
||||
w.WriteHeader(428)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(206)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
cfg := testCfg(hostPort(srv), "readkey", "writekey")
|
||||
b, err := NewBackend("bunny", "", cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("NewBackend: %v", err)
|
||||
}
|
||||
|
||||
state, transient, err := b.Probe(context.Background(), sha)
|
||||
if err != nil {
|
||||
t.Fatalf("Probe: %v", err)
|
||||
}
|
||||
if state != Present || transient {
|
||||
t.Fatalf("expected Present/non-transient after retry, got %v/%v", state, transient)
|
||||
}
|
||||
if calls != 2 {
|
||||
t.Fatalf("expected 2 requests (428 then 206), got %d", calls)
|
||||
}
|
||||
if len(*slept) != 1 || (*slept)[0] != time.Second {
|
||||
t.Fatalf("expected one 1s backoff, got %v", *slept)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProbe404NoRetry(t *testing.T) {
|
||||
slept := stubProbeSleep(t)
|
||||
sha := shaOf("gone-blob")
|
||||
|
||||
f := newFakeBunny()
|
||||
srv := httptest.NewServer(f.handler())
|
||||
defer srv.Close()
|
||||
|
||||
cfg := testCfg(hostPort(srv), "readkey", "writekey")
|
||||
b, err := NewBackend("bunny", "", cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("NewBackend: %v", err)
|
||||
}
|
||||
|
||||
state, transient, err := b.Probe(context.Background(), sha)
|
||||
if err != nil {
|
||||
t.Fatalf("Probe: %v", err)
|
||||
}
|
||||
if state != Absent || transient {
|
||||
t.Fatalf("expected Absent/non-transient, got %v/%v", state, transient)
|
||||
}
|
||||
if got := len(f.requestsSnapshot()); got != 1 {
|
||||
t.Fatalf("expected exactly 1 request for 404, got %d", got)
|
||||
}
|
||||
if len(*slept) != 0 {
|
||||
t.Fatalf("expected zero backoffs for 404, got %v", *slept)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProbeUsesRangeGetNotHead(t *testing.T) {
|
||||
sha := shaOf("present-blob")
|
||||
f := newFakeBunny()
|
||||
f.blobs[sha] = []byte("hello")
|
||||
srv := httptest.NewServer(f.handler())
|
||||
defer srv.Close()
|
||||
|
||||
cfg := testCfg(hostPort(srv), "readkey", "writekey")
|
||||
b, err := NewBackend("bunny", "", cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("NewBackend: %v", err)
|
||||
}
|
||||
|
||||
state, transient, err := b.Probe(context.Background(), sha)
|
||||
if err != nil {
|
||||
t.Fatalf("Probe: %v", err)
|
||||
}
|
||||
if state != Present || transient {
|
||||
t.Fatalf("expected Present/non-transient, got %v/%v", state, transient)
|
||||
}
|
||||
|
||||
reqs := f.requestsSnapshot()
|
||||
if len(reqs) != 1 {
|
||||
t.Fatalf("expected exactly 1 request, got %d", len(reqs))
|
||||
}
|
||||
if reqs[0].Method != "GET" {
|
||||
t.Fatalf("expected GET, got %s", reqs[0].Method)
|
||||
}
|
||||
if reqs[0].Range != "bytes=0-0" {
|
||||
t.Fatalf("expected Range bytes=0-0, got %q", reqs[0].Range)
|
||||
}
|
||||
if f.headCount != 0 {
|
||||
t.Fatalf("expected zero HEAD requests, got %d", f.headCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProbeStates(t *testing.T) {
|
||||
stubProbeSleep(t)
|
||||
cases := []struct {
|
||||
status int
|
||||
wantState ProbeState
|
||||
wantTransient bool
|
||||
}{
|
||||
{200, Present, false},
|
||||
{206, Present, false},
|
||||
{404, Absent, false},
|
||||
{428, Unconfirmed, true},
|
||||
{503, Unconfirmed, true},
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
t.Run(fmt.Sprintf("status_%d", c.status), func(t *testing.T) {
|
||||
sha := shaOf(fmt.Sprintf("blob-%d", c.status))
|
||||
f := newFakeBunny()
|
||||
f.statusFor[sha] = c.status
|
||||
srv := httptest.NewServer(f.handler())
|
||||
defer srv.Close()
|
||||
|
||||
cfg := testCfg(hostPort(srv), "readkey", "writekey")
|
||||
b, err := NewBackend("bunny", "", cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("NewBackend: %v", err)
|
||||
}
|
||||
|
||||
state, transient, err := b.Probe(context.Background(), sha)
|
||||
if err != nil && c.status < 500 {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if state != c.wantState {
|
||||
t.Fatalf("status %d: expected state %v, got %v", c.status, c.wantState, state)
|
||||
}
|
||||
if transient != c.wantTransient {
|
||||
t.Fatalf("status %d: expected transient=%v, got %v", c.status, c.wantTransient, transient)
|
||||
}
|
||||
if f.headCount != 0 {
|
||||
t.Fatalf("expected zero HEAD requests, got %d", f.headCount)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBunnyPutSequence(t *testing.T) {
|
||||
t.Run("absent sha", func(t *testing.T) {
|
||||
sha := shaOf("new-blob")
|
||||
f := newFakeBunny()
|
||||
srv := httptest.NewServer(f.handler())
|
||||
defer srv.Close()
|
||||
|
||||
cfg := testCfg(hostPort(srv), "readkey", "writekey")
|
||||
b, err := NewBackend("bunny", "", cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("NewBackend: %v", err)
|
||||
}
|
||||
|
||||
dir := t.TempDir()
|
||||
srcPath := filepath.Join(dir, "src")
|
||||
if err := os.WriteFile(srcPath, []byte("new-blob-content"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := b.Put(context.Background(), sha, srcPath); err != nil {
|
||||
t.Fatalf("Put: %v", err)
|
||||
}
|
||||
|
||||
reqs := f.requestsSnapshot()
|
||||
if len(reqs) != 2 {
|
||||
t.Fatalf("expected 2 requests (probe, put), got %d: %+v", len(reqs), reqs)
|
||||
}
|
||||
if reqs[0].Method != "GET" || reqs[0].Range != "bytes=0-0" {
|
||||
t.Fatalf("expected first request to be range probe GET, got %+v", reqs[0])
|
||||
}
|
||||
if reqs[1].Method != "PUT" {
|
||||
t.Fatalf("expected second request to be PUT, got %+v", reqs[1])
|
||||
}
|
||||
if got := reqs[1].Headers.Get("Checksum"); got != strings.ToUpper(sha) {
|
||||
t.Fatalf("expected Checksum %s, got %s", strings.ToUpper(sha), got)
|
||||
}
|
||||
if got := reqs[1].Headers.Get("AccessKey"); got != "writekey" {
|
||||
t.Fatalf("expected AccessKey writekey, got %s", got)
|
||||
}
|
||||
if f.headCount != 0 {
|
||||
t.Fatalf("expected zero HEAD requests, got %d", f.headCount)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("present sha", func(t *testing.T) {
|
||||
sha := shaOf("existing-blob")
|
||||
f := newFakeBunny()
|
||||
f.blobs[sha] = []byte("existing-blob-content")
|
||||
srv := httptest.NewServer(f.handler())
|
||||
defer srv.Close()
|
||||
|
||||
cfg := testCfg(hostPort(srv), "readkey", "writekey")
|
||||
b, err := NewBackend("bunny", "", cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("NewBackend: %v", err)
|
||||
}
|
||||
|
||||
dir := t.TempDir()
|
||||
srcPath := filepath.Join(dir, "src")
|
||||
if err := os.WriteFile(srcPath, []byte("existing-blob-content"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := b.Put(context.Background(), sha, srcPath); err != nil {
|
||||
t.Fatalf("Put: %v", err)
|
||||
}
|
||||
|
||||
reqs := f.requestsSnapshot()
|
||||
if len(reqs) != 1 {
|
||||
t.Fatalf("expected probe-only (1 request) since present, got %d: %+v", len(reqs), reqs)
|
||||
}
|
||||
if reqs[0].Method != "GET" {
|
||||
t.Fatalf("expected GET probe, got %+v", reqs[0])
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestReadWriteKeySplit(t *testing.T) {
|
||||
sha := shaOf("split-blob")
|
||||
f := newFakeBunny()
|
||||
srv := httptest.NewServer(f.handler())
|
||||
defer srv.Close()
|
||||
|
||||
cfg := testCfg(hostPort(srv), "readkeyXXX", "writekeyYYY")
|
||||
b, err := NewBackend("bunny", "", cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("NewBackend: %v", err)
|
||||
}
|
||||
|
||||
dir := t.TempDir()
|
||||
srcPath := filepath.Join(dir, "src")
|
||||
if err := os.WriteFile(srcPath, []byte("split-blob-content"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := b.Put(context.Background(), sha, srcPath); err != nil {
|
||||
t.Fatalf("Put: %v", err)
|
||||
}
|
||||
|
||||
reqs := f.requestsSnapshot()
|
||||
if len(reqs) != 2 {
|
||||
t.Fatalf("expected 2 requests, got %d", len(reqs))
|
||||
}
|
||||
if got := reqs[0].Headers.Get("AccessKey"); got != "readkeyXXX" {
|
||||
t.Fatalf("probe expected AccessKey readkeyXXX, got %s", got)
|
||||
}
|
||||
if got := reqs[1].Headers.Get("AccessKey"); got != "writekeyYYY" {
|
||||
t.Fatalf("put expected AccessKey writekeyYYY, got %s", got)
|
||||
}
|
||||
if reqs[0].Headers.Get("AccessKey") == reqs[1].Headers.Get("AccessKey") {
|
||||
t.Fatalf("expected distinct read/write keys")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBunnyNoReadKeyFailsClosed(t *testing.T) {
|
||||
// Grep-based tripwire: no non-test depot source may reference stdin/tty.
|
||||
dir := "."
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadDir: %v", err)
|
||||
}
|
||||
for _, e := range entries {
|
||||
name := e.Name()
|
||||
if e.IsDir() || !strings.HasSuffix(name, ".go") || strings.HasSuffix(name, "_test.go") {
|
||||
continue
|
||||
}
|
||||
data, err := os.ReadFile(filepath.Join(dir, name))
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile %s: %v", name, err)
|
||||
}
|
||||
s := string(data)
|
||||
if strings.Contains(s, "os.Stdin") || strings.Contains(s, "bufio.NewReader(os.Stdin)") || strings.Contains(s, "/dev/tty") {
|
||||
t.Fatalf("%s references stdin/tty; credential prompts are banned", name)
|
||||
}
|
||||
}
|
||||
|
||||
// Confirm undrained pipe: swap in an os.Pipe as stdin-equivalent to prove
|
||||
// no read occurs.
|
||||
r, w, err := os.Pipe()
|
||||
if err != nil {
|
||||
t.Fatalf("Pipe: %v", err)
|
||||
}
|
||||
oldStdin := os.Stdin
|
||||
os.Stdin = r
|
||||
defer func() { os.Stdin = oldStdin; r.Close() }()
|
||||
|
||||
cfg := testCfg("storage.example.com", "", "writekey")
|
||||
_, err = NewBackend("bunny", "", cfg)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for empty ReadKey")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "ReadKey") && !strings.Contains(err.Error(), "BUNNY_STORAGE_READ_PASSWORD") && !strings.Contains(err.Error(), "BUNNY_STORAGE_PASSWORD") {
|
||||
t.Fatalf("expected error to mention missing env var, got: %v", err)
|
||||
}
|
||||
|
||||
w.Close()
|
||||
// If code read from stdin it would have blocked already (pipe with no
|
||||
// writer-side data yet); assert the pipe is still open/undrained by
|
||||
// writing now and reading it back ourselves.
|
||||
if _, err := w.Write([]byte("x")); err == nil {
|
||||
t.Fatal("expected write to closed pipe writer to fail")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBunnyNoStorageHostFailsClosed(t *testing.T) {
|
||||
cfg := testCfg("", "readkey", "writekey")
|
||||
_, err := NewBackend("bunny", "", cfg)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for empty StorageHost")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "StorageHost") && !strings.Contains(err.Error(), "BUNNY_STORAGE_HOST") {
|
||||
t.Fatalf("expected error to mention missing StorageHost/BUNNY_STORAGE_HOST, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetHashMismatchDeletes(t *testing.T) {
|
||||
sha := shaOf("get-mismatch-blob")
|
||||
f := newFakeBunny()
|
||||
f.blobs[sha] = []byte("wrong content")
|
||||
srv := httptest.NewServer(f.handler())
|
||||
defer srv.Close()
|
||||
|
||||
cfg := testCfg(hostPort(srv), "readkey", "writekey")
|
||||
// force CDN unreachable so Get falls back to storage
|
||||
cfg.CDNBase = "http://127.0.0.1:1"
|
||||
b, err := NewBackend("bunny", "", cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("NewBackend: %v", err)
|
||||
}
|
||||
|
||||
dir := t.TempDir()
|
||||
dest := filepath.Join(dir, "dest")
|
||||
|
||||
err = b.Get(context.Background(), sha, dest)
|
||||
if err == nil {
|
||||
t.Fatal("expected hash mismatch error")
|
||||
}
|
||||
if _, statErr := os.Stat(dest); !os.IsNotExist(statErr) {
|
||||
t.Fatalf("expected dest to be removed on mismatch, stat err: %v", statErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCDNPutReadOnly(t *testing.T) {
|
||||
cfg := testCfg("storage.example.com", "readkey", "writekey")
|
||||
b, err := NewBackend("cdn", "", cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("NewBackend: %v", err)
|
||||
}
|
||||
|
||||
dir := t.TempDir()
|
||||
srcPath := filepath.Join(dir, "src")
|
||||
if err := os.WriteFile(srcPath, []byte("data"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
err = b.Put(context.Background(), shaOf("anything"), srcPath)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for cdn Put (read-only)")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "read-only") {
|
||||
t.Fatalf("expected read-only error, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestNoHeadInSources is the anti-HEAD regression tripwire from field note 3:
|
||||
// grep-assert no http.MethodHead/.Head( usage in non-test depot sources.
|
||||
func TestNoHeadInSources(t *testing.T) {
|
||||
dir := "."
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadDir: %v", err)
|
||||
}
|
||||
for _, e := range entries {
|
||||
name := e.Name()
|
||||
if e.IsDir() || !strings.HasSuffix(name, ".go") || strings.HasSuffix(name, "_test.go") {
|
||||
continue
|
||||
}
|
||||
data, err := os.ReadFile(filepath.Join(dir, name))
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile %s: %v", name, err)
|
||||
}
|
||||
s := string(data)
|
||||
if strings.Contains(s, "http.MethodHead") || strings.Contains(s, ".Head(") {
|
||||
t.Fatalf("%s references HEAD probe (banned)", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestLocalBackendViaFactory sanity-checks NewBackend("local", ...).
|
||||
func TestLocalBackendViaFactory(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
b, err := NewBackend("local", root, Config{})
|
||||
if err != nil {
|
||||
t.Fatalf("NewBackend: %v", err)
|
||||
}
|
||||
lb, ok := b.(*LocalBackend)
|
||||
if !ok {
|
||||
t.Fatalf("expected *LocalBackend, got %T", b)
|
||||
}
|
||||
if lb.Root != root {
|
||||
t.Fatalf("expected Root=%s, got %s", root, lb.Root)
|
||||
}
|
||||
|
||||
if _, err := NewBackend("local", "", Config{}); err == nil {
|
||||
t.Fatal("expected error for empty local root")
|
||||
}
|
||||
|
||||
if _, err := NewBackend("nonsense", "", Config{}); err == nil {
|
||||
t.Fatal("expected error for unknown backend name")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,479 @@
|
||||
package depot
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/rand"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
exitOK = 0
|
||||
exitDrift = 1
|
||||
exitUnconfirmed = 2
|
||||
exitUsage = 64
|
||||
exitInternal = 70
|
||||
)
|
||||
|
||||
// Run executes a depot subcommand. args[0] is the subcommand
|
||||
// (status|push|verify|get|pull); returns the process exit code.
|
||||
func Run(args []string, stdout, stderr io.Writer, getenv func(string) string) int {
|
||||
if len(args) == 0 {
|
||||
printRunUsage(stderr)
|
||||
return exitUsage
|
||||
}
|
||||
rest := args[1:]
|
||||
switch args[0] {
|
||||
case "status":
|
||||
return runStatus(rest, stdout, stderr, getenv)
|
||||
case "push":
|
||||
return runPush(rest, stdout, stderr, getenv)
|
||||
case "verify":
|
||||
return runVerify(rest, stdout, stderr, getenv)
|
||||
case "get":
|
||||
return runGet(rest, stdout, stderr, getenv)
|
||||
case "pull":
|
||||
return runPull(rest, stdout, stderr, getenv)
|
||||
default:
|
||||
fmt.Fprintf(stderr, "depot: unknown subcommand %q\n\n", args[0])
|
||||
printRunUsage(stderr)
|
||||
return exitUsage
|
||||
}
|
||||
}
|
||||
|
||||
func printRunUsage(w io.Writer) {
|
||||
fmt.Fprint(w, `usage:
|
||||
depot status [--manifests DIR] [--source LOCAL_DEPOT] --target bunny|cdn|local
|
||||
depot push [--manifests DIR] --source LOCAL_DEPOT --target bunny
|
||||
depot verify [--manifests DIR] --target bunny|cdn [--sample N]
|
||||
depot get <sha> <dest> --target cdn|bunny|local
|
||||
depot pull [--manifests DIR] --dest DIR --target cdn|bunny
|
||||
|
||||
--target local uses the DEPOT_DIR environment variable as the local root.
|
||||
`)
|
||||
}
|
||||
|
||||
// errUsage marks errors that are the caller's fault (usage, exit 64) rather
|
||||
// than internal/backend failures (exit 70).
|
||||
var errUsage = errors.New("usage error")
|
||||
|
||||
// resolveBackend builds the named backend, resolving "local" against DEPOT_DIR.
|
||||
func resolveBackend(target string, getenv func(string) string, cfg Config) (Backend, error) {
|
||||
root := ""
|
||||
if target == "local" {
|
||||
root = getenv("DEPOT_DIR")
|
||||
if root == "" {
|
||||
return nil, fmt.Errorf("--target local requires DEPOT_DIR to be set: %w", errUsage)
|
||||
}
|
||||
}
|
||||
return NewBackend(target, root, cfg)
|
||||
}
|
||||
|
||||
// backendErrExit maps a resolveBackend error onto the exit contract.
|
||||
func backendErrExit(err error) int {
|
||||
if errors.Is(err, errUsage) {
|
||||
return exitUsage
|
||||
}
|
||||
return exitInternal
|
||||
}
|
||||
|
||||
func printSweepStatus(stdout io.Writer, referenced int, res SweepResult) {
|
||||
fmt.Fprintf(stdout, "referenced=%d present=%d missing=%d unconfirmed=%d\n",
|
||||
referenced, len(res.Present), len(res.Missing), len(res.Unconfirmed))
|
||||
for _, sha := range res.Missing {
|
||||
fmt.Fprintf(stdout, "missing %s\n", sha)
|
||||
}
|
||||
for _, sha := range res.Unconfirmed {
|
||||
fmt.Fprintf(stdout, "unconfirmed %s\n", sha)
|
||||
}
|
||||
}
|
||||
|
||||
func sweepExitCode(res SweepResult) int {
|
||||
if len(res.Missing) > 0 {
|
||||
return exitDrift
|
||||
}
|
||||
if len(res.Unconfirmed) > 0 {
|
||||
return exitUnconfirmed
|
||||
}
|
||||
return exitOK
|
||||
}
|
||||
|
||||
func shaKeys(m map[string]int64) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for sha := range m {
|
||||
out = append(out, sha)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
func runStatus(args []string, stdout, stderr io.Writer, getenv func(string) string) int {
|
||||
fs := flag.NewFlagSet("status", flag.ContinueOnError)
|
||||
fs.SetOutput(stderr)
|
||||
manifests := fs.String("manifests", "assets", "directory of *.yml manifests")
|
||||
_ = fs.String("source", "", "local depot root (unused for status target=local; see DEPOT_DIR)")
|
||||
target := fs.String("target", "", "bunny|cdn|local")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return exitUsage
|
||||
}
|
||||
if *target == "" {
|
||||
fmt.Fprintln(stderr, "depot status: --target is required")
|
||||
return exitUsage
|
||||
}
|
||||
|
||||
cfg := LoadConfig(getenv)
|
||||
backend, err := resolveBackend(*target, getenv, cfg)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "depot status:", err)
|
||||
return backendErrExit(err)
|
||||
}
|
||||
|
||||
shaSizes, err := ReferencedSHAs(*manifests)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "depot status:", err)
|
||||
return exitInternal
|
||||
}
|
||||
shas := shaKeys(shaSizes)
|
||||
|
||||
res, err := Sweep(context.Background(), backend, shas, cfg, stderr)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "depot status:", err)
|
||||
return exitInternal
|
||||
}
|
||||
|
||||
printSweepStatus(stdout, len(shas), res)
|
||||
return sweepExitCode(res)
|
||||
}
|
||||
|
||||
func runPush(args []string, stdout, stderr io.Writer, getenv func(string) string) int {
|
||||
fs := flag.NewFlagSet("push", flag.ContinueOnError)
|
||||
fs.SetOutput(stderr)
|
||||
manifests := fs.String("manifests", "assets", "directory of *.yml manifests")
|
||||
source := fs.String("source", "", "local depot root to read blobs from")
|
||||
target := fs.String("target", "", "must be bunny")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return exitUsage
|
||||
}
|
||||
if *target != "bunny" {
|
||||
fmt.Fprintln(stderr, "depot push: --target must be bunny")
|
||||
return exitUsage
|
||||
}
|
||||
if *source == "" {
|
||||
fmt.Fprintln(stderr, "depot push: --source is required")
|
||||
return exitUsage
|
||||
}
|
||||
|
||||
cfg := LoadConfig(getenv)
|
||||
backend, err := NewBackend("bunny", "", cfg)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "depot push:", err)
|
||||
return exitInternal
|
||||
}
|
||||
|
||||
shaSizes, err := ReferencedSHAs(*manifests)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "depot push:", err)
|
||||
return exitInternal
|
||||
}
|
||||
shas := shaKeys(shaSizes)
|
||||
|
||||
res, err := Sweep(context.Background(), backend, shas, cfg, stderr)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "depot push:", err)
|
||||
return exitInternal
|
||||
}
|
||||
|
||||
toUpload := append(append([]string(nil), res.Missing...), res.Unconfirmed...)
|
||||
sort.Strings(toUpload)
|
||||
|
||||
var (
|
||||
mu sync.Mutex
|
||||
uploaded int
|
||||
failed int
|
||||
transportErr error
|
||||
)
|
||||
|
||||
work := func(sha string) {
|
||||
srcPath := filepath.Join(*source, BlobKey(sha))
|
||||
if _, statErr := os.Stat(srcPath); statErr != nil {
|
||||
mu.Lock()
|
||||
failed++
|
||||
mu.Unlock()
|
||||
fmt.Fprintf(stderr, "push: missing source blob %s\n", sha)
|
||||
return
|
||||
}
|
||||
if err := backend.Put(context.Background(), sha, srcPath); err != nil {
|
||||
mu.Lock()
|
||||
failed++
|
||||
if transportErr == nil {
|
||||
transportErr = err
|
||||
}
|
||||
mu.Unlock()
|
||||
fmt.Fprintf(stderr, "push: upload %s: %v\n", sha, err)
|
||||
return
|
||||
}
|
||||
mu.Lock()
|
||||
uploaded++
|
||||
mu.Unlock()
|
||||
}
|
||||
|
||||
runWorkers(cfg.Jobs, toUpload, work)
|
||||
|
||||
fmt.Fprintf(stdout, "uploaded=%d failed=%d\n", uploaded, failed)
|
||||
|
||||
if transportErr != nil {
|
||||
return exitInternal
|
||||
}
|
||||
if failed > 0 {
|
||||
return exitDrift
|
||||
}
|
||||
return exitOK
|
||||
}
|
||||
|
||||
func runVerify(args []string, stdout, stderr io.Writer, getenv func(string) string) int {
|
||||
fs := flag.NewFlagSet("verify", flag.ContinueOnError)
|
||||
fs.SetOutput(stderr)
|
||||
manifests := fs.String("manifests", "assets", "directory of *.yml manifests")
|
||||
target := fs.String("target", "", "bunny|cdn")
|
||||
sample := fs.Int("sample", 3, "number of present blobs to spot-check by download (0=skip)")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return exitUsage
|
||||
}
|
||||
if *target != "bunny" && *target != "cdn" {
|
||||
fmt.Fprintln(stderr, "depot verify: --target must be bunny or cdn")
|
||||
return exitUsage
|
||||
}
|
||||
|
||||
cfg := LoadConfig(getenv)
|
||||
backend, err := NewBackend(*target, "", cfg)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "depot verify:", err)
|
||||
return exitInternal
|
||||
}
|
||||
|
||||
shaSizes, err := ReferencedSHAs(*manifests)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "depot verify:", err)
|
||||
return exitInternal
|
||||
}
|
||||
shas := shaKeys(shaSizes)
|
||||
|
||||
res, err := Sweep(context.Background(), backend, shas, cfg, stderr)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "depot verify:", err)
|
||||
return exitInternal
|
||||
}
|
||||
|
||||
printSweepStatus(stdout, len(shas), res)
|
||||
if code := sweepExitCode(res); code != exitOK {
|
||||
return code
|
||||
}
|
||||
|
||||
if *sample <= 0 || len(res.Present) == 0 {
|
||||
return exitOK
|
||||
}
|
||||
|
||||
n := *sample
|
||||
if n > len(res.Present) {
|
||||
n = len(res.Present)
|
||||
}
|
||||
rng := rand.New(rand.NewSource(time.Now().UnixNano()))
|
||||
picks := rng.Perm(len(res.Present))[:n]
|
||||
|
||||
tmpDir, err := os.MkdirTemp("", "depot-verify-")
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "depot verify:", err)
|
||||
return exitInternal
|
||||
}
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
for _, idx := range picks {
|
||||
sha := res.Present[idx]
|
||||
dest := filepath.Join(tmpDir, sha)
|
||||
if err := backend.Get(context.Background(), sha, dest); err != nil {
|
||||
fmt.Fprintf(stderr, "depot verify: sample %s: %v\n", sha, err)
|
||||
return exitDrift
|
||||
}
|
||||
}
|
||||
return exitOK
|
||||
}
|
||||
|
||||
func runGet(args []string, stdout, stderr io.Writer, getenv func(string) string) int {
|
||||
fs := flag.NewFlagSet("get", flag.ContinueOnError)
|
||||
fs.SetOutput(stderr)
|
||||
target := fs.String("target", "", "cdn|bunny|local")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return exitUsage
|
||||
}
|
||||
positional := fs.Args()
|
||||
if len(positional) != 2 {
|
||||
fmt.Fprintln(stderr, "depot get: usage: depot get <sha> <dest> --target cdn|bunny|local")
|
||||
return exitUsage
|
||||
}
|
||||
sha, dest := positional[0], positional[1]
|
||||
if !ValidSHA(sha) {
|
||||
fmt.Fprintf(stderr, "depot get: invalid sha %q\n", sha)
|
||||
return exitUsage
|
||||
}
|
||||
if *target == "" {
|
||||
fmt.Fprintln(stderr, "depot get: --target is required")
|
||||
return exitUsage
|
||||
}
|
||||
|
||||
cfg := LoadConfig(getenv)
|
||||
backend, err := resolveBackend(*target, getenv, cfg)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "depot get:", err)
|
||||
return backendErrExit(err)
|
||||
}
|
||||
|
||||
if err := backend.Get(context.Background(), sha, dest); err != nil {
|
||||
fmt.Fprintln(stderr, "depot get:", err)
|
||||
return exitInternal
|
||||
}
|
||||
return exitOK
|
||||
}
|
||||
|
||||
func runPull(args []string, stdout, stderr io.Writer, getenv func(string) string) int {
|
||||
fs := flag.NewFlagSet("pull", flag.ContinueOnError)
|
||||
fs.SetOutput(stderr)
|
||||
manifests := fs.String("manifests", "assets", "directory of *.yml manifests")
|
||||
dest := fs.String("dest", "", "local directory to pull blobs into")
|
||||
target := fs.String("target", "", "cdn|bunny")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return exitUsage
|
||||
}
|
||||
if *target != "bunny" && *target != "cdn" {
|
||||
fmt.Fprintln(stderr, "depot pull: --target must be bunny or cdn")
|
||||
return exitUsage
|
||||
}
|
||||
if *dest == "" {
|
||||
fmt.Fprintln(stderr, "depot pull: --dest is required")
|
||||
return exitUsage
|
||||
}
|
||||
|
||||
cfg := LoadConfig(getenv)
|
||||
backend, err := NewBackend(*target, "", cfg)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "depot pull:", err)
|
||||
return exitInternal
|
||||
}
|
||||
|
||||
shaSizes, err := ReferencedSHAs(*manifests)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "depot pull:", err)
|
||||
return exitInternal
|
||||
}
|
||||
shas := shaKeys(shaSizes)
|
||||
|
||||
// Split into "already present and correct locally" (skip) vs "needs a
|
||||
// probe/download decision".
|
||||
var present int
|
||||
var toCheck []string
|
||||
for _, sha := range shas {
|
||||
destPath := filepath.Join(*dest, BlobKey(sha))
|
||||
if localFileMatchesSHA(destPath, sha) {
|
||||
present++
|
||||
continue
|
||||
}
|
||||
toCheck = append(toCheck, sha)
|
||||
}
|
||||
|
||||
// Sweep the source to distinguish "not there" (exit 1, listed) from
|
||||
// "there, download it".
|
||||
res, err := Sweep(context.Background(), backend, toCheck, cfg, stderr)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "depot pull:", err)
|
||||
return exitInternal
|
||||
}
|
||||
|
||||
var (
|
||||
mu sync.Mutex
|
||||
pulled int
|
||||
anyFail bool
|
||||
)
|
||||
work := func(sha string) {
|
||||
destPath := filepath.Join(*dest, BlobKey(sha))
|
||||
if err := backend.Get(context.Background(), sha, destPath); err != nil {
|
||||
mu.Lock()
|
||||
anyFail = true
|
||||
mu.Unlock()
|
||||
fmt.Fprintf(stderr, "pull: %s: %v\n", sha, err)
|
||||
return
|
||||
}
|
||||
mu.Lock()
|
||||
pulled++
|
||||
mu.Unlock()
|
||||
}
|
||||
runWorkers(cfg.Jobs, res.Present, work)
|
||||
|
||||
// ponytail: unconfirmed source state (probe budget exhausted / flaky
|
||||
// responses) is treated as a download failure rather than a third exit
|
||||
// path; if that proves too coarse in practice, give pull its own
|
||||
// unconfirmed accounting like status.
|
||||
if len(res.Unconfirmed) > 0 {
|
||||
anyFail = true
|
||||
for _, sha := range res.Unconfirmed {
|
||||
fmt.Fprintf(stderr, "pull: %s: unconfirmed at source\n", sha)
|
||||
}
|
||||
}
|
||||
|
||||
for _, sha := range res.Missing {
|
||||
fmt.Fprintf(stdout, "missing %s\n", sha)
|
||||
}
|
||||
|
||||
fmt.Fprintf(stdout, "pulled=%d present=%d\n", pulled, present)
|
||||
|
||||
if anyFail {
|
||||
return exitInternal
|
||||
}
|
||||
if len(res.Missing) > 0 {
|
||||
return exitDrift
|
||||
}
|
||||
return exitOK
|
||||
}
|
||||
|
||||
// localFileMatchesSHA reports whether path exists and hashes to sha.
|
||||
func localFileMatchesSHA(path, sha string) bool {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
defer f.Close()
|
||||
h := sha256.New()
|
||||
if _, err := io.Copy(h, f); err != nil {
|
||||
return false
|
||||
}
|
||||
return fmt.Sprintf("%x", h.Sum(nil)) == sha
|
||||
}
|
||||
|
||||
// runWorkers runs work(sha) for each sha in items using up to jobs goroutines.
|
||||
func runWorkers(jobs int, items []string, work func(sha string)) {
|
||||
if jobs < 1 {
|
||||
jobs = 1
|
||||
}
|
||||
ch := make(chan string)
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < jobs; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for sha := range ch {
|
||||
work(sha)
|
||||
}
|
||||
}()
|
||||
}
|
||||
for _, sha := range items {
|
||||
ch <- sha
|
||||
}
|
||||
close(ch)
|
||||
wg.Wait()
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
package depot
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func testGetenv(vals map[string]string) func(string) string {
|
||||
return func(k string) string { return vals[k] }
|
||||
}
|
||||
|
||||
func TestRunUsage(t *testing.T) {
|
||||
t.Run("no args", func(t *testing.T) {
|
||||
var out, errb bytes.Buffer
|
||||
code := Run(nil, &out, &errb, testGetenv(nil))
|
||||
if code != 64 {
|
||||
t.Fatalf("expected 64, got %d (stderr=%s)", code, errb.String())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("unknown subcommand", func(t *testing.T) {
|
||||
var out, errb bytes.Buffer
|
||||
code := Run([]string{"bogus"}, &out, &errb, testGetenv(nil))
|
||||
if code != 64 {
|
||||
t.Fatalf("expected 64, got %d (stderr=%s)", code, errb.String())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("status --target local without DEPOT_DIR", func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writeManifest(t, dir, shaOf("blob-a"), 5)
|
||||
var out, errb bytes.Buffer
|
||||
code := Run([]string{"status", "--manifests", dir, "--target", "local"}, &out, &errb, testGetenv(nil))
|
||||
if code != 64 {
|
||||
t.Fatalf("expected 64, got %d (stderr=%s)", code, errb.String())
|
||||
}
|
||||
if !bytesContains(errb.String(), "DEPOT_DIR") {
|
||||
t.Fatalf("expected stderr to mention DEPOT_DIR, got %s", errb.String())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("push --target cdn rejected", func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
var out, errb bytes.Buffer
|
||||
code := Run([]string{"push", "--manifests", dir, "--source", dir, "--target", "cdn"}, &out, &errb, testGetenv(nil))
|
||||
if code != 64 {
|
||||
t.Fatalf("expected 64, got %d (stderr=%s)", code, errb.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestGetInvalidSHA(t *testing.T) {
|
||||
var out, errb bytes.Buffer
|
||||
dir := t.TempDir()
|
||||
code := Run([]string{"get", "not-a-sha", dir + "/dest", "--target", "local"}, &out, &errb, testGetenv(map[string]string{"DEPOT_DIR": dir}))
|
||||
if code != 64 {
|
||||
t.Fatalf("expected 64, got %d (stderr=%s)", code, errb.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestNoKeyStatusBunnyFailsClosedNoStdin(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writeManifest(t, dir, shaOf("blob-a"), 5)
|
||||
|
||||
var out, errb bytes.Buffer
|
||||
code := Run([]string{"status", "--manifests", dir, "--target", "bunny"}, &out, &errb, testGetenv(nil))
|
||||
if code != 70 {
|
||||
t.Fatalf("expected 70, got %d (stdout=%s stderr=%s)", code, out.String(), errb.String())
|
||||
}
|
||||
if !bytesContains(errb.String(), "BUNNY_STORAGE_HOST") {
|
||||
t.Fatalf("expected stderr to mention BUNNY_STORAGE_HOST, got %s", errb.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatusExitCodes(t *testing.T) {
|
||||
t.Run("drift", func(t *testing.T) {
|
||||
f := newFakeBunny()
|
||||
srv := httptest.NewServer(f.handler())
|
||||
defer srv.Close()
|
||||
|
||||
sha := shaOf("missing-blob")
|
||||
dir := t.TempDir()
|
||||
writeManifest(t, dir, sha, 5)
|
||||
|
||||
var out, errb bytes.Buffer
|
||||
code := Run([]string{"status", "--manifests", dir, "--target", "bunny"}, &out, &errb,
|
||||
testGetenv(map[string]string{
|
||||
"BUNNY_STORAGE_HOST": hostPort(srv),
|
||||
"BUNNY_STORAGE_READ_PASSWORD": "readkey",
|
||||
}))
|
||||
if code != 1 {
|
||||
t.Fatalf("expected 1, got %d (stdout=%s stderr=%s)", code, out.String(), errb.String())
|
||||
}
|
||||
if !bytesContains(out.String(), "missing="+"1") {
|
||||
t.Fatalf("expected missing=1 in output, got %s", out.String())
|
||||
}
|
||||
if !bytesContains(out.String(), "missing "+sha) {
|
||||
t.Fatalf("expected missing line for sha, got %s", out.String())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("unconfirmed-only", func(t *testing.T) {
|
||||
stubProbeSleep(t)
|
||||
f := newFakeBunny()
|
||||
sha := shaOf("flaky-blob")
|
||||
f.statusFor[sha] = 428
|
||||
srv := httptest.NewServer(f.handler())
|
||||
defer srv.Close()
|
||||
|
||||
dir := t.TempDir()
|
||||
writeManifest(t, dir, sha, 5)
|
||||
|
||||
var out, errb bytes.Buffer
|
||||
code := Run([]string{"status", "--manifests", dir, "--target", "bunny"}, &out, &errb,
|
||||
testGetenv(map[string]string{
|
||||
"BUNNY_STORAGE_HOST": hostPort(srv),
|
||||
"BUNNY_STORAGE_READ_PASSWORD": "readkey",
|
||||
"DEPOT_CONFIRM_RETRIES": "1",
|
||||
}))
|
||||
if code != 2 {
|
||||
t.Fatalf("expected 2, got %d (stdout=%s stderr=%s)", code, out.String(), errb.String())
|
||||
}
|
||||
if !bytesContains(out.String(), "unconfirmed="+"1") {
|
||||
t.Fatalf("expected unconfirmed=1 in output, got %s", out.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestPushTransportErrorCountsFailed(t *testing.T) {
|
||||
// Probes 404 (blob absent), PUTs 500 (transport-level upload failure).
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method == http.MethodPut {
|
||||
w.WriteHeader(500)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(404)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
sha := shaOf("push-fail-blob")
|
||||
manifestsDir := t.TempDir()
|
||||
writeManifest(t, manifestsDir, sha, 14)
|
||||
sourceDir := t.TempDir()
|
||||
blobPath := filepath.Join(sourceDir, BlobKey(sha))
|
||||
if err := os.MkdirAll(filepath.Dir(blobPath), 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(blobPath, []byte("push-fail-blob"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var out, errb bytes.Buffer
|
||||
code := Run([]string{"push", "--manifests", manifestsDir, "--source", sourceDir, "--target", "bunny"}, &out, &errb,
|
||||
testGetenv(map[string]string{
|
||||
"BUNNY_STORAGE_HOST": hostPort(srv),
|
||||
"BUNNY_STORAGE_PASSWORD": "writekey",
|
||||
}))
|
||||
if code != 70 {
|
||||
t.Fatalf("expected 70, got %d (stdout=%s stderr=%s)", code, out.String(), errb.String())
|
||||
}
|
||||
if !bytesContains(out.String(), "uploaded=0 failed=1") {
|
||||
t.Fatalf("expected uploaded=0 failed=1, got %s", out.String())
|
||||
}
|
||||
}
|
||||
|
||||
func bytesContains(s, substr string) bool {
|
||||
return bytes.Contains([]byte(s), []byte(substr))
|
||||
}
|
||||
|
||||
func writeManifest(t *testing.T, dir, sha string, size int64) {
|
||||
t.Helper()
|
||||
content := fmt.Sprintf("assets:\n - path: foo\n sha256: %s\n size: %d\n", sha, size)
|
||||
if err := os.WriteFile(filepath.Join(dir, "manifest.yml"), []byte(content), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
package depot
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// confirmSleep is the backoff sleeper for the serial confirm phase;
|
||||
// overridden in tests.
|
||||
var confirmSleep = time.Sleep
|
||||
|
||||
// SweepResult partitions the swept shas by confirmed state. Unconfirmed is
|
||||
// distinct from Missing and must never be collapsed into it: it means the
|
||||
// probe budget was exhausted, not that the blob is known absent.
|
||||
type SweepResult struct {
|
||||
Present []string
|
||||
Missing []string
|
||||
Unconfirmed []string
|
||||
}
|
||||
|
||||
type probeOutcome struct {
|
||||
sha string
|
||||
state ProbeState
|
||||
transient bool
|
||||
}
|
||||
|
||||
// Sweep probes every sha against b: one parallel pass (cfg.ProbeJobs workers)
|
||||
// followed by a serial confirm phase over every non-Present result (unless
|
||||
// the non-Present count exceeds cfg.ConfirmMax, in which case all of them
|
||||
// are reported Unconfirmed with zero re-probes). Progress is reported to
|
||||
// progress roughly every 1000 blobs.
|
||||
func Sweep(ctx context.Context, b Backend, shas []string, cfg Config, progress io.Writer) (SweepResult, error) {
|
||||
sorted := append([]string(nil), shas...)
|
||||
sort.Strings(sorted)
|
||||
|
||||
outcomes := make([]probeOutcome, len(sorted))
|
||||
jobs := cfg.ProbeJobs
|
||||
if jobs < 1 {
|
||||
jobs = 1
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
work := make(chan int)
|
||||
var probed int64
|
||||
var mu sync.Mutex
|
||||
var firstErr error
|
||||
|
||||
worker := func() {
|
||||
defer wg.Done()
|
||||
for i := range work {
|
||||
state, transient, err := b.Probe(ctx, sorted[i])
|
||||
mu.Lock()
|
||||
if err != nil && !transient {
|
||||
if firstErr == nil {
|
||||
firstErr = fmt.Errorf("probe %s: %w", sorted[i], err)
|
||||
}
|
||||
} else {
|
||||
outcomes[i] = probeOutcome{sha: sorted[i], state: state, transient: transient}
|
||||
}
|
||||
probed++
|
||||
n := probed
|
||||
mu.Unlock()
|
||||
if n%1000 == 0 {
|
||||
fmt.Fprintf(progress, "probed %d/%d\n", n, len(sorted))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for w := 0; w < jobs; w++ {
|
||||
wg.Add(1)
|
||||
go worker()
|
||||
}
|
||||
for i := range sorted {
|
||||
work <- i
|
||||
}
|
||||
close(work)
|
||||
wg.Wait()
|
||||
|
||||
if firstErr != nil {
|
||||
return SweepResult{}, firstErr
|
||||
}
|
||||
|
||||
var res SweepResult
|
||||
var pending []probeOutcome
|
||||
for _, o := range outcomes {
|
||||
switch o.state {
|
||||
case Present:
|
||||
res.Present = append(res.Present, o.sha)
|
||||
case Absent:
|
||||
res.Missing = append(res.Missing, o.sha)
|
||||
default:
|
||||
pending = append(pending, o)
|
||||
}
|
||||
}
|
||||
|
||||
if cfg.ConfirmMax > 0 && len(pending) > cfg.ConfirmMax {
|
||||
for _, o := range pending {
|
||||
res.Unconfirmed = append(res.Unconfirmed, o.sha)
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
for _, o := range pending {
|
||||
state, err := confirm(ctx, b, o.sha, cfg.ConfirmRetries)
|
||||
if err != nil {
|
||||
return SweepResult{}, err
|
||||
}
|
||||
switch state {
|
||||
case Present:
|
||||
res.Present = append(res.Present, o.sha)
|
||||
case Absent:
|
||||
res.Missing = append(res.Missing, o.sha)
|
||||
default:
|
||||
res.Unconfirmed = append(res.Unconfirmed, o.sha)
|
||||
}
|
||||
}
|
||||
|
||||
sort.Strings(res.Present)
|
||||
sort.Strings(res.Missing)
|
||||
sort.Strings(res.Unconfirmed)
|
||||
|
||||
if len(sorted)%1000 != 0 {
|
||||
fmt.Fprintf(progress, "probed %d/%d\n", len(sorted), len(sorted))
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// confirm re-probes sha serially up to retries attempts with linear backoff
|
||||
// (1s, 2s, ... between attempts). A 2xx result confirms Present, a 404/410
|
||||
// confirms Missing (Absent) immediately. If it is still transient after all
|
||||
// attempts, the state is left Unconfirmed rather than guessed as Missing.
|
||||
func confirm(ctx context.Context, b Backend, sha string, retries int) (ProbeState, error) {
|
||||
if retries < 1 {
|
||||
retries = 1
|
||||
}
|
||||
var last ProbeState = Unconfirmed
|
||||
for attempt := 1; attempt <= retries; attempt++ {
|
||||
state, transient, err := b.Probe(ctx, sha)
|
||||
if err != nil && !transient {
|
||||
return Unconfirmed, fmt.Errorf("confirm %s: %w", sha, err)
|
||||
}
|
||||
if state == Present || state == Absent {
|
||||
return state, nil
|
||||
}
|
||||
last = Unconfirmed
|
||||
if attempt < retries {
|
||||
confirmSleep(time.Duration(attempt) * time.Second)
|
||||
}
|
||||
}
|
||||
return last, nil
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
package depot
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"sort"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// stubBackend lets tests script Probe behavior per call.
|
||||
type stubBackend struct {
|
||||
probe func(ctx context.Context, sha string) (ProbeState, bool, error)
|
||||
}
|
||||
|
||||
func (s *stubBackend) Name() string { return "stub" }
|
||||
func (s *stubBackend) Probe(ctx context.Context, sha string) (ProbeState, bool, error) {
|
||||
return s.probe(ctx, sha)
|
||||
}
|
||||
func (s *stubBackend) Get(ctx context.Context, sha, dest string) error { return nil }
|
||||
func (s *stubBackend) Put(ctx context.Context, sha, src string) error { return nil }
|
||||
|
||||
func sweepTestCfg() Config {
|
||||
return Config{ProbeJobs: 4, ConfirmRetries: 3, ConfirmMax: 200}
|
||||
}
|
||||
|
||||
func TestSweepAllPresent(t *testing.T) {
|
||||
b := &stubBackend{probe: func(ctx context.Context, sha string) (ProbeState, bool, error) {
|
||||
return Present, false, nil
|
||||
}}
|
||||
shas := []string{"bbb", "aaa", "ccc"}
|
||||
res, err := Sweep(context.Background(), b, shas, sweepTestCfg(), io.Discard)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
want := []string{"aaa", "bbb", "ccc"}
|
||||
sort.Strings(want)
|
||||
if len(res.Present) != 3 || len(res.Missing) != 0 || len(res.Unconfirmed) != 0 {
|
||||
t.Fatalf("got %+v", res)
|
||||
}
|
||||
for i, sha := range res.Present {
|
||||
if sha != want[i] {
|
||||
t.Fatalf("expected sorted output, got %v", res.Present)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSweepMissing(t *testing.T) {
|
||||
b := &stubBackend{probe: func(ctx context.Context, sha string) (ProbeState, bool, error) {
|
||||
return Absent, false, nil
|
||||
}}
|
||||
shas := []string{"aaa"}
|
||||
res, err := Sweep(context.Background(), b, shas, sweepTestCfg(), io.Discard)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if len(res.Missing) != 1 || res.Missing[0] != "aaa" {
|
||||
t.Fatalf("got %+v", res)
|
||||
}
|
||||
if len(res.Present) != 0 || len(res.Unconfirmed) != 0 {
|
||||
t.Fatalf("got %+v", res)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSweepThrottledUnconfirmed(t *testing.T) {
|
||||
var sleeps []time.Duration
|
||||
orig := confirmSleep
|
||||
confirmSleep = func(d time.Duration) { sleeps = append(sleeps, d) }
|
||||
defer func() { confirmSleep = orig }()
|
||||
|
||||
b := &stubBackend{probe: func(ctx context.Context, sha string) (ProbeState, bool, error) {
|
||||
return Unconfirmed, true, nil
|
||||
}}
|
||||
shas := []string{"aaa"}
|
||||
cfg := sweepTestCfg()
|
||||
res, err := Sweep(context.Background(), b, shas, cfg, io.Discard)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if len(res.Unconfirmed) != 1 || res.Unconfirmed[0] != "aaa" {
|
||||
t.Fatalf("got %+v", res)
|
||||
}
|
||||
if len(res.Missing) != 0 {
|
||||
t.Fatalf("must never collapse into missing: %+v", res)
|
||||
}
|
||||
if len(sleeps) != 2 || sleeps[0] != 1*time.Second || sleeps[1] != 2*time.Second {
|
||||
t.Fatalf("expected sleeps [1s 2s], got %v", sleeps)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSweepTransientRecovers(t *testing.T) {
|
||||
orig := confirmSleep
|
||||
confirmSleep = func(d time.Duration) {}
|
||||
defer func() { confirmSleep = orig }()
|
||||
|
||||
var calls int32
|
||||
b := &stubBackend{probe: func(ctx context.Context, sha string) (ProbeState, bool, error) {
|
||||
n := atomic.AddInt32(&calls, 1)
|
||||
if n == 1 {
|
||||
return Unconfirmed, true, nil // initial parallel probe: 428
|
||||
}
|
||||
return Present, false, nil // serial re-probe: 206
|
||||
}}
|
||||
res, err := Sweep(context.Background(), b, []string{"aaa"}, sweepTestCfg(), io.Discard)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if len(res.Present) != 1 || res.Present[0] != "aaa" {
|
||||
t.Fatalf("got %+v", res)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSweepConfirmMaxCap(t *testing.T) {
|
||||
orig := confirmSleep
|
||||
var sleepCalls int32
|
||||
confirmSleep = func(d time.Duration) { atomic.AddInt32(&sleepCalls, 1) }
|
||||
defer func() { confirmSleep = orig }()
|
||||
|
||||
b := &stubBackend{probe: func(ctx context.Context, sha string) (ProbeState, bool, error) {
|
||||
return Unconfirmed, true, nil
|
||||
}}
|
||||
shas := []string{"a", "b", "c", "d", "e"}
|
||||
cfg := sweepTestCfg()
|
||||
cfg.ConfirmMax = 2
|
||||
res, err := Sweep(context.Background(), b, shas, cfg, io.Discard)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if len(res.Unconfirmed) != 5 {
|
||||
t.Fatalf("expected all 5 unconfirmed, got %+v", res)
|
||||
}
|
||||
if len(res.Missing) != 0 || len(res.Present) != 0 {
|
||||
t.Fatalf("got %+v", res)
|
||||
}
|
||||
if sleepCalls != 0 {
|
||||
t.Fatalf("expected zero serial re-probes/sleeps when over ConfirmMax, got %d calls", sleepCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSweepParallelBounded(t *testing.T) {
|
||||
var inFlight int32
|
||||
var maxInFlight int32
|
||||
shas := make([]string, 100)
|
||||
for i := range shas {
|
||||
shas[i] = string(rune('a'+i%26)) + string(rune('A'+i/26))
|
||||
}
|
||||
b := &stubBackend{probe: func(ctx context.Context, sha string) (ProbeState, bool, error) {
|
||||
n := atomic.AddInt32(&inFlight, 1)
|
||||
for {
|
||||
cur := atomic.LoadInt32(&maxInFlight)
|
||||
if n <= cur || atomic.CompareAndSwapInt32(&maxInFlight, cur, n) {
|
||||
break
|
||||
}
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
atomic.AddInt32(&inFlight, -1)
|
||||
return Present, false, nil
|
||||
}}
|
||||
cfg := sweepTestCfg()
|
||||
cfg.ProbeJobs = 4
|
||||
_, err := Sweep(context.Background(), b, shas, cfg, io.Discard)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if maxInFlight > 4 {
|
||||
t.Fatalf("expected max concurrency <= 4, got %d", maxInFlight)
|
||||
}
|
||||
}
|
||||
+253
-44
@@ -2,21 +2,23 @@
|
||||
// builders shared by the `crucible` dispatcher and the standalone
|
||||
// `crucible-<name>` shims.
|
||||
//
|
||||
// Cutover state (Phase 5 → 6): the internal pipeline/topdata/erf/wiki/music
|
||||
// Cutover state (Phase 5 → 6): the internal pipeline/topdata/erf/wiki
|
||||
// packages migrated from gitea/sow-tools now live in this tree, so wired
|
||||
// builders delegate to internal/app's legacy command surface (mapped per
|
||||
// docs/command-surface.md). A builder with no migrated logic yet (depot) keeps
|
||||
// the Wired=false fail-closed path: exit 70, never a faked artifact.
|
||||
// docs/command-surface.md). depot is wired but bypasses that legacy surface
|
||||
// entirely, delegating straight to internal/depot.Run. A builder with no
|
||||
// migrated logic yet keeps the Wired=false fail-closed path: exit 70, never a
|
||||
// faked artifact.
|
||||
package dispatch
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"text/tabwriter"
|
||||
|
||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/app"
|
||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/buildinfo"
|
||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/depot"
|
||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/menu"
|
||||
)
|
||||
|
||||
@@ -30,25 +32,59 @@ const (
|
||||
|
||||
// Builder is one tool surface in the Crucible suite.
|
||||
type Builder struct {
|
||||
Name string // canonical dispatcher subcommand, e.g. "module"
|
||||
Bin string // standalone binary name, e.g. "crucible-module"
|
||||
Summary string // one-line description
|
||||
Legacy []string // nwn-tool commands this subsumes (migration aid; see docs/command-surface.md)
|
||||
Extra []string // additional callable subcommands beyond Legacy (e.g. cross-listed "music")
|
||||
Wired bool // true once the builder delegates to migrated internal/app logic
|
||||
Name string // canonical dispatcher subcommand, e.g. "module"
|
||||
Bin string // standalone binary name, e.g. "crucible-module"
|
||||
Summary string // one-line description
|
||||
Commands []Command // visible commands plus hidden compatibility aliases
|
||||
Wired bool // true once the builder delegates to migrated internal/app logic
|
||||
}
|
||||
|
||||
// subcommands returns every legacy command a wired builder accepts: the
|
||||
// canonical Legacy set plus any cross-listed Extra commands.
|
||||
func (b Builder) subcommands() []string { return append(append([]string{}, b.Legacy...), b.Extra...) }
|
||||
// Command is one public builder action. AppCommand names the unchanged
|
||||
// internal/app implementation command. Aliases remain callable for compatibility
|
||||
// but are omitted from routine help and the interactive menu.
|
||||
type Command struct {
|
||||
Name string
|
||||
Summary string
|
||||
AppCommand string
|
||||
Usage string
|
||||
Options []string
|
||||
Aliases []CommandAlias
|
||||
}
|
||||
|
||||
func (b Builder) accepts(sub string) bool {
|
||||
for _, s := range b.subcommands() {
|
||||
if s == sub {
|
||||
return true
|
||||
type CommandAlias struct {
|
||||
Name string
|
||||
AppCommand string
|
||||
}
|
||||
|
||||
func (b Builder) subcommands() []string {
|
||||
commands := make([]string, 0, len(b.Commands))
|
||||
for _, command := range b.Commands {
|
||||
commands = append(commands, command.Name)
|
||||
}
|
||||
return commands
|
||||
}
|
||||
|
||||
func (b Builder) command(name string) (Command, string, bool) {
|
||||
for _, command := range b.Commands {
|
||||
if command.Name == name {
|
||||
return command, command.AppCommand, true
|
||||
}
|
||||
for _, alias := range command.Aliases {
|
||||
if alias.Name == name {
|
||||
target := alias.AppCommand
|
||||
if target == "" {
|
||||
target = command.AppCommand
|
||||
}
|
||||
return command, target, true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
return Command{}, "", false
|
||||
}
|
||||
|
||||
func (b Builder) accepts(sub string) bool {
|
||||
_, _, ok := b.command(sub)
|
||||
return ok
|
||||
}
|
||||
|
||||
// Registry is the single source of truth for the Crucible command surface.
|
||||
@@ -57,41 +93,180 @@ var Registry = []Builder{
|
||||
{
|
||||
Name: "depot",
|
||||
Bin: "crucible-depot",
|
||||
Summary: "verify/move content-addressed depot blobs (SeaweedFS)",
|
||||
Legacy: nil,
|
||||
Wired: false, // no migrated logic yet; fails closed (exit 70)
|
||||
Summary: "content-addressed asset depot (local/cdn/bunny)",
|
||||
Commands: []Command{
|
||||
{Name: "status", Summary: "report referenced-vs-present drift against a backend", Usage: "crucible depot status [--manifests DIR] [--source DIR] --target bunny|cdn|local"},
|
||||
{Name: "push", Summary: "upload referenced-but-absent blobs from a local depot", Usage: "crucible depot push [--manifests DIR] --source DIR --target bunny"},
|
||||
{Name: "verify", Summary: "existence sweep plus sampled download re-hash", Usage: "crucible depot verify [--manifests DIR] --target bunny|cdn [--sample N]"},
|
||||
{Name: "get", Summary: "fetch one blob with sha re-verify", Usage: "crucible depot get <sha> <dest> --target cdn|bunny|local"},
|
||||
{Name: "pull", Summary: "incremental verified pull of every referenced blob", Usage: "crucible depot pull [--manifests DIR] --dest DIR --target cdn|bunny"},
|
||||
},
|
||||
Wired: true,
|
||||
},
|
||||
{
|
||||
Name: "hak",
|
||||
Bin: "crucible-hak",
|
||||
Summary: "pack/unpack ERF/HAK archives + hak manifests",
|
||||
Legacy: []string{"build-haks", "apply-hak-manifest"},
|
||||
Extra: []string{"music"}, // music BMUs are packed into HAKs (command-surface.md)
|
||||
Wired: true,
|
||||
Commands: []Command{
|
||||
{
|
||||
Name: "build",
|
||||
Summary: "build configured HAK archives and their manifest",
|
||||
AppCommand: "build-haks",
|
||||
Usage: "crucible hak build [options]",
|
||||
Options: []string{
|
||||
"--hak <name> build selected configured HAKs",
|
||||
"--archive <name> build selected archive members",
|
||||
"--source-manifest <path> read a generated source manifest",
|
||||
"--content-addressed-root <path> resolve source-manifest blobs here",
|
||||
"--plan-only plan outputs without writing archives",
|
||||
"--quiet | --verbose | --debug select output detail",
|
||||
},
|
||||
Aliases: []CommandAlias{{Name: "build-haks"}},
|
||||
},
|
||||
{
|
||||
Name: "manifest",
|
||||
Summary: "apply a generated HAK list to the module source",
|
||||
AppCommand: "apply-hak-manifest",
|
||||
Usage: "crucible hak manifest [manifest-path]",
|
||||
Aliases: []CommandAlias{{Name: "apply-hak-manifest"}},
|
||||
},
|
||||
},
|
||||
Wired: true,
|
||||
},
|
||||
{
|
||||
Name: "module",
|
||||
Bin: "crucible-module",
|
||||
Summary: "build/extract/validate/compare the .mod",
|
||||
Legacy: []string{"build", "build-module", "extract", "validate", "compare"},
|
||||
// apply-hak-manifest is canonically a hak command but reachable here too;
|
||||
// music is folded into the module build pipeline (command-surface.md).
|
||||
Extra: []string{"apply-hak-manifest", "music"},
|
||||
Commands: []Command{
|
||||
{
|
||||
Name: "build",
|
||||
Summary: "build the module and configured project outputs",
|
||||
AppCommand: "build",
|
||||
Usage: "crucible module build [--quiet|--verbose|--debug]",
|
||||
Aliases: []CommandAlias{{Name: "build-module", AppCommand: "build-module"}},
|
||||
},
|
||||
{
|
||||
Name: "extract",
|
||||
Summary: "extract built archives into configured source trees",
|
||||
AppCommand: "extract",
|
||||
Usage: "crucible module extract [resource ...]",
|
||||
},
|
||||
{
|
||||
Name: "validate",
|
||||
Summary: "validate project layout, config, and source inventory",
|
||||
AppCommand: "validate",
|
||||
Usage: "crucible module validate",
|
||||
},
|
||||
{
|
||||
Name: "compare",
|
||||
Summary: "compare source content with built module and HAK archives",
|
||||
AppCommand: "compare",
|
||||
Usage: "crucible module compare",
|
||||
},
|
||||
{
|
||||
Name: "manifest",
|
||||
Summary: "apply a generated HAK list to the module source",
|
||||
AppCommand: "apply-hak-manifest",
|
||||
Usage: "crucible module manifest [manifest-path]",
|
||||
Aliases: []CommandAlias{{Name: "apply-hak-manifest"}},
|
||||
},
|
||||
},
|
||||
Wired: true,
|
||||
},
|
||||
{
|
||||
Name: "topdata",
|
||||
Bin: "crucible-topdata",
|
||||
Summary: "compile 2da/tlk topdata + packages",
|
||||
Legacy: []string{"validate-topdata", "build-topdata", "build-top-package", "compare-topdata", "convert-topdata"},
|
||||
Wired: true,
|
||||
Commands: []Command{
|
||||
{
|
||||
Name: "validate",
|
||||
Summary: "validate topdata layout and canonical JSON compatibility",
|
||||
AppCommand: "validate-topdata",
|
||||
Usage: "crucible topdata validate",
|
||||
Aliases: []CommandAlias{{Name: "validate-topdata"}},
|
||||
},
|
||||
{
|
||||
Name: "build",
|
||||
Summary: "compile topdata and refresh the configured package",
|
||||
AppCommand: "build-topdata",
|
||||
Usage: "crucible topdata build [--force] [--wiki] [--skip-lfs]",
|
||||
Options: []string{
|
||||
"--force rebuild outputs even when current",
|
||||
"--wiki build wiki drafts after compilation",
|
||||
"--skip-lfs skip Git LFS materialization",
|
||||
},
|
||||
Aliases: []CommandAlias{{Name: "build-topdata"}},
|
||||
},
|
||||
{
|
||||
Name: "package",
|
||||
Summary: "package cached topdata outputs into HAK and TLK files",
|
||||
AppCommand: "build-top-package",
|
||||
Usage: "crucible topdata package [--force] [--skip-lfs]",
|
||||
Options: []string{
|
||||
"--force rebuild outputs even when current",
|
||||
"--skip-lfs skip Git LFS materialization",
|
||||
},
|
||||
Aliases: []CommandAlias{{Name: "build-top-package"}},
|
||||
},
|
||||
{
|
||||
Name: "compare",
|
||||
Summary: "compare current output with a fresh native build",
|
||||
AppCommand: "compare-topdata",
|
||||
Usage: "crucible topdata compare",
|
||||
Aliases: []CommandAlias{{Name: "compare-topdata"}},
|
||||
},
|
||||
{
|
||||
Name: "convert",
|
||||
Summary: "convert between 2DA and native JSON/module formats",
|
||||
AppCommand: "convert-topdata",
|
||||
Usage: "crucible topdata convert <2da-to-json|2da-to-module|json-to-2da> ...",
|
||||
Options: []string{
|
||||
"2da-to-json <input.2da> <output.json>",
|
||||
"2da-to-module [flags] <input.2da> [output.json]",
|
||||
"json-to-2da <input.json> <output.2da>",
|
||||
},
|
||||
Aliases: []CommandAlias{{Name: "convert-topdata"}},
|
||||
},
|
||||
},
|
||||
Wired: true,
|
||||
},
|
||||
{
|
||||
Name: "wiki",
|
||||
Bin: "crucible-wiki",
|
||||
Summary: "render + deploy mechanical wiki pages",
|
||||
Legacy: []string{"build-wiki", "deploy-wiki"},
|
||||
Wired: true,
|
||||
Commands: []Command{
|
||||
{
|
||||
Name: "build",
|
||||
Summary: "render wiki page drafts from compiled topdata",
|
||||
AppCommand: "build-wiki",
|
||||
Usage: "crucible wiki build [--force]",
|
||||
Options: []string{"--force rebuild drafts even when current"},
|
||||
Aliases: []CommandAlias{{Name: "build-wiki"}},
|
||||
},
|
||||
{
|
||||
Name: "deploy",
|
||||
Summary: "deploy generated wiki pages to NodeBB",
|
||||
AppCommand: "deploy-wiki",
|
||||
Usage: "crucible wiki deploy [options]",
|
||||
Options: []string{
|
||||
"--source-dir <path> read generated pages here",
|
||||
"--endpoint <url> override the NodeBB endpoint",
|
||||
"--token <token> authenticate with an API token",
|
||||
"--username/--password <value> authenticate with credentials",
|
||||
"--version <version> label the deployed revision",
|
||||
"--namespace <name> deploy selected namespaces",
|
||||
"--category <namespace=id> override a namespace category",
|
||||
"--manifest <path> write deployment state here",
|
||||
"--stale-policy <policy> report, archive, or purge",
|
||||
"--dry-run report changes without writing",
|
||||
"--create allow missing pages to be created",
|
||||
"--force update unchanged pages",
|
||||
"--reset-managed-namespaces reset managed namespace state",
|
||||
},
|
||||
Aliases: []CommandAlias{{Name: "deploy-wiki"}},
|
||||
},
|
||||
},
|
||||
Wired: true,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -129,10 +304,14 @@ func menuItems() []menu.Item {
|
||||
continue
|
||||
}
|
||||
for _, sub := range b.subcommands() {
|
||||
command, _, _ := b.command(sub)
|
||||
items = append(items, menu.Item{
|
||||
Label: b.Name + " " + sub,
|
||||
Desc: b.Summary,
|
||||
Desc: command.Summary,
|
||||
Args: []string{b.Name, sub},
|
||||
Usage: command.Usage,
|
||||
Options: append([]string(nil),
|
||||
command.Options...),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -196,6 +375,12 @@ func runBuilder(name string, args []string, out, errw io.Writer) int {
|
||||
return exitOK
|
||||
}
|
||||
}
|
||||
if b.Name == "depot" && b.Wired {
|
||||
// depot parses its own subcommand (status/push/verify/get/pull) and
|
||||
// has its own richer exit contract (0/1/2/64/70), so it bypasses the
|
||||
// b.Commands/delegateLegacy routing entirely.
|
||||
return depot.Run(args, out, errw, os.Getenv)
|
||||
}
|
||||
if !b.Wired {
|
||||
// No migrated logic yet (depot): fail closed, never fake an artifact.
|
||||
fmt.Fprintf(errw, unwiredMsg, b.Name, b.Bin)
|
||||
@@ -207,14 +392,21 @@ func runBuilder(name string, args []string, out, errw io.Writer) int {
|
||||
return exitUsage
|
||||
}
|
||||
sub := args[0]
|
||||
if !b.accepts(sub) {
|
||||
command, appCommand, ok := b.command(sub)
|
||||
if !ok {
|
||||
fmt.Fprintf(errw, "crucible %s: unknown subcommand %q\n\n", b.Name, sub)
|
||||
builderHelp(errw, b)
|
||||
return exitUsage
|
||||
}
|
||||
// Delegate to the migrated legacy command surface. args[0] is already the
|
||||
// legacy command name, so it is forwarded unchanged.
|
||||
return delegateLegacy(args, errw)
|
||||
if len(args) > 1 {
|
||||
switch args[1] {
|
||||
case "-h", "--help", "help":
|
||||
commandHelp(out, command)
|
||||
return exitOK
|
||||
}
|
||||
}
|
||||
legacyArgs := append([]string{appCommand}, args[1:]...)
|
||||
return delegateLegacy(legacyArgs, errw)
|
||||
}
|
||||
|
||||
// delegateLegacy runs a migrated nwn-tool command via internal/app and maps its
|
||||
@@ -250,17 +442,15 @@ func usage(w io.Writer) {
|
||||
fmt.Fprintf(w, " config [args] inspect/validate effective configuration\n")
|
||||
fmt.Fprintf(w, " changelog [args] generate the release changelog\n")
|
||||
fmt.Fprintf(w, "\nBuilders read all inputs from the project tree (nwn-tool.yaml + data/); no\n")
|
||||
fmt.Fprintf(w, "NWN install is required. If a builder ever needs an NWN root, it must be\n")
|
||||
fmt.Fprintf(w, "passed explicitly via NWN_ROOT; crucible never guesses from $HOME. See\n")
|
||||
fmt.Fprintf(w, "NWN install is required. A builder that one day needs NWN game data\n")
|
||||
fmt.Fprintf(w, "auto-detects the install; it never requires a hand-set path. See\n")
|
||||
fmt.Fprintf(w, "docs/consumer-contract.md.\n")
|
||||
}
|
||||
|
||||
func list(w io.Writer) {
|
||||
tw := tabwriter.NewWriter(w, 0, 2, 2, ' ', 0)
|
||||
for _, b := range Registry {
|
||||
fmt.Fprintf(tw, " %s\t%s\t%s\n", b.Name, b.Bin, b.Summary)
|
||||
fmt.Fprintf(w, "%s\t%s\t%s\n", b.Name, b.Bin, b.Summary)
|
||||
}
|
||||
tw.Flush()
|
||||
}
|
||||
|
||||
func builderHelp(w io.Writer, b Builder) {
|
||||
@@ -272,8 +462,27 @@ func builderHelp(w io.Writer, b Builder) {
|
||||
}
|
||||
fmt.Fprintf(w, "usage: crucible %s <subcommand> [args] (or: %s <subcommand> [args])\n\n", b.Name, b.Bin)
|
||||
fmt.Fprintf(w, "subcommands:\n")
|
||||
for _, c := range b.subcommands() {
|
||||
fmt.Fprintf(w, " %s\n", c)
|
||||
width := 0
|
||||
for _, command := range b.Commands {
|
||||
if len(command.Name) > width {
|
||||
width = len(command.Name)
|
||||
}
|
||||
}
|
||||
for _, command := range b.Commands {
|
||||
fmt.Fprintf(w, " %-*s %s\n", width, command.Name, command.Summary)
|
||||
}
|
||||
fmt.Fprintf(w, "\nstatus: wired to migrated nwn-tool logic. See docs/command-surface.md.\n")
|
||||
}
|
||||
|
||||
func commandHelp(w io.Writer, command Command) {
|
||||
fmt.Fprintf(w, "%s\n", command.Summary)
|
||||
if command.Usage != "" {
|
||||
fmt.Fprintf(w, "\nusage: %s\n", command.Usage)
|
||||
}
|
||||
if len(command.Options) > 0 {
|
||||
fmt.Fprintln(w, "\noptions:")
|
||||
for _, option := range command.Options {
|
||||
fmt.Fprintf(w, " %s\n", option)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -25,22 +25,54 @@ func TestHelpAndNoArgs(t *testing.T) {
|
||||
if code := run([]string{"help"}, &out, &errw); code != exitOK {
|
||||
t.Fatalf("help exit=%d want %d", code, exitOK)
|
||||
}
|
||||
if !strings.Contains(out.String(), "builders:") {
|
||||
t.Fatalf("help output missing builders section:\n%s", out.String())
|
||||
for _, builder := range Registry {
|
||||
if !strings.Contains(out.String(), builder.Name) {
|
||||
t.Errorf("help output missing builder %q:\n%s", builder.Name, out.String())
|
||||
}
|
||||
}
|
||||
// No args is a usage error (exit 64) but still prints help.
|
||||
out.Reset()
|
||||
if code := run(nil, &out, &errw); code != exitUsage {
|
||||
t.Fatalf("no-args exit=%d want %d", code, exitUsage)
|
||||
}
|
||||
if out.Len() == 0 {
|
||||
t.Fatal("no-args usage error should include help")
|
||||
}
|
||||
}
|
||||
|
||||
func TestListCoversRegistry(t *testing.T) {
|
||||
var out bytes.Buffer
|
||||
list(&out)
|
||||
for _, b := range Registry {
|
||||
if !strings.Contains(out.String(), b.Name) || !strings.Contains(out.String(), b.Bin) {
|
||||
t.Fatalf("list missing %s/%s:\n%s", b.Name, b.Bin, out.String())
|
||||
lines := strings.Split(strings.TrimSpace(out.String()), "\n")
|
||||
if len(lines) != len(Registry) {
|
||||
t.Fatalf("list returned %d rows for %d builders:\n%s", len(lines), len(Registry), out.String())
|
||||
}
|
||||
builders := make(map[string]Builder, len(Registry))
|
||||
for _, builder := range Registry {
|
||||
builders[builder.Name] = builder
|
||||
}
|
||||
seen := make(map[string]bool, len(lines))
|
||||
for _, line := range lines {
|
||||
fields := strings.Split(strings.TrimSpace(line), "\t")
|
||||
if len(fields) != 3 {
|
||||
t.Fatalf("list row must be name<TAB>bin<TAB>summary, got %q", line)
|
||||
}
|
||||
builder, ok := builders[fields[0]]
|
||||
if !ok {
|
||||
t.Errorf("list returned unregistered builder %q", fields[0])
|
||||
continue
|
||||
}
|
||||
if fields[1] != builder.Bin {
|
||||
t.Errorf("builder %q listed binary %q, want %q", builder.Name, fields[1], builder.Bin)
|
||||
}
|
||||
if strings.TrimSpace(fields[2]) == "" {
|
||||
t.Errorf("builder %q listed an empty summary", builder.Name)
|
||||
}
|
||||
seen[fields[0]] = true
|
||||
}
|
||||
for _, builder := range Registry {
|
||||
if !seen[builder.Name] {
|
||||
t.Errorf("list missing builder %q", builder.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -50,8 +82,8 @@ func TestUnknownBuilderFailsUsage(t *testing.T) {
|
||||
if code := run([]string{"frobnicate"}, &out, &errw); code != exitUsage {
|
||||
t.Fatalf("unknown builder exit=%d want %d", code, exitUsage)
|
||||
}
|
||||
if !strings.Contains(errw.String(), "unknown builder") {
|
||||
t.Fatalf("unknown builder stderr=%q", errw.String())
|
||||
if errw.Len() == 0 {
|
||||
t.Fatal("unknown builder should explain the usage error")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -65,8 +97,8 @@ func TestUnwiredBuilderFailsClosed(t *testing.T) {
|
||||
if code := run([]string{b.Name}, &out, &errw); code != exitUnwired {
|
||||
t.Errorf("crucible %s: exit=%d want %d (must fail closed)", b.Name, code, exitUnwired)
|
||||
}
|
||||
if !strings.Contains(errw.String(), "not wired") {
|
||||
t.Errorf("crucible %s: stderr missing fail-closed message: %q", b.Name, errw.String())
|
||||
if errw.Len() == 0 {
|
||||
t.Errorf("crucible %s: missing fail-closed explanation", b.Name)
|
||||
}
|
||||
// Via standalone shim path.
|
||||
out.Reset()
|
||||
@@ -93,8 +125,8 @@ func TestWiredBuilderRejectsBadInvocation(t *testing.T) {
|
||||
if code := runBuilder(b.Name, []string{"frobnicate"}, &out, &errw); code != exitUsage {
|
||||
t.Errorf("crucible %s frobnicate: exit=%d want %d", b.Name, code, exitUsage)
|
||||
}
|
||||
if !strings.Contains(errw.String(), "unknown subcommand") {
|
||||
t.Errorf("crucible %s frobnicate: stderr=%q", b.Name, errw.String())
|
||||
if errw.Len() == 0 {
|
||||
t.Errorf("crucible %s frobnicate: missing usage explanation", b.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -105,34 +137,153 @@ func TestBuilderHelpIsOK(t *testing.T) {
|
||||
if code := runBuilder(b.Name, []string{"--help"}, &out, &errw); code != exitOK {
|
||||
t.Errorf("%s --help: exit=%d want %d", b.Name, code, exitOK)
|
||||
}
|
||||
if !strings.Contains(out.String(), b.Summary) {
|
||||
t.Errorf("%s --help: missing summary", b.Name)
|
||||
if !strings.Contains(out.String(), b.Name) || !strings.Contains(out.String(), b.Bin) {
|
||||
t.Errorf("%s --help: missing builder identity", b.Name)
|
||||
}
|
||||
for _, subcommand := range b.subcommands() {
|
||||
if !strings.Contains(out.String(), subcommand) {
|
||||
t.Errorf("%s --help: missing subcommand %q", b.Name, subcommand)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Every legacy command named in command-surface.md must have exactly one
|
||||
// canonical home (Builder.Legacy), so the migrated surface has no gaps or
|
||||
// ambiguous homes.
|
||||
func TestLegacyCommandsHaveExactlyOneHome(t *testing.T) {
|
||||
legacy := []string{
|
||||
"build", "build-module", "extract", "validate", "compare",
|
||||
"build-haks", "apply-hak-manifest",
|
||||
"validate-topdata", "build-topdata", "build-top-package", "compare-topdata", "convert-topdata",
|
||||
"build-wiki", "deploy-wiki",
|
||||
func TestCanonicalCommandSurface(t *testing.T) {
|
||||
want := map[string][]string{
|
||||
"depot": {"status", "push", "verify", "get", "pull"},
|
||||
"hak": {"build", "manifest"},
|
||||
"module": {"build", "extract", "validate", "compare", "manifest"},
|
||||
"topdata": {"validate", "build", "package", "compare", "convert"},
|
||||
"wiki": {"build", "deploy"},
|
||||
}
|
||||
for _, cmd := range legacy {
|
||||
homes := 0
|
||||
for _, b := range Registry {
|
||||
for _, c := range b.Legacy {
|
||||
if c == cmd {
|
||||
homes++
|
||||
for _, builder := range Registry {
|
||||
got := builder.subcommands()
|
||||
expected, ok := want[builder.Name]
|
||||
if !ok {
|
||||
t.Fatalf("unexpected builder %q", builder.Name)
|
||||
}
|
||||
if strings.Join(got, ",") != strings.Join(expected, ",") {
|
||||
t.Errorf("%s commands = %v, want %v", builder.Name, got, expected)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryCommandNamesAndAliasesAreUnambiguous(t *testing.T) {
|
||||
for _, builder := range Registry {
|
||||
seen := map[string]bool{}
|
||||
for _, command := range builder.Commands {
|
||||
// depot parses its own subcommands and bypasses AppCommand routing
|
||||
// entirely (see the depot special-case in runBuilder), so its
|
||||
// Commands carry no AppCommand.
|
||||
requireAppCommand := builder.Name != "depot"
|
||||
if command.Name == "" || command.Summary == "" || command.Usage == "" || (requireAppCommand && command.AppCommand == "") {
|
||||
t.Errorf("%s has incomplete command metadata: %#v", builder.Name, command)
|
||||
}
|
||||
if seen[command.Name] {
|
||||
t.Errorf("%s repeats command name %q", builder.Name, command.Name)
|
||||
}
|
||||
seen[command.Name] = true
|
||||
for _, alias := range command.Aliases {
|
||||
if alias.Name == "" {
|
||||
t.Errorf("%s %s has an empty alias", builder.Name, command.Name)
|
||||
continue
|
||||
}
|
||||
if seen[alias.Name] {
|
||||
t.Errorf("%s repeats command or alias %q", builder.Name, alias.Name)
|
||||
}
|
||||
seen[alias.Name] = true
|
||||
}
|
||||
}
|
||||
if homes != 1 {
|
||||
t.Errorf("legacy command %q has %d canonical homes, want 1", cmd, homes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMusicCommandsAreNotAccepted(t *testing.T) {
|
||||
for _, builderName := range []string{"hak", "module"} {
|
||||
builder, ok := find(builderName)
|
||||
if !ok {
|
||||
t.Fatalf("missing builder %q", builderName)
|
||||
}
|
||||
if builder.accepts("music") {
|
||||
t.Errorf("%s must not accept the removed music command", builderName)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHiddenAliasesPreserveImplementationTargets(t *testing.T) {
|
||||
tests := []struct {
|
||||
builder string
|
||||
alias string
|
||||
target string
|
||||
}{
|
||||
{"hak", "build-haks", "build-haks"},
|
||||
{"hak", "apply-hak-manifest", "apply-hak-manifest"},
|
||||
{"module", "build-module", "build-module"},
|
||||
{"module", "apply-hak-manifest", "apply-hak-manifest"},
|
||||
{"topdata", "validate-topdata", "validate-topdata"},
|
||||
{"topdata", "build-topdata", "build-topdata"},
|
||||
{"topdata", "build-top-package", "build-top-package"},
|
||||
{"topdata", "compare-topdata", "compare-topdata"},
|
||||
{"topdata", "convert-topdata", "convert-topdata"},
|
||||
{"wiki", "build-wiki", "build-wiki"},
|
||||
{"wiki", "deploy-wiki", "deploy-wiki"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.builder+"/"+tt.alias, func(t *testing.T) {
|
||||
builder, ok := find(tt.builder)
|
||||
if !ok {
|
||||
t.Fatalf("missing builder %q", tt.builder)
|
||||
}
|
||||
command, target, ok := builder.command(tt.alias)
|
||||
if !ok {
|
||||
t.Fatalf("hidden alias %q is not accepted", tt.alias)
|
||||
}
|
||||
if target != tt.target {
|
||||
t.Fatalf("alias %q target = %q, want %q", tt.alias, target, tt.target)
|
||||
}
|
||||
if command.Name == tt.alias {
|
||||
t.Fatalf("alias %q must not be a visible command", tt.alias)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuilderHelpHidesCompatibilityAliases(t *testing.T) {
|
||||
aliases := map[string][]string{
|
||||
"hak": {"build-haks", "apply-hak-manifest"},
|
||||
"module": {"build-module", "apply-hak-manifest"},
|
||||
"topdata": {"validate-topdata", "build-topdata", "build-top-package", "compare-topdata", "convert-topdata"},
|
||||
"wiki": {"build-wiki", "deploy-wiki"},
|
||||
}
|
||||
for builderName, hidden := range aliases {
|
||||
var out, errw bytes.Buffer
|
||||
if code := runBuilder(builderName, []string{"--help"}, &out, &errw); code != exitOK {
|
||||
t.Fatalf("%s help exit = %d", builderName, code)
|
||||
}
|
||||
for _, alias := range hidden {
|
||||
if strings.Contains(out.String(), alias) {
|
||||
t.Errorf("%s help exposed hidden alias %q:\n%s", builderName, alias, out.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCommandHelpUsesCanonicalGuidanceWithoutLoadingProject(t *testing.T) {
|
||||
var out, errw bytes.Buffer
|
||||
if code := runBuilder("topdata", []string{"build", "--help"}, &out, &errw); code != exitOK {
|
||||
t.Fatalf("command help exit = %d, stderr:\n%s", code, errw.String())
|
||||
}
|
||||
for _, want := range []string{
|
||||
"compile topdata",
|
||||
"usage: crucible topdata build",
|
||||
"--force",
|
||||
"--skip-lfs",
|
||||
} {
|
||||
if !strings.Contains(out.String(), want) {
|
||||
t.Errorf("command help missing %q:\n%s", want, out.String())
|
||||
}
|
||||
}
|
||||
if errw.Len() != 0 {
|
||||
t.Fatalf("command help must not load a project or emit errors:\n%s", errw.String())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -142,15 +293,35 @@ func TestMenuItemsSkipsUnwiredIncludesWired(t *testing.T) {
|
||||
t.Fatal("expected menu items")
|
||||
}
|
||||
sawModuleBuild := false
|
||||
sawDepotStatus := false
|
||||
for _, it := range items {
|
||||
if it.Args[0] == "depot" {
|
||||
t.Errorf("unwired builder %q must not appear in the menu", it.Args[0])
|
||||
}
|
||||
if len(it.Args) == 2 && it.Args[0] == "module" && it.Args[1] == "build" {
|
||||
sawModuleBuild = true
|
||||
}
|
||||
if len(it.Args) == 2 && it.Args[0] == "depot" && it.Args[1] == "status" {
|
||||
sawDepotStatus = true
|
||||
}
|
||||
}
|
||||
if !sawModuleBuild {
|
||||
t.Error("expected 'module build' in the menu")
|
||||
}
|
||||
if !sawDepotStatus {
|
||||
t.Error("expected wired builder 'depot status' in the menu")
|
||||
}
|
||||
}
|
||||
|
||||
// TestDepotRoutesToDepotRun asserts the depot special-case in runBuilder
|
||||
// reaches depot.Run rather than the generic Commands/delegateLegacy path or
|
||||
// the unwired fail-closed path. With no manifests dir present, depot.Run's
|
||||
// status command fails resolving the backend/manifest (exit 70), but the
|
||||
// stderr text must come from depot, never the dispatcher's unwiredMsg.
|
||||
func TestDepotRoutesToDepotRun(t *testing.T) {
|
||||
var out, errw bytes.Buffer
|
||||
code := runBuilder("depot", []string{"status", "--target", "local"}, &out, &errw)
|
||||
if errw.Len() == 0 {
|
||||
t.Fatalf("depot status with no manifests dir: expected an error from depot.Run, got no stderr (exit=%d)", code)
|
||||
}
|
||||
if strings.Contains(errw.String(), "not wired yet") {
|
||||
t.Fatalf("depot status: stderr shows the dispatcher's unwired message, want depot.Run's own error:\n%s", errw.String())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -123,6 +123,7 @@ var extensionTypes = map[string]uint16{
|
||||
"mtr": 0x0818,
|
||||
"jpg": 0x081C,
|
||||
"lod": 0x081E,
|
||||
"gif": 0x081F,
|
||||
"png": 0x0820,
|
||||
"lyt": 0x0BB8,
|
||||
"vis": 0x0BB9,
|
||||
@@ -188,6 +189,7 @@ var typeExtensions = map[uint16]string{
|
||||
0x0818: "mtr",
|
||||
0x081C: "jpg",
|
||||
0x081E: "lod",
|
||||
0x081F: "gif",
|
||||
0x0820: "png",
|
||||
0x0BB8: "lyt",
|
||||
0x0BB9: "vis",
|
||||
|
||||
+23
-6
@@ -14,9 +14,11 @@ import (
|
||||
|
||||
// Item is one selectable command.
|
||||
type Item struct {
|
||||
Label string // e.g. "module build"
|
||||
Desc string // e.g. "build/extract/validate/compare the .mod"
|
||||
Args []string // dispatcher args, e.g. {"module", "build"}
|
||||
Label string // e.g. "module build"
|
||||
Desc string // e.g. "build the configured module"
|
||||
Args []string // dispatcher args, e.g. {"module", "build"}
|
||||
Usage string
|
||||
Options []string
|
||||
}
|
||||
|
||||
// ANSI styling; terminals that don't support it simply ignore the codes.
|
||||
@@ -35,9 +37,15 @@ func Select(out io.Writer, in io.Reader, items []Item) ([]string, error) {
|
||||
return nil, fmt.Errorf("no commands available")
|
||||
}
|
||||
fmt.Fprintf(out, "%s%sCrucible%s — pick a command:\n\n", cBold, cCyan, cReset)
|
||||
labelWidth := 0
|
||||
for _, item := range items {
|
||||
if len(item.Label) > labelWidth {
|
||||
labelWidth = len(item.Label)
|
||||
}
|
||||
}
|
||||
for i, it := range items {
|
||||
fmt.Fprintf(out, " %s%2d%s %s%-24s%s %s%s%s\n",
|
||||
cCyan, i+1, cReset, cBold, it.Label, cReset, cDim, it.Desc, cReset)
|
||||
fmt.Fprintf(out, " %s%2d%s %s%-*s%s %s%s%s\n",
|
||||
cCyan, i+1, cReset, cBold, labelWidth, it.Label, cReset, cDim, it.Desc, cReset)
|
||||
}
|
||||
fmt.Fprintf(out, " %sq%s quit\n\nselection: ", cCyan, cReset)
|
||||
|
||||
@@ -52,7 +60,16 @@ func Select(out io.Writer, in io.Reader, items []Item) ([]string, error) {
|
||||
return nil, fmt.Errorf("invalid selection %q", choice)
|
||||
}
|
||||
it := items[n-1]
|
||||
fmt.Fprintf(out, "extra args for %q (optional): ", it.Label)
|
||||
if it.Usage != "" {
|
||||
fmt.Fprintf(out, "\nusage: %s\n", it.Usage)
|
||||
}
|
||||
if len(it.Options) > 0 {
|
||||
fmt.Fprintln(out, "options:")
|
||||
for _, option := range it.Options {
|
||||
fmt.Fprintf(out, " %s\n", option)
|
||||
}
|
||||
}
|
||||
fmt.Fprint(out, "\narguments (press Enter to use defaults): ")
|
||||
argLine, _ := r.ReadString('\n')
|
||||
extra := strings.Fields(strings.TrimSpace(argLine))
|
||||
return append(append([]string{}, it.Args...), extra...), nil
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
package menu
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"reflect"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
@@ -45,3 +47,58 @@ func TestSelectInvalidErrors(t *testing.T) {
|
||||
t.Fatal("out-of-range selection should error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectAlignsDescriptionsAfterLongestLabel(t *testing.T) {
|
||||
items := []Item{
|
||||
{Label: "hak build", Desc: "build haks", Args: []string{"hak", "build"}},
|
||||
{Label: "topdata exceptionally-long-command", Desc: "long command", Args: []string{"topdata", "long"}},
|
||||
}
|
||||
var out bytes.Buffer
|
||||
if _, err := Select(&out, strings.NewReader("q\n"), items); err != nil {
|
||||
t.Fatalf("select: %v", err)
|
||||
}
|
||||
text := stripANSI(out.String())
|
||||
var positions []int
|
||||
for _, line := range strings.Split(text, "\n") {
|
||||
for _, desc := range []string{"build haks", "long command"} {
|
||||
if index := strings.Index(line, desc); index >= 0 {
|
||||
positions = append(positions, index)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(positions) != 2 {
|
||||
t.Fatalf("expected two menu descriptions, got %d:\n%s", len(positions), text)
|
||||
}
|
||||
if positions[0] != positions[1] {
|
||||
t.Fatalf("descriptions are not aligned: columns %v\n%s", positions, text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectShowsCommandGuidanceBeforeArgumentsPrompt(t *testing.T) {
|
||||
items := []Item{{
|
||||
Label: "topdata build",
|
||||
Desc: "compile topdata",
|
||||
Args: []string{"topdata", "build"},
|
||||
Usage: "crucible topdata build [--force]",
|
||||
Options: []string{"--force rebuild outputs"},
|
||||
}}
|
||||
var out bytes.Buffer
|
||||
if _, err := Select(&out, strings.NewReader("1\n\n"), items); err != nil {
|
||||
t.Fatalf("select: %v", err)
|
||||
}
|
||||
text := stripANSI(out.String())
|
||||
for _, want := range []string{
|
||||
"usage: crucible topdata build [--force]",
|
||||
"options:",
|
||||
"--force rebuild outputs",
|
||||
"arguments (press Enter to use defaults):",
|
||||
} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Errorf("selected command guidance missing %q:\n%s", want, text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func stripANSI(value string) string {
|
||||
return regexp.MustCompile(`\x1b\[[0-9;]*m`).ReplaceAllString(value, "")
|
||||
}
|
||||
|
||||
@@ -1,47 +0,0 @@
|
||||
package music
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type DatasetConfig struct {
|
||||
Source string `json:"source" yaml:"source"`
|
||||
Prefix string `json:"prefix,omitempty" yaml:"prefix,omitempty"`
|
||||
StageRoot string `json:"stage_root,omitempty" yaml:"stage_root,omitempty"`
|
||||
CreditsRoot string `json:"credits_root,omitempty" yaml:"credits_root,omitempty"`
|
||||
}
|
||||
|
||||
type DefaultsConfig struct {
|
||||
StageRoot string `json:"stage_root,omitempty" yaml:"stage_root,omitempty"`
|
||||
CreditsRoot string `json:"credits_root,omitempty" yaml:"credits_root,omitempty"`
|
||||
CreditsOverlay string `json:"credits_overlay,omitempty" yaml:"credits_overlay,omitempty"`
|
||||
MaxStemLength *int `json:"max_stem_length,omitempty" yaml:"max_stem_length,omitempty"`
|
||||
}
|
||||
|
||||
func ValidateDatasetSource(field, source string) error {
|
||||
trimmed := strings.TrimSpace(source)
|
||||
if trimmed == "" {
|
||||
return fmt.Errorf("%s.source is required", field)
|
||||
}
|
||||
if strings.Contains(trimmed, "\x00") {
|
||||
return fmt.Errorf("%s.source must not contain NUL bytes", field)
|
||||
}
|
||||
clean := filepath.Clean(filepath.FromSlash(trimmed))
|
||||
if clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) {
|
||||
return fmt.Errorf("%s.source must not escape project root", field)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func ValidateDatasetID(id string) error {
|
||||
trimmed := strings.TrimSpace(id)
|
||||
if trimmed == "" {
|
||||
return fmt.Errorf("dataset id must not be empty")
|
||||
}
|
||||
if strings.Contains(trimmed, "/") || strings.Contains(trimmed, "\\") {
|
||||
return fmt.Errorf("dataset id %q must not contain path separators", trimmed)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,292 +0,0 @@
|
||||
package music
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func ParseCreditsOverlay(path string) (CreditsOverlay, error) {
|
||||
entries, err := ParseCreditsMarkdown(path)
|
||||
if err != nil {
|
||||
return CreditsOverlay{}, err
|
||||
}
|
||||
overlay := CreditsOverlay{
|
||||
ByOriginal: make(map[string]CreditsEntry),
|
||||
ByOutput: make(map[string]CreditsEntry),
|
||||
Entries: entries,
|
||||
}
|
||||
for _, entry := range entries {
|
||||
if entry.OriginalFile != "" {
|
||||
key := strings.ToLower(strings.TrimSpace(entry.OriginalFile))
|
||||
if _, exists := overlay.ByOriginal[key]; exists {
|
||||
return CreditsOverlay{}, fmt.Errorf("duplicate manual credits row for original file %s", entry.OriginalFile)
|
||||
}
|
||||
overlay.ByOriginal[key] = entry
|
||||
}
|
||||
if entry.OutputFile != "" {
|
||||
key := strings.ToLower(strings.TrimSpace(entry.OutputFile))
|
||||
if _, exists := overlay.ByOutput[key]; exists {
|
||||
return CreditsOverlay{}, fmt.Errorf("duplicate manual credits row for output file %s", entry.OutputFile)
|
||||
}
|
||||
overlay.ByOutput[key] = entry
|
||||
}
|
||||
}
|
||||
return overlay, nil
|
||||
}
|
||||
|
||||
func (o CreditsOverlay) Match(originalFile, outputFile string) *CreditsEntry {
|
||||
if entry, ok := o.ByOutput[strings.ToLower(strings.TrimSpace(outputFile))]; ok {
|
||||
entryCopy := entry
|
||||
return &entryCopy
|
||||
}
|
||||
if entry, ok := o.ByOriginal[strings.ToLower(strings.TrimSpace(originalFile))]; ok {
|
||||
entryCopy := entry
|
||||
return &entryCopy
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func ValidateOverlayEntries(dir string, overlay CreditsOverlay, generated []CreditsEntry) error {
|
||||
if len(overlay.Entries) == 0 {
|
||||
return nil
|
||||
}
|
||||
validOriginal := make(map[string]struct{}, len(generated))
|
||||
validOutput := make(map[string]struct{}, len(generated))
|
||||
for _, entry := range generated {
|
||||
validOriginal[strings.ToLower(entry.OriginalFile)] = struct{}{}
|
||||
validOutput[strings.ToLower(entry.OutputFile)] = struct{}{}
|
||||
}
|
||||
for _, entry := range overlay.Entries {
|
||||
if entry.OriginalFile != "" {
|
||||
if _, ok := validOriginal[strings.ToLower(entry.OriginalFile)]; !ok {
|
||||
return fmt.Errorf("manual credits entry in %s references unknown original file %s", dir, entry.OriginalFile)
|
||||
}
|
||||
}
|
||||
if entry.OutputFile != "" {
|
||||
if _, ok := validOutput[strings.ToLower(entry.OutputFile)]; !ok {
|
||||
return fmt.Errorf("manual credits entry in %s references unknown output file %s", dir, entry.OutputFile)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func ParseCreditsMarkdown(path string) ([]CreditsEntry, error) {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
lines := strings.Split(strings.ReplaceAll(string(raw), "\r\n", "\n"), "\n")
|
||||
header := []string(nil)
|
||||
entries := make([]CreditsEntry, 0)
|
||||
for _, line := range lines {
|
||||
line = strings.TrimSpace(line)
|
||||
if !strings.HasPrefix(line, "|") {
|
||||
continue
|
||||
}
|
||||
cells := ParseMarkdownTableRow(line)
|
||||
if len(cells) == 0 {
|
||||
continue
|
||||
}
|
||||
if header == nil {
|
||||
header = make([]string, len(cells))
|
||||
for i, cell := range cells {
|
||||
header[i] = NormalizeCreditsHeader(cell)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if IsMarkdownSeparatorRow(cells) {
|
||||
continue
|
||||
}
|
||||
entry := CreditsEntry{}
|
||||
for i, key := range header {
|
||||
if i >= len(cells) {
|
||||
continue
|
||||
}
|
||||
value := CleanCreditsCell(cells[i])
|
||||
switch key {
|
||||
case "artist":
|
||||
entry.Artist = value
|
||||
case "title":
|
||||
entry.Title = value
|
||||
case "output_file":
|
||||
entry.OutputFile = value
|
||||
case "original_file":
|
||||
entry.OriginalFile = value
|
||||
case "album":
|
||||
entry.Album = value
|
||||
case "date":
|
||||
entry.Date = value
|
||||
case "rights":
|
||||
entry.Rights = value
|
||||
case "notes":
|
||||
entry.Notes = value
|
||||
}
|
||||
}
|
||||
if entry.Artist == "" && entry.Title == "" && entry.OutputFile == "" && entry.OriginalFile == "" {
|
||||
continue
|
||||
}
|
||||
entries = append(entries, entry)
|
||||
}
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
func ParseMarkdownTableRow(line string) []string {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
trimmed = strings.TrimPrefix(trimmed, "|")
|
||||
trimmed = strings.TrimSuffix(trimmed, "|")
|
||||
parts := strings.Split(trimmed, "|")
|
||||
cells := make([]string, 0, len(parts))
|
||||
for _, part := range parts {
|
||||
cells = append(cells, strings.TrimSpace(strings.ReplaceAll(part, `\|`, "|")))
|
||||
}
|
||||
return cells
|
||||
}
|
||||
|
||||
func IsMarkdownSeparatorRow(cells []string) bool {
|
||||
if len(cells) == 0 {
|
||||
return false
|
||||
}
|
||||
for _, cell := range cells {
|
||||
cell = strings.TrimSpace(cell)
|
||||
cell = strings.ReplaceAll(cell, "-", "")
|
||||
cell = strings.ReplaceAll(cell, ":", "")
|
||||
if cell != "" {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func NormalizeCreditsHeader(value string) string {
|
||||
switch strings.ToLower(CleanCreditsCell(value)) {
|
||||
case "artist":
|
||||
return "artist"
|
||||
case "title":
|
||||
return "title"
|
||||
case "output file":
|
||||
return "output_file"
|
||||
case "original file":
|
||||
return "original_file"
|
||||
case "album":
|
||||
return "album"
|
||||
case "date":
|
||||
return "date"
|
||||
case "license / copyright":
|
||||
return "rights"
|
||||
case "notes":
|
||||
return "notes"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func CleanCreditsCell(value string) string {
|
||||
value = strings.TrimSpace(value)
|
||||
value = strings.Trim(value, "`")
|
||||
value = strings.ReplaceAll(value, "<br>", "\n")
|
||||
return strings.TrimSpace(value)
|
||||
}
|
||||
|
||||
func RenderCreditsMarkdown(entries []CreditsEntry) string {
|
||||
var builder strings.Builder
|
||||
builder.WriteString(CreditsHeader)
|
||||
for _, entry := range entries {
|
||||
builder.WriteString("| ")
|
||||
builder.WriteString(EscapeMarkdownCell(entry.Artist))
|
||||
builder.WriteString(" | ")
|
||||
builder.WriteString(EscapeMarkdownCell(entry.Title))
|
||||
builder.WriteString(" | ")
|
||||
builder.WriteString(EscapeMarkdownCell(WrapCodeCell(entry.OutputFile)))
|
||||
builder.WriteString(" | ")
|
||||
builder.WriteString(EscapeMarkdownCell(WrapCodeCell(entry.OriginalFile)))
|
||||
builder.WriteString(" | ")
|
||||
builder.WriteString(EscapeMarkdownCell(entry.Album))
|
||||
builder.WriteString(" | ")
|
||||
builder.WriteString(EscapeMarkdownCell(entry.Date))
|
||||
builder.WriteString(" | ")
|
||||
builder.WriteString(EscapeMarkdownCell(entry.Rights))
|
||||
builder.WriteString(" | ")
|
||||
builder.WriteString(EscapeMarkdownCell(entry.Notes))
|
||||
builder.WriteString(" |\n")
|
||||
}
|
||||
return builder.String()
|
||||
}
|
||||
|
||||
func WrapCodeCell(value string) string {
|
||||
if strings.TrimSpace(value) == "" {
|
||||
return ""
|
||||
}
|
||||
return "`" + value + "`"
|
||||
}
|
||||
|
||||
func EscapeMarkdownCell(value string) string {
|
||||
value = strings.ReplaceAll(value, `\`, `\\`)
|
||||
value = strings.ReplaceAll(value, "|", `\|`)
|
||||
value = strings.ReplaceAll(value, "\n", "<br>")
|
||||
return value
|
||||
}
|
||||
|
||||
func ApplyCreditsOverlay(entry *CreditsEntry, overlay *CreditsEntry) {
|
||||
if overlay == nil {
|
||||
return
|
||||
}
|
||||
entry.Artist = FirstNonEmpty(overlay.Artist, entry.Artist)
|
||||
entry.Title = FirstNonEmpty(overlay.Title, entry.Title)
|
||||
entry.Album = FirstNonEmpty(overlay.Album, entry.Album)
|
||||
entry.Date = FirstNonEmpty(overlay.Date, entry.Date)
|
||||
entry.Rights = FirstNonEmpty(overlay.Rights, entry.Rights)
|
||||
entry.Notes = FirstNonEmpty(overlay.Notes, entry.Notes)
|
||||
}
|
||||
|
||||
func SyncGeneratedCreditsArtifacts(root string, desiredFiles map[string][]byte) (int, error) {
|
||||
changedFiles := 0
|
||||
existingFiles := make(map[string]struct{})
|
||||
if err := filepath.WalkDir(root, func(path string, d os.DirEntry, walkErr error) error {
|
||||
if walkErr != nil {
|
||||
if os.IsNotExist(walkErr) {
|
||||
return nil
|
||||
}
|
||||
return walkErr
|
||||
}
|
||||
if d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
existingFiles[path] = struct{}{}
|
||||
return nil
|
||||
}); err != nil {
|
||||
return 0, fmt.Errorf("scan credits dir: %w", err)
|
||||
}
|
||||
|
||||
for path, content := range desiredFiles {
|
||||
current, err := os.ReadFile(path)
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
return 0, fmt.Errorf("read credits artifact %s: %w", path, err)
|
||||
}
|
||||
if err == nil && string(current) == string(content) {
|
||||
delete(existingFiles, path)
|
||||
continue
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
return 0, fmt.Errorf("create generated credits dir: %w", err)
|
||||
}
|
||||
if err := os.WriteFile(path, content, 0o644); err != nil {
|
||||
return 0, fmt.Errorf("write credits artifact %s: %w", path, err)
|
||||
}
|
||||
changedFiles++
|
||||
delete(existingFiles, path)
|
||||
}
|
||||
|
||||
for path := range existingFiles {
|
||||
if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
|
||||
return 0, fmt.Errorf("remove stale credits artifact %s: %w", path, err)
|
||||
}
|
||||
changedFiles++
|
||||
}
|
||||
return changedFiles, nil
|
||||
}
|
||||
@@ -1,62 +0,0 @@
|
||||
package music
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func ProbeMetadata(ffprobePath, sourcePath string) (Metadata, error) {
|
||||
cmd := exec.Command(ffprobePath, "-v", "quiet", "-print_format", "json", "-show_format", sourcePath)
|
||||
output, err := cmd.Output()
|
||||
if err != nil {
|
||||
return Metadata{}, err
|
||||
}
|
||||
var payload struct {
|
||||
Format struct {
|
||||
Tags map[string]string `json:"tags"`
|
||||
} `json:"format"`
|
||||
}
|
||||
if err := json.Unmarshal(output, &payload); err != nil {
|
||||
return Metadata{}, err
|
||||
}
|
||||
tags := make(map[string]string, len(payload.Format.Tags))
|
||||
for key, value := range payload.Format.Tags {
|
||||
tags[strings.ToLower(strings.TrimSpace(key))] = strings.TrimSpace(value)
|
||||
}
|
||||
return Metadata{
|
||||
Title: FirstNonEmpty(tags["title"]),
|
||||
Artist: FirstNonEmpty(tags["artist"], tags["album_artist"], tags["composer"]),
|
||||
Album: FirstNonEmpty(tags["album"]),
|
||||
Date: FirstNonEmpty(tags["date"], tags["year"]),
|
||||
Comment: FirstNonEmpty(tags["comment"], tags["description"]),
|
||||
Copyright: FirstNonEmpty(tags["copyright"]),
|
||||
License: FirstNonEmpty(tags["license"], tags["license_url"]),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func ConvertSource(ffmpegPath, sourcePath, outputPath string) error {
|
||||
cmd := exec.Command(
|
||||
ffmpegPath, "-y",
|
||||
"-i", sourcePath,
|
||||
"-vn",
|
||||
"-map_metadata", "-1",
|
||||
"-ar", "44100",
|
||||
"-ac", "2",
|
||||
"-b:a", "192k",
|
||||
"-codec:a", "libmp3lame",
|
||||
"-write_xing", "0",
|
||||
"-f", "mp3",
|
||||
outputPath,
|
||||
)
|
||||
output, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
message := strings.TrimSpace(string(output))
|
||||
if message == "" {
|
||||
message = err.Error()
|
||||
}
|
||||
return fmt.Errorf("%s", message)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,239 +0,0 @@
|
||||
package music
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
MaxStemLen = 16
|
||||
CreditsHeader = "# NWN Music Pack Credits\n\nOriginal rights remain with the respective composers and licensors.\n\n## Processed Files\n\n| Artist | Title | Output File | Original File | Album | Date | License / Copyright | Notes |\n|---|---|---|---|---|---|---|---|\n"
|
||||
)
|
||||
|
||||
var (
|
||||
defaultConvertExtensions = []string{
|
||||
".flac",
|
||||
".m4a",
|
||||
".mp3",
|
||||
".ogg",
|
||||
".wav",
|
||||
}
|
||||
ConvertExtensions = map[string]struct{}{
|
||||
".flac": {},
|
||||
".m4a": {},
|
||||
".mp3": {},
|
||||
".ogg": {},
|
||||
".wav": {},
|
||||
}
|
||||
PassthroughExtensions = map[string]struct{}{
|
||||
".bmu": {},
|
||||
".wav": {},
|
||||
}
|
||||
DropWords = map[string]struct{}{
|
||||
"mp3": {}, "wav": {}, "flac": {}, "ogg": {}, "m4a": {},
|
||||
"music": {}, "track": {}, "audio": {}, "song": {}, "soundtrack": {},
|
||||
"final": {}, "version": {}, "ver": {}, "v": {}, "mix": {}, "master": {}, "remaster": {},
|
||||
"free": {}, "royalty": {}, "copyright": {}, "background": {}, "bgm": {},
|
||||
"loop": {}, "loops": {}, "loopable": {}, "seamless": {},
|
||||
"official": {}, "download": {}, "preview": {},
|
||||
"the": {}, "a": {}, "an": {}, "and": {}, "of": {}, "to": {}, "in": {}, "for": {}, "with": {}, "by": {},
|
||||
}
|
||||
BracketRE = regexp.MustCompile(`\[[^\]]*\]|\([^)]*\)|\{[^}]*\}`)
|
||||
SplitRE = regexp.MustCompile(`[^A-Za-z0-9]+`)
|
||||
YearRE = regexp.MustCompile(`^\d{4,}$`)
|
||||
VowelRE = regexp.MustCompile(`[aeiou]`)
|
||||
)
|
||||
|
||||
func DefaultConvertExtensions() []string {
|
||||
return append([]string(nil), defaultConvertExtensions...)
|
||||
}
|
||||
|
||||
type Metadata struct {
|
||||
Title string
|
||||
Artist string
|
||||
Album string
|
||||
Date string
|
||||
Rights string
|
||||
Notes string
|
||||
Comment string
|
||||
Copyright string
|
||||
License string
|
||||
}
|
||||
|
||||
type CreditsEntry struct {
|
||||
Artist string `json:"artist"`
|
||||
Title string `json:"title"`
|
||||
OutputFile string `json:"output_file"`
|
||||
OriginalFile string `json:"original_file"`
|
||||
Album string `json:"album"`
|
||||
Date string `json:"date"`
|
||||
Rights string `json:"rights"`
|
||||
Notes string `json:"notes"`
|
||||
}
|
||||
|
||||
type CreditsSource struct {
|
||||
Path string `json:"path"`
|
||||
Kind string `json:"kind"`
|
||||
Entries []CreditsEntry `json:"entries"`
|
||||
}
|
||||
|
||||
type CreditsInventory struct {
|
||||
Sources []CreditsSource `json:"sources"`
|
||||
}
|
||||
|
||||
type CreditsOverlay struct {
|
||||
ByOriginal map[string]CreditsEntry
|
||||
ByOutput map[string]CreditsEntry
|
||||
Entries []CreditsEntry
|
||||
}
|
||||
|
||||
func ResolveFFmpeg() (string, error) {
|
||||
return ResolveFFmpegWithConfig("")
|
||||
}
|
||||
|
||||
func ResolveFFmpegWithConfig(configuredPath string) (string, error) {
|
||||
if configured := strings.TrimSpace(configuredPath); configured != "" {
|
||||
return configured, nil
|
||||
}
|
||||
if configured := strings.TrimSpace(os.Getenv("SOW_FFMPEG")); configured != "" {
|
||||
return configured, nil
|
||||
}
|
||||
path, err := exec.LookPath("ffmpeg")
|
||||
if err != nil {
|
||||
return "", ffmpegMissingErr("ffmpeg")
|
||||
}
|
||||
return path, nil
|
||||
}
|
||||
|
||||
func ResolveFFprobe() (string, error) {
|
||||
return ResolveFFprobeWithConfig("")
|
||||
}
|
||||
|
||||
func ResolveFFprobeWithConfig(configuredPath string) (string, error) {
|
||||
if configured := strings.TrimSpace(configuredPath); configured != "" {
|
||||
return configured, nil
|
||||
}
|
||||
if configured := strings.TrimSpace(os.Getenv("SOW_FFPROBE")); configured != "" {
|
||||
return configured, nil
|
||||
}
|
||||
path, err := exec.LookPath("ffprobe")
|
||||
if err != nil {
|
||||
return "", ffmpegMissingErr("ffprobe")
|
||||
}
|
||||
return path, nil
|
||||
}
|
||||
|
||||
// ffmpegMissingErr renders an actionable, cross-platform "tool not found"
|
||||
// message. Only the music builder needs ffmpeg/ffprobe; every other builder
|
||||
// runs with zero external dependencies.
|
||||
func ffmpegMissingErr(tool string) error {
|
||||
return fmt.Errorf(`%[1]s not found on PATH.
|
||||
Windows: winget install Gyan.FFmpeg
|
||||
macOS: brew install ffmpeg
|
||||
Linux: apt install ffmpeg (or your distro's package manager)
|
||||
Only the music builder needs %[1]s; set SOW_%[2]s to a full path to override.`,
|
||||
tool, strings.ToUpper(tool))
|
||||
}
|
||||
|
||||
func IsMusicAssetPath(rel string) bool {
|
||||
rel = filepath.ToSlash(strings.TrimSpace(rel))
|
||||
return rel == "envi/music" || strings.HasPrefix(rel, "envi/music/")
|
||||
}
|
||||
|
||||
func PathIsUnder(root, rel string) bool {
|
||||
root = TrimSlashes(filepath.ToSlash(root))
|
||||
rel = TrimSlashes(filepath.ToSlash(rel))
|
||||
if root == "" || rel == "" {
|
||||
return false
|
||||
}
|
||||
return rel == root || strings.HasPrefix(rel, root+"/")
|
||||
}
|
||||
|
||||
func PrefixForDir(prefixes map[string]string, dir string) string {
|
||||
dir = TrimSlashes(filepath.ToSlash(dir))
|
||||
bestPrefix := ""
|
||||
bestLen := -1
|
||||
keys := make([]string, 0, len(prefixes))
|
||||
for key := range prefixes {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, key := range keys {
|
||||
value := prefixes[key]
|
||||
normalizedKey := TrimSlashes(filepath.ToSlash(key))
|
||||
if normalizedKey == "" {
|
||||
continue
|
||||
}
|
||||
if dir != normalizedKey && !strings.HasPrefix(dir, normalizedKey+"/") {
|
||||
continue
|
||||
}
|
||||
if len(normalizedKey) > bestLen {
|
||||
bestLen = len(normalizedKey)
|
||||
bestPrefix = SanitizePrefix(value)
|
||||
}
|
||||
}
|
||||
return bestPrefix
|
||||
}
|
||||
|
||||
func SanitizePrefix(prefix string) string {
|
||||
prefix = strings.ToLower(strings.TrimSpace(prefix))
|
||||
var builder strings.Builder
|
||||
for _, r := range prefix {
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z':
|
||||
builder.WriteRune(r)
|
||||
case r >= '0' && r <= '9':
|
||||
builder.WriteRune(r)
|
||||
case r == '_':
|
||||
builder.WriteRune(r)
|
||||
}
|
||||
}
|
||||
result := builder.String()
|
||||
if strings.TrimSpace(prefix) != "" && strings.HasSuffix(strings.TrimSpace(prefix), "_") && !strings.HasSuffix(result, "_") {
|
||||
result += "_"
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func TrimSlashes(value string) string {
|
||||
return strings.Trim(strings.TrimSpace(value), "/")
|
||||
}
|
||||
|
||||
func Min(a, b int) int {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func Max(a, b int) int {
|
||||
if a > b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func FirstNonEmpty(values ...string) string {
|
||||
for _, value := range values {
|
||||
if strings.TrimSpace(value) != "" {
|
||||
return strings.TrimSpace(value)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func JoinNonEmpty(sep string, values ...string) string {
|
||||
parts := make([]string, 0, len(values))
|
||||
for _, value := range values {
|
||||
value = strings.TrimSpace(value)
|
||||
if value != "" {
|
||||
parts = append(parts, value)
|
||||
}
|
||||
}
|
||||
return strings.Join(parts, sep)
|
||||
}
|
||||
@@ -1,417 +0,0 @@
|
||||
package music
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestUniqueNameHandlesShortStemCollisions(t *testing.T) {
|
||||
used := map[string]struct{}{
|
||||
"mus_wg_mystc": {},
|
||||
}
|
||||
|
||||
got := UniqueName("mus_wg_mystc", used)
|
||||
if got != "mus_wg_mystc_1" {
|
||||
t.Fatalf("unexpected collision result: got %q want %q", got, "mus_wg_mystc_1")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUniqueNameNoCollision(t *testing.T) {
|
||||
used := map[string]struct{}{}
|
||||
got := UniqueName("new_stem", used)
|
||||
if got != "new_stem" {
|
||||
t.Fatalf("expected 'new_stem', got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUniqueNameMultipleCollisions(t *testing.T) {
|
||||
used := map[string]struct{}{
|
||||
"stem": {},
|
||||
"stem_1": {},
|
||||
"stem_2": {},
|
||||
}
|
||||
got := UniqueName("stem", used)
|
||||
if got != "stem_3" {
|
||||
t.Fatalf("expected 'stem_3', got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateStem(t *testing.T) {
|
||||
used := map[string]struct{}{
|
||||
"existing": {},
|
||||
}
|
||||
stem, err := GenerateStem("My Cool Track.mp3", "mus_", used)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if stem == "" {
|
||||
t.Fatal("expected non-empty stem")
|
||||
}
|
||||
if len(stem) > MaxStemLen {
|
||||
t.Fatalf("stem %q exceeds max length %d", stem, MaxStemLen)
|
||||
}
|
||||
if _, exists := used[stem]; !exists {
|
||||
t.Fatal("expected stem to be registered in used set")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateStemPrefixTooLong(t *testing.T) {
|
||||
_, err := GenerateStem("test.mp3", "this_prefix_is_way_too_long_for_stem_", nil)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for too-long prefix")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSlugWords(t *testing.T) {
|
||||
words := SlugWords("My Cool Music Track (Official) [HD]")
|
||||
if len(words) == 0 {
|
||||
t.Fatal("expected non-empty words")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSlugWordsStripsBrackets(t *testing.T) {
|
||||
words := SlugWords("Song [Explicit] (Remix)")
|
||||
for _, w := range words {
|
||||
if w == "explicit" || w == "remix" {
|
||||
t.Fatalf("bracket content should be removed, got word %q", w)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSanitizePrefix(t *testing.T) {
|
||||
if got := SanitizePrefix("Mus_WG_"); got != "mus_wg_" {
|
||||
t.Fatalf("unexpected prefix: %q", got)
|
||||
}
|
||||
if got := SanitizePrefix(" Test "); got != "test" {
|
||||
t.Fatalf("unexpected prefix: %q", got)
|
||||
}
|
||||
if got := SanitizePrefix(""); got != "" {
|
||||
t.Fatalf("expected empty prefix, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReserveName(t *testing.T) {
|
||||
used := map[string]struct{}{}
|
||||
if err := ReserveName("testname", used); err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if _, exists := used["testname"]; !exists {
|
||||
t.Fatal("expected name to be reserved")
|
||||
}
|
||||
if err := ReserveName("testname", used); err == nil {
|
||||
t.Fatal("expected error for duplicate name")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateManualOutputFile(t *testing.T) {
|
||||
if err := ValidateManualOutputFile("test.bmu"); err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if err := ValidateManualOutputFile("test.txt"); err == nil {
|
||||
t.Fatal("expected error for non-bmu extension")
|
||||
}
|
||||
if err := ValidateManualOutputFile(".bmu"); err == nil {
|
||||
t.Fatal("expected error for empty stem")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseCreditsMarkdownMissingFile(t *testing.T) {
|
||||
entries, err := ParseCreditsMarkdown("nonexistent.md")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error for missing file: %v", err)
|
||||
}
|
||||
if entries != nil {
|
||||
t.Fatalf("expected nil entries for missing file, got %v", entries)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseCreditsMarkdownRealContent(t *testing.T) {
|
||||
content := "# Header\n\n| Artist | Title | Output File | Original File | Album | Date | License / Copyright | Notes |\n|---|---|---|---|---|---|---|---|\n| Test Artist | Test Title | `output.bmu` | `original.mp3` | Test Album | 2024 | MIT | test |\n"
|
||||
path := filepath.Join(t.TempDir(), "CREDITS.md")
|
||||
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
||||
t.Fatalf("write test credits: %v", err)
|
||||
}
|
||||
|
||||
entries, err := ParseCreditsMarkdown(path)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if len(entries) != 1 {
|
||||
t.Fatalf("expected 1 entry, got %d", len(entries))
|
||||
}
|
||||
if entries[0].Artist != "Test Artist" {
|
||||
t.Fatalf("expected 'Test Artist', got %q", entries[0].Artist)
|
||||
}
|
||||
if entries[0].Title != "Test Title" {
|
||||
t.Fatalf("expected 'Test Title', got %q", entries[0].Title)
|
||||
}
|
||||
if entries[0].OutputFile != "output.bmu" {
|
||||
t.Fatalf("expected 'output.bmu', got %q", entries[0].OutputFile)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderCreditsMarkdown(t *testing.T) {
|
||||
entries := []CreditsEntry{
|
||||
{
|
||||
Artist: "Test Artist",
|
||||
Title: "Test Title",
|
||||
OutputFile: "output.bmu",
|
||||
OriginalFile: "original.mp3",
|
||||
Album: "Test Album",
|
||||
Date: "2024",
|
||||
Rights: "MIT",
|
||||
Notes: "test note",
|
||||
},
|
||||
}
|
||||
|
||||
result := RenderCreditsMarkdown(entries)
|
||||
if !strings.Contains(result, "Test Artist") {
|
||||
t.Fatal("expected rendered output to contain artist")
|
||||
}
|
||||
if !strings.Contains(result, "output.bmu") {
|
||||
t.Fatal("expected rendered output to contain output file")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseCreditsOverlay(t *testing.T) {
|
||||
content := "# Header\n\n| Artist | Title | Output File | Original File |\n|---|---|---|---|\n| Overlay Artist | Ovr Title | `override.bmu` | `original.mp3` |\n"
|
||||
path := filepath.Join(t.TempDir(), "CREDITS.md")
|
||||
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
||||
t.Fatalf("write test overlay: %v", err)
|
||||
}
|
||||
|
||||
overlay, err := ParseCreditsOverlay(path)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if len(overlay.Entries) != 1 {
|
||||
t.Fatalf("expected 1 overlay entry, got %d", len(overlay.Entries))
|
||||
}
|
||||
|
||||
matched := overlay.Match("original.mp3", "")
|
||||
if matched == nil {
|
||||
t.Fatal("expected overlay match by original file")
|
||||
}
|
||||
if matched.Artist != "Overlay Artist" {
|
||||
t.Fatalf("expected 'Overlay Artist', got %q", matched.Artist)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyCreditsOverlay(t *testing.T) {
|
||||
entry := &CreditsEntry{
|
||||
Artist: "Original Artist",
|
||||
Title: "Original Title",
|
||||
}
|
||||
overlay := &CreditsEntry{
|
||||
Artist: "Overlay Artist",
|
||||
}
|
||||
|
||||
ApplyCreditsOverlay(entry, overlay)
|
||||
if entry.Artist != "Overlay Artist" {
|
||||
t.Fatalf("expected 'Overlay Artist', got %q", entry.Artist)
|
||||
}
|
||||
if entry.Title != "Original Title" {
|
||||
t.Fatalf("expected 'Original Title' preserved, got %q", entry.Title)
|
||||
}
|
||||
|
||||
ApplyCreditsOverlay(entry, nil)
|
||||
if entry.Artist != "Overlay Artist" {
|
||||
t.Fatalf("expected nil overlay to be no-op, got %q", entry.Artist)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateOverlayEntries(t *testing.T) {
|
||||
generated := []CreditsEntry{
|
||||
{OriginalFile: "track1.mp3", OutputFile: "track1.bmu"},
|
||||
}
|
||||
overlay := CreditsOverlay{
|
||||
Entries: []CreditsEntry{
|
||||
{OriginalFile: "track1.mp3"},
|
||||
},
|
||||
}
|
||||
if err := ValidateOverlayEntries("test", overlay, generated); err != nil {
|
||||
t.Fatalf("unexpected validation error: %v", err)
|
||||
}
|
||||
|
||||
badOverlay := CreditsOverlay{
|
||||
Entries: []CreditsEntry{
|
||||
{OriginalFile: "nonexistent.mp3"},
|
||||
},
|
||||
}
|
||||
if err := ValidateOverlayEntries("test", badOverlay, generated); err == nil {
|
||||
t.Fatal("expected validation error for unknown original file")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsMusicAssetPath(t *testing.T) {
|
||||
if !IsMusicAssetPath("envi/music") {
|
||||
t.Fatal("expected 'envi/music' to be music path")
|
||||
}
|
||||
if !IsMusicAssetPath("envi/music/westgate") {
|
||||
t.Fatal("expected 'envi/music/westgate' to be music path")
|
||||
}
|
||||
if IsMusicAssetPath("envi/textures") {
|
||||
t.Fatal("expected 'envi/textures' not to be music path")
|
||||
}
|
||||
if IsMusicAssetPath("") {
|
||||
t.Fatal("expected empty string not to be music path")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFirstNonEmpty(t *testing.T) {
|
||||
if got := FirstNonEmpty("", "hello", "world"); got != "hello" {
|
||||
t.Fatalf("expected 'hello', got %q", got)
|
||||
}
|
||||
if got := FirstNonEmpty("", "", ""); got != "" {
|
||||
t.Fatalf("expected empty, got %q", got)
|
||||
}
|
||||
if got := FirstNonEmpty("first"); got != "first" {
|
||||
t.Fatalf("expected 'first', got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestJoinNonEmpty(t *testing.T) {
|
||||
if got := JoinNonEmpty(", ", "a", "", "b"); got != "a, b" {
|
||||
t.Fatalf("expected 'a, b', got %q", got)
|
||||
}
|
||||
if got := JoinNonEmpty(", "); got != "" {
|
||||
t.Fatalf("expected empty, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrimSlashes(t *testing.T) {
|
||||
if got := TrimSlashes("/foo/bar/"); got != "foo/bar" {
|
||||
t.Fatalf("expected 'foo/bar', got %q", got)
|
||||
}
|
||||
if got := TrimSlashes("///"); got != "" {
|
||||
t.Fatalf("expected empty, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEscapeMarkdownCell(t *testing.T) {
|
||||
result := EscapeMarkdownCell("a|b\nc")
|
||||
if !strings.Contains(result, `\|`) {
|
||||
t.Fatal("expected pipe to be escaped")
|
||||
}
|
||||
if !strings.Contains(result, "<br>") {
|
||||
t.Fatal("expected newline to be replaced with <br>")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWrapCodeCell(t *testing.T) {
|
||||
if got := WrapCodeCell("test.bmu"); got != "`test.bmu`" {
|
||||
t.Fatalf("expected '`test.bmu`', got %q", got)
|
||||
}
|
||||
if got := WrapCodeCell(""); got != "" {
|
||||
t.Fatalf("expected empty string, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSyncGeneratedCreditsArtifactsWritesNewFiles(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
desired := map[string][]byte{
|
||||
filepath.Join(root, "test.md"): []byte("content"),
|
||||
}
|
||||
changed, err := SyncGeneratedCreditsArtifacts(root, desired)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if changed != 1 {
|
||||
t.Fatalf("expected 1 changed file, got %d", changed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSyncGeneratedCreditsArtifactsNoChanges(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
path := filepath.Join(root, "test.md")
|
||||
if err := os.WriteFile(path, []byte("content"), 0o644); err != nil {
|
||||
t.Fatalf("write file: %v", err)
|
||||
}
|
||||
desired := map[string][]byte{
|
||||
path: []byte("content"),
|
||||
}
|
||||
changed, err := SyncGeneratedCreditsArtifacts(root, desired)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if changed != 0 {
|
||||
t.Fatalf("expected 0 changed files, got %d", changed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSyncGeneratedCreditsArtifactsRemovesStale(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
stalePath := filepath.Join(root, "stale.md")
|
||||
if err := os.WriteFile(stalePath, []byte("stale"), 0o644); err != nil {
|
||||
t.Fatalf("write stale file: %v", err)
|
||||
}
|
||||
desired := map[string][]byte{}
|
||||
changed, err := SyncGeneratedCreditsArtifacts(root, desired)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if changed != 1 {
|
||||
t.Fatalf("expected 1 removed file, got %d", changed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrefixForDir(t *testing.T) {
|
||||
prefixes := map[string]string{
|
||||
"envi/music/westgate": "mus_wg_",
|
||||
}
|
||||
if got := PrefixForDir(prefixes, "envi/music/westgate"); got != "mus_wg_" {
|
||||
t.Fatalf("expected 'mus_wg_', got %q", got)
|
||||
}
|
||||
if got := PrefixForDir(prefixes, "envi/music/other"); got != "" {
|
||||
t.Fatalf("expected empty prefix, got %q", got)
|
||||
}
|
||||
if got := PrefixForDir(nil, "envi/music"); got != "" {
|
||||
t.Fatalf("expected empty prefix for nil map, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaxMin(t *testing.T) {
|
||||
if got := Min(3, 5); got != 3 {
|
||||
t.Fatalf("Min(3,5) = %d, want 3", got)
|
||||
}
|
||||
if got := Max(3, 5); got != 5 {
|
||||
t.Fatalf("Max(3,5) = %d, want 5", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveFFmpegEnv(t *testing.T) {
|
||||
t.Setenv("SOW_FFMPEG", "/custom/ffmpeg")
|
||||
path, err := ResolveFFmpeg()
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if path != "/custom/ffmpeg" {
|
||||
t.Fatalf("expected '/custom/ffmpeg', got %q", path)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveFFprobeEnv(t *testing.T) {
|
||||
t.Setenv("SOW_FFPROBE", "/custom/ffprobe")
|
||||
path, err := ResolveFFprobe()
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if path != "/custom/ffprobe" {
|
||||
t.Fatalf("expected '/custom/ffprobe', got %q", path)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFFmpegMissingErrHasInstallHints(t *testing.T) {
|
||||
msg := ffmpegMissingErr("ffmpeg").Error()
|
||||
for _, want := range []string{"ffmpeg", "winget", "brew", "apt", "SOW_FFMPEG"} {
|
||||
if !strings.Contains(msg, want) {
|
||||
t.Errorf("ffmpeg error missing %q; got:\n%s", want, msg)
|
||||
}
|
||||
}
|
||||
probe := ffmpegMissingErr("ffprobe").Error()
|
||||
if !strings.Contains(probe, "SOW_FFPROBE") {
|
||||
t.Errorf("ffprobe error should mention SOW_FFPROBE; got:\n%s", probe)
|
||||
}
|
||||
}
|
||||
@@ -1,197 +0,0 @@
|
||||
package music
|
||||
|
||||
import (
|
||||
"crypto/sha1"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func GenerateStem(fileName, prefix string, used map[string]struct{}) (string, error) {
|
||||
return GenerateStemWithMax(fileName, prefix, MaxStemLen, used)
|
||||
}
|
||||
|
||||
func GenerateStemWithMax(fileName, prefix string, maxStemLen int, used map[string]struct{}) (string, error) {
|
||||
if maxStemLen <= 0 {
|
||||
maxStemLen = MaxStemLen
|
||||
}
|
||||
maxCore := maxStemLen - len(prefix)
|
||||
if maxCore < 3 {
|
||||
return "", fmt.Errorf("prefix too long: %q", prefix)
|
||||
}
|
||||
base := strings.TrimSuffix(fileName, filepath.Ext(fileName))
|
||||
words := SlugWords(base)
|
||||
core := MakeMusicCore(words, maxCore)
|
||||
if core == "" {
|
||||
sum := sha1.Sum([]byte(fileName))
|
||||
core = fmt.Sprintf("%x", sum[:])[:maxCore]
|
||||
}
|
||||
stem := strings.TrimRight((prefix + core)[:Min(maxStemLen, len(prefix)+len(core))], "_")
|
||||
if stem == "" {
|
||||
return "", fmt.Errorf("could not derive music stem for %s", fileName)
|
||||
}
|
||||
return UniqueNameWithMax(stem, maxStemLen, used), nil
|
||||
}
|
||||
|
||||
func SlugWords(value string) []string {
|
||||
value = BracketRE.ReplaceAllString(value, " ")
|
||||
value = strings.ReplaceAll(value, "&", " and ")
|
||||
value = strings.ReplaceAll(value, "'", "")
|
||||
value = strings.ReplaceAll(value, "\u2019", "")
|
||||
value = strings.ToLower(value)
|
||||
var ascii strings.Builder
|
||||
for _, r := range value {
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z':
|
||||
ascii.WriteRune(r)
|
||||
case r >= '0' && r <= '9':
|
||||
ascii.WriteRune(r)
|
||||
default:
|
||||
ascii.WriteRune(' ')
|
||||
}
|
||||
}
|
||||
rawWords := SplitRE.Split(ascii.String(), -1)
|
||||
words := make([]string, 0, len(rawWords))
|
||||
for _, word := range rawWords {
|
||||
if word == "" {
|
||||
continue
|
||||
}
|
||||
if _, drop := DropWords[word]; drop {
|
||||
continue
|
||||
}
|
||||
if YearRE.MatchString(word) {
|
||||
continue
|
||||
}
|
||||
words = append(words, word)
|
||||
}
|
||||
return words
|
||||
}
|
||||
|
||||
func MakeMusicCore(words []string, maxCore int) string {
|
||||
core := ""
|
||||
for _, word := range words {
|
||||
candidate := word
|
||||
if core != "" {
|
||||
candidate = core + "_" + word
|
||||
}
|
||||
if len(candidate) <= maxCore {
|
||||
core = candidate
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
if core != "" {
|
||||
return core
|
||||
}
|
||||
abbrev := make([]string, 0, Min(4, len(words)))
|
||||
for _, word := range words {
|
||||
if len(abbrev) == 4 {
|
||||
break
|
||||
}
|
||||
abbrev = append(abbrev, AbbreviateWord(word, 5))
|
||||
}
|
||||
for _, word := range abbrev {
|
||||
candidate := word
|
||||
if core != "" {
|
||||
candidate = core + "_" + word
|
||||
}
|
||||
if len(candidate) <= maxCore {
|
||||
core = candidate
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
if core != "" {
|
||||
return core
|
||||
}
|
||||
if len(words) > 0 {
|
||||
return strings.Trim(words[0][:Min(maxCore, len(words[0]))], "_")
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func AbbreviateWord(word string, maxLen int) string {
|
||||
if len(word) <= maxLen {
|
||||
return word
|
||||
}
|
||||
consonants := VowelRE.ReplaceAllString(word, "")
|
||||
candidate := word[:1]
|
||||
if len(consonants) > 1 {
|
||||
candidate += consonants[1:]
|
||||
}
|
||||
if len(candidate) >= 3 {
|
||||
return candidate[:Min(maxLen, len(candidate))]
|
||||
}
|
||||
return word[:Min(maxLen, len(word))]
|
||||
}
|
||||
|
||||
func UniqueName(stem string, used map[string]struct{}) string {
|
||||
return UniqueNameWithMax(stem, MaxStemLen, used)
|
||||
}
|
||||
|
||||
func UniqueNameWithMax(stem string, maxStemLen int, used map[string]struct{}) string {
|
||||
if maxStemLen <= 0 {
|
||||
maxStemLen = MaxStemLen
|
||||
}
|
||||
if _, exists := used[stem]; !exists {
|
||||
used[stem] = struct{}{}
|
||||
return stem
|
||||
}
|
||||
for index := 1; ; index++ {
|
||||
suffix := fmt.Sprintf("_%d", index)
|
||||
prefixLen := Min(len(stem), Max(0, maxStemLen-len(suffix)))
|
||||
candidate := strings.TrimRight(stem[:prefixLen], "_") + suffix
|
||||
if _, exists := used[candidate]; exists {
|
||||
continue
|
||||
}
|
||||
used[candidate] = struct{}{}
|
||||
return candidate
|
||||
}
|
||||
}
|
||||
|
||||
func ReserveName(stem string, used map[string]struct{}) error {
|
||||
stem = strings.ToLower(strings.TrimSpace(stem))
|
||||
if stem == "" {
|
||||
return fmt.Errorf("output filename is empty")
|
||||
}
|
||||
if _, exists := used[stem]; exists {
|
||||
return fmt.Errorf("output filename %s collides with an existing asset", stem)
|
||||
}
|
||||
used[stem] = struct{}{}
|
||||
return nil
|
||||
}
|
||||
|
||||
func ValidateManualOutputFile(name string) error {
|
||||
return ValidateManualOutputFileWithRules(name, ".bmu", MaxStemLen)
|
||||
}
|
||||
|
||||
func ValidateManualOutputFileWithRules(name, extension string, maxStemLen int) error {
|
||||
name = strings.ToLower(strings.TrimSpace(name))
|
||||
extension = strings.ToLower(strings.TrimSpace(extension))
|
||||
if extension == "" {
|
||||
extension = ".bmu"
|
||||
}
|
||||
if maxStemLen <= 0 {
|
||||
maxStemLen = MaxStemLen
|
||||
}
|
||||
if !strings.HasSuffix(name, extension) {
|
||||
return fmt.Errorf("output file must end with %s", extension)
|
||||
}
|
||||
stem := strings.TrimSuffix(name, extension)
|
||||
if stem == "" {
|
||||
return fmt.Errorf("output file stem is empty")
|
||||
}
|
||||
if len(stem) > maxStemLen {
|
||||
return fmt.Errorf("output file stem %q exceeds %d characters", stem, maxStemLen)
|
||||
}
|
||||
for _, r := range stem {
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z':
|
||||
case r >= '0' && r <= '9':
|
||||
case r == '_':
|
||||
default:
|
||||
return fmt.Errorf("output file stem %q contains unsupported character %q", stem, string(r))
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
+15
-69
@@ -26,8 +26,6 @@ type BuildResult struct {
|
||||
HAKPaths []string
|
||||
Manifest string
|
||||
AutogenManifestPaths []string
|
||||
CreditsArtifactPaths []string
|
||||
CreditsSummary CreditsRefreshSummary
|
||||
HAKSummary HAKArchiveSummary
|
||||
Resources int
|
||||
HAKAssets int
|
||||
@@ -36,21 +34,6 @@ type BuildResult struct {
|
||||
TopPackageFiles int
|
||||
}
|
||||
|
||||
type CreditsRefreshSummary struct {
|
||||
ScannedDirs []string
|
||||
TrackCount int
|
||||
ArtifactPaths []string
|
||||
GeneratedPaths []string
|
||||
InventoryPath string
|
||||
ChangedFiles int
|
||||
Mappings []MusicFileMapping
|
||||
}
|
||||
|
||||
type MusicFileMapping struct {
|
||||
Source string
|
||||
Output string
|
||||
}
|
||||
|
||||
type HAKArchiveSummary struct {
|
||||
Total int
|
||||
Reused int
|
||||
@@ -110,8 +93,6 @@ type BuildHAKOptions struct {
|
||||
ArchiveNames []string
|
||||
SourceManifestPath string
|
||||
ContentAddressedRoot string
|
||||
SkipMusic bool
|
||||
MusicDatasetIDs []string
|
||||
}
|
||||
|
||||
func Build(p *project.Project) (BuildResult, error) {
|
||||
@@ -241,26 +222,11 @@ func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bo
|
||||
if err != nil {
|
||||
return BuildResult{}, err
|
||||
}
|
||||
musicAssets := &preparedMusicAssets{SkipSourceRel: map[string]struct{}{}}
|
||||
if !opts.SkipMusic {
|
||||
musicAssets, err = prepareMusicAssetsWithOptions(p, musicPrepareOptions{
|
||||
datasetIDs: opts.MusicDatasetIDs,
|
||||
write: writeArchives,
|
||||
})
|
||||
if err != nil {
|
||||
return BuildResult{}, err
|
||||
}
|
||||
}
|
||||
defer func() {
|
||||
if cleanupErr := cleanupPreparedMusicAssets(musicAssets); cleanupErr != nil && err == nil {
|
||||
err = cleanupErr
|
||||
}
|
||||
}()
|
||||
generated2DAAssets, err := topdata.BuildGenerated2DAAssets(p, progress)
|
||||
if err != nil {
|
||||
return BuildResult{}, err
|
||||
}
|
||||
assetResources, err := collectAssetResources(p, false, allowedAssets, musicAssets, generated2DAAssets)
|
||||
assetResources, err := collectAssetResources(p, false, allowedAssets, generated2DAAssets)
|
||||
if err != nil {
|
||||
return BuildResult{}, err
|
||||
}
|
||||
@@ -277,8 +243,6 @@ func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bo
|
||||
}
|
||||
|
||||
result = BuildResult{HAKAssets: len(assetResources)}
|
||||
result.CreditsArtifactPaths = append(result.CreditsArtifactPaths, musicAssets.Artifacts...)
|
||||
result.CreditsSummary = musicAssets.Summary
|
||||
if err := writeAutogenManifestOutputs(progress, autogenManifests); err != nil {
|
||||
return BuildResult{}, err
|
||||
}
|
||||
@@ -410,8 +374,8 @@ func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bo
|
||||
}
|
||||
|
||||
// buildDirectHAKs packages HAKs directly from the content-addressed source blob
|
||||
// cache. It skips music preparation, generated 2da discovery, git creation-time
|
||||
// lookups, LFS discovery, and any materialized asset tree scan.
|
||||
// cache. It skips generated 2da discovery, git creation-time lookups, LFS
|
||||
// discovery, and any materialized asset tree scan.
|
||||
func buildDirectHAKs(p *project.Project, progress ProgressFunc, writeArchives bool, manifest *SourceBuildManifest, opts BuildHAKOptions) (BuildResult, error) {
|
||||
progressf(progress, "Collecting asset resources...")
|
||||
assetResources, err := collectDirectSourceResources(manifest, opts.ContentAddressedRoot)
|
||||
@@ -651,7 +615,7 @@ func collectModuleResources(p *project.Project, moduleHakOrder []string) ([]erf.
|
||||
return moduleResources, nil
|
||||
}
|
||||
|
||||
func collectAssetResources(p *project.Project, requireContent bool, allowed map[string]struct{}, musicAssets *preparedMusicAssets, generated2DAAssets []topdata.Generated2DAAsset) ([]assetResource, error) {
|
||||
func collectAssetResources(p *project.Project, requireContent bool, allowed map[string]struct{}, generated2DAAssets []topdata.Generated2DAAsset) ([]assetResource, error) {
|
||||
var hakResources []assetResource
|
||||
assetCreatedAt, err := collectGitAssetCreationTimes(p)
|
||||
if err != nil {
|
||||
@@ -661,18 +625,19 @@ func collectAssetResources(p *project.Project, requireContent bool, allowed map[
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
assetsRelPath, err := filepath.Rel(p.Root, p.AssetsDir())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("resolve assets dir relative path: %w", err)
|
||||
// A module-only project (consumes prebuilt HAKs, has no paths.assets) has no
|
||||
// asset files to collect — AssetFiles is empty and AssetsDir() is "". Skip the
|
||||
// relative-path resolve, which would otherwise fail with `Rel: can't make ""`.
|
||||
assetsRelPath := ""
|
||||
if p.AssetsDir() != "" {
|
||||
assetsRelPath, err = filepath.Rel(p.Root, p.AssetsDir())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("resolve assets dir relative path: %w", err)
|
||||
}
|
||||
assetsRelPath = filepath.ToSlash(assetsRelPath)
|
||||
}
|
||||
assetsRelPath = filepath.ToSlash(assetsRelPath)
|
||||
|
||||
for _, rel := range p.Inventory.AssetFiles {
|
||||
if musicAssets != nil {
|
||||
if _, skip := musicAssets.SkipSourceRel[filepath.ToSlash(rel)]; skip {
|
||||
continue
|
||||
}
|
||||
}
|
||||
if len(allowed) > 0 {
|
||||
if _, ok := allowed[filepath.ToSlash(rel)]; !ok {
|
||||
continue
|
||||
@@ -700,16 +665,6 @@ func collectAssetResources(p *project.Project, requireContent bool, allowed map[
|
||||
ContentID: resourceInfo.ContentID,
|
||||
})
|
||||
}
|
||||
if musicAssets != nil {
|
||||
for _, asset := range musicAssets.Generated {
|
||||
if len(allowed) > 0 {
|
||||
if _, ok := allowed[filepath.ToSlash(asset.Rel)]; !ok {
|
||||
continue
|
||||
}
|
||||
}
|
||||
hakResources = append(hakResources, asset)
|
||||
}
|
||||
}
|
||||
for _, asset := range generated2DAAssets {
|
||||
rel := filepath.ToSlash(asset.Rel)
|
||||
if len(allowed) > 0 {
|
||||
@@ -1611,20 +1566,11 @@ func plannedModuleHAKOrder(p *project.Project) (order []string, err error) {
|
||||
return append([]string(nil), manifest.ModuleHAKs...), nil
|
||||
}
|
||||
|
||||
musicAssets, err := prepareMusicAssets(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() {
|
||||
if cleanupErr := cleanupPreparedMusicAssets(musicAssets); cleanupErr != nil && err == nil {
|
||||
err = cleanupErr
|
||||
}
|
||||
}()
|
||||
generated2DAAssets, err := topdata.BuildGenerated2DAAssets(p, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
assetResources, err := collectAssetResources(p, false, nil, musicAssets, generated2DAAssets)
|
||||
assetResources, err := collectAssetResources(p, false, nil, generated2DAAssets)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -223,7 +223,7 @@ func ensureBuildIsCurrent(p *project.Project, modulePath string, hakPaths []stri
|
||||
}
|
||||
|
||||
if !newestSource.IsZero() && newestSource.After(oldestArchive) {
|
||||
return fmt.Errorf("built archives are older than source file %s; run build-module or build-haks before compare", newestPath)
|
||||
return fmt.Errorf("built archives are older than source file %s; run crucible module build or crucible hak build before compare", newestPath)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -501,6 +501,9 @@ func writeManagedFile(path string, data []byte) (writeState, error) {
|
||||
if bytes.Equal(existing, data) {
|
||||
return writeSkipped, nil
|
||||
}
|
||||
if areaGFFJSONEqualIgnoringRootVersion(path, existing, data) {
|
||||
return writeSkipped, nil
|
||||
}
|
||||
if err := os.WriteFile(path, data, 0o644); err != nil {
|
||||
return writeSkipped, fmt.Errorf("overwrite %s: %w", path, err)
|
||||
}
|
||||
@@ -516,6 +519,38 @@ func writeManagedFile(path string, data []byte) (writeState, error) {
|
||||
return writeNew, nil
|
||||
}
|
||||
|
||||
func areaGFFJSONEqualIgnoringRootVersion(path string, existing, extracted []byte) bool {
|
||||
if !strings.HasSuffix(strings.ToLower(filepath.Base(path)), ".are.json") {
|
||||
return false
|
||||
}
|
||||
|
||||
var left, right gff.Document
|
||||
if err := json.Unmarshal(existing, &left); err != nil {
|
||||
return false
|
||||
}
|
||||
if err := json.Unmarshal(extracted, &right); err != nil {
|
||||
return false
|
||||
}
|
||||
removeRootField(&left.Root, "Version")
|
||||
removeRootField(&right.Root, "Version")
|
||||
|
||||
leftRaw, err := json.Marshal(left)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
rightRaw, err := json.Marshal(right)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return bytes.Equal(leftRaw, rightRaw)
|
||||
}
|
||||
|
||||
func removeRootField(s *gff.Struct, label string) {
|
||||
s.Fields = slices.DeleteFunc(s.Fields, func(field gff.Field) bool {
|
||||
return field.Label == label
|
||||
})
|
||||
}
|
||||
|
||||
func cleanupStaleFiles(p *project.Project, desired map[string]struct{}, scope extractionScope) (int, []error) {
|
||||
candidates := make([]string, 0, len(p.Inventory.SourceFiles)+len(p.Inventory.ScriptFiles)+len(p.Inventory.AssetFiles))
|
||||
if scope.Source && safeCleanupRoot(p.SourceDir(), p.Root) {
|
||||
|
||||
@@ -1,524 +0,0 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf"
|
||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/music"
|
||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
|
||||
)
|
||||
|
||||
type preparedMusicAssets struct {
|
||||
Generated []assetResource
|
||||
SkipSourceRel map[string]struct{}
|
||||
Artifacts []string
|
||||
TempRoots []string
|
||||
Summary CreditsRefreshSummary
|
||||
}
|
||||
|
||||
type MusicBuildOptions struct {
|
||||
DatasetIDs []string
|
||||
Write bool
|
||||
}
|
||||
|
||||
type musicPrepareOptions struct {
|
||||
datasetIDs []string
|
||||
write bool
|
||||
}
|
||||
|
||||
type musicSourceGroup struct {
|
||||
DatasetID string
|
||||
Dataset project.EffectiveMusicDataset
|
||||
SourceDir string
|
||||
OutputDir string
|
||||
Sources []string
|
||||
}
|
||||
|
||||
type musicCreditGroup struct {
|
||||
CreditsRoot string
|
||||
SourceDir string
|
||||
Entries []music.CreditsEntry
|
||||
}
|
||||
|
||||
func musicRootPath(p *project.Project, path string) string {
|
||||
if strings.TrimSpace(path) == "" {
|
||||
return p.Root
|
||||
}
|
||||
if filepath.IsAbs(path) {
|
||||
return path
|
||||
}
|
||||
return filepath.Join(p.Root, filepath.FromSlash(path))
|
||||
}
|
||||
|
||||
func effectiveMusicDatasets(p *project.Project, onlyIDs []string) (map[string]project.EffectiveMusicDataset, error) {
|
||||
effective := p.EffectiveConfig()
|
||||
wanted := map[string]struct{}{}
|
||||
for _, id := range onlyIDs {
|
||||
id = strings.TrimSpace(id)
|
||||
if id != "" {
|
||||
wanted[id] = struct{}{}
|
||||
}
|
||||
}
|
||||
datasets := map[string]project.EffectiveMusicDataset{}
|
||||
for id, dataset := range effective.Music.Datasets {
|
||||
if len(wanted) > 0 {
|
||||
if _, ok := wanted[id]; !ok {
|
||||
continue
|
||||
}
|
||||
delete(wanted, id)
|
||||
}
|
||||
datasets[id] = dataset
|
||||
}
|
||||
if len(wanted) > 0 {
|
||||
unknown := make([]string, 0, len(wanted))
|
||||
for id := range wanted {
|
||||
unknown = append(unknown, id)
|
||||
}
|
||||
sort.Strings(unknown)
|
||||
return nil, fmt.Errorf("unknown music dataset(s): %s", strings.Join(unknown, ", "))
|
||||
}
|
||||
if len(datasets) == 0 && len(wanted) == 0 {
|
||||
for _, rel := range p.Inventory.AssetFiles {
|
||||
ext := strings.ToLower(filepath.Ext(rel))
|
||||
if !music.PathIsUnder("envi/music", rel) {
|
||||
continue
|
||||
}
|
||||
if _, ok := extensionSet(effective.Music.ConvertExtensions)[ext]; ok {
|
||||
datasets["legacy_envi_music"] = project.EffectiveMusicDataset{
|
||||
Source: "envi/music",
|
||||
Output: "envi/music",
|
||||
StageRoot: effective.Music.StageRoot,
|
||||
CreditsRoot: effective.Music.CreditsRoot,
|
||||
NamingScheme: effective.Music.NamingScheme,
|
||||
OutputExtension: effective.Music.OutputExtension,
|
||||
MaxStemLength: effective.Music.MaxStemLength,
|
||||
PackageMode: "hak_asset",
|
||||
ConvertExtensions: effective.Music.ConvertExtensions,
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return datasets, nil
|
||||
}
|
||||
|
||||
func extensionSet(values []string) map[string]struct{} {
|
||||
out := make(map[string]struct{}, len(values))
|
||||
for _, value := range values {
|
||||
value = strings.ToLower(strings.TrimSpace(value))
|
||||
if value != "" {
|
||||
out[value] = struct{}{}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func datasetForMusicSource(datasets map[string]project.EffectiveMusicDataset, rel, ext string) (string, project.EffectiveMusicDataset, bool) {
|
||||
bestID := ""
|
||||
var best project.EffectiveMusicDataset
|
||||
bestLen := -1
|
||||
for id, dataset := range datasets {
|
||||
if _, ok := extensionSet(dataset.ConvertExtensions)[ext]; !ok {
|
||||
continue
|
||||
}
|
||||
if !music.PathIsUnder(dataset.Source, rel) {
|
||||
continue
|
||||
}
|
||||
if len(dataset.Source) > bestLen {
|
||||
bestID = id
|
||||
best = dataset
|
||||
bestLen = len(dataset.Source)
|
||||
}
|
||||
}
|
||||
return bestID, best, bestID != ""
|
||||
}
|
||||
|
||||
func plannedMusicAsset(outputRel, sourceRel string) (assetResource, error) {
|
||||
extension := strings.TrimPrefix(strings.ToLower(filepath.Ext(outputRel)), ".")
|
||||
resourceType, ok := erf.HAKResourceTypeForExtension(extension)
|
||||
if !ok {
|
||||
return assetResource{}, fmt.Errorf("unsupported generated music resource extension %q", filepath.Ext(outputRel))
|
||||
}
|
||||
name := strings.ToLower(strings.TrimSuffix(filepath.Base(outputRel), filepath.Ext(outputRel)))
|
||||
return assetResource{
|
||||
Rel: outputRel,
|
||||
Resource: erf.Resource{
|
||||
Name: name,
|
||||
Type: resourceType,
|
||||
},
|
||||
ContentID: "music-plan:" + filepath.ToSlash(sourceRel),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func BuildMusic(p *project.Project, opts MusicBuildOptions) (BuildResult, error) {
|
||||
prepared, err := prepareMusicAssetsWithOptions(p, musicPrepareOptions{datasetIDs: opts.DatasetIDs, write: opts.Write})
|
||||
if err != nil {
|
||||
return BuildResult{}, err
|
||||
}
|
||||
if err := cleanupPreparedMusicAssets(prepared); err != nil {
|
||||
return BuildResult{}, err
|
||||
}
|
||||
return BuildResult{
|
||||
CreditsArtifactPaths: prepared.Artifacts,
|
||||
CreditsSummary: prepared.Summary,
|
||||
HAKAssets: len(prepared.Generated),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func prepareMusicAssets(p *project.Project) (*preparedMusicAssets, error) {
|
||||
return prepareMusicAssetsWithOptions(p, musicPrepareOptions{write: true})
|
||||
}
|
||||
|
||||
func prepareMusicAssetsWithOptions(p *project.Project, opts musicPrepareOptions) (*preparedMusicAssets, error) {
|
||||
usedNames := make(map[string]struct{})
|
||||
groupsByKey := make(map[string]*musicSourceGroup)
|
||||
datasets, err := effectiveMusicDatasets(p, opts.datasetIDs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, rel := range p.Inventory.AssetFiles {
|
||||
rel = filepath.ToSlash(rel)
|
||||
ext := strings.ToLower(filepath.Ext(rel))
|
||||
name := strings.ToLower(strings.TrimSuffix(filepath.Base(rel), filepath.Ext(rel)))
|
||||
if datasetID, dataset, ok := datasetForMusicSource(datasets, rel, ext); ok {
|
||||
sourceDir := filepath.ToSlash(filepath.Dir(rel))
|
||||
relDir, err := filepath.Rel(filepath.FromSlash(dataset.Source), filepath.FromSlash(sourceDir))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("resolve music source directory %s: %w", sourceDir, err)
|
||||
}
|
||||
relDir = filepath.ToSlash(relDir)
|
||||
outputDir := dataset.Output
|
||||
if relDir != "." && relDir != "" {
|
||||
outputDir = filepath.ToSlash(filepath.Join(outputDir, filepath.FromSlash(relDir)))
|
||||
}
|
||||
key := datasetID + "\x00" + sourceDir
|
||||
group := groupsByKey[key]
|
||||
if group == nil {
|
||||
group = &musicSourceGroup{
|
||||
DatasetID: datasetID,
|
||||
Dataset: dataset,
|
||||
SourceDir: sourceDir,
|
||||
OutputDir: outputDir,
|
||||
}
|
||||
groupsByKey[key] = group
|
||||
}
|
||||
group.Sources = append(group.Sources, rel)
|
||||
continue
|
||||
}
|
||||
if name != "" {
|
||||
usedNames[name] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
result := &preparedMusicAssets{
|
||||
SkipSourceRel: make(map[string]struct{}),
|
||||
}
|
||||
if len(groupsByKey) == 0 {
|
||||
if !opts.write {
|
||||
return result, nil
|
||||
}
|
||||
writeResult, err := writeCreditsArtifacts(p, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result.Artifacts = writeResult.Artifacts
|
||||
result.Summary = writeResult.Summary
|
||||
return result, nil
|
||||
}
|
||||
|
||||
groups := make([]*musicSourceGroup, 0, len(groupsByKey))
|
||||
for _, group := range groupsByKey {
|
||||
sort.Strings(group.Sources)
|
||||
groups = append(groups, group)
|
||||
}
|
||||
sort.Slice(groups, func(i, j int) bool {
|
||||
if groups[i].SourceDir != groups[j].SourceDir {
|
||||
return groups[i].SourceDir < groups[j].SourceDir
|
||||
}
|
||||
return groups[i].DatasetID < groups[j].DatasetID
|
||||
})
|
||||
|
||||
creditGroups := make([]musicCreditGroup, 0, len(groups))
|
||||
for _, group := range groups {
|
||||
result.Summary.ScannedDirs = append(result.Summary.ScannedDirs, group.SourceDir)
|
||||
result.Summary.TrackCount += len(group.Sources)
|
||||
}
|
||||
ffmpegPath := ""
|
||||
ffprobePath := ""
|
||||
if opts.write {
|
||||
var err error
|
||||
ffmpegPath, err = music.ResolveFFmpegWithConfig(p.MusicFFmpegPath())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ffprobePath, err = music.ResolveFFprobeWithConfig(p.MusicFFprobePath())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
for _, group := range groups {
|
||||
stageRoot := musicRootPath(p, group.Dataset.StageRoot)
|
||||
result.TempRoots = append(result.TempRoots, stageRoot)
|
||||
prefix := music.SanitizePrefix(group.Dataset.Prefix)
|
||||
overlayPath := filepath.Join(p.AssetsDir(), filepath.FromSlash(group.SourceDir), p.EffectiveConfig().Music.CreditsOverlay)
|
||||
overlay, err := music.ParseCreditsOverlay(overlayPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse credits overlay %s: %w", overlayPath, err)
|
||||
}
|
||||
|
||||
entries := make([]music.CreditsEntry, 0, len(group.Sources))
|
||||
for _, rel := range group.Sources {
|
||||
base := filepath.Base(rel)
|
||||
manual := overlay.Match(base, "")
|
||||
outputFile := ""
|
||||
switch {
|
||||
case manual != nil && strings.TrimSpace(manual.OutputFile) != "":
|
||||
outputFile = strings.ToLower(strings.TrimSpace(manual.OutputFile))
|
||||
if err := music.ValidateManualOutputFileWithRules(outputFile, group.Dataset.OutputExtension, group.Dataset.MaxStemLength); err != nil {
|
||||
return nil, fmt.Errorf("manual credits output for %s: %w", rel, err)
|
||||
}
|
||||
if err := music.ReserveName(strings.TrimSuffix(outputFile, filepath.Ext(outputFile)), usedNames); err != nil {
|
||||
return nil, fmt.Errorf("manual credits output for %s: %w", rel, err)
|
||||
}
|
||||
default:
|
||||
stem, err := music.GenerateStemWithMax(base, prefix, group.Dataset.MaxStemLength, usedNames)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("generate music name for %s: %w", rel, err)
|
||||
}
|
||||
outputFile = stem + group.Dataset.OutputExtension
|
||||
}
|
||||
outputRel := filepath.ToSlash(filepath.Join(group.OutputDir, outputFile))
|
||||
result.Summary.Mappings = append(result.Summary.Mappings, MusicFileMapping{
|
||||
Source: rel,
|
||||
Output: outputRel,
|
||||
})
|
||||
result.SkipSourceRel[rel] = struct{}{}
|
||||
|
||||
if !opts.write {
|
||||
if group.Dataset.PackageMode == "hak_asset" {
|
||||
asset, err := plannedMusicAsset(outputRel, rel)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result.Generated = append(result.Generated, asset)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
metadata, err := music.ProbeMetadata(ffprobePath, filepath.Join(p.AssetsDir(), filepath.FromSlash(rel)))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("probe music metadata for %s: %w", rel, err)
|
||||
}
|
||||
entry := music.CreditsEntry{
|
||||
Artist: metadata.Artist,
|
||||
Title: music.FirstNonEmpty(metadata.Title, strings.TrimSuffix(base, filepath.Ext(base))),
|
||||
OutputFile: outputFile,
|
||||
OriginalFile: base,
|
||||
Album: metadata.Album,
|
||||
Date: metadata.Date,
|
||||
Rights: music.FirstNonEmpty(metadata.Rights, music.JoinNonEmpty("; ", metadata.License, metadata.Copyright)),
|
||||
Notes: music.FirstNonEmpty(metadata.Notes, metadata.Comment),
|
||||
}
|
||||
music.ApplyCreditsOverlay(&entry, manual)
|
||||
|
||||
stagePath := filepath.Join(stageRoot, filepath.FromSlash(outputRel))
|
||||
if err := os.MkdirAll(filepath.Dir(stagePath), 0o755); err != nil {
|
||||
return nil, fmt.Errorf("create music stage dir: %w", err)
|
||||
}
|
||||
if err := music.ConvertSource(ffmpegPath, filepath.Join(p.AssetsDir(), filepath.FromSlash(rel)), stagePath); err != nil {
|
||||
return nil, fmt.Errorf("convert music %s: %w", rel, err)
|
||||
}
|
||||
if group.Dataset.PackageMode == "hak_asset" {
|
||||
resourceInfo, err := assetResourceFromPath(stagePath, outputRel, lfsAssetInfo{}, false)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load converted music %s: %w", outputRel, err)
|
||||
}
|
||||
info, err := os.Stat(stagePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("stat converted music %s: %w", stagePath, err)
|
||||
}
|
||||
result.Generated = append(result.Generated, assetResource{
|
||||
Rel: outputRel,
|
||||
Resource: resourceInfo.Resource,
|
||||
Size: erf.ArchiveSize([]erf.Resource{resourceInfo.Resource}),
|
||||
CreatedAt: info.ModTime(),
|
||||
ContentID: resourceInfo.ContentID,
|
||||
})
|
||||
}
|
||||
entries = append(entries, entry)
|
||||
}
|
||||
if opts.write {
|
||||
if err := music.ValidateOverlayEntries(group.SourceDir, overlay, entries); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
creditGroups = append(creditGroups, musicCreditGroup{
|
||||
CreditsRoot: group.Dataset.CreditsRoot,
|
||||
SourceDir: group.SourceDir,
|
||||
Entries: entries,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
if !opts.write {
|
||||
sort.Slice(result.Generated, func(i, j int) bool {
|
||||
return compareResourceKeys(result.Generated[i].Resource, result.Generated[j].Resource) < 0
|
||||
})
|
||||
return result, nil
|
||||
}
|
||||
|
||||
writeResult, err := writeCreditsArtifacts(p, creditGroups)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
writeResult.Summary.ScannedDirs = append(writeResult.Summary.ScannedDirs, result.Summary.ScannedDirs...)
|
||||
writeResult.Summary.TrackCount = result.Summary.TrackCount
|
||||
result.Artifacts = writeResult.Artifacts
|
||||
result.Summary = writeResult.Summary
|
||||
sort.Slice(result.Generated, func(i, j int) bool {
|
||||
return compareResourceKeys(result.Generated[i].Resource, result.Generated[j].Resource) < 0
|
||||
})
|
||||
return result, nil
|
||||
}
|
||||
|
||||
type creditsArtifactWriteResult struct {
|
||||
Artifacts []string
|
||||
Summary CreditsRefreshSummary
|
||||
}
|
||||
|
||||
func writeCreditsArtifacts(p *project.Project, generated []musicCreditGroup) (creditsArtifactWriteResult, error) {
|
||||
defaultCreditsRoot := p.CreditsDir()
|
||||
sources := make([]music.CreditsSource, 0)
|
||||
desiredByRoot := map[string]map[string][]byte{}
|
||||
summary := CreditsRefreshSummary{}
|
||||
if generated != nil {
|
||||
sort.Slice(generated, func(i, j int) bool {
|
||||
if generated[i].SourceDir != generated[j].SourceDir {
|
||||
return generated[i].SourceDir < generated[j].SourceDir
|
||||
}
|
||||
return generated[i].CreditsRoot < generated[j].CreditsRoot
|
||||
})
|
||||
for _, group := range generated {
|
||||
creditsRoot := musicRootPath(p, group.CreditsRoot)
|
||||
if desiredByRoot[creditsRoot] == nil {
|
||||
desiredByRoot[creditsRoot] = map[string][]byte{}
|
||||
}
|
||||
entries := append([]music.CreditsEntry(nil), group.Entries...)
|
||||
sort.Slice(entries, func(i, j int) bool {
|
||||
if strings.ToLower(entries[i].Artist) != strings.ToLower(entries[j].Artist) {
|
||||
return strings.ToLower(entries[i].Artist) < strings.ToLower(entries[j].Artist)
|
||||
}
|
||||
if strings.ToLower(entries[i].Title) != strings.ToLower(entries[j].Title) {
|
||||
return strings.ToLower(entries[i].Title) < strings.ToLower(entries[j].Title)
|
||||
}
|
||||
return strings.ToLower(entries[i].OutputFile) < strings.ToLower(entries[j].OutputFile)
|
||||
})
|
||||
outputPath := filepath.Join(creditsRoot, filepath.FromSlash(group.SourceDir), p.EffectiveConfig().Music.CreditsOverlay)
|
||||
desiredByRoot[creditsRoot][outputPath] = []byte(music.RenderCreditsMarkdown(entries))
|
||||
summary.GeneratedPaths = append(summary.GeneratedPaths, outputPath)
|
||||
for _, entry := range entries {
|
||||
summary.Mappings = append(summary.Mappings, MusicFileMapping{
|
||||
Source: entry.OriginalFile,
|
||||
Output: entry.OutputFile,
|
||||
})
|
||||
}
|
||||
sources = append(sources, music.CreditsSource{
|
||||
Path: filepath.ToSlash(outputPath),
|
||||
Kind: "generated_music",
|
||||
Entries: entries,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
err := filepath.WalkDir(p.AssetsDir(), func(path string, d os.DirEntry, walkErr error) error {
|
||||
if walkErr != nil {
|
||||
return walkErr
|
||||
}
|
||||
if d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
if strings.ToLower(d.Name()) != "credits.md" {
|
||||
return nil
|
||||
}
|
||||
entries, err := music.ParseCreditsMarkdown(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse credits %s: %w", path, err)
|
||||
}
|
||||
rel, err := filepath.Rel(p.Root, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sources = append(sources, music.CreditsSource{
|
||||
Path: filepath.ToSlash(rel),
|
||||
Kind: "authored",
|
||||
Entries: entries,
|
||||
})
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return creditsArtifactWriteResult{}, err
|
||||
}
|
||||
|
||||
sort.Slice(sources, func(i, j int) bool { return sources[i].Path < sources[j].Path })
|
||||
payload, err := json.MarshalIndent(music.CreditsInventory{Sources: sources}, "", " ")
|
||||
if err != nil {
|
||||
return creditsArtifactWriteResult{}, fmt.Errorf("marshal credits inventory: %w", err)
|
||||
}
|
||||
payload = append(payload, '\n')
|
||||
if len(desiredByRoot) == 0 {
|
||||
desiredByRoot[defaultCreditsRoot] = map[string][]byte{}
|
||||
}
|
||||
artifacts := make([]string, 0)
|
||||
changedFiles := 0
|
||||
roots := make([]string, 0, len(desiredByRoot))
|
||||
for root := range desiredByRoot {
|
||||
roots = append(roots, root)
|
||||
}
|
||||
sort.Strings(roots)
|
||||
for _, creditsRoot := range roots {
|
||||
desiredFiles := desiredByRoot[creditsRoot]
|
||||
inventoryPath := filepath.Join(creditsRoot, "credits.json")
|
||||
desiredFiles[inventoryPath] = payload
|
||||
if err := os.MkdirAll(creditsRoot, 0o755); err != nil {
|
||||
return creditsArtifactWriteResult{}, fmt.Errorf("create credits dir: %w", err)
|
||||
}
|
||||
changed, err := music.SyncGeneratedCreditsArtifacts(creditsRoot, desiredFiles)
|
||||
if err != nil {
|
||||
return creditsArtifactWriteResult{}, err
|
||||
}
|
||||
changedFiles += changed
|
||||
for path := range desiredFiles {
|
||||
artifacts = append(artifacts, path)
|
||||
}
|
||||
if summary.InventoryPath == "" {
|
||||
summary.InventoryPath = inventoryPath
|
||||
}
|
||||
}
|
||||
sort.Strings(artifacts)
|
||||
summary.ArtifactPaths = append(summary.ArtifactPaths, artifacts...)
|
||||
summary.ChangedFiles = changedFiles
|
||||
return creditsArtifactWriteResult{
|
||||
Artifacts: artifacts,
|
||||
Summary: summary,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func cleanupPreparedMusicAssets(prepared *preparedMusicAssets) error {
|
||||
if prepared == nil {
|
||||
return nil
|
||||
}
|
||||
for _, root := range prepared.TempRoots {
|
||||
if strings.TrimSpace(root) == "" {
|
||||
continue
|
||||
}
|
||||
if err := os.RemoveAll(root); err != nil {
|
||||
return fmt.Errorf("remove temporary music artifact root %s: %w", root, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
+149
-392
@@ -134,6 +134,113 @@ func TestBuildThenExtract(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractSkipsAreaWhenOnlyRootVersionChanges(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
mustMkdir(t, filepath.Join(root, "src", "module"))
|
||||
mustMkdir(t, filepath.Join(root, "src", "areas"))
|
||||
mustMkdir(t, filepath.Join(root, "assets"))
|
||||
mustMkdir(t, filepath.Join(root, "build"))
|
||||
|
||||
mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{
|
||||
"module": {
|
||||
"name": "Test Module",
|
||||
"resref": "testmod"
|
||||
},
|
||||
"paths": {
|
||||
"source": "src",
|
||||
"assets": "assets",
|
||||
"build": "build"
|
||||
}
|
||||
}
|
||||
`)
|
||||
mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{
|
||||
"file_type": "IFO ",
|
||||
"file_version": "V3.2",
|
||||
"root": {
|
||||
"struct_type": 0,
|
||||
"fields": [
|
||||
{
|
||||
"label": "Mod_Name",
|
||||
"type": "CExoString",
|
||||
"value": "Test Module"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
`)
|
||||
mustWriteFile(t, filepath.Join(root, "src", "areas", "area001.are.json"), `{
|
||||
"file_type": "ARE ",
|
||||
"file_version": "V3.2",
|
||||
"root": {
|
||||
"struct_type": 0,
|
||||
"fields": [
|
||||
{
|
||||
"label": "Version",
|
||||
"type": "DWord",
|
||||
"value": 2
|
||||
},
|
||||
{
|
||||
"label": "Tag",
|
||||
"type": "CExoString",
|
||||
"value": "area001"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
`)
|
||||
|
||||
p, err := project.Load(root)
|
||||
if err != nil {
|
||||
t.Fatalf("load project: %v", err)
|
||||
}
|
||||
if err := p.ValidateLayout(); err != nil {
|
||||
t.Fatalf("validate layout: %v", err)
|
||||
}
|
||||
if err := p.Scan(); err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
if _, err := BuildModule(p); err != nil {
|
||||
t.Fatalf("build module: %v", err)
|
||||
}
|
||||
|
||||
mustWriteFile(t, filepath.Join(root, "src", "areas", "area001.are.json"), `{
|
||||
"file_type": "ARE ",
|
||||
"file_version": "V3.2",
|
||||
"root": {
|
||||
"struct_type": 0,
|
||||
"fields": [
|
||||
{
|
||||
"label": "Version",
|
||||
"type": "DWord",
|
||||
"value": 1
|
||||
},
|
||||
{
|
||||
"label": "Tag",
|
||||
"type": "CExoString",
|
||||
"value": "area001"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
`)
|
||||
if err := p.Scan(); err != nil {
|
||||
t.Fatalf("rescan: %v", err)
|
||||
}
|
||||
|
||||
result, err := Extract(p)
|
||||
if err != nil {
|
||||
t.Fatalf("extract: %v", err)
|
||||
}
|
||||
if result.Overwritten != 0 {
|
||||
t.Fatalf("Version-only area extraction must not overwrite, got %d overwritten", result.Overwritten)
|
||||
}
|
||||
|
||||
document := readGFFJSON(t, filepath.Join(root, "src", "areas", "area001.are.json"))
|
||||
if got, want := fieldValue(t, document.Root, "Version"), gff.DWordValue(1); got != want {
|
||||
t.Fatalf("expected existing Version %#v to remain, got %#v", want, got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractReadsHAKAssets(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
mustMkdir(t, filepath.Join(root, "src"))
|
||||
@@ -1893,7 +2000,7 @@ func TestPlanHAKsWritesManifestWithoutArchives(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildHAKsWritesConfiguredHeadVisualeffectsAutogenManifest(t *testing.T) {
|
||||
func TestBuildHAKsWritesConfiguredAccessoryVisualeffectsAutogenManifest(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
mustMkdir(t, filepath.Join(root, "src"))
|
||||
mustMkdir(t, filepath.Join(root, "assets", "vfxs", "chest_accessories"))
|
||||
@@ -1980,7 +2087,7 @@ func TestBuildHAKsWritesConfiguredHeadVisualeffectsAutogenManifest(t *testing.T)
|
||||
|
||||
headRaw, err := os.ReadFile(filepath.Join(root, "build", "sow-accessory-vfx-manifest.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("read head visualeffects manifest: %v", err)
|
||||
t.Fatalf("read accessory visualeffects manifest: %v", err)
|
||||
}
|
||||
text := string(headRaw)
|
||||
for _, want := range []string{
|
||||
@@ -1998,12 +2105,12 @@ func TestBuildHAKsWritesConfiguredHeadVisualeffectsAutogenManifest(t *testing.T)
|
||||
`"model_stem": "hfx_hair_bangs"`,
|
||||
} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Fatalf("expected head visualeffects manifest to contain %q, got:\n%s", want, text)
|
||||
t.Fatalf("expected accessory visualeffects manifest to contain %q, got:\n%s", want, text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildHAKsWritesFolderCategoryInHeadVisualeffectsAutogenManifest(t *testing.T) {
|
||||
func TestBuildHAKsWritesFolderCategoryInAccessoryVisualeffectsAutogenManifest(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
mustMkdir(t, filepath.Join(root, "src"))
|
||||
mustMkdir(t, filepath.Join(root, "assets", "vfxs", "head_features", "sinfar", "ears_plt"))
|
||||
@@ -2082,7 +2189,7 @@ func TestBuildHAKsWritesFolderCategoryInHeadVisualeffectsAutogenManifest(t *test
|
||||
}
|
||||
headRaw, err := os.ReadFile(filepath.Join(root, "build", "sow-accessory-vfx-manifest.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("read head visualeffects manifest: %v", err)
|
||||
t.Fatalf("read accessory visualeffects manifest: %v", err)
|
||||
}
|
||||
text := string(headRaw)
|
||||
for _, want := range []string{
|
||||
@@ -2095,7 +2202,7 @@ func TestBuildHAKsWritesFolderCategoryInHeadVisualeffectsAutogenManifest(t *test
|
||||
`"category_from": "immediate_parent"`,
|
||||
} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Fatalf("expected head visualeffects manifest to contain %q, got:\n%s", want, text)
|
||||
t.Fatalf("expected accessory visualeffects manifest to contain %q, got:\n%s", want, text)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2929,7 +3036,7 @@ func TestPlanHAKChunksPutsNewestAssetsInLastChunk(t *testing.T) {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
|
||||
assets, err := collectAssetResources(p, false, nil, nil, nil)
|
||||
assets, err := collectAssetResources(p, false, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("collect asset resources: %v", err)
|
||||
}
|
||||
@@ -3086,7 +3193,7 @@ func TestCollectLFSAssetInfoUsesGitCommonDirForWorktree(t *testing.T) {
|
||||
func runGitCommitTest(t *testing.T, dir, timestamp, message string) {
|
||||
t.Helper()
|
||||
runGitTest(t, dir, "add", ".")
|
||||
cmd := exec.Command("git", "commit", "-m", message)
|
||||
cmd := exec.Command("git", "-c", "commit.gpgsign=false", "-c", "tag.gpgsign=false", "commit", "-m", message)
|
||||
cmd.Dir = dir
|
||||
cmd.Env = append(os.Environ(),
|
||||
"GIT_AUTHOR_DATE="+timestamp,
|
||||
@@ -3100,7 +3207,7 @@ func runGitCommitTest(t *testing.T, dir, timestamp, message string) {
|
||||
|
||||
func runGitTest(t *testing.T, dir string, args ...string) {
|
||||
t.Helper()
|
||||
cmd := exec.Command("git", args...)
|
||||
cmd := exec.Command("git", append([]string{"-c", "commit.gpgsign=false", "-c", "tag.gpgsign=false"}, args...)...)
|
||||
if dir != "" {
|
||||
cmd.Dir = dir
|
||||
}
|
||||
@@ -3676,168 +3783,38 @@ func TestBuildHAKsFailsForDuplicateResourcesInSameHAK(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildHAKsConvertsMusicSourcesAndWritesCreditsArtifacts(t *testing.T) {
|
||||
// A module-only project (consumes prebuilt HAKs, no paths.assets) must build via
|
||||
// the full `crucible module build` (pipeline.Build) without an assets tree. HAK
|
||||
// collection used to crash on the unset assets dir
|
||||
// (`lstat : no such file or directory` / `Rel: can't make "" relative`).
|
||||
func TestBuildModuleOnlyProjectWithoutAssetsDir(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
mustMkdir(t, filepath.Join(root, "assets", "envi", "music", "westgate"))
|
||||
mustMkdir(t, filepath.Join(root, "src", "module"))
|
||||
mustMkdir(t, filepath.Join(root, "build"))
|
||||
|
||||
mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{
|
||||
"module": {
|
||||
"name": "Test Assets",
|
||||
"resref": "testassets"
|
||||
},
|
||||
"paths": {
|
||||
"assets": "assets",
|
||||
"build": "build"
|
||||
},
|
||||
"music": {
|
||||
"prefixes": {
|
||||
"envi/music/westgate": "mus_wg_"
|
||||
}
|
||||
},
|
||||
"haks": [
|
||||
{
|
||||
"name": "envi",
|
||||
"priority": 1,
|
||||
"max_bytes": 1048576,
|
||||
"split": false,
|
||||
"include": ["envi/**"]
|
||||
}
|
||||
]
|
||||
}
|
||||
`)
|
||||
|
||||
mustWriteFile(t, filepath.Join(root, "assets", "envi", "music", "westgate", "AleandAnecdotes.mp3"), "source-mp3")
|
||||
mustWriteFile(t, filepath.Join(root, "assets", "envi", "music", "westgate", "CREDITS.md"), "# NWN Music Pack Credits\n\n| Artist | Title | Output File | Original File | Album | Date | License / Copyright | Notes |\n|---|---|---|---|---|---|---|---|\n| Darren Curtis | Ale and Anecdotes | `mus_wg_andnc.bmu` | `AleandAnecdotes.mp3` | Darren's Commercially Free Music | | Darren Curtis Music | |\n")
|
||||
|
||||
ffprobePath := filepath.Join(root, "ffprobe")
|
||||
mustWriteFile(t, ffprobePath, "#!/bin/sh\nprintf '%s\\n' '{\"format\":{\"tags\":{\"artist\":\"Unknown\",\"title\":\"Broken Metadata\"}}}'\n")
|
||||
if err := os.Chmod(ffprobePath, 0o755); err != nil {
|
||||
t.Fatalf("chmod ffprobe: %v", err)
|
||||
}
|
||||
|
||||
ffmpegPath := filepath.Join(root, "ffmpeg")
|
||||
mustWriteFile(t, ffmpegPath, "#!/bin/sh\nfor last; do :; done\nprintf 'bmu-data' > \"$last\"\n")
|
||||
if err := os.Chmod(ffmpegPath, 0o755); err != nil {
|
||||
t.Fatalf("chmod ffmpeg: %v", err)
|
||||
}
|
||||
|
||||
t.Setenv("SOW_FFMPEG", ffmpegPath)
|
||||
t.Setenv("SOW_FFPROBE", ffprobePath)
|
||||
|
||||
p, err := project.Load(root)
|
||||
if err != nil {
|
||||
t.Fatalf("load project: %v", err)
|
||||
}
|
||||
if err := p.ValidateLayout(); err != nil {
|
||||
t.Fatalf("validate layout: %v", err)
|
||||
}
|
||||
if err := p.Scan(); err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
|
||||
result, err := BuildHAKs(p)
|
||||
if err != nil {
|
||||
t.Fatalf("build haks: %v", err)
|
||||
}
|
||||
if len(result.CreditsArtifactPaths) == 0 {
|
||||
t.Fatal("expected credits artifacts to be written")
|
||||
}
|
||||
|
||||
manifestRaw, err := os.ReadFile(filepath.Join(root, "build", "haks.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("read manifest: %v", err)
|
||||
}
|
||||
if !strings.Contains(string(manifestRaw), "envi/music/westgate/mus_wg_andnc.bmu") {
|
||||
t.Fatalf("expected converted bmu in manifest, got %s", string(manifestRaw))
|
||||
}
|
||||
if strings.Contains(string(manifestRaw), "AleandAnecdotes.mp3") {
|
||||
t.Fatalf("did not expect source mp3 in manifest, got %s", string(manifestRaw))
|
||||
}
|
||||
|
||||
creditsRaw, err := os.ReadFile(filepath.Join(root, ".cache", "credits", "envi", "music", "westgate", "CREDITS.md"))
|
||||
if err != nil {
|
||||
t.Fatalf("read generated credits: %v", err)
|
||||
}
|
||||
creditsText := string(creditsRaw)
|
||||
if !strings.Contains(creditsText, "Darren Curtis") || !strings.Contains(creditsText, "mus_wg_andnc.bmu") {
|
||||
t.Fatalf("unexpected generated credits contents: %s", creditsText)
|
||||
}
|
||||
if strings.Contains(creditsText, "Broken Metadata") {
|
||||
t.Fatalf("manual credits overlay should override bad metadata: %s", creditsText)
|
||||
}
|
||||
|
||||
inventoryRaw, err := os.ReadFile(filepath.Join(root, ".cache", "credits", "credits.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("read credits inventory: %v", err)
|
||||
}
|
||||
inventoryText := string(inventoryRaw)
|
||||
if !strings.Contains(inventoryText, "assets/envi/music/westgate/CREDITS.md") {
|
||||
t.Fatalf("expected authored credits source in inventory: %s", inventoryText)
|
||||
}
|
||||
if !strings.Contains(inventoryText, ".cache/credits/envi/music/westgate/CREDITS.md") {
|
||||
t.Fatalf("expected generated credits source in inventory: %s", inventoryText)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(root, ".cache", "music")); !os.IsNotExist(err) {
|
||||
t.Fatalf("expected temporary music staging to be cleaned, got err=%v", err)
|
||||
}
|
||||
|
||||
secondResult, err := BuildHAKs(p)
|
||||
if err != nil {
|
||||
t.Fatalf("build haks second pass: %v", err)
|
||||
}
|
||||
if secondResult.CreditsSummary.ChangedFiles != 0 {
|
||||
t.Fatalf("expected second credits refresh to be unchanged, got %d changed files", secondResult.CreditsSummary.ChangedFiles)
|
||||
}
|
||||
if secondResult.CreditsSummary.TrackCount != 1 {
|
||||
t.Fatalf("expected one tracked music conversion, got %d", secondResult.CreditsSummary.TrackCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildHAKsUsesExplicitMusicDatasetOutsideLegacyRoot(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
mustMkdir(t, filepath.Join(root, "assets", "audio", "westgate"))
|
||||
mustMkdir(t, filepath.Join(root, "build"))
|
||||
|
||||
ffprobePath := filepath.Join(root, "tools", "ffprobe")
|
||||
ffmpegPath := filepath.Join(root, "tools", "ffmpeg")
|
||||
mustMkdir(t, filepath.Dir(ffprobePath))
|
||||
mustWriteFile(t, ffprobePath, "#!/bin/sh\nprintf '%s\\n' '{\"format\":{\"tags\":{\"artist\":\"Dataset Artist\",\"title\":\"Dataset Title\"}}}'\n")
|
||||
mustWriteFile(t, ffmpegPath, "#!/bin/sh\nfor last; do :; done\nprintf 'dataset-bmu' > \"$last\"\n")
|
||||
if err := os.Chmod(ffprobePath, 0o755); err != nil {
|
||||
t.Fatalf("chmod ffprobe: %v", err)
|
||||
}
|
||||
if err := os.Chmod(ffmpegPath, 0o755); err != nil {
|
||||
t.Fatalf("chmod ffmpeg: %v", err)
|
||||
}
|
||||
|
||||
mustWriteFile(t, filepath.Join(root, "nwn-tool.yaml"), `
|
||||
module:
|
||||
name: Test Assets
|
||||
resref: testassets
|
||||
name: Test Module
|
||||
resref: testmod
|
||||
paths:
|
||||
assets: assets
|
||||
source: src
|
||||
build: build
|
||||
music:
|
||||
tools:
|
||||
ffmpeg: tools/ffmpeg
|
||||
ffprobe: tools/ffprobe
|
||||
defaults:
|
||||
credits_root: custom-credits
|
||||
datasets:
|
||||
westgate_audio:
|
||||
source: audio/westgate
|
||||
output: generated/music
|
||||
prefix: wg_
|
||||
haks:
|
||||
- name: music
|
||||
priority: 1
|
||||
max_bytes: 1048576
|
||||
split: false
|
||||
include:
|
||||
- generated/music/**
|
||||
`)
|
||||
mustWriteFile(t, filepath.Join(root, "assets", "audio", "westgate", "Theme Song.mp3"), "source-mp3")
|
||||
mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{
|
||||
"file_type": "IFO ",
|
||||
"file_version": "V3.2",
|
||||
"root": {
|
||||
"struct_type": 0,
|
||||
"fields": [
|
||||
{
|
||||
"label": "Mod_Name",
|
||||
"type": "CExoString",
|
||||
"value": "Test Module"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
`)
|
||||
|
||||
p, err := project.Load(root)
|
||||
if err != nil {
|
||||
@@ -3849,238 +3826,18 @@ haks:
|
||||
if err := p.Scan(); err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
if p.AssetsDir() != "" {
|
||||
t.Fatalf("test precondition: expected unset assets dir, got %q", p.AssetsDir())
|
||||
}
|
||||
|
||||
result, err := BuildHAKs(p)
|
||||
result, err := Build(p)
|
||||
if err != nil {
|
||||
t.Fatalf("build haks: %v", err)
|
||||
t.Fatalf("full build of module-only project: %v", err)
|
||||
}
|
||||
if result.CreditsSummary.TrackCount != 1 {
|
||||
t.Fatalf("expected one music track, got %#v", result.CreditsSummary)
|
||||
if _, err := os.Stat(result.ModulePath); err != nil {
|
||||
t.Fatalf("expected built .mod at %s: %v", result.ModulePath, err)
|
||||
}
|
||||
manifestRaw, err := os.ReadFile(filepath.Join(root, "build", "haks.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("read manifest: %v", err)
|
||||
}
|
||||
manifestText := string(manifestRaw)
|
||||
if !strings.Contains(manifestText, "generated/music/wg_theme.bmu") {
|
||||
t.Fatalf("expected dataset output resource in manifest, got:\n%s", manifestText)
|
||||
}
|
||||
if strings.Contains(manifestText, "audio/westgate/Theme Song.mp3") {
|
||||
t.Fatalf("did not expect source mp3 in manifest, got:\n%s", manifestText)
|
||||
}
|
||||
creditsRaw, err := os.ReadFile(filepath.Join(root, "custom-credits", "audio", "westgate", "CREDITS.md"))
|
||||
if err != nil {
|
||||
t.Fatalf("read generated dataset credits: %v", err)
|
||||
}
|
||||
if !strings.Contains(string(creditsRaw), "Dataset Artist") || !strings.Contains(string(creditsRaw), "wg_theme.bmu") {
|
||||
t.Fatalf("unexpected generated credits:\n%s", string(creditsRaw))
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildHAKsConvertsFLACMusicSourcesByDefault(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
mustMkdir(t, filepath.Join(root, "assets", "audio", "westgate"))
|
||||
mustMkdir(t, filepath.Join(root, "build"))
|
||||
|
||||
ffprobePath := filepath.Join(root, "tools", "ffprobe")
|
||||
ffmpegPath := filepath.Join(root, "tools", "ffmpeg")
|
||||
mustMkdir(t, filepath.Dir(ffprobePath))
|
||||
mustWriteFile(t, ffprobePath, "#!/bin/sh\nprintf '%s\\n' '{\"format\":{\"tags\":{\"artist\":\"FLAC Artist\",\"title\":\"FLAC Title\"}}}'\n")
|
||||
mustWriteFile(t, ffmpegPath, "#!/bin/sh\nfor last; do :; done\nprintf 'dataset-bmu' > \"$last\"\n")
|
||||
if err := os.Chmod(ffprobePath, 0o755); err != nil {
|
||||
t.Fatalf("chmod ffprobe: %v", err)
|
||||
}
|
||||
if err := os.Chmod(ffmpegPath, 0o755); err != nil {
|
||||
t.Fatalf("chmod ffmpeg: %v", err)
|
||||
}
|
||||
|
||||
mustWriteFile(t, filepath.Join(root, "nwn-tool.yaml"), `
|
||||
module:
|
||||
name: Test Assets
|
||||
resref: testassets
|
||||
paths:
|
||||
assets: assets
|
||||
build: build
|
||||
music:
|
||||
tools:
|
||||
ffmpeg: tools/ffmpeg
|
||||
ffprobe: tools/ffprobe
|
||||
datasets:
|
||||
westgate_audio:
|
||||
source: audio/westgate
|
||||
output: generated/music
|
||||
prefix: wg_
|
||||
haks:
|
||||
- name: music
|
||||
priority: 1
|
||||
max_bytes: 1048576
|
||||
split: false
|
||||
include:
|
||||
- generated/music/**
|
||||
`)
|
||||
mustWriteFile(t, filepath.Join(root, "assets", "audio", "westgate", "Theme Song.flac"), "source-flac")
|
||||
|
||||
p, err := project.Load(root)
|
||||
if err != nil {
|
||||
t.Fatalf("load project: %v", err)
|
||||
}
|
||||
if err := p.ValidateLayout(); err != nil {
|
||||
t.Fatalf("validate layout: %v", err)
|
||||
}
|
||||
if err := p.Scan(); err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
|
||||
result, err := BuildHAKs(p)
|
||||
if err != nil {
|
||||
t.Fatalf("build haks: %v", err)
|
||||
}
|
||||
if result.CreditsSummary.TrackCount != 1 {
|
||||
t.Fatalf("expected one music track, got %#v", result.CreditsSummary)
|
||||
}
|
||||
manifestRaw, err := os.ReadFile(filepath.Join(root, "build", "haks.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("read manifest: %v", err)
|
||||
}
|
||||
manifestText := string(manifestRaw)
|
||||
if !strings.Contains(manifestText, "generated/music/wg_theme.bmu") {
|
||||
t.Fatalf("expected generated FLAC music in manifest, got:\n%s", manifestText)
|
||||
}
|
||||
if strings.Contains(manifestText, "audio/westgate/Theme Song.flac") {
|
||||
t.Fatalf("did not expect source flac in manifest, got:\n%s", manifestText)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildHAKsUsesYAMLMusicConvertExtensionsForDiscovery(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
mustMkdir(t, filepath.Join(root, "assets", "audio", "westgate"))
|
||||
mustMkdir(t, filepath.Join(root, "build"))
|
||||
|
||||
ffprobePath := filepath.Join(root, "tools", "ffprobe")
|
||||
ffmpegPath := filepath.Join(root, "tools", "ffmpeg")
|
||||
mustMkdir(t, filepath.Dir(ffprobePath))
|
||||
mustWriteFile(t, ffprobePath, "#!/bin/sh\nprintf '%s\\n' '{\"format\":{\"tags\":{\"artist\":\"AAC Artist\",\"title\":\"AAC Title\"}}}'\n")
|
||||
mustWriteFile(t, ffmpegPath, "#!/bin/sh\nfor last; do :; done\nprintf 'dataset-bmu' > \"$last\"\n")
|
||||
if err := os.Chmod(ffprobePath, 0o755); err != nil {
|
||||
t.Fatalf("chmod ffprobe: %v", err)
|
||||
}
|
||||
if err := os.Chmod(ffmpegPath, 0o755); err != nil {
|
||||
t.Fatalf("chmod ffmpeg: %v", err)
|
||||
}
|
||||
|
||||
mustWriteFile(t, filepath.Join(root, "nwn-tool.yaml"), `
|
||||
module:
|
||||
name: Test Assets
|
||||
resref: testassets
|
||||
paths:
|
||||
assets: assets
|
||||
build: build
|
||||
music:
|
||||
defaults:
|
||||
convert_extensions:
|
||||
- .aac
|
||||
tools:
|
||||
ffmpeg: tools/ffmpeg
|
||||
ffprobe: tools/ffprobe
|
||||
datasets:
|
||||
westgate_audio:
|
||||
source: audio/westgate
|
||||
output: generated/music
|
||||
prefix: wg_
|
||||
haks:
|
||||
- name: music
|
||||
priority: 1
|
||||
max_bytes: 1048576
|
||||
split: false
|
||||
include:
|
||||
- generated/music/**
|
||||
`)
|
||||
mustWriteFile(t, filepath.Join(root, "assets", "audio", "westgate", "Theme Song.aac"), "source-aac")
|
||||
|
||||
p, err := project.Load(root)
|
||||
if err != nil {
|
||||
t.Fatalf("load project: %v", err)
|
||||
}
|
||||
if err := p.ValidateLayout(); err != nil {
|
||||
t.Fatalf("validate layout: %v", err)
|
||||
}
|
||||
if err := p.Scan(); err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
|
||||
result, err := BuildHAKs(p)
|
||||
if err != nil {
|
||||
t.Fatalf("build haks: %v", err)
|
||||
}
|
||||
if result.CreditsSummary.TrackCount != 1 {
|
||||
t.Fatalf("expected one music track discovered through YAML convert_extensions, got %#v", result.CreditsSummary)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlanHAKsDoesNotTranscodeMusicSources(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
mustMkdir(t, filepath.Join(root, "assets", "audio", "westgate"))
|
||||
mustMkdir(t, filepath.Join(root, "build"))
|
||||
|
||||
ffmpegPath := filepath.Join(root, "tools", "ffmpeg")
|
||||
mustMkdir(t, filepath.Dir(ffmpegPath))
|
||||
mustWriteFile(t, ffmpegPath, "#!/bin/sh\necho should-not-run >&2\nexit 42\n")
|
||||
if err := os.Chmod(ffmpegPath, 0o755); err != nil {
|
||||
t.Fatalf("chmod ffmpeg: %v", err)
|
||||
}
|
||||
|
||||
mustWriteFile(t, filepath.Join(root, "nwn-tool.yaml"), `
|
||||
module:
|
||||
name: Test Assets
|
||||
resref: testassets
|
||||
paths:
|
||||
assets: assets
|
||||
build: build
|
||||
music:
|
||||
tools:
|
||||
ffmpeg: tools/ffmpeg
|
||||
ffprobe: tools/ffprobe
|
||||
datasets:
|
||||
westgate_audio:
|
||||
source: audio/westgate
|
||||
output: generated/music
|
||||
prefix: wg_
|
||||
haks:
|
||||
- name: music
|
||||
priority: 1
|
||||
max_bytes: 1048576
|
||||
split: false
|
||||
include:
|
||||
- generated/music/**
|
||||
`)
|
||||
mustWriteFile(t, filepath.Join(root, "assets", "audio", "westgate", "Theme Song.mp3"), "source-mp3")
|
||||
|
||||
p, err := project.Load(root)
|
||||
if err != nil {
|
||||
t.Fatalf("load project: %v", err)
|
||||
}
|
||||
if err := p.ValidateLayout(); err != nil {
|
||||
t.Fatalf("validate layout: %v", err)
|
||||
}
|
||||
if err := p.Scan(); err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
|
||||
result, err := PlanHAKs(p)
|
||||
if err != nil {
|
||||
t.Fatalf("plan haks: %v", err)
|
||||
}
|
||||
if result.CreditsSummary.TrackCount != 1 {
|
||||
t.Fatalf("expected one planned music track, got %#v", result.CreditsSummary)
|
||||
}
|
||||
manifestRaw, err := os.ReadFile(filepath.Join(root, "build", "haks.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("read manifest: %v", err)
|
||||
}
|
||||
if !strings.Contains(string(manifestRaw), "generated/music/wg_theme.bmu") {
|
||||
t.Fatalf("expected planned generated music in manifest, got:\n%s", string(manifestRaw))
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(root, ".cache", "music")); !os.IsNotExist(err) {
|
||||
t.Fatalf("plan-only should not stage music, stat err=%v", err)
|
||||
if result.HAKAssets != 0 {
|
||||
t.Fatalf("module-only project has no assets, expected 0 HAK assets, got %d", result.HAKAssets)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,8 +8,6 @@ import (
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/music"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -21,11 +19,6 @@ const (
|
||||
DefaultHAKArchiveTemplate = "{hak.name}.hak"
|
||||
DefaultSourceJSONPattern = "{resref}.{extension}.json"
|
||||
DefaultValidationProfile = "nwn_module"
|
||||
DefaultMusicStageRoot = "{paths.cache}/music"
|
||||
DefaultCreditsRoot = "{paths.cache}/credits"
|
||||
DefaultCreditsOverlayFile = "CREDITS.md"
|
||||
DefaultMusicNamingScheme = "nwn_bmu"
|
||||
DefaultMusicOutputExtension = ".bmu"
|
||||
DefaultTopDataBuild = "{paths.build}/topdata"
|
||||
DefaultTopDataCompiled2DADir = "2da"
|
||||
DefaultTopDataCompiledTLK = "sow_tlk.tlk"
|
||||
@@ -85,7 +78,6 @@ type EffectiveConfig struct {
|
||||
Generated GeneratedConfig `json:"generated_assets" yaml:"generated_assets"`
|
||||
Inventory InventoryConfig `json:"inventory" yaml:"inventory"`
|
||||
Validation ValidationConfig `json:"validation" yaml:"validation"`
|
||||
Music EffectiveMusicConfig `json:"music" yaml:"music"`
|
||||
TopData EffectiveTopDataConfig `json:"topdata" yaml:"topdata"`
|
||||
Extract ExtractConfig `json:"extract" yaml:"extract"`
|
||||
Autogen EffectiveAutogenConfig `json:"autogen" yaml:"autogen"`
|
||||
@@ -108,32 +100,6 @@ type EffectiveOutputConfig struct {
|
||||
HAKArchive string `json:"hak_archive" yaml:"hak_archive"`
|
||||
}
|
||||
|
||||
type EffectiveMusicConfig struct {
|
||||
Prefixes map[string]string `json:"prefixes" yaml:"prefixes"`
|
||||
Tools MusicToolsConfig `json:"tools" yaml:"tools"`
|
||||
StageRoot string `json:"stage_root" yaml:"stage_root"`
|
||||
CreditsRoot string `json:"credits_root" yaml:"credits_root"`
|
||||
CreditsOverlay string `json:"credits_overlay" yaml:"credits_overlay"`
|
||||
MaxStemLength int `json:"max_stem_length" yaml:"max_stem_length"`
|
||||
NamingScheme string `json:"naming_scheme" yaml:"naming_scheme"`
|
||||
OutputExtension string `json:"output_extension" yaml:"output_extension"`
|
||||
ConvertExtensions []string `json:"convert_extensions" yaml:"convert_extensions"`
|
||||
Datasets map[string]EffectiveMusicDataset `json:"datasets,omitempty" yaml:"datasets,omitempty"`
|
||||
}
|
||||
|
||||
type EffectiveMusicDataset struct {
|
||||
Source string `json:"source" yaml:"source"`
|
||||
Output string `json:"output" yaml:"output"`
|
||||
Prefix string `json:"prefix,omitempty" yaml:"prefix,omitempty"`
|
||||
StageRoot string `json:"stage_root,omitempty" yaml:"stage_root,omitempty"`
|
||||
CreditsRoot string `json:"credits_root,omitempty" yaml:"credits_root,omitempty"`
|
||||
NamingScheme string `json:"naming_scheme" yaml:"naming_scheme"`
|
||||
OutputExtension string `json:"output_extension" yaml:"output_extension"`
|
||||
MaxStemLength int `json:"max_stem_length" yaml:"max_stem_length"`
|
||||
PackageMode string `json:"package_mode" yaml:"package_mode"`
|
||||
ConvertExtensions []string `json:"convert_extensions" yaml:"convert_extensions"`
|
||||
}
|
||||
|
||||
type EffectiveTopDataConfig struct {
|
||||
Source string `json:"source" yaml:"source"`
|
||||
Build string `json:"build" yaml:"build"`
|
||||
@@ -275,81 +241,6 @@ func (p *Project) EffectiveConfig() EffectiveConfig {
|
||||
extract.ConsumeArchives = &defaultConsume
|
||||
}
|
||||
|
||||
musicStageRoot := expandPathTemplate(DefaultMusicStageRoot, paths)
|
||||
musicCreditsRoot := expandPathTemplate(DefaultCreditsRoot, paths)
|
||||
musicCreditsOverlay := DefaultCreditsOverlayFile
|
||||
musicMaxStemLength := 16
|
||||
musicNamingScheme := DefaultMusicNamingScheme
|
||||
musicOutputExtension := DefaultMusicOutputExtension
|
||||
musicConvertExtensions := music.DefaultConvertExtensions()
|
||||
if d := p.Config.Music.Defaults; d != nil {
|
||||
if d.StageRoot != "" {
|
||||
musicStageRoot = expandPathTemplate(d.StageRoot, paths)
|
||||
}
|
||||
if d.CreditsRoot != "" {
|
||||
musicCreditsRoot = expandPathTemplate(d.CreditsRoot, paths)
|
||||
}
|
||||
if d.CreditsOverlay != "" {
|
||||
musicCreditsOverlay = d.CreditsOverlay
|
||||
}
|
||||
if d.MaxStemLength != nil {
|
||||
musicMaxStemLength = *d.MaxStemLength
|
||||
}
|
||||
if d.NamingScheme != "" {
|
||||
musicNamingScheme = d.NamingScheme
|
||||
}
|
||||
if d.OutputExtension != "" {
|
||||
musicOutputExtension = normalizeExtension(d.OutputExtension)
|
||||
}
|
||||
if len(d.ConvertExtensions) > 0 {
|
||||
musicConvertExtensions = normalizeExtensionSlice(d.ConvertExtensions)
|
||||
}
|
||||
}
|
||||
musicPrefixes := cloneStringMap(p.Config.Music.Prefixes)
|
||||
if len(musicPrefixes) == 0 {
|
||||
musicPrefixes = make(map[string]string, len(p.Config.Music.Datasets))
|
||||
for _, ds := range p.Config.Music.Datasets {
|
||||
if ds.Source != "" && ds.Prefix != "" {
|
||||
musicPrefixes[ds.Source] = ds.Prefix
|
||||
}
|
||||
}
|
||||
}
|
||||
effectiveDatasets := make(map[string]EffectiveMusicDataset, len(p.Config.Music.Datasets))
|
||||
for id, ds := range p.Config.Music.Datasets {
|
||||
dsStageRoot := ds.StageRoot
|
||||
if dsStageRoot == "" {
|
||||
dsStageRoot = musicStageRoot
|
||||
}
|
||||
dsCreditsRoot := ds.CreditsRoot
|
||||
if dsCreditsRoot == "" {
|
||||
dsCreditsRoot = musicCreditsRoot
|
||||
}
|
||||
dsOutput := strings.TrimSpace(ds.Output)
|
||||
if dsOutput == "" {
|
||||
dsOutput = ds.Source
|
||||
}
|
||||
dsNamingScheme := defaultString(ds.NamingScheme, musicNamingScheme)
|
||||
dsOutputExtension := musicOutputExtension
|
||||
if ds.OutputExtension != "" {
|
||||
dsOutputExtension = normalizeExtension(ds.OutputExtension)
|
||||
}
|
||||
dsConvertExtensions := musicConvertExtensions
|
||||
if len(ds.ConvertExtensions) > 0 {
|
||||
dsConvertExtensions = normalizeExtensionSlice(ds.ConvertExtensions)
|
||||
}
|
||||
effectiveDatasets[id] = EffectiveMusicDataset{
|
||||
Source: ds.Source,
|
||||
Output: dsOutput,
|
||||
Prefix: ds.Prefix,
|
||||
StageRoot: dsStageRoot,
|
||||
CreditsRoot: dsCreditsRoot,
|
||||
NamingScheme: dsNamingScheme,
|
||||
OutputExtension: dsOutputExtension,
|
||||
MaxStemLength: musicMaxStemLength,
|
||||
PackageMode: defaultString(ds.PackageMode, "hak_asset"),
|
||||
ConvertExtensions: dsConvertExtensions,
|
||||
}
|
||||
}
|
||||
effective := EffectiveConfig{
|
||||
ConfigSource: p.ConfigSource,
|
||||
Module: p.Config.Module,
|
||||
@@ -359,20 +250,8 @@ func (p *Project) EffectiveConfig() EffectiveConfig {
|
||||
Generated: generated,
|
||||
Inventory: inventory,
|
||||
Validation: validation,
|
||||
Music: EffectiveMusicConfig{
|
||||
Prefixes: musicPrefixes,
|
||||
Tools: p.Config.Music.Tools,
|
||||
StageRoot: musicStageRoot,
|
||||
CreditsRoot: musicCreditsRoot,
|
||||
CreditsOverlay: musicCreditsOverlay,
|
||||
MaxStemLength: musicMaxStemLength,
|
||||
NamingScheme: musicNamingScheme,
|
||||
OutputExtension: musicOutputExtension,
|
||||
ConvertExtensions: musicConvertExtensions,
|
||||
Datasets: effectiveDatasets,
|
||||
},
|
||||
TopData: top,
|
||||
Extract: extract,
|
||||
TopData: top,
|
||||
Extract: extract,
|
||||
Autogen: EffectiveAutogenConfig{
|
||||
Producers: slices.Clone(p.Config.Autogen.Producers),
|
||||
Consumers: slices.Clone(p.Config.Autogen.Consumers),
|
||||
@@ -517,13 +396,6 @@ func markMissingDefaults(provenance ConfigProvenance) {
|
||||
"validation.profile": DefaultValidationProfile,
|
||||
"validation.builtin_script_prefixes": "NWN built-in script prefixes",
|
||||
"validation.required_fields": "NWN required GFF fields",
|
||||
"music.stage_root": DefaultMusicStageRoot,
|
||||
"music.credits_root": DefaultCreditsRoot,
|
||||
"music.credits_overlay": DefaultCreditsOverlayFile,
|
||||
"music.max_stem_length": "16",
|
||||
"music.naming_scheme": DefaultMusicNamingScheme,
|
||||
"music.output_extension": DefaultMusicOutputExtension,
|
||||
"music.convert_extensions": strings.Join(music.DefaultConvertExtensions(), ", "),
|
||||
"topdata.build": DefaultTopDataBuild,
|
||||
"topdata.compiled_2da_dir": DefaultTopDataCompiled2DADir,
|
||||
"topdata.compiled_tlk": DefaultTopDataCompiledTLK,
|
||||
@@ -690,8 +562,6 @@ func activeOverrides(effective EffectiveConfig) []ConfigOverride {
|
||||
"autogen.cache.parts_manifest_refresh": effective.Autogen.Cache.PartsManifestRefreshEnv,
|
||||
"autogen.release_source.server_url": effective.Autogen.ReleaseSource.ServerURLEnv,
|
||||
"autogen.release_source.repo": effective.Autogen.ReleaseSource.RepoEnv,
|
||||
"music.ffmpeg": "SOW_FFMPEG",
|
||||
"music.ffprobe": "SOW_FFPROBE",
|
||||
"wiki.endpoint": "NODEBB_API_ENDPOINT",
|
||||
"wiki.token": "NODEBB_API_TOKEN",
|
||||
"wiki.categories": "NODEBB_WIKI_CATEGORIES",
|
||||
|
||||
+106
-239
@@ -31,7 +31,7 @@ var SourceExtensions = []string{
|
||||
}
|
||||
|
||||
var AssetExtensions = []string{
|
||||
".2da", ".bik", ".bmp", ".bmu", ".dds", ".dwk", ".gr2", ".itp", ".jpg", ".lod", ".lyt", ".mdb", ".mdl", ".mdx", ".mp3", ".mtr", ".ogg", ".plt", ".png", ".pwk", ".set", ".shd", ".tga", ".txi", ".uti", ".vis", ".wav", ".wlk", ".wok", ".xml",
|
||||
".2da", ".bik", ".bmp", ".bmu", ".dds", ".dwk", ".gr2", ".itp", ".jpg", ".lod", ".lyt", ".mdb", ".mdl", ".mdx", ".mtr", ".plt", ".png", ".pwk", ".set", ".shd", ".tga", ".txi", ".uti", ".vis", ".wav", ".wlk", ".wok", ".xml",
|
||||
}
|
||||
|
||||
var BuiltinScriptPrefixes = []string{
|
||||
@@ -88,7 +88,6 @@ type Config struct {
|
||||
HAKs []HAKConfig `json:"haks" yaml:"haks"`
|
||||
Inventory InventoryConfig `json:"inventory" yaml:"inventory"`
|
||||
Validation ValidationConfig `json:"validation" yaml:"validation"`
|
||||
Music MusicConfig `json:"music" yaml:"music"`
|
||||
TopData TopDataConfig `json:"topdata" yaml:"topdata"`
|
||||
Extract ExtractConfig `json:"extract" yaml:"extract"`
|
||||
Autogen AutogenConfig `json:"autogen" yaml:"autogen"`
|
||||
@@ -153,40 +152,6 @@ type HAKConfig struct {
|
||||
Include []string `json:"include" yaml:"include"`
|
||||
}
|
||||
|
||||
type MusicConfig struct {
|
||||
Prefixes map[string]string `json:"prefixes,omitempty" yaml:"prefixes,omitempty"`
|
||||
Tools MusicToolsConfig `json:"tools,omitempty" yaml:"tools,omitempty"`
|
||||
Defaults *MusicDefaultsConfig `json:"defaults,omitempty" yaml:"defaults,omitempty"`
|
||||
Datasets map[string]MusicDatasetConfig `json:"datasets,omitempty" yaml:"datasets,omitempty"`
|
||||
}
|
||||
|
||||
type MusicToolsConfig struct {
|
||||
FFmpeg string `json:"ffmpeg,omitempty" yaml:"ffmpeg,omitempty"`
|
||||
FFprobe string `json:"ffprobe,omitempty" yaml:"ffprobe,omitempty"`
|
||||
}
|
||||
|
||||
type MusicDefaultsConfig struct {
|
||||
StageRoot string `json:"stage_root,omitempty" yaml:"stage_root,omitempty"`
|
||||
CreditsRoot string `json:"credits_root,omitempty" yaml:"credits_root,omitempty"`
|
||||
CreditsOverlay string `json:"credits_overlay,omitempty" yaml:"credits_overlay,omitempty"`
|
||||
MaxStemLength *int `json:"max_stem_length,omitempty" yaml:"max_stem_length,omitempty"`
|
||||
NamingScheme string `json:"naming_scheme,omitempty" yaml:"naming_scheme,omitempty"`
|
||||
OutputExtension string `json:"output_extension,omitempty" yaml:"output_extension,omitempty"`
|
||||
ConvertExtensions []string `json:"convert_extensions,omitempty" yaml:"convert_extensions,omitempty"`
|
||||
}
|
||||
|
||||
type MusicDatasetConfig struct {
|
||||
Source string `json:"source" yaml:"source"`
|
||||
Output string `json:"output,omitempty" yaml:"output,omitempty"`
|
||||
Prefix string `json:"prefix,omitempty" yaml:"prefix,omitempty"`
|
||||
StageRoot string `json:"stage_root,omitempty" yaml:"stage_root,omitempty"`
|
||||
CreditsRoot string `json:"credits_root,omitempty" yaml:"credits_root,omitempty"`
|
||||
NamingScheme string `json:"naming_scheme,omitempty" yaml:"naming_scheme,omitempty"`
|
||||
OutputExtension string `json:"output_extension,omitempty" yaml:"output_extension,omitempty"`
|
||||
PackageMode string `json:"package_mode,omitempty" yaml:"package_mode,omitempty"`
|
||||
ConvertExtensions []string `json:"convert_extensions,omitempty" yaml:"convert_extensions,omitempty"`
|
||||
}
|
||||
|
||||
type TopDataConfig struct {
|
||||
Source string `json:"source" yaml:"source"`
|
||||
Build string `json:"build" yaml:"build"`
|
||||
@@ -344,43 +309,45 @@ type AutogenReleaseSourceConfig struct {
|
||||
}
|
||||
|
||||
type AutogenProducerConfig struct {
|
||||
ID string `json:"id" yaml:"id"`
|
||||
Root string `json:"root" yaml:"root"`
|
||||
Include []string `json:"include" yaml:"include"`
|
||||
Derive AutogenDeriveConfig `json:"derive" yaml:"derive"`
|
||||
Manifest AutogenManifestConfig `json:"manifest" yaml:"manifest"`
|
||||
HeadVisualeffects HeadVisualeffectsConfig `json:"accessory_visualeffects" yaml:"accessory_visualeffects"`
|
||||
ID string `json:"id" yaml:"id"`
|
||||
Root string `json:"root" yaml:"root"`
|
||||
Include []string `json:"include" yaml:"include"`
|
||||
Derive AutogenDeriveConfig `json:"derive" yaml:"derive"`
|
||||
Manifest AutogenManifestConfig `json:"manifest" yaml:"manifest"`
|
||||
AccessoryVisualeffects AccessoryVisualeffectsConfig `json:"accessory_visualeffects" yaml:"accessory_visualeffects"`
|
||||
}
|
||||
|
||||
type AutogenConsumerConfig struct {
|
||||
ID string `json:"id" yaml:"id"`
|
||||
Producer string `json:"producer" yaml:"producer"`
|
||||
Dataset string `json:"dataset" yaml:"dataset"`
|
||||
Mode string `json:"mode" yaml:"mode"`
|
||||
Optional bool `json:"optional,omitempty" yaml:"optional,omitempty"`
|
||||
Root string `json:"root" yaml:"root"`
|
||||
Include []string `json:"include" yaml:"include"`
|
||||
Derive AutogenDeriveConfig `json:"derive" yaml:"derive"`
|
||||
PartsRows PartsRowsConfig `json:"parts_rows" yaml:"parts_rows"`
|
||||
HeadVisualeffects HeadVisualeffectsConfig `json:"accessory_visualeffects" yaml:"accessory_visualeffects"`
|
||||
Manifest AutogenManifestConfig `json:"manifest" yaml:"manifest"`
|
||||
LocalOverrideRoot string `json:"local_override_root,omitempty" yaml:"local_override_root,omitempty"`
|
||||
ID string `json:"id" yaml:"id"`
|
||||
Producer string `json:"producer" yaml:"producer"`
|
||||
Dataset string `json:"dataset" yaml:"dataset"`
|
||||
Mode string `json:"mode" yaml:"mode"`
|
||||
Optional bool `json:"optional,omitempty" yaml:"optional,omitempty"`
|
||||
Root string `json:"root" yaml:"root"`
|
||||
Include []string `json:"include" yaml:"include"`
|
||||
Derive AutogenDeriveConfig `json:"derive" yaml:"derive"`
|
||||
PartsRows PartsRowsConfig `json:"parts_rows" yaml:"parts_rows"`
|
||||
AccessoryVisualeffects AccessoryVisualeffectsConfig `json:"accessory_visualeffects" yaml:"accessory_visualeffects"`
|
||||
Manifest AutogenManifestConfig `json:"manifest" yaml:"manifest"`
|
||||
LocalOverrideRoot string `json:"local_override_root,omitempty" yaml:"local_override_root,omitempty"`
|
||||
ManifestFile string `json:"manifest_file,omitempty" yaml:"manifest_file,omitempty"`
|
||||
Source AutogenSourceConfig `json:"source,omitempty" yaml:"source,omitempty"`
|
||||
}
|
||||
|
||||
type HeadVisualeffectsConfig struct {
|
||||
Groups map[string]HeadVisualeffectGroupConfig `json:"groups,omitempty" yaml:"groups"`
|
||||
GroupTokenSource string `json:"group_token_source,omitempty" yaml:"group_token_source"`
|
||||
CategoryFrom string `json:"category_from,omitempty" yaml:"category_from"`
|
||||
Delimiter string `json:"delimiter,omitempty" yaml:"delimiter"`
|
||||
Case string `json:"case,omitempty" yaml:"case"`
|
||||
StripModelPrefixes []string `json:"strip_model_prefixes,omitempty" yaml:"strip_model_prefixes"`
|
||||
KeyFormat string `json:"key_format,omitempty" yaml:"key_format"`
|
||||
LabelFormat string `json:"label_format,omitempty" yaml:"label_format"`
|
||||
ModelColumn string `json:"model_column,omitempty" yaml:"model_column"`
|
||||
RowDefaults map[string]string `json:"row_defaults,omitempty" yaml:"row_defaults"`
|
||||
type AccessoryVisualeffectsConfig struct {
|
||||
Groups map[string]AccessoryVisualeffectGroupConfig `json:"groups,omitempty" yaml:"groups"`
|
||||
GroupTokenSource string `json:"group_token_source,omitempty" yaml:"group_token_source"`
|
||||
CategoryFrom string `json:"category_from,omitempty" yaml:"category_from"`
|
||||
Delimiter string `json:"delimiter,omitempty" yaml:"delimiter"`
|
||||
Case string `json:"case,omitempty" yaml:"case"`
|
||||
StripModelPrefixes []string `json:"strip_model_prefixes,omitempty" yaml:"strip_model_prefixes"`
|
||||
KeyFormat string `json:"key_format,omitempty" yaml:"key_format"`
|
||||
LabelFormat string `json:"label_format,omitempty" yaml:"label_format"`
|
||||
ModelColumn string `json:"model_column,omitempty" yaml:"model_column"`
|
||||
RowDefaults map[string]string `json:"row_defaults,omitempty" yaml:"row_defaults"`
|
||||
}
|
||||
|
||||
type HeadVisualeffectGroupConfig struct {
|
||||
type AccessoryVisualeffectGroupConfig struct {
|
||||
Prefix string `json:"prefix" yaml:"prefix"`
|
||||
ModelColumn string `json:"model_column,omitempty" yaml:"model_column"`
|
||||
ModelColumns []string `json:"model_columns,omitempty" yaml:"model_columns"`
|
||||
@@ -415,6 +382,17 @@ type AutogenDeriveConfig struct {
|
||||
GroupFrom string `json:"group_from,omitempty" yaml:"group_from,omitempty"`
|
||||
}
|
||||
|
||||
type AutogenSourceConfig struct {
|
||||
Kind string `json:"kind,omitempty" yaml:"kind,omitempty"`
|
||||
CDNBase string `json:"cdn_base,omitempty" yaml:"cdn_base,omitempty"`
|
||||
CDNBaseEnv string `json:"cdn_base_env,omitempty" yaml:"cdn_base_env,omitempty"`
|
||||
ChannelsPath string `json:"channels_path,omitempty" yaml:"channels_path,omitempty"`
|
||||
ManifestPath string `json:"manifest_path,omitempty" yaml:"manifest_path,omitempty"`
|
||||
ReleaseMarkerPath string `json:"release_marker_path,omitempty" yaml:"release_marker_path,omitempty"`
|
||||
ChannelEnv string `json:"channel_env,omitempty" yaml:"channel_env,omitempty"`
|
||||
OfflineOverrideEnv string `json:"offline_override_env,omitempty" yaml:"offline_override_env,omitempty"`
|
||||
}
|
||||
|
||||
type AutogenManifestConfig struct {
|
||||
ReleaseTag string `json:"release_tag" yaml:"release_tag"`
|
||||
AssetName string `json:"asset_name" yaml:"asset_name"`
|
||||
@@ -640,6 +618,7 @@ func (p *Project) ValidateLayout() error {
|
||||
failures = append(failures, validateTopDataRowGeneration(effective.TopData.RowGeneration)...)
|
||||
failures = append(failures, validateTopDataRowExtensions(effective.TopData.RowExtensions)...)
|
||||
failures = append(failures, validateTopDataClassFeatInjections(effective.TopData.ClassFeatInjections)...)
|
||||
failures = append(failures, validateAutogenConsumerSources(effective.Autogen.Consumers)...)
|
||||
failures = append(failures, validateRelativePath("topdata.compiled_2da_dir", effective.TopData.Compiled2DADir)...)
|
||||
failures = append(failures, validateRelativePath("topdata.compiled_tlk", effective.TopData.CompiledTLK)...)
|
||||
failures = append(failures, validateRelativePath("topdata.wiki.output_root", effective.TopData.Wiki.OutputRoot)...)
|
||||
@@ -736,8 +715,6 @@ func (p *Project) ValidateLayout() error {
|
||||
}
|
||||
}
|
||||
failures = append(failures, validateAutogenConfig(p.Config.Autogen)...)
|
||||
failures = append(failures, validateMusicConfig(p.Config.Music)...)
|
||||
|
||||
hakNames := map[string]struct{}{}
|
||||
for index, hak := range p.Config.HAKs {
|
||||
name := strings.TrimSpace(hak.Name)
|
||||
@@ -774,6 +751,13 @@ func (p *Project) ValidateLayout() error {
|
||||
}
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
// An output dir that does not exist yet is fine: the builder creates
|
||||
// it (MkdirAll) before writing. A bare clone must validate/build from
|
||||
// a clean tree with no pre-step (R2/parity) — only a path that exists
|
||||
// but is NOT a directory is a real error.
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
continue
|
||||
}
|
||||
failures = append(failures, fmt.Errorf("%s: %w", label, err))
|
||||
continue
|
||||
}
|
||||
@@ -1038,46 +1022,9 @@ func (p *Project) Scan() error {
|
||||
assetDir := p.AssetsDir()
|
||||
var assetFiles []string
|
||||
if assetDir != "" && filepath.Clean(assetDir) != filepath.Clean(p.Root) {
|
||||
musicSourceExts := make(map[string]struct{})
|
||||
for _, ext := range effective.Music.ConvertExtensions {
|
||||
musicSourceExts[ext] = struct{}{}
|
||||
}
|
||||
musicSourceRoots := make(map[string]map[string]struct{})
|
||||
for _, ds := range effective.Music.Datasets {
|
||||
if strings.TrimSpace(ds.Source) == "" {
|
||||
continue
|
||||
}
|
||||
exts := musicSourceRoots[ds.Source]
|
||||
if exts == nil {
|
||||
exts = make(map[string]struct{})
|
||||
musicSourceRoots[ds.Source] = exts
|
||||
}
|
||||
for _, ext := range ds.ConvertExtensions {
|
||||
exts[ext] = struct{}{}
|
||||
}
|
||||
for ext := range musicSourceExts {
|
||||
exts[ext] = struct{}{}
|
||||
}
|
||||
}
|
||||
var err error
|
||||
assetFiles, _, err = scanDir(assetDir, func(path string) bool {
|
||||
rel, err := filepath.Rel(assetDir, path)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
rel = filepath.ToSlash(rel)
|
||||
ext := strings.ToLower(filepath.Ext(rel))
|
||||
if slices.Contains(effective.Inventory.AssetExtensions, ext) {
|
||||
return true
|
||||
}
|
||||
for root, exts := range musicSourceRoots {
|
||||
if !pathIsUnder(root, rel) {
|
||||
continue
|
||||
}
|
||||
_, ok := exts[ext]
|
||||
return ok
|
||||
}
|
||||
return false
|
||||
return slices.Contains(effective.Inventory.AssetExtensions, strings.ToLower(filepath.Ext(path)))
|
||||
})
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
@@ -1322,33 +1269,6 @@ func (p *Project) TopDataPackageTLKPath() string {
|
||||
return filepath.Join(p.BuildDir(), p.TopDataPackageTLKName())
|
||||
}
|
||||
|
||||
func (p *Project) MusicStageDir() string {
|
||||
return p.rootPath(p.EffectiveConfig().Music.StageRoot)
|
||||
}
|
||||
|
||||
func (p *Project) CreditsDir() string {
|
||||
return p.rootPath(p.EffectiveConfig().Music.CreditsRoot)
|
||||
}
|
||||
|
||||
func (p *Project) MusicToolPath(path string) string {
|
||||
path = strings.TrimSpace(path)
|
||||
if path == "" {
|
||||
return ""
|
||||
}
|
||||
if filepath.IsAbs(path) {
|
||||
return path
|
||||
}
|
||||
return filepath.Join(p.Root, filepath.FromSlash(path))
|
||||
}
|
||||
|
||||
func (p *Project) MusicFFmpegPath() string {
|
||||
return p.MusicToolPath(p.EffectiveConfig().Music.Tools.FFmpeg)
|
||||
}
|
||||
|
||||
func (p *Project) MusicFFprobePath() string {
|
||||
return p.MusicToolPath(p.EffectiveConfig().Music.Tools.FFprobe)
|
||||
}
|
||||
|
||||
func (p *Project) AutogenCacheDir() string {
|
||||
return p.rootPath(p.EffectiveConfig().Autogen.Cache.Root)
|
||||
}
|
||||
@@ -1438,36 +1358,6 @@ func normalizeConfig(cfg *Config) {
|
||||
for i := range cfg.Autogen.Consumers {
|
||||
cfg.Autogen.Consumers[i].Include = normalizeStringSlice(cfg.Autogen.Consumers[i].Include)
|
||||
}
|
||||
normalizeMusicConfig(&cfg.Music)
|
||||
}
|
||||
|
||||
func normalizeMusicConfig(music *MusicConfig) {
|
||||
if music.Defaults != nil {
|
||||
music.Defaults.ConvertExtensions = normalizeExtensionSlice(music.Defaults.ConvertExtensions)
|
||||
music.Defaults.OutputExtension = normalizeExtension(music.Defaults.OutputExtension)
|
||||
}
|
||||
if len(music.Prefixes) > 0 && len(music.Datasets) == 0 {
|
||||
datasets := make(map[string]MusicDatasetConfig, len(music.Prefixes))
|
||||
for dir, prefix := range music.Prefixes {
|
||||
source := trimConfigSlashes(filepath.ToSlash(strings.TrimSpace(dir)))
|
||||
if source == "" {
|
||||
continue
|
||||
}
|
||||
key := strings.ReplaceAll(source, "/", "_")
|
||||
datasets[key] = MusicDatasetConfig{
|
||||
Source: source,
|
||||
Prefix: strings.TrimSpace(prefix),
|
||||
}
|
||||
}
|
||||
music.Datasets = datasets
|
||||
}
|
||||
for id, ds := range music.Datasets {
|
||||
ds.Source = trimConfigSlashes(filepath.ToSlash(strings.TrimSpace(ds.Source)))
|
||||
ds.Output = trimConfigSlashes(filepath.ToSlash(strings.TrimSpace(ds.Output)))
|
||||
ds.OutputExtension = normalizeExtension(ds.OutputExtension)
|
||||
ds.ConvertExtensions = normalizeExtensionSlice(ds.ConvertExtensions)
|
||||
music.Datasets[id] = ds
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeStringSlice(input []string) []string {
|
||||
@@ -1625,8 +1515,8 @@ func validateAutogenConfig(cfg AutogenConfig) []error {
|
||||
failures = append(failures, validateGlobList(fieldPrefix+".include", producer.Include)...)
|
||||
failures = append(failures, validateAutogenDeriveConfig(fieldPrefix+".derive", producer.Derive)...)
|
||||
failures = append(failures, validateAutogenManifestConfig(fieldPrefix+".manifest", producer.Manifest)...)
|
||||
if headVisualeffectsConfigConfigured(producer.HeadVisualeffects) {
|
||||
failures = append(failures, validateHeadVisualeffectsConfig(fieldPrefix+".accessory_visualeffects", producer.HeadVisualeffects)...)
|
||||
if accessoryVisualeffectsConfigConfigured(producer.AccessoryVisualeffects) {
|
||||
failures = append(failures, validateAccessoryVisualeffectsConfig(fieldPrefix+".accessory_visualeffects", producer.AccessoryVisualeffects)...)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1665,23 +1555,62 @@ func validateAutogenConfig(cfg AutogenConfig) []error {
|
||||
default:
|
||||
failures = append(failures, fmt.Errorf("%s.mode %q is not supported", fieldPrefix, consumer.Mode))
|
||||
}
|
||||
if strings.TrimSpace(consumer.Root) == "" {
|
||||
failures = append(failures, fmt.Errorf("%s.root is required", fieldPrefix))
|
||||
if strings.TrimSpace(consumer.Mode) == "accessory_visualeffects" && accessoryVisualeffectsConfigConfigured(consumer.AccessoryVisualeffects) {
|
||||
failures = append(failures, validateAccessoryVisualeffectsConfig(fieldPrefix+".accessory_visualeffects", consumer.AccessoryVisualeffects)...)
|
||||
}
|
||||
if len(consumer.Include) == 0 {
|
||||
failures = append(failures, fmt.Errorf("%s.include must contain at least one glob", fieldPrefix))
|
||||
// A manifest_file consumer reads a pre-resolved local manifest, and a
|
||||
// cdn_channel consumer resolves entirely from Source.* at runtime, so
|
||||
// neither reads the released-source fields (root/include/derive/manifest)
|
||||
// at build time — don't require them. See resolveAutogenConsumerManifest.
|
||||
if strings.TrimSpace(consumer.ManifestFile) == "" &&
|
||||
strings.TrimSpace(consumer.Source.Kind) != "cdn_channel" {
|
||||
if strings.TrimSpace(consumer.Root) == "" {
|
||||
failures = append(failures, fmt.Errorf("%s.root is required", fieldPrefix))
|
||||
}
|
||||
if len(consumer.Include) == 0 {
|
||||
failures = append(failures, fmt.Errorf("%s.include must contain at least one glob", fieldPrefix))
|
||||
}
|
||||
failures = append(failures, validateGlobList(fieldPrefix+".include", consumer.Include)...)
|
||||
failures = append(failures, validateAutogenDeriveConfig(fieldPrefix+".derive", consumer.Derive)...)
|
||||
failures = append(failures, validateAutogenManifestConfig(fieldPrefix+".manifest", consumer.Manifest)...)
|
||||
}
|
||||
failures = append(failures, validateGlobList(fieldPrefix+".include", consumer.Include)...)
|
||||
failures = append(failures, validateAutogenDeriveConfig(fieldPrefix+".derive", consumer.Derive)...)
|
||||
if strings.TrimSpace(consumer.Mode) == "accessory_visualeffects" && headVisualeffectsConfigConfigured(consumer.HeadVisualeffects) {
|
||||
failures = append(failures, validateHeadVisualeffectsConfig(fieldPrefix+".accessory_visualeffects", consumer.HeadVisualeffects)...)
|
||||
}
|
||||
failures = append(failures, validateAutogenManifestConfig(fieldPrefix+".manifest", consumer.Manifest)...)
|
||||
}
|
||||
|
||||
return failures
|
||||
}
|
||||
|
||||
func validateAutogenConsumerSources(consumers []AutogenConsumerConfig) []error {
|
||||
var failures []error
|
||||
partsRows := 0
|
||||
for _, c := range consumers {
|
||||
if strings.TrimSpace(c.Mode) == "parts_rows" {
|
||||
partsRows++
|
||||
}
|
||||
}
|
||||
if partsRows > 1 {
|
||||
failures = append(failures, fmt.Errorf("at most one autogen consumer may use mode parts_rows; found %d (override precedence would be ambiguous)", partsRows))
|
||||
}
|
||||
for _, c := range consumers {
|
||||
if strings.TrimSpace(c.Source.Kind) != "cdn_channel" {
|
||||
continue
|
||||
}
|
||||
label := strings.TrimSpace(c.ID)
|
||||
if label == "" {
|
||||
label = "<unnamed>"
|
||||
}
|
||||
if strings.TrimSpace(c.Source.ChannelsPath) == "" {
|
||||
failures = append(failures, fmt.Errorf("autogen consumer %s: source.channels_path is required for kind cdn_channel", label))
|
||||
}
|
||||
manifestPath := strings.TrimSpace(c.Source.ManifestPath)
|
||||
if manifestPath == "" {
|
||||
failures = append(failures, fmt.Errorf("autogen consumer %s: source.manifest_path is required for kind cdn_channel", label))
|
||||
} else if !strings.Contains(manifestPath, "{tag}") {
|
||||
failures = append(failures, fmt.Errorf("autogen consumer %s: source.manifest_path must contain {tag}", label))
|
||||
}
|
||||
}
|
||||
return failures
|
||||
}
|
||||
|
||||
func validateGeneratedConfig(cfg GeneratedConfig) []error {
|
||||
var failures []error
|
||||
seen := map[string]struct{}{}
|
||||
@@ -1736,7 +1665,7 @@ func validateGeneratedConfig(cfg GeneratedConfig) []error {
|
||||
return failures
|
||||
}
|
||||
|
||||
func headVisualeffectsConfigConfigured(cfg HeadVisualeffectsConfig) bool {
|
||||
func accessoryVisualeffectsConfigConfigured(cfg AccessoryVisualeffectsConfig) bool {
|
||||
return len(cfg.Groups) > 0 ||
|
||||
strings.TrimSpace(cfg.GroupTokenSource) != "" ||
|
||||
strings.TrimSpace(cfg.CategoryFrom) != "" ||
|
||||
@@ -1749,7 +1678,7 @@ func headVisualeffectsConfigConfigured(cfg HeadVisualeffectsConfig) bool {
|
||||
len(cfg.RowDefaults) > 0
|
||||
}
|
||||
|
||||
func validateHeadVisualeffectsConfig(fieldPrefix string, cfg HeadVisualeffectsConfig) []error {
|
||||
func validateAccessoryVisualeffectsConfig(fieldPrefix string, cfg AccessoryVisualeffectsConfig) []error {
|
||||
var failures []error
|
||||
groupTokenSource := strings.TrimSpace(cfg.GroupTokenSource)
|
||||
switch groupTokenSource {
|
||||
@@ -1931,68 +1860,6 @@ func validatePartsRowsFormat(format string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateMusicConfig(cfg MusicConfig) []error {
|
||||
var failures []error
|
||||
if cfg.Defaults != nil {
|
||||
if cfg.Defaults.MaxStemLength != nil && *cfg.Defaults.MaxStemLength < 3 {
|
||||
failures = append(failures, errors.New("music.defaults.max_stem_length must be at least 3"))
|
||||
}
|
||||
if cfg.Defaults.NamingScheme != "" && cfg.Defaults.NamingScheme != "nwn_bmu" && cfg.Defaults.NamingScheme != "passthrough" {
|
||||
failures = append(failures, fmt.Errorf("music.defaults.naming_scheme %q is not supported", cfg.Defaults.NamingScheme))
|
||||
}
|
||||
if cfg.Defaults.OutputExtension != "" && !strings.HasPrefix(cfg.Defaults.OutputExtension, ".") {
|
||||
failures = append(failures, errors.New("music.defaults.output_extension must start with ."))
|
||||
}
|
||||
}
|
||||
normalizedRoots := map[string]string{}
|
||||
for root, prefix := range cfg.Prefixes {
|
||||
normalizedRoot := trimConfigSlashes(filepath.ToSlash(strings.TrimSpace(root)))
|
||||
if normalizedRoot == "" {
|
||||
failures = append(failures, errors.New("music.prefixes contains an empty root"))
|
||||
continue
|
||||
}
|
||||
if prior, exists := normalizedRoots[normalizedRoot]; exists {
|
||||
failures = append(failures, fmt.Errorf("music.prefixes roots %q and %q normalize to the same path", prior, root))
|
||||
}
|
||||
normalizedRoots[normalizedRoot] = root
|
||||
if strings.TrimSpace(prefix) == "" {
|
||||
failures = append(failures, fmt.Errorf("music.prefixes[%q] must not be empty", root))
|
||||
}
|
||||
}
|
||||
seenSources := map[string]string{}
|
||||
for id, ds := range cfg.Datasets {
|
||||
if strings.TrimSpace(id) == "" {
|
||||
failures = append(failures, errors.New("music.datasets contains an empty key"))
|
||||
} else if strings.Contains(id, "/") || strings.Contains(id, "\\") {
|
||||
failures = append(failures, fmt.Errorf("music.datasets id %q must not contain path separators", id))
|
||||
}
|
||||
source := trimConfigSlashes(filepath.ToSlash(strings.TrimSpace(ds.Source)))
|
||||
if source == "" {
|
||||
failures = append(failures, fmt.Errorf("music.datasets[%q].source is required", id))
|
||||
} else if pathEscapesRoot(source) {
|
||||
failures = append(failures, fmt.Errorf("music.datasets[%q].source must not escape project root", id))
|
||||
} else if prior, exists := seenSources[source]; exists {
|
||||
failures = append(failures, fmt.Errorf("music.datasets[%q].source %q conflicts with dataset %q", id, ds.Source, prior))
|
||||
} else {
|
||||
seenSources[source] = id
|
||||
}
|
||||
output := trimConfigSlashes(filepath.ToSlash(strings.TrimSpace(ds.Output)))
|
||||
if output != "" && pathEscapesRoot(output) {
|
||||
failures = append(failures, fmt.Errorf("music.datasets[%q].output must not escape project root", id))
|
||||
}
|
||||
if ds.NamingScheme != "" && ds.NamingScheme != "nwn_bmu" && ds.NamingScheme != "passthrough" {
|
||||
failures = append(failures, fmt.Errorf("music.datasets[%q].naming_scheme %q is not supported", id, ds.NamingScheme))
|
||||
}
|
||||
if ds.PackageMode != "" && ds.PackageMode != "hak_asset" && ds.PackageMode != "none" {
|
||||
failures = append(failures, fmt.Errorf("music.datasets[%q].package_mode %q is not supported", id, ds.PackageMode))
|
||||
}
|
||||
if ds.OutputExtension != "" && !strings.HasPrefix(ds.OutputExtension, ".") {
|
||||
failures = append(failures, fmt.Errorf("music.datasets[%q].output_extension must start with .", id))
|
||||
}
|
||||
}
|
||||
return failures
|
||||
}
|
||||
|
||||
func pathEscapesRoot(path string) bool {
|
||||
clean := filepath.Clean(filepath.FromSlash(path))
|
||||
return clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator))
|
||||
|
||||
+175
-111
@@ -46,7 +46,7 @@ haks:
|
||||
}
|
||||
}
|
||||
|
||||
func TestEffectiveConfigAppliesVisibleToolkitDefaults(t *testing.T) {
|
||||
func TestEffectiveConfigReportsConfiguredAndDefaultProvenance(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
writeProjectFile(t, filepath.Join(root, ConfigFile), `
|
||||
module:
|
||||
@@ -62,35 +62,13 @@ paths:
|
||||
}
|
||||
|
||||
effective := proj.EffectiveConfig()
|
||||
if got, want := effective.Paths.Build, "build"; got != want {
|
||||
t.Fatalf("expected default build path %q, got %q", want, got)
|
||||
for _, field := range []string{"module.name", "paths.build"} {
|
||||
if strings.TrimSpace(effective.Provenance[field].Source) == "" {
|
||||
t.Errorf("expected %s to report provenance", field)
|
||||
}
|
||||
}
|
||||
if got, want := effective.Outputs.HAKManifest, "haks.json"; got != want {
|
||||
t.Fatalf("expected default HAK manifest %q, got %q", want, got)
|
||||
}
|
||||
if got, want := strings.Join(effective.Extract.Archives, ","), "testmod.mod"; got != want {
|
||||
t.Fatalf("expected default extract archives %q, got %q", want, got)
|
||||
}
|
||||
if effective.Extract.ConsumeArchives == nil || *effective.Extract.ConsumeArchives {
|
||||
t.Fatalf("expected default extract consume_archives false, got %#v", effective.Extract.ConsumeArchives)
|
||||
}
|
||||
if got, want := effective.TopData.PackageHAK, "sow_top.hak"; got != want {
|
||||
t.Fatalf("expected default topdata HAK %q, got %q", want, got)
|
||||
}
|
||||
if got, want := strings.Join(effective.Validation.BuiltinScriptPrefixes, ","), "ga_,gc_,gen_,gui_,nw_,nwg_,ta_,x0_,x1_,x2_,x3_"; got != want {
|
||||
t.Fatalf("expected default built-in script prefixes %q, got %q", want, got)
|
||||
}
|
||||
if got := strings.Join(effective.Validation.RequiredFields["ifo"], ","); got != "Mod_Name" {
|
||||
t.Fatalf("expected default IFO required fields, got %#v", effective.Validation.RequiredFields)
|
||||
}
|
||||
if got, want := strings.Join(effective.Music.ConvertExtensions, ","), ".flac,.m4a,.mp3,.ogg,.wav"; got != want {
|
||||
t.Fatalf("expected default music convert extensions %q, got %q", want, got)
|
||||
}
|
||||
if prov := effective.Provenance["paths.build"]; prov.Source != "toolkit default" {
|
||||
t.Fatalf("expected paths.build toolkit default provenance, got %#v", prov)
|
||||
}
|
||||
if prov := effective.Provenance["module.name"]; prov.Source != "yaml" {
|
||||
t.Fatalf("expected module.name YAML provenance, got %#v", prov)
|
||||
if effective.Module.Name != "Test Module" {
|
||||
t.Fatalf("configured module name was not preserved: %q", effective.Module.Name)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -116,12 +94,16 @@ validation:
|
||||
if err != nil {
|
||||
t.Fatalf("Load returned error: %v", err)
|
||||
}
|
||||
got := strings.Join(proj.EffectiveConfig().Validation.BuiltinScriptPrefixes, ",")
|
||||
if want := "custom_,nw_"; got != want {
|
||||
t.Fatalf("expected configured validation prefixes %q, got %q", want, got)
|
||||
effective := proj.EffectiveConfig()
|
||||
for _, prefix := range []string{"custom_", "nw_"} {
|
||||
if !slices.Contains(effective.Validation.BuiltinScriptPrefixes, prefix) {
|
||||
t.Errorf("configured validation prefixes missing %q: %#v", prefix, effective.Validation.BuiltinScriptPrefixes)
|
||||
}
|
||||
}
|
||||
if got := strings.Join(proj.EffectiveConfig().Validation.RequiredFields["ifo"], ","); got != "Mod_Hak,Mod_Name" {
|
||||
t.Fatalf("expected configured required fields, got %#v", proj.EffectiveConfig().Validation.RequiredFields)
|
||||
for _, field := range []string{"Mod_Hak", "Mod_Name"} {
|
||||
if !slices.Contains(effective.Validation.RequiredFields["ifo"], field) {
|
||||
t.Errorf("configured required fields missing %q: %#v", field, effective.Validation.RequiredFields)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -402,50 +384,30 @@ func TestValidateLayoutRejectsInvalidTopDataClassFeatInjections(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestScanUsesYAMLMusicConvertExtensionsWithinConfiguredDatasetRoots(t *testing.T) {
|
||||
func TestDefaultAssetExtensionsKeepAuthoredBMUWithoutConversionSources(t *testing.T) {
|
||||
for _, extension := range []string{".bmu", ".wav"} {
|
||||
if !slices.Contains(AssetExtensions, extension) {
|
||||
t.Errorf("default asset extensions must retain %s", extension)
|
||||
}
|
||||
}
|
||||
for _, extension := range []string{".mp3", ".ogg"} {
|
||||
if slices.Contains(AssetExtensions, extension) {
|
||||
t.Errorf("default asset extensions must not include conversion source %s", extension)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadRejectsRemovedMusicConfig(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
if err := os.MkdirAll(filepath.Join(root, "assets", "audio", "westgate"), 0o755); err != nil {
|
||||
t.Fatalf("mkdir westgate audio: %v", err)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(root, "assets", "audio", "other"), 0o755); err != nil {
|
||||
t.Fatalf("mkdir other audio: %v", err)
|
||||
}
|
||||
writeProjectFile(t, filepath.Join(root, ConfigFile), `
|
||||
module:
|
||||
name: Test Module
|
||||
resref: testmod
|
||||
paths:
|
||||
source: src
|
||||
assets: assets
|
||||
music:
|
||||
defaults:
|
||||
convert_extensions:
|
||||
- .flac
|
||||
datasets:
|
||||
westgate_audio:
|
||||
source: audio/westgate
|
||||
convert_extensions:
|
||||
- .aac
|
||||
datasets: {}
|
||||
`)
|
||||
writeProjectFile(t, filepath.Join(root, "assets", "audio", "westgate", "theme.aac"), "aac")
|
||||
writeProjectFile(t, filepath.Join(root, "assets", "audio", "westgate", "theme.flac"), "flac")
|
||||
writeProjectFile(t, filepath.Join(root, "assets", "audio", "other", "ignored.aac"), "aac")
|
||||
|
||||
proj, err := Load(root)
|
||||
if err != nil {
|
||||
t.Fatalf("Load returned error: %v", err)
|
||||
}
|
||||
if err := proj.Scan(); err != nil {
|
||||
t.Fatalf("Scan returned error: %v", err)
|
||||
}
|
||||
if !slices.Contains(proj.Inventory.AssetFiles, "audio/westgate/theme.aac") {
|
||||
t.Fatalf("expected configured dataset AAC file to be discovered, got %#v", proj.Inventory.AssetFiles)
|
||||
}
|
||||
if !slices.Contains(proj.Inventory.AssetFiles, "audio/westgate/theme.flac") {
|
||||
t.Fatalf("expected configured dataset FLAC file to be discovered, got %#v", proj.Inventory.AssetFiles)
|
||||
}
|
||||
if slices.Contains(proj.Inventory.AssetFiles, "audio/other/ignored.aac") {
|
||||
t.Fatalf("did not expect non-dataset AAC file to be discovered, got %#v", proj.Inventory.AssetFiles)
|
||||
if _, err := Load(root); err == nil || !strings.Contains(err.Error(), "music") {
|
||||
t.Fatalf("expected strict YAML decoding to reject removed music config, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -853,8 +815,8 @@ paths:
|
||||
if !bytes.Equal(first, second) {
|
||||
t.Fatalf("effective config JSON is not deterministic")
|
||||
}
|
||||
if !bytes.Contains(first, []byte(`"hak_manifest": "haks.json"`)) {
|
||||
t.Fatalf("effective config JSON missing HAK manifest default: %s", first)
|
||||
if !bytes.Contains(first, []byte(`"name": "Test Module"`)) {
|
||||
t.Fatalf("effective config JSON missing configured module name: %s", first)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1131,6 +1093,52 @@ func TestValidateLayoutAllowsMissingAssetsDir(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// paths.build is an OUTPUT dir the builder creates (MkdirAll) before writing, so
|
||||
// a bare clone with no build dir yet must still validate/build with no pre-step
|
||||
// (R2/parity). Only a build path that exists but is not a directory is an error.
|
||||
func TestValidateLayoutAllowsMissingBuildDir(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
mkdirAll(t, filepath.Join(root, "src"))
|
||||
// deliberately do NOT create build/
|
||||
|
||||
proj := &Project{
|
||||
Root: root,
|
||||
Config: Config{
|
||||
Module: ModuleConfig{Name: "Test", ResRef: "test"},
|
||||
Paths: PathConfig{Source: "src", Build: "build"},
|
||||
},
|
||||
}
|
||||
|
||||
if err := proj.ValidateLayout(); err != nil {
|
||||
t.Fatalf("ValidateLayout returned error for missing build dir: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(root, "build")); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatalf("expected missing build dir to remain absent, got err=%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A build path that exists but is a regular file is still a hard error.
|
||||
func TestValidateLayoutRejectsBuildDirThatIsAFile(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
mkdirAll(t, filepath.Join(root, "src"))
|
||||
if err := os.WriteFile(filepath.Join(root, "build"), []byte("i am a file, not a dir"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
proj := &Project{
|
||||
Root: root,
|
||||
Config: Config{
|
||||
Module: ModuleConfig{Name: "Test", ResRef: "test"},
|
||||
Paths: PathConfig{Source: "src", Build: "build"},
|
||||
},
|
||||
}
|
||||
|
||||
err := proj.ValidateLayout()
|
||||
if err == nil || !strings.Contains(err.Error(), "paths.build must be a directory") {
|
||||
t.Fatalf("expected paths.build-must-be-a-directory error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateLayoutRejectsInvalidTopDataPackageOutputNames(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
mkdirAll(t, filepath.Join(root, "src"))
|
||||
@@ -1331,7 +1339,7 @@ func TestValidateLayoutRejectsInvalidAutogenConsumerConfig(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateLayoutRejectsInvalidHeadVisualeffectsConfig(t *testing.T) {
|
||||
func TestValidateLayoutRejectsInvalidAccessoryVisualeffectsConfig(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
mkdirAll(t, filepath.Join(root, "src"))
|
||||
mkdirAll(t, filepath.Join(root, "build"))
|
||||
@@ -1356,8 +1364,8 @@ func TestValidateLayoutRejectsInvalidHeadVisualeffectsConfig(t *testing.T) {
|
||||
AssetName: "sow-accessory-vfx-manifest.json",
|
||||
CacheName: "sow-accessory-vfx-manifest.json",
|
||||
},
|
||||
HeadVisualeffects: HeadVisualeffectsConfig{
|
||||
Groups: map[string]HeadVisualeffectGroupConfig{
|
||||
AccessoryVisualeffects: AccessoryVisualeffectsConfig{
|
||||
Groups: map[string]AccessoryVisualeffectGroupConfig{
|
||||
"head_jewels": {
|
||||
ModelColumns: []string{"Imp Root M Node"},
|
||||
},
|
||||
@@ -1373,7 +1381,7 @@ func TestValidateLayoutRejectsInvalidHeadVisualeffectsConfig(t *testing.T) {
|
||||
|
||||
err := proj.ValidateLayout()
|
||||
if err == nil {
|
||||
t.Fatal("expected head visualeffects validation error")
|
||||
t.Fatal("expected accessory visualeffects validation error")
|
||||
}
|
||||
for _, needle := range []string{
|
||||
"autogen.consumers[0].accessory_visualeffects.groups[head_jewels].prefix",
|
||||
@@ -1387,7 +1395,7 @@ func TestValidateLayoutRejectsInvalidHeadVisualeffectsConfig(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateLayoutRejectsInvalidHeadVisualeffectsNamingConfig(t *testing.T) {
|
||||
func TestValidateLayoutRejectsInvalidAccessoryVisualeffectsNamingConfig(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
mkdirAll(t, filepath.Join(root, "src"))
|
||||
mkdirAll(t, filepath.Join(root, "build"))
|
||||
@@ -1412,8 +1420,8 @@ func TestValidateLayoutRejectsInvalidHeadVisualeffectsNamingConfig(t *testing.T)
|
||||
AssetName: "sow-accessory-vfx-manifest.json",
|
||||
CacheName: "sow-accessory-vfx-manifest.json",
|
||||
},
|
||||
HeadVisualeffects: HeadVisualeffectsConfig{
|
||||
Groups: map[string]HeadVisualeffectGroupConfig{
|
||||
AccessoryVisualeffects: AccessoryVisualeffectsConfig{
|
||||
Groups: map[string]AccessoryVisualeffectGroupConfig{
|
||||
"head_features": {},
|
||||
},
|
||||
GroupTokenSource: "unsupported",
|
||||
@@ -1429,7 +1437,7 @@ func TestValidateLayoutRejectsInvalidHeadVisualeffectsNamingConfig(t *testing.T)
|
||||
|
||||
err := proj.ValidateLayout()
|
||||
if err == nil {
|
||||
t.Fatal("expected head visualeffects naming validation error")
|
||||
t.Fatal("expected accessory visualeffects naming validation error")
|
||||
}
|
||||
for _, needle := range []string{
|
||||
"autogen.consumers[0].accessory_visualeffects.group_token_source",
|
||||
@@ -1443,7 +1451,52 @@ func TestValidateLayoutRejectsInvalidHeadVisualeffectsNamingConfig(t *testing.T)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadRejectsLegacyHeadVisualeffectsNamingFields(t *testing.T) {
|
||||
func TestValidateLayoutAcceptsCDNChannelConsumerWithoutReleasedSourceFields(t *testing.T) {
|
||||
// cdn_channel consumers resolve entirely from Source.* at runtime; they must
|
||||
// not be required to supply root/include/derive/manifest fields.
|
||||
root := t.TempDir()
|
||||
mkdirAll(t, filepath.Join(root, "src"))
|
||||
mkdirAll(t, filepath.Join(root, "build"))
|
||||
|
||||
proj := &Project{
|
||||
Root: root,
|
||||
Config: Config{
|
||||
Module: ModuleConfig{Name: "Test", ResRef: "test"},
|
||||
Paths: PathConfig{Source: "src", Build: "build"},
|
||||
Autogen: AutogenConfig{
|
||||
Consumers: []AutogenConsumerConfig{
|
||||
{
|
||||
ID: "accessory_visualeffects",
|
||||
Producer: "accessory_visualeffects",
|
||||
Dataset: "visualeffects",
|
||||
Mode: "accessory_visualeffects",
|
||||
Optional: true,
|
||||
AccessoryVisualeffects: AccessoryVisualeffectsConfig{
|
||||
Groups: map[string]AccessoryVisualeffectGroupConfig{
|
||||
"head_accessories": {Prefix: "head_acc_"},
|
||||
"chest_accessories": {Prefix: "chest_acc_"},
|
||||
"head_decorations": {Prefix: "head_dec_"},
|
||||
"head_features": {Prefix: "head_feat_"},
|
||||
},
|
||||
},
|
||||
Source: AutogenSourceConfig{
|
||||
Kind: "cdn_channel",
|
||||
ChannelsPath: "releases/haks/channels.json",
|
||||
ManifestPath: "releases/haks/{tag}/vfxs.yml",
|
||||
},
|
||||
// Intentionally omitted: Root, Include, Derive, Manifest
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
if err := proj.ValidateLayout(); err != nil {
|
||||
t.Fatalf("cdn_channel consumer should pass ValidateLayout without released-source fields, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadRejectsLegacyAccessoryVisualeffectsNamingFields(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
writeProjectFile(t, filepath.Join(root, ConfigFile), `
|
||||
module:
|
||||
@@ -1477,7 +1530,7 @@ autogen:
|
||||
|
||||
_, err := Load(root)
|
||||
if err == nil {
|
||||
t.Fatal("expected legacy head visualeffects naming fields to be rejected")
|
||||
t.Fatal("expected legacy accessory visualeffects naming fields to be rejected")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "legacy_groups") {
|
||||
t.Fatalf("expected legacy_groups unknown field error, got %v", err)
|
||||
@@ -1707,34 +1760,6 @@ func TestValidateLayoutRejectsDuplicateHAKNames(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateLayoutRejectsAmbiguousMusicPrefixes(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
mkdirAll(t, filepath.Join(root, "assets"))
|
||||
mkdirAll(t, filepath.Join(root, "build"))
|
||||
|
||||
proj := &Project{
|
||||
Root: root,
|
||||
Config: Config{
|
||||
Module: ModuleConfig{Name: "Test", ResRef: "test"},
|
||||
Paths: PathConfig{Assets: "assets", Build: "build"},
|
||||
Music: MusicConfig{
|
||||
Prefixes: map[string]string{
|
||||
"envi/music": "mus_",
|
||||
"/envi/music": "mus2_",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
err := proj.ValidateLayout()
|
||||
if err == nil {
|
||||
t.Fatal("expected music prefix validation error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "normalize to the same path") {
|
||||
t.Fatalf("expected ambiguous prefix validation error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScanAllowsMissingAssetsDir(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
mkdirAll(t, filepath.Join(root, "src"))
|
||||
@@ -1814,6 +1839,45 @@ func TestCloneWithHAKNamesRejectsUnknownHAKs(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateAutogenConsumerSourcesCDNChannel(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
source AutogenSourceConfig
|
||||
wantFail bool
|
||||
}{
|
||||
{"valid", AutogenSourceConfig{Kind: "cdn_channel", ChannelsPath: "releases/haks/channels.json", ManifestPath: "releases/haks/{tag}/vfxs.yml"}, false},
|
||||
{"missing channels_path", AutogenSourceConfig{Kind: "cdn_channel", ManifestPath: "releases/haks/{tag}/vfxs.yml"}, true},
|
||||
{"missing manifest_path", AutogenSourceConfig{Kind: "cdn_channel", ChannelsPath: "releases/haks/channels.json"}, true},
|
||||
{"manifest_path lacks {tag}", AutogenSourceConfig{Kind: "cdn_channel", ChannelsPath: "releases/haks/channels.json", ManifestPath: "releases/haks/vfxs.yml"}, true},
|
||||
{"empty kind is ignored", AutogenSourceConfig{}, false},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
failures := validateAutogenConsumerSources([]AutogenConsumerConfig{{ID: "x", Source: tc.source}})
|
||||
if tc.wantFail && len(failures) == 0 {
|
||||
t.Fatalf("expected validation failure, got none")
|
||||
}
|
||||
if !tc.wantFail && len(failures) != 0 {
|
||||
t.Fatalf("expected no failure, got %v", failures)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateRejectsMultiplePartsRowsConsumers(t *testing.T) {
|
||||
consumers := []AutogenConsumerConfig{
|
||||
{ID: "parts-a", Mode: "parts_rows"},
|
||||
{ID: "parts-b", Mode: "parts_rows"},
|
||||
}
|
||||
if failures := validateAutogenConsumerSources(consumers); len(failures) == 0 {
|
||||
t.Fatal("expected error for two parts_rows consumers")
|
||||
}
|
||||
single := []AutogenConsumerConfig{{ID: "parts", Mode: "parts_rows"}}
|
||||
if failures := validateAutogenConsumerSources(single); len(failures) != 0 {
|
||||
t.Fatalf("single parts_rows consumer must validate, got %v", failures)
|
||||
}
|
||||
}
|
||||
|
||||
func loadAndValidate(root string) error {
|
||||
proj, err := Load(root)
|
||||
if err != nil {
|
||||
|
||||
@@ -65,6 +65,33 @@ effective output while keeping project policy in topdata authoring files.
|
||||
`position` defaults to `append`. The module uses `prepend` for class feat
|
||||
injections to keep generated policy rows before per-class authored rows.
|
||||
|
||||
## Global Injection Grammar
|
||||
|
||||
Each injection entry may use only `row`, `require_present`, `any_present`, and
|
||||
`unless_present`. `when_present` is invalid and rejected; it is not an alias.
|
||||
|
||||
```json
|
||||
{
|
||||
"row": {
|
||||
"FeatIndex": { "id": "feat:example" }
|
||||
},
|
||||
"require_present": [{ "field": "FeatIndex", "id": "feat:required" }],
|
||||
"any_present": [
|
||||
{ "field": "FeatIndex", "id": "masterfeats:metamagic" },
|
||||
{ "field": "FeatIndex", "id": "masterfeats:combat" }
|
||||
],
|
||||
"unless_present": [{ "field": "FeatIndex", "id": "feat:example" }]
|
||||
}
|
||||
```
|
||||
|
||||
- `require_present` means every listed reference must exist.
|
||||
- `any_present` means at least one listed reference must exist, and the array
|
||||
must not be empty.
|
||||
- `unless_present` means none of the listed references may exist.
|
||||
- Every present group on an entry must pass.
|
||||
- Conditions observe the current rows in injection order, so earlier injections
|
||||
can affect later conditions.
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
- Class feat output contains the same effective shared feats as the prior YAML
|
||||
|
||||
+426
-71
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
@@ -13,10 +14,13 @@ import (
|
||||
"time"
|
||||
|
||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
const autogenManifestCacheMaxAge = time.Hour
|
||||
|
||||
const defaultBunnyCDNBase = "https://cdn-a7f3k9.westgate.pw"
|
||||
|
||||
// errAutogenManifestUnavailable marks a released autogen manifest that could not
|
||||
// be located or fetched (network failure, missing release/asset, empty payload,
|
||||
// or an undeterminable sow-assets repo). It is deliberately distinct from an
|
||||
@@ -35,7 +39,7 @@ type autogenManifest struct {
|
||||
Repo string `json:"repo"`
|
||||
Ref string `json:"ref"`
|
||||
GeneratedAt string `json:"generated_at"`
|
||||
HeadVisualeffects *project.HeadVisualeffectsConfig `json:"accessory_visualeffects,omitempty"`
|
||||
AccessoryVisualeffects *project.AccessoryVisualeffectsConfig `json:"accessory_visualeffects,omitempty"`
|
||||
Entries []autogenManifestEntry `json:"entries"`
|
||||
}
|
||||
|
||||
@@ -100,7 +104,7 @@ func applyAutogenConsumers(p *project.Project, collected []nativeCollectedDatase
|
||||
case "parts_rows":
|
||||
result = augmentWithAutogeneratedParts(result, autogenPartsInventory(entries))
|
||||
case "accessory_visualeffects":
|
||||
result, err = augmentWithAutogeneratedHeadVisualeffects(result, entries, consumer, manifest.HeadVisualeffects)
|
||||
result, err = augmentWithAutogeneratedAccessoryVisualeffects(result, entries, consumer, manifest.AccessoryVisualeffects)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -178,17 +182,22 @@ func preserveAutogenConsumerLockEntries(collected []nativeCollectedDataset, cons
|
||||
func autogenConsumerManagedLockKeyMatcher(consumer project.AutogenConsumerConfig) func(string) bool {
|
||||
switch consumer.Mode {
|
||||
case "accessory_visualeffects":
|
||||
policy := resolveHeadVisualeffectsPolicy(consumer, nil)
|
||||
delimiter := policy.Delimiter
|
||||
if delimiter == "" {
|
||||
delimiter = "/"
|
||||
dataset := strings.TrimSpace(consumer.Dataset)
|
||||
if dataset == "" {
|
||||
dataset = "visualeffects"
|
||||
}
|
||||
prefix := "visualeffects:"
|
||||
prefix := dataset + ":"
|
||||
// Build the group set from the resolved policy so consumers that omit
|
||||
// AccessoryVisualeffects.Groups still match the four default group names.
|
||||
// NOTE: this matcher assumes the default folder_name+preserve group
|
||||
// representation. Keys produced under case:lower or
|
||||
// group_token_source:prefix would not match and would escape pruning;
|
||||
// no production config uses those combinations.
|
||||
policy := resolveAccessoryVisualeffectsPolicy(consumer, nil)
|
||||
groups := make(map[string]struct{}, len(policy.Groups))
|
||||
for group, groupPolicy := range policy.Groups {
|
||||
token := applyHeadVisualeffectCase(headVisualeffectGroupToken(group, groupPolicy, policy), policy)
|
||||
if strings.TrimSpace(token) != "" {
|
||||
groups[token] = struct{}{}
|
||||
for group := range policy.Groups {
|
||||
if group = strings.TrimSpace(group); group != "" {
|
||||
groups[group] = struct{}{}
|
||||
}
|
||||
}
|
||||
if len(groups) == 0 {
|
||||
@@ -199,7 +208,7 @@ func autogenConsumerManagedLockKeyMatcher(consumer project.AutogenConsumerConfig
|
||||
return false
|
||||
}
|
||||
rest := key[len(prefix):]
|
||||
idx := strings.Index(rest, delimiter)
|
||||
idx := strings.Index(rest, "/")
|
||||
if idx <= 0 {
|
||||
return false
|
||||
}
|
||||
@@ -225,6 +234,12 @@ func autogenConsumerTargetsDataset(dataset nativeCollectedDataset, consumer proj
|
||||
}
|
||||
|
||||
func resolveAutogenConsumerManifest(p *project.Project, consumer project.AutogenConsumerConfig, progress func(string)) (*autogenManifest, error) {
|
||||
if strings.TrimSpace(consumer.Source.Kind) == "cdn_channel" {
|
||||
return resolveCDNChannelManifest(p, consumer, consumer.Source, progress)
|
||||
}
|
||||
if manifestFile := strings.TrimSpace(consumer.ManifestFile); manifestFile != "" {
|
||||
return readAutogenConsumerManifestFile(p, consumer, manifestFile, progress)
|
||||
}
|
||||
overrideRoot, err := resolveAutogenLocalOverrideRoot(p, consumer)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -242,6 +257,301 @@ func resolveAutogenConsumerManifest(p *project.Project, consumer project.Autogen
|
||||
return resolveReleasedAutogenManifest(p, consumer, progress)
|
||||
}
|
||||
|
||||
// readAutogenConsumerManifestFile reads a pre-resolved autogenManifest JSON from
|
||||
// a local file (relative paths resolve against the project root). A missing file
|
||||
// is an ignorable-optional condition — it returns errAutogenManifestUnavailable
|
||||
// so an optional consumer fails open and preserves its pinned lock IDs. A present
|
||||
// but malformed file, or an id that disagrees with the consumer's producer, is a
|
||||
// hard error.
|
||||
func readAutogenConsumerManifestFile(p *project.Project, consumer project.AutogenConsumerConfig, manifestFile string, progress func(string)) (*autogenManifest, error) {
|
||||
path := manifestFile
|
||||
if !filepath.IsAbs(path) {
|
||||
path = filepath.Join(p.Root, path)
|
||||
}
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil, unavailableAutogenManifest(fmt.Errorf("manifest file %s not found", path))
|
||||
}
|
||||
return nil, fmt.Errorf("read autogen manifest file %s: %w", path, err)
|
||||
}
|
||||
var manifest autogenManifest
|
||||
if err := json.Unmarshal(raw, &manifest); err != nil {
|
||||
return nil, fmt.Errorf("parse autogen manifest file %s: %w", path, err)
|
||||
}
|
||||
if manifest.ID != "" && manifest.ID != consumer.Producer {
|
||||
return nil, fmt.Errorf("autogen manifest file %s declared id %q, expected %q", path, manifest.ID, consumer.Producer)
|
||||
}
|
||||
if progress != nil {
|
||||
progress(fmt.Sprintf("Reading local autogen manifest for %s from %s...", consumer.ID, path))
|
||||
}
|
||||
return &manifest, nil
|
||||
}
|
||||
|
||||
// resolveCDNChannelManifest resolves the accessory-VFX model list anonymously
|
||||
// from the asset CDN: channel pointer -> tag -> per-tag vfxs.yml. It ports the
|
||||
// fail policy of the deleted resolve-accessory-vfx.sh: unreachable inputs fail
|
||||
// OPEN (errAutogenManifestUnavailable -> optional consumer preserves lock IDs);
|
||||
// a published-but-broken release (vfxs.yml 404 while haks.json present) or a
|
||||
// malformed vfxs.yml HARD fail.
|
||||
func resolveCDNChannelManifest(p *project.Project, consumer project.AutogenConsumerConfig, src project.AutogenSourceConfig, progress func(string)) (*autogenManifest, error) {
|
||||
if progress == nil {
|
||||
progress = func(string) {}
|
||||
}
|
||||
|
||||
// Manifest basename + provenance label are derived from config, not hardcoded,
|
||||
// so the same resolver serves vfxs.yml (accessory VFX) and part.yml (parts).
|
||||
manifestBase := filepath.Base(filepath.FromSlash(strings.TrimSpace(src.ManifestPath)))
|
||||
if manifestBase == "." || manifestBase == "" || manifestBase == string(filepath.Separator) {
|
||||
manifestBase = "manifest.yml"
|
||||
}
|
||||
label := strings.TrimSpace(consumer.ID)
|
||||
if label == "" {
|
||||
label = "autogen"
|
||||
}
|
||||
|
||||
// 1. Offline / air-gapped override: a manifest file path or a manifest-repo
|
||||
// checkout root containing assets/<manifestBase>. Skips the network entirely.
|
||||
if envName := strings.TrimSpace(src.OfflineOverrideEnv); envName != "" {
|
||||
if override := strings.TrimSpace(os.Getenv(envName)); override != "" {
|
||||
manifestPath := override
|
||||
if info, err := os.Stat(override); err == nil && info.IsDir() {
|
||||
manifestPath = filepath.Join(override, "assets", manifestBase)
|
||||
}
|
||||
raw, err := os.ReadFile(manifestPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("offline %s override %s: %w", manifestBase, manifestPath, err)
|
||||
}
|
||||
entries, err := filterCDNChannelEntries(raw, consumer)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
progress(fmt.Sprintf("Using offline %s override for %s from %s...", manifestBase, consumer.ID, manifestPath))
|
||||
return &autogenManifest{ID: consumer.Producer, Ref: "local", Entries: entries}, nil
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Channel: explicit env, else derived from the git tag.
|
||||
channel := strings.TrimSpace(os.Getenv(strings.TrimSpace(src.ChannelEnv)))
|
||||
if channel == "" {
|
||||
channel = deriveAssetChannel(p.Root)
|
||||
}
|
||||
|
||||
// 3. CDN base: baked default, then env override, then config literal (most specific wins).
|
||||
cdnBase := defaultBunnyCDNBase
|
||||
if envName := strings.TrimSpace(src.CDNBaseEnv); envName != "" {
|
||||
if v := strings.TrimSpace(os.Getenv(envName)); v != "" {
|
||||
cdnBase = v
|
||||
}
|
||||
}
|
||||
if v := strings.TrimSpace(src.CDNBase); v != "" {
|
||||
cdnBase = v
|
||||
}
|
||||
cdnBase = strings.TrimRight(cdnBase, "/")
|
||||
|
||||
join := func(path, tag string) string {
|
||||
return cdnBase + "/" + strings.TrimLeft(strings.ReplaceAll(path, "{tag}", tag), "/")
|
||||
}
|
||||
|
||||
// 4. channels.json -> tag. Unreachable / non-JSON / channel-absent = fail open.
|
||||
channelsURL := join(src.ChannelsPath, "")
|
||||
status, body, err := httpGetStatus(channelsURL)
|
||||
if err != nil {
|
||||
return nil, unavailableAutogenManifest(fmt.Errorf("channels.json unreachable %s: %w", channelsURL, err))
|
||||
}
|
||||
if status != http.StatusOK {
|
||||
return nil, unavailableAutogenManifest(fmt.Errorf("channels.json HTTP %d %s", status, channelsURL))
|
||||
}
|
||||
var channels map[string]string
|
||||
if err := json.Unmarshal(body, &channels); err != nil {
|
||||
return nil, unavailableAutogenManifest(fmt.Errorf("channels.json not JSON %s: %w", channelsURL, err))
|
||||
}
|
||||
tag := strings.TrimSpace(channels[channel])
|
||||
if tag == "" {
|
||||
return nil, unavailableAutogenManifest(fmt.Errorf("channel %q absent in channels.json (%s)", channel, channelsURL))
|
||||
}
|
||||
progress(fmt.Sprintf("%s: channel %s -> tag %s", label, channel, tag))
|
||||
|
||||
// 5. per-tag manifest for the tag.
|
||||
manifestURL := join(src.ManifestPath, tag)
|
||||
vstatus, vbody, err := httpGetStatus(manifestURL)
|
||||
if err != nil {
|
||||
return nil, unavailableAutogenManifest(fmt.Errorf("%s unreachable %s: %w", manifestBase, manifestURL, err))
|
||||
}
|
||||
switch vstatus {
|
||||
case http.StatusOK:
|
||||
// fall through to parse
|
||||
case http.StatusNotFound:
|
||||
// 404 + release marker present = published-but-broken release: HARD fail
|
||||
// (this is the v0.1.4 silent-drop bug). 404 + marker absent = no published
|
||||
// release for this tag: fail open.
|
||||
if marker := strings.TrimSpace(src.ReleaseMarkerPath); marker != "" {
|
||||
markerURL := join(marker, tag)
|
||||
if mstatus, _, merr := httpGetStatus(markerURL); merr == nil && mstatus == http.StatusOK {
|
||||
return nil, fmt.Errorf("release %s has %s but no %s (%s) — rows would silently vanish; backfill/republish %s for %s", tag, marker, manifestBase, manifestURL, manifestBase, tag)
|
||||
}
|
||||
}
|
||||
return nil, unavailableAutogenManifest(fmt.Errorf("%s 404 %s (no published release for %s)", manifestBase, manifestURL, tag))
|
||||
default:
|
||||
return nil, unavailableAutogenManifest(fmt.Errorf("%s HTTP %d %s", manifestBase, vstatus, manifestURL))
|
||||
}
|
||||
|
||||
entries, err := filterCDNChannelEntries(vbody, consumer)
|
||||
if err != nil {
|
||||
return nil, err // malformed manifest = HARD fail
|
||||
}
|
||||
progress(fmt.Sprintf("%s: resolved %d model entries from %s", label, len(entries), manifestURL))
|
||||
return &autogenManifest{ID: consumer.Producer, Ref: tag, Entries: entries}, nil
|
||||
}
|
||||
|
||||
// filterCDNChannelEntries dispatches per-tag manifest parsing on the consumer
|
||||
// mode. parts_rows consumes part.yml (the parts inventory contract); every other
|
||||
// mode consumes vfxs.yml (the accessory-VFX contract).
|
||||
func filterCDNChannelEntries(raw []byte, consumer project.AutogenConsumerConfig) ([]autogenManifestEntry, error) {
|
||||
switch strings.TrimSpace(consumer.Mode) {
|
||||
case "parts_rows":
|
||||
return filterPartsCDNChannelEntries(raw)
|
||||
default:
|
||||
return filterVFXCDNChannelEntries(raw, consumer)
|
||||
}
|
||||
}
|
||||
|
||||
// filterPartsCDNChannelEntries parses part.yml and keeps restype==mdl assets
|
||||
// under part/<supported-category>/.../<stem><digits>.mdl. The asset category is
|
||||
// the path segment after part/; the row ID is the trailing decimal of the
|
||||
// filename stem. Body/race/gender/left-right variants dedup by (category,rowID).
|
||||
// Zero accepted rows is a HARD fail (silent-drop guard).
|
||||
func filterPartsCDNChannelEntries(raw []byte) ([]autogenManifestEntry, error) {
|
||||
var manifest struct {
|
||||
Assets *[]struct {
|
||||
Path string `yaml:"path"`
|
||||
Restype string `yaml:"restype"`
|
||||
} `yaml:"assets"`
|
||||
}
|
||||
if err := yaml.Unmarshal(raw, &manifest); err != nil {
|
||||
return nil, fmt.Errorf("malformed part.yml (cannot parse): %w", err)
|
||||
}
|
||||
if manifest.Assets == nil {
|
||||
return nil, fmt.Errorf("malformed part.yml (no assets array)")
|
||||
}
|
||||
seen := map[string]struct{}{}
|
||||
var entries []autogenManifestEntry
|
||||
for _, a := range *manifest.Assets {
|
||||
if a.Restype != "mdl" {
|
||||
continue
|
||||
}
|
||||
path := filepath.ToSlash(strings.TrimSpace(a.Path))
|
||||
if !strings.HasPrefix(path, "part/") {
|
||||
continue
|
||||
}
|
||||
rel := strings.TrimPrefix(path, "part/")
|
||||
segs := strings.Split(rel, "/")
|
||||
if len(segs) < 2 {
|
||||
continue
|
||||
}
|
||||
category := segs[0]
|
||||
if !isSupportedPartCategory(category) {
|
||||
continue // ignores _masters, cloak, head, helm, tail, wings
|
||||
}
|
||||
stem := strings.TrimSuffix(segs[len(segs)-1], ".mdl")
|
||||
match := trailingNumberRegex.FindString(stem)
|
||||
if match == "" {
|
||||
return nil, fmt.Errorf("part.yml: supported-category model %q has no trailing row number", path)
|
||||
}
|
||||
rowID, err := strconv.Atoi(match)
|
||||
if err != nil || rowID == 0 {
|
||||
return nil, fmt.Errorf("part.yml: supported-category model %q resolves to invalid row id %q", path, match)
|
||||
}
|
||||
key := category + "/" + strconv.Itoa(rowID)
|
||||
if _, dup := seen[key]; dup {
|
||||
continue
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
entries = append(entries, autogenManifestEntry{
|
||||
Source: rel, Group: category, ModelStem: stem, RowID: rowID,
|
||||
})
|
||||
}
|
||||
if len(entries) == 0 {
|
||||
return nil, fmt.Errorf("part.yml: zero supported part rows after filtering")
|
||||
}
|
||||
slices.SortFunc(entries, func(a, b autogenManifestEntry) int {
|
||||
return strings.Compare(a.Source, b.Source)
|
||||
})
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// filterVFXCDNChannelEntries parses vfxs.yml and keeps restype==mdl assets under
|
||||
// vfxs/<group>/ for the consumer's 4 accessory groups, stripping the leading
|
||||
// vfxs/ from each source path. A present-but-malformed manifest (unparseable, or
|
||||
// no assets array) is an error; an empty assets array yields zero entries.
|
||||
func filterVFXCDNChannelEntries(raw []byte, consumer project.AutogenConsumerConfig) ([]autogenManifestEntry, error) {
|
||||
var manifest struct {
|
||||
Assets *[]struct {
|
||||
Path string `yaml:"path"`
|
||||
Restype string `yaml:"restype"`
|
||||
} `yaml:"assets"`
|
||||
}
|
||||
if err := yaml.Unmarshal(raw, &manifest); err != nil {
|
||||
return nil, fmt.Errorf("malformed vfxs.yml (cannot parse): %w", err)
|
||||
}
|
||||
if manifest.Assets == nil {
|
||||
return nil, fmt.Errorf("malformed vfxs.yml (no assets array)")
|
||||
}
|
||||
groups := make(map[string]struct{}, len(consumer.AccessoryVisualeffects.Groups))
|
||||
for g := range consumer.AccessoryVisualeffects.Groups {
|
||||
if g = strings.TrimSpace(g); g != "" {
|
||||
groups[g] = struct{}{}
|
||||
}
|
||||
}
|
||||
var entries []autogenManifestEntry
|
||||
for _, a := range *manifest.Assets {
|
||||
if a.Restype != "mdl" {
|
||||
continue
|
||||
}
|
||||
path := filepath.ToSlash(strings.TrimSpace(a.Path))
|
||||
if !strings.HasPrefix(path, "vfxs/") {
|
||||
continue
|
||||
}
|
||||
rel := strings.TrimPrefix(path, "vfxs/")
|
||||
parts := strings.Split(rel, "/")
|
||||
if len(parts) < 2 {
|
||||
continue
|
||||
}
|
||||
if _, ok := groups[parts[0]]; !ok {
|
||||
continue
|
||||
}
|
||||
entry := autogenManifestEntry{
|
||||
Source: rel,
|
||||
Group: parts[0],
|
||||
ModelStem: strings.TrimSuffix(parts[len(parts)-1], ".mdl"),
|
||||
}
|
||||
if len(parts) > 2 {
|
||||
entry.Subgroup = strings.Join(parts[1:len(parts)-1], "/")
|
||||
}
|
||||
entries = append(entries, entry)
|
||||
}
|
||||
slices.SortFunc(entries, func(a, b autogenManifestEntry) int {
|
||||
return strings.Compare(a.Source, b.Source)
|
||||
})
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// deriveAssetChannel maps the current git tag (or GITHUB_REF_NAME) to a channel:
|
||||
// a prerelease tag (v*-*) -> testing; a stable tag (v*) -> current; anything
|
||||
// else (branch/PR/dev) -> current. Mirrors resolve-accessory-vfx.sh.
|
||||
func deriveAssetChannel(root string) string {
|
||||
ref := strings.TrimSpace(os.Getenv("GITHUB_REF_NAME"))
|
||||
if ref == "" {
|
||||
if out, err := gitOutput(root, "describe", "--tags", "--exact-match"); err == nil {
|
||||
ref = strings.TrimSpace(out)
|
||||
}
|
||||
}
|
||||
if strings.HasPrefix(ref, "v") && strings.Contains(ref, "-") {
|
||||
return "testing"
|
||||
}
|
||||
return "current"
|
||||
}
|
||||
|
||||
func resolveAutogenLocalOverrideRoot(p *project.Project, consumer project.AutogenConsumerConfig) (string, error) {
|
||||
root := strings.TrimSpace(consumer.LocalOverrideRoot)
|
||||
if root != "" {
|
||||
@@ -782,12 +1092,12 @@ func augmentWithAutogeneratedCachedModels(collected []nativeCollectedDataset, en
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func augmentWithAutogeneratedHeadVisualeffects(collected []nativeCollectedDataset, entries []autogenManifestEntry, consumer project.AutogenConsumerConfig, manifestPolicy *project.HeadVisualeffectsConfig) ([]nativeCollectedDataset, error) {
|
||||
func augmentWithAutogeneratedAccessoryVisualeffects(collected []nativeCollectedDataset, entries []autogenManifestEntry, consumer project.AutogenConsumerConfig, manifestPolicy *project.AccessoryVisualeffectsConfig) ([]nativeCollectedDataset, error) {
|
||||
if len(entries) == 0 {
|
||||
return collected, nil
|
||||
}
|
||||
|
||||
policy := resolveHeadVisualeffectsPolicy(consumer, manifestPolicy)
|
||||
policy := resolveAccessoryVisualeffectsPolicy(consumer, manifestPolicy)
|
||||
result := append([]nativeCollectedDataset(nil), collected...)
|
||||
for i, dataset := range result {
|
||||
if dataset.Dataset.Name != "visualeffects" {
|
||||
@@ -836,30 +1146,61 @@ func augmentWithAutogeneratedHeadVisualeffects(collected []nativeCollectedDatase
|
||||
return rowID
|
||||
}
|
||||
|
||||
liveLockKeys := make(map[string]struct{}, len(entries))
|
||||
for _, entry := range entries {
|
||||
key, label, modelStem, groupPolicy, ok := headVisualeffectIdentity(dataset.Dataset.Name, entry, policy)
|
||||
key, label, modelStem, groupPolicy, ok := accessoryVisualeffectIdentity(dataset.Dataset.Name, entry, policy)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
lockKey := accessoryVisualeffectLockKey(dataset.Dataset.Name, entry)
|
||||
liveLockKeys[lockKey] = struct{}{}
|
||||
|
||||
if existing, exists := rowByKey[key]; exists {
|
||||
applyDiscoveredHeadVisualeffectDefaults(existing, dataset.Columns, modelStem, label, policy, groupPolicy)
|
||||
applyDiscoveredAccessoryVisualeffectDefaults(existing, dataset.Columns, modelStem, label, policy, groupPolicy)
|
||||
if _, pinned := lockData[lockKey]; !pinned {
|
||||
if rowID, ok := existing["id"].(int); ok {
|
||||
lockData[lockKey] = rowID
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
rowID, ok := lockData[key]
|
||||
rowID, ok := lockData[lockKey]
|
||||
if !ok {
|
||||
if preservedRowID, preserved := historicalLockData[key]; preserved {
|
||||
rowID = preservedRowID
|
||||
} else {
|
||||
switch {
|
||||
case hasKey(historicalLockData, lockKey):
|
||||
rowID = historicalLockData[lockKey]
|
||||
case hasKey(historicalLockData, key):
|
||||
// One-time cutover: adopt the ID from the old config-derived key
|
||||
// so IDs carry over with zero shift on the key-scheme switch. The
|
||||
// stale old key is pruned below.
|
||||
rowID = historicalLockData[key]
|
||||
default:
|
||||
rowID = allocateNextID()
|
||||
}
|
||||
lockData[key] = rowID
|
||||
lockData[lockKey] = rowID
|
||||
}
|
||||
newRow := createDefaultHeadVisualeffectRow(dataset.Columns, rowID, key, label, modelStem, policy, groupPolicy)
|
||||
newRow := createDefaultAccessoryVisualeffectRow(dataset.Columns, rowID, key, label, modelStem, policy, groupPolicy)
|
||||
rows = append(rows, newRow)
|
||||
rowByKey[key] = newRow
|
||||
}
|
||||
|
||||
// Stale pruning: drop managed lock keys (this consumer's accessory keys,
|
||||
// incl. now-superseded old config-scheme keys) that no live entry resolves
|
||||
// to, freeing their IDs for first-free reuse. Safe here because the
|
||||
// augmentor only runs on a successful, non-empty resolution; the fail-open
|
||||
// paths skip it entirely and never reach this code.
|
||||
if managed := autogenConsumerManagedLockKeyMatcher(consumer); managed != nil {
|
||||
for lk := range lockData {
|
||||
if _, live := liveLockKeys[lk]; live {
|
||||
continue
|
||||
}
|
||||
if managed(lk) {
|
||||
delete(lockData, lk)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
slices.SortFunc(rows, func(a, b map[string]any) int {
|
||||
return a["id"].(int) - b["id"].(int)
|
||||
})
|
||||
@@ -878,8 +1219,22 @@ func nextAvailableAutogenID(used map[int]struct{}) int {
|
||||
}
|
||||
}
|
||||
|
||||
type headVisualeffectsPolicy struct {
|
||||
Groups map[string]headVisualeffectsGroupPolicy
|
||||
// accessoryVisualeffectLockKey is the model-anchored autogen lock identity:
|
||||
// the dataset namespace prefix plus the model source path from vfxs.yml (leading
|
||||
// vfxs/ already stripped on entry.Source). Config-derived presentation parts
|
||||
// (group token, category, delimiter, case, prefix-stripping) are deliberately
|
||||
// excluded so a config-only edit never reshuffles row IDs.
|
||||
func accessoryVisualeffectLockKey(dataset string, entry autogenManifestEntry) string {
|
||||
return dataset + ":" + strings.TrimPrefix(filepath.ToSlash(strings.TrimSpace(entry.Source)), "vfxs/")
|
||||
}
|
||||
|
||||
func hasKey(m map[string]int, key string) bool {
|
||||
_, ok := m[key]
|
||||
return ok
|
||||
}
|
||||
|
||||
type accessoryVisualeffectsPolicy struct {
|
||||
Groups map[string]accessoryVisualeffectsGroupPolicy
|
||||
GroupTokenSource string
|
||||
CategoryFrom string
|
||||
Delimiter string
|
||||
@@ -891,22 +1246,22 @@ type headVisualeffectsPolicy struct {
|
||||
RowDefaults map[string]string
|
||||
}
|
||||
|
||||
type headVisualeffectsGroupPolicy struct {
|
||||
type accessoryVisualeffectsGroupPolicy struct {
|
||||
Prefix string
|
||||
ModelColumns []string
|
||||
RowDefaults map[string]string
|
||||
}
|
||||
|
||||
func resolveHeadVisualeffectsPolicy(consumer project.AutogenConsumerConfig, manifestPolicy *project.HeadVisualeffectsConfig) headVisualeffectsPolicy {
|
||||
policy := defaultHeadVisualeffectsPolicy()
|
||||
func resolveAccessoryVisualeffectsPolicy(consumer project.AutogenConsumerConfig, manifestPolicy *project.AccessoryVisualeffectsConfig) accessoryVisualeffectsPolicy {
|
||||
policy := defaultAccessoryVisualeffectsPolicy()
|
||||
if manifestPolicy != nil {
|
||||
applyHeadVisualeffectsConfig(&policy, *manifestPolicy)
|
||||
applyAccessoryVisualeffectsConfig(&policy, *manifestPolicy)
|
||||
}
|
||||
applyHeadVisualeffectsConfig(&policy, consumer.HeadVisualeffects)
|
||||
applyAccessoryVisualeffectsConfig(&policy, consumer.AccessoryVisualeffects)
|
||||
return policy
|
||||
}
|
||||
|
||||
func applyHeadVisualeffectsConfig(policy *headVisualeffectsPolicy, cfg project.HeadVisualeffectsConfig) {
|
||||
func applyAccessoryVisualeffectsConfig(policy *accessoryVisualeffectsPolicy, cfg project.AccessoryVisualeffectsConfig) {
|
||||
for group, groupCfg := range cfg.Groups {
|
||||
group = strings.TrimSpace(group)
|
||||
prefix := strings.TrimSpace(groupCfg.Prefix)
|
||||
@@ -917,13 +1272,13 @@ func applyHeadVisualeffectsConfig(policy *headVisualeffectsPolicy, cfg project.H
|
||||
if prefix != "" {
|
||||
groupPolicy.Prefix = prefix
|
||||
}
|
||||
if columns := normalizeHeadVisualeffectsModelColumns(groupCfg.ModelColumn, groupCfg.ModelColumns); len(columns) > 0 {
|
||||
if columns := normalizeAccessoryVisualeffectsModelColumns(groupCfg.ModelColumn, groupCfg.ModelColumns); len(columns) > 0 {
|
||||
groupPolicy.ModelColumns = columns
|
||||
}
|
||||
if groupPolicy.RowDefaults == nil {
|
||||
groupPolicy.RowDefaults = map[string]string{}
|
||||
}
|
||||
mergeStringMap(groupPolicy.RowDefaults, normalizeHeadVisualeffectsRowDefaults(groupCfg.RowDefaults))
|
||||
mergeStringMap(groupPolicy.RowDefaults, normalizeAccessoryVisualeffectsRowDefaults(groupCfg.RowDefaults))
|
||||
policy.Groups[group] = groupPolicy
|
||||
}
|
||||
if strings.TrimSpace(cfg.GroupTokenSource) != "" {
|
||||
@@ -950,12 +1305,12 @@ func applyHeadVisualeffectsConfig(policy *headVisualeffectsPolicy, cfg project.H
|
||||
if strings.TrimSpace(cfg.ModelColumn) != "" {
|
||||
policy.ModelColumn = strings.TrimSpace(cfg.ModelColumn)
|
||||
}
|
||||
mergeStringMap(policy.RowDefaults, normalizeHeadVisualeffectsRowDefaults(cfg.RowDefaults))
|
||||
mergeStringMap(policy.RowDefaults, normalizeAccessoryVisualeffectsRowDefaults(cfg.RowDefaults))
|
||||
}
|
||||
|
||||
func defaultHeadVisualeffectsPolicy() headVisualeffectsPolicy {
|
||||
return headVisualeffectsPolicy{
|
||||
Groups: map[string]headVisualeffectsGroupPolicy{
|
||||
func defaultAccessoryVisualeffectsPolicy() accessoryVisualeffectsPolicy {
|
||||
return accessoryVisualeffectsPolicy{
|
||||
Groups: map[string]accessoryVisualeffectsGroupPolicy{
|
||||
"chest_accessories": {},
|
||||
"head_accessories": {},
|
||||
"head_decorations": {},
|
||||
@@ -977,7 +1332,7 @@ func defaultHeadVisualeffectsPolicy() headVisualeffectsPolicy {
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeHeadVisualeffectsModelColumns(modelColumn string, modelColumns []string) []string {
|
||||
func normalizeAccessoryVisualeffectsModelColumns(modelColumn string, modelColumns []string) []string {
|
||||
var result []string
|
||||
if column := strings.TrimSpace(modelColumn); column != "" {
|
||||
result = append(result, column)
|
||||
@@ -994,7 +1349,7 @@ func normalizeHeadVisualeffectsModelColumns(modelColumn string, modelColumns []s
|
||||
return result
|
||||
}
|
||||
|
||||
func normalizeHeadVisualeffectsRowDefaults(defaults map[string]string) map[string]string {
|
||||
func normalizeAccessoryVisualeffectsRowDefaults(defaults map[string]string) map[string]string {
|
||||
result := map[string]string{}
|
||||
for column, value := range defaults {
|
||||
column = strings.TrimSpace(column)
|
||||
@@ -1012,44 +1367,44 @@ func mergeStringMap(target map[string]string, source map[string]string) {
|
||||
}
|
||||
}
|
||||
|
||||
func headVisualeffectIdentity(dataset string, entry autogenManifestEntry, policy headVisualeffectsPolicy) (string, string, string, headVisualeffectsGroupPolicy, bool) {
|
||||
entry = normalizeHeadVisualeffectEntry(entry)
|
||||
func accessoryVisualeffectIdentity(dataset string, entry autogenManifestEntry, policy accessoryVisualeffectsPolicy) (string, string, string, accessoryVisualeffectsGroupPolicy, bool) {
|
||||
entry = normalizeAccessoryVisualeffectEntry(entry)
|
||||
group, ok := policy.Groups[entry.Group]
|
||||
if !ok || strings.TrimSpace(entry.ModelStem) == "" {
|
||||
return "", "", "", headVisualeffectsGroupPolicy{}, false
|
||||
return "", "", "", accessoryVisualeffectsGroupPolicy{}, false
|
||||
}
|
||||
modelStem := strings.TrimSpace(entry.ModelStem)
|
||||
stem := stripHeadVisualeffectModelPrefix(modelStem, policy)
|
||||
stem := stripAccessoryVisualeffectModelPrefix(modelStem, policy)
|
||||
stem = strings.TrimSpace(stem)
|
||||
if stem == "" {
|
||||
return "", "", "", headVisualeffectsGroupPolicy{}, false
|
||||
return "", "", "", accessoryVisualeffectsGroupPolicy{}, false
|
||||
}
|
||||
category := headVisualeffectCategory(entry, policy)
|
||||
groupToken := headVisualeffectGroupToken(entry.Group, group, policy)
|
||||
category := accessoryVisualeffectCategory(entry, policy)
|
||||
groupToken := accessoryVisualeffectGroupToken(entry.Group, group, policy)
|
||||
values := map[string]string{
|
||||
"dataset": dataset,
|
||||
"group": applyHeadVisualeffectCase(groupToken, policy),
|
||||
"group": applyAccessoryVisualeffectCase(groupToken, policy),
|
||||
"group_raw": entry.Group,
|
||||
"prefix": applyHeadVisualeffectCase(group.Prefix, policy),
|
||||
"category": applyHeadVisualeffectCase(category, policy),
|
||||
"prefix": applyAccessoryVisualeffectCase(group.Prefix, policy),
|
||||
"category": applyAccessoryVisualeffectCase(category, policy),
|
||||
"category_upper": strings.ToUpper(category),
|
||||
"category_segment": headVisualeffectDelimitedSegment(applyHeadVisualeffectCase(category, policy), policy.Delimiter),
|
||||
"category_segment_upper": headVisualeffectDelimitedSegment(strings.ToUpper(category), policy.Delimiter),
|
||||
"subgroup": applyHeadVisualeffectCase(entry.Subgroup, policy),
|
||||
"stem": applyHeadVisualeffectCase(stem, policy),
|
||||
"category_segment": accessoryVisualeffectDelimitedSegment(applyAccessoryVisualeffectCase(category, policy), policy.Delimiter),
|
||||
"category_segment_upper": accessoryVisualeffectDelimitedSegment(strings.ToUpper(category), policy.Delimiter),
|
||||
"subgroup": applyAccessoryVisualeffectCase(entry.Subgroup, policy),
|
||||
"stem": applyAccessoryVisualeffectCase(stem, policy),
|
||||
"stem_upper": strings.ToUpper(stem),
|
||||
"model_stem": modelStem,
|
||||
"delimiter": policy.Delimiter,
|
||||
}
|
||||
key := expandHeadVisualeffectsFormat(policy.KeyFormat, values)
|
||||
label := expandHeadVisualeffectsFormat(policy.LabelFormat, values)
|
||||
key := expandAccessoryVisualeffectsFormat(policy.KeyFormat, values)
|
||||
label := expandAccessoryVisualeffectsFormat(policy.LabelFormat, values)
|
||||
if strings.TrimSpace(key) == "" || strings.TrimSpace(label) == "" {
|
||||
return "", "", "", headVisualeffectsGroupPolicy{}, false
|
||||
return "", "", "", accessoryVisualeffectsGroupPolicy{}, false
|
||||
}
|
||||
return key, label, modelStem, group, true
|
||||
}
|
||||
|
||||
func normalizeHeadVisualeffectEntry(entry autogenManifestEntry) autogenManifestEntry {
|
||||
func normalizeAccessoryVisualeffectEntry(entry autogenManifestEntry) autogenManifestEntry {
|
||||
entry.Group = strings.TrimSpace(entry.Group)
|
||||
if strings.TrimSpace(entry.Group) == "" && strings.TrimSpace(entry.Source) != "" {
|
||||
parts := strings.Split(filepath.ToSlash(entry.Source), "/")
|
||||
@@ -1072,7 +1427,7 @@ func normalizeHeadVisualeffectEntry(entry autogenManifestEntry) autogenManifestE
|
||||
return entry
|
||||
}
|
||||
|
||||
func stripHeadVisualeffectModelPrefix(modelStem string, policy headVisualeffectsPolicy) string {
|
||||
func stripAccessoryVisualeffectModelPrefix(modelStem string, policy accessoryVisualeffectsPolicy) string {
|
||||
stem := modelStem
|
||||
for _, prefix := range policy.StripModelPrefixes {
|
||||
prefix = strings.TrimSpace(prefix)
|
||||
@@ -1084,7 +1439,7 @@ func stripHeadVisualeffectModelPrefix(modelStem string, policy headVisualeffects
|
||||
return stem
|
||||
}
|
||||
|
||||
func headVisualeffectGroupToken(groupName string, group headVisualeffectsGroupPolicy, policy headVisualeffectsPolicy) string {
|
||||
func accessoryVisualeffectGroupToken(groupName string, group accessoryVisualeffectsGroupPolicy, policy accessoryVisualeffectsPolicy) string {
|
||||
if policy.GroupTokenSource == "folder_name" {
|
||||
return groupName
|
||||
}
|
||||
@@ -1094,7 +1449,7 @@ func headVisualeffectGroupToken(groupName string, group headVisualeffectsGroupPo
|
||||
return groupName
|
||||
}
|
||||
|
||||
func headVisualeffectCategory(entry autogenManifestEntry, policy headVisualeffectsPolicy) string {
|
||||
func accessoryVisualeffectCategory(entry autogenManifestEntry, policy accessoryVisualeffectsPolicy) string {
|
||||
switch policy.CategoryFrom {
|
||||
case "immediate_parent":
|
||||
return strings.TrimSpace(entry.Category)
|
||||
@@ -1105,14 +1460,14 @@ func headVisualeffectCategory(entry autogenManifestEntry, policy headVisualeffec
|
||||
}
|
||||
}
|
||||
|
||||
func headVisualeffectDelimitedSegment(value, delimiter string) string {
|
||||
func accessoryVisualeffectDelimitedSegment(value, delimiter string) string {
|
||||
if strings.TrimSpace(value) == "" {
|
||||
return ""
|
||||
}
|
||||
return value + delimiter
|
||||
}
|
||||
|
||||
func applyHeadVisualeffectCase(value string, policy headVisualeffectsPolicy) string {
|
||||
func applyAccessoryVisualeffectCase(value string, policy accessoryVisualeffectsPolicy) string {
|
||||
switch policy.Case {
|
||||
case "lower":
|
||||
return strings.ToLower(value)
|
||||
@@ -1123,7 +1478,7 @@ func applyHeadVisualeffectCase(value string, policy headVisualeffectsPolicy) str
|
||||
}
|
||||
}
|
||||
|
||||
func expandHeadVisualeffectsFormat(format string, values map[string]string) string {
|
||||
func expandAccessoryVisualeffectsFormat(format string, values map[string]string) string {
|
||||
result := format
|
||||
for key, value := range values {
|
||||
result = strings.ReplaceAll(result, "{"+key+"}", value)
|
||||
@@ -1131,7 +1486,7 @@ func expandHeadVisualeffectsFormat(format string, values map[string]string) stri
|
||||
return result
|
||||
}
|
||||
|
||||
func createDefaultHeadVisualeffectRow(columns []string, rowID int, key, label, modelStem string, policy headVisualeffectsPolicy, group headVisualeffectsGroupPolicy) map[string]any {
|
||||
func createDefaultAccessoryVisualeffectRow(columns []string, rowID int, key, label, modelStem string, policy accessoryVisualeffectsPolicy, group accessoryVisualeffectsGroupPolicy) map[string]any {
|
||||
row := map[string]any{
|
||||
"id": rowID,
|
||||
"key": key,
|
||||
@@ -1146,13 +1501,13 @@ func createDefaultHeadVisualeffectRow(columns []string, rowID int, key, label, m
|
||||
for column, value := range group.RowDefaults {
|
||||
row[column] = value
|
||||
}
|
||||
for _, column := range headVisualeffectModelColumns(policy, group) {
|
||||
for _, column := range accessoryVisualeffectModelColumns(policy, group) {
|
||||
row[column] = modelStem
|
||||
}
|
||||
return row
|
||||
}
|
||||
|
||||
func applyDiscoveredHeadVisualeffectDefaults(row map[string]any, columns []string, modelStem, label string, policy headVisualeffectsPolicy, group headVisualeffectsGroupPolicy) {
|
||||
func applyDiscoveredAccessoryVisualeffectDefaults(row map[string]any, columns []string, modelStem, label string, policy accessoryVisualeffectsPolicy, group accessoryVisualeffectsGroupPolicy) {
|
||||
if isNullLikeValue(row["Label"]) {
|
||||
row["Label"] = label
|
||||
}
|
||||
@@ -1166,7 +1521,7 @@ func applyDiscoveredHeadVisualeffectDefaults(row map[string]any, columns []strin
|
||||
row[column] = value
|
||||
}
|
||||
}
|
||||
for _, column := range headVisualeffectModelColumns(policy, group) {
|
||||
for _, column := range accessoryVisualeffectModelColumns(policy, group) {
|
||||
if isNullLikeValue(row[column]) {
|
||||
row[column] = modelStem
|
||||
}
|
||||
@@ -1178,7 +1533,7 @@ func applyDiscoveredHeadVisualeffectDefaults(row map[string]any, columns []strin
|
||||
}
|
||||
}
|
||||
|
||||
func headVisualeffectModelColumns(policy headVisualeffectsPolicy, group headVisualeffectsGroupPolicy) []string {
|
||||
func accessoryVisualeffectModelColumns(policy accessoryVisualeffectsPolicy, group accessoryVisualeffectsGroupPolicy) []string {
|
||||
if len(group.ModelColumns) > 0 {
|
||||
return group.ModelColumns
|
||||
}
|
||||
@@ -1196,14 +1551,14 @@ func formatAutogenManifest(root string, producer project.AutogenProducerConfig,
|
||||
GeneratedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
Entries: entries,
|
||||
}
|
||||
if autogenHeadVisualeffectsConfigConfigured(producer.HeadVisualeffects) {
|
||||
cfg := producer.HeadVisualeffects
|
||||
manifest.HeadVisualeffects = &cfg
|
||||
if autogenAccessoryVisualeffectsConfigConfigured(producer.AccessoryVisualeffects) {
|
||||
cfg := producer.AccessoryVisualeffects
|
||||
manifest.AccessoryVisualeffects = &cfg
|
||||
}
|
||||
return json.MarshalIndent(manifest, "", " ")
|
||||
}
|
||||
|
||||
func autogenHeadVisualeffectsConfigConfigured(cfg project.HeadVisualeffectsConfig) bool {
|
||||
func autogenAccessoryVisualeffectsConfigConfigured(cfg project.AccessoryVisualeffectsConfig) bool {
|
||||
return len(cfg.Groups) > 0 ||
|
||||
strings.TrimSpace(cfg.GroupTokenSource) != "" ||
|
||||
strings.TrimSpace(cfg.CategoryFrom) != "" ||
|
||||
|
||||
@@ -0,0 +1,229 @@
|
||||
package topdata
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
|
||||
)
|
||||
|
||||
func cdnConsumer() project.AutogenConsumerConfig {
|
||||
return project.AutogenConsumerConfig{
|
||||
ID: "accessory_visualeffects", Producer: "accessory_visualeffects",
|
||||
Dataset: "visualeffects", Mode: "accessory_visualeffects", Optional: true,
|
||||
AccessoryVisualeffects: project.AccessoryVisualeffectsConfig{
|
||||
Groups: map[string]project.AccessoryVisualeffectGroupConfig{
|
||||
"head_accessories": {}, "chest_accessories": {},
|
||||
"head_decorations": {}, "head_features": {},
|
||||
},
|
||||
},
|
||||
Source: project.AutogenSourceConfig{
|
||||
Kind: "cdn_channel",
|
||||
ChannelsPath: "releases/haks/channels.json",
|
||||
ManifestPath: "releases/haks/{tag}/vfxs.yml",
|
||||
ReleaseMarkerPath: "releases/haks/{tag}/haks.json",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// cdnServer serves channels.json, vfxs.yml, and haks.json from in-memory maps.
|
||||
// A key absent from the map returns 404.
|
||||
func cdnServer(t *testing.T, files map[string]string) *httptest.Server {
|
||||
t.Helper()
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
body, ok := files[r.URL.Path]
|
||||
if !ok {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
fmt.Fprint(w, body)
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
return srv
|
||||
}
|
||||
|
||||
func TestResolveCDNChannelHappyPath(t *testing.T) {
|
||||
root := testProjectRoot(t)
|
||||
srv := cdnServer(t, map[string]string{
|
||||
"/releases/haks/channels.json": `{"current":"v1.2.3"}`,
|
||||
"/releases/haks/v1.2.3/vfxs.yml": `assets:
|
||||
- path: vfxs/head_accessories/hat/hfx_bandana.mdl
|
||||
restype: mdl
|
||||
sha256: aaa
|
||||
- path: vfxs/chest_accessories/cape/cfx_cloak.mdl
|
||||
restype: mdl
|
||||
sha256: bbb
|
||||
- path: vfxs/head_accessories/hat/hfx_bandana.tga
|
||||
restype: tga
|
||||
sha256: ccc
|
||||
- path: vfxs/weapons/sword.mdl
|
||||
restype: mdl
|
||||
sha256: ddd
|
||||
`,
|
||||
})
|
||||
t.Setenv("BUNNY_CDN_BASE", srv.URL)
|
||||
t.Setenv("SOW_TOPDATA_ASSET_CHANNEL", "current")
|
||||
|
||||
p := testProject(root)
|
||||
c := cdnConsumer()
|
||||
c.Source.CDNBaseEnv = "BUNNY_CDN_BASE"
|
||||
c.Source.ChannelEnv = "SOW_TOPDATA_ASSET_CHANNEL"
|
||||
|
||||
m, err := resolveCDNChannelManifest(p, c, c.Source, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("resolve failed: %v", err)
|
||||
}
|
||||
if len(m.Entries) != 2 {
|
||||
t.Fatalf("want 2 mdl entries under target groups, got %d: %#v", len(m.Entries), m.Entries)
|
||||
}
|
||||
// Sorted by source: chest_accessories/... before head_accessories/...
|
||||
if m.Entries[0].Source != "chest_accessories/cape/cfx_cloak.mdl" {
|
||||
t.Fatalf("unexpected first entry: %#v", m.Entries[0])
|
||||
}
|
||||
if m.Entries[1].Source != "head_accessories/hat/hfx_bandana.mdl" ||
|
||||
m.Entries[1].Group != "head_accessories" ||
|
||||
m.Entries[1].Subgroup != "hat" ||
|
||||
m.Entries[1].ModelStem != "hfx_bandana" {
|
||||
t.Fatalf("unexpected derived entry: %#v", m.Entries[1])
|
||||
}
|
||||
if m.Ref != "v1.2.3" {
|
||||
t.Fatalf("want ref v1.2.3, got %q", m.Ref)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveCDNChannelFailOpen(t *testing.T) {
|
||||
root := testProjectRoot(t)
|
||||
c := cdnConsumer()
|
||||
c.Source.CDNBaseEnv = "BUNNY_CDN_BASE"
|
||||
c.Source.ChannelEnv = "SOW_TOPDATA_ASSET_CHANNEL"
|
||||
t.Setenv("SOW_TOPDATA_ASSET_CHANNEL", "current")
|
||||
|
||||
t.Run("channels.json unreachable", func(t *testing.T) {
|
||||
t.Setenv("BUNNY_CDN_BASE", "http://127.0.0.1:0") // unroutable
|
||||
_, err := resolveCDNChannelManifest(testProject(root), c, c.Source, nil)
|
||||
if err == nil || !errorIsUnavailable(err) {
|
||||
t.Fatalf("want fail-open (unavailable), got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("channel absent", func(t *testing.T) {
|
||||
srv := cdnServer(t, map[string]string{"/releases/haks/channels.json": `{"testing":"v9"}`})
|
||||
t.Setenv("BUNNY_CDN_BASE", srv.URL)
|
||||
_, err := resolveCDNChannelManifest(testProject(root), c, c.Source, nil)
|
||||
if err == nil || !errorIsUnavailable(err) {
|
||||
t.Fatalf("want fail-open (unavailable), got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("vfxs 404 no release marker", func(t *testing.T) {
|
||||
srv := cdnServer(t, map[string]string{"/releases/haks/channels.json": `{"current":"v1"}`})
|
||||
t.Setenv("BUNNY_CDN_BASE", srv.URL)
|
||||
_, err := resolveCDNChannelManifest(testProject(root), c, c.Source, nil)
|
||||
if err == nil || !errorIsUnavailable(err) {
|
||||
t.Fatalf("want fail-open (unavailable), got %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestResolveCDNChannelHardFail(t *testing.T) {
|
||||
root := testProjectRoot(t)
|
||||
c := cdnConsumer()
|
||||
c.Source.CDNBaseEnv = "BUNNY_CDN_BASE"
|
||||
c.Source.ChannelEnv = "SOW_TOPDATA_ASSET_CHANNEL"
|
||||
t.Setenv("SOW_TOPDATA_ASSET_CHANNEL", "current")
|
||||
|
||||
t.Run("broken release: vfxs 404 + haks.json present", func(t *testing.T) {
|
||||
srv := cdnServer(t, map[string]string{
|
||||
"/releases/haks/channels.json": `{"current":"v1"}`,
|
||||
"/releases/haks/v1/haks.json": `{}`,
|
||||
})
|
||||
t.Setenv("BUNNY_CDN_BASE", srv.URL)
|
||||
_, err := resolveCDNChannelManifest(testProject(root), c, c.Source, nil)
|
||||
if err == nil || errorIsUnavailable(err) {
|
||||
t.Fatalf("want HARD fail, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("malformed vfxs.yml: no assets array", func(t *testing.T) {
|
||||
srv := cdnServer(t, map[string]string{
|
||||
"/releases/haks/channels.json": `{"current":"v1"}`,
|
||||
"/releases/haks/v1/vfxs.yml": "generated_at: 2026-01-01\n",
|
||||
})
|
||||
t.Setenv("BUNNY_CDN_BASE", srv.URL)
|
||||
_, err := resolveCDNChannelManifest(testProject(root), c, c.Source, nil)
|
||||
if err == nil || errorIsUnavailable(err) {
|
||||
t.Fatalf("want HARD fail, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("malformed vfxs.yml: syntactically broken YAML", func(t *testing.T) {
|
||||
srv := cdnServer(t, map[string]string{
|
||||
"/releases/haks/channels.json": `{"current":"v1"}`,
|
||||
"/releases/haks/v1/vfxs.yml": "assets:\n - path: [unterminated",
|
||||
})
|
||||
t.Setenv("BUNNY_CDN_BASE", srv.URL)
|
||||
_, err := resolveCDNChannelManifest(testProject(root), c, c.Source, nil)
|
||||
if err == nil || errorIsUnavailable(err) {
|
||||
t.Fatalf("want HARD fail on broken YAML, got %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestResolveCDNChannelOfflineOverride(t *testing.T) {
|
||||
root := testProjectRoot(t)
|
||||
vfxs := filepath.Join(root, "vfxs.yml")
|
||||
writeFile(t, vfxs, `assets:
|
||||
- path: vfxs/head_features/scar/hfx_scar.mdl
|
||||
restype: mdl
|
||||
sha256: zzz
|
||||
`)
|
||||
t.Setenv("SOW_VFXS_MANIFEST", vfxs)
|
||||
c := cdnConsumer()
|
||||
c.Source.OfflineOverrideEnv = "SOW_VFXS_MANIFEST"
|
||||
|
||||
m, err := resolveCDNChannelManifest(testProject(root), c, c.Source, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("offline override failed: %v", err)
|
||||
}
|
||||
if len(m.Entries) != 1 || m.Entries[0].Source != "head_features/scar/hfx_scar.mdl" {
|
||||
t.Fatalf("unexpected offline entries: %#v", m.Entries)
|
||||
}
|
||||
}
|
||||
|
||||
// TestResolveCDNChannelOfflineUsesManifestBasename proves the resolver derives
|
||||
// the offline checkout filename from manifest_path's basename (part.yml here),
|
||||
// not a hardcoded assets/vfxs.yml, and routes parts_rows through the parts filter.
|
||||
func TestResolveCDNChannelOfflineUsesManifestBasename(t *testing.T) {
|
||||
root := testProjectRoot(t)
|
||||
dir := filepath.Join(root, "manifest-checkout")
|
||||
if err := os.MkdirAll(filepath.Join(dir, "assets"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
writeFile(t, filepath.Join(dir, "assets", "part.yml"),
|
||||
"assets:\n - {path: part/belt/m/pfa0_belt017.mdl, restype: mdl}\n")
|
||||
t.Setenv("SOW_PART_MANIFEST", dir)
|
||||
|
||||
c := projectPartsConsumer()
|
||||
c.Source = project.AutogenSourceConfig{
|
||||
Kind: "cdn_channel",
|
||||
ManifestPath: "releases/haks/{tag}/part.yml",
|
||||
OfflineOverrideEnv: "SOW_PART_MANIFEST",
|
||||
}
|
||||
|
||||
m, err := resolveCDNChannelManifest(testProject(root), c, c.Source, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("resolve: %v", err)
|
||||
}
|
||||
if len(m.Entries) != 1 || m.Entries[0].RowID != 17 || m.Entries[0].Group != "belt" {
|
||||
t.Fatalf("unexpected entries: %+v", m.Entries)
|
||||
}
|
||||
}
|
||||
|
||||
func errorIsUnavailable(err error) bool {
|
||||
return errors.Is(err, errAutogenManifestUnavailable)
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
package topdata
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
|
||||
)
|
||||
|
||||
func accConsumer() project.AutogenConsumerConfig {
|
||||
return project.AutogenConsumerConfig{
|
||||
ID: "accessory_visualeffects", Producer: "accessory_visualeffects",
|
||||
Dataset: "visualeffects", Mode: "accessory_visualeffects", Optional: true,
|
||||
AccessoryVisualeffects: project.AccessoryVisualeffectsConfig{
|
||||
Groups: map[string]project.AccessoryVisualeffectGroupConfig{
|
||||
"head_accessories": {ModelColumns: []string{"Imp_HeadCon_Node"}},
|
||||
},
|
||||
GroupTokenSource: "folder_name", CategoryFrom: "immediate_parent",
|
||||
Delimiter: "/", Case: "preserve",
|
||||
StripModelPrefixes: []string{"hfx_"},
|
||||
KeyFormat: "{dataset}:{group}{delimiter}{category_segment}{stem}",
|
||||
LabelFormat: "{group}{delimiter}{category_segment}{stem}",
|
||||
ModelColumn: "Imp_HeadCon_Node",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func vfxEntry() autogenManifestEntry {
|
||||
return autogenManifestEntry{
|
||||
Source: "head_accessories/hat/hfx_bandana.mdl", Group: "head_accessories",
|
||||
Subgroup: "hat", ModelStem: "hfx_bandana",
|
||||
}
|
||||
}
|
||||
|
||||
func vfxDataset(lock map[string]int) nativeCollectedDataset {
|
||||
return nativeCollectedDataset{
|
||||
Dataset: nativeDataset{Name: "visualeffects", Kind: nativeDatasetBase},
|
||||
Columns: []string{"Label", "Imp_HeadCon_Node"},
|
||||
Rows: nil,
|
||||
LockData: lock,
|
||||
}
|
||||
}
|
||||
|
||||
func TestAccessoryLockKeyIsModelAnchored(t *testing.T) {
|
||||
got := accessoryVisualeffectLockKey("visualeffects", vfxEntry())
|
||||
want := "visualeffects:head_accessories/hat/hfx_bandana.mdl"
|
||||
if got != want {
|
||||
t.Fatalf("lock key = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// A config-only change (here: a different key_format) must NOT move the ID,
|
||||
// because identity is the model path, not the presentation key.
|
||||
func TestConfigOnlyChangeKeepsLockID(t *testing.T) {
|
||||
lockKey := "visualeffects:head_accessories/hat/hfx_bandana.mdl"
|
||||
collected := []nativeCollectedDataset{vfxDataset(map[string]int{lockKey: 555})}
|
||||
|
||||
c := accConsumer()
|
||||
c.AccessoryVisualeffects.KeyFormat = "{dataset}:{stem}" // config-only change
|
||||
got, err := augmentWithAutogeneratedAccessoryVisualeffects(collected, []autogenManifestEntry{vfxEntry()}, c, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("augment failed: %v", err)
|
||||
}
|
||||
if got[0].LockData[lockKey] != 555 {
|
||||
t.Fatalf("expected ID preserved at 555, got %#v", got[0].LockData)
|
||||
}
|
||||
}
|
||||
|
||||
// A model re-export (same path) keeps its ID — content change != identity change.
|
||||
func TestModelReExportKeepsLockID(t *testing.T) {
|
||||
lockKey := "visualeffects:head_accessories/hat/hfx_bandana.mdl"
|
||||
collected := []nativeCollectedDataset{vfxDataset(map[string]int{lockKey: 777})}
|
||||
got, err := augmentWithAutogeneratedAccessoryVisualeffects(collected, []autogenManifestEntry{vfxEntry()}, accConsumer(), nil)
|
||||
if err != nil {
|
||||
t.Fatalf("augment failed: %v", err)
|
||||
}
|
||||
if got[0].LockData[lockKey] != 777 {
|
||||
t.Fatalf("expected ID 777 preserved, got %#v", got[0].LockData)
|
||||
}
|
||||
}
|
||||
|
||||
// A removed model frees its old ID; the surviving model keeps its lock entry.
|
||||
func TestModelRemovalPrunesStaleLockKey(t *testing.T) {
|
||||
keep := "visualeffects:head_accessories/hat/hfx_bandana.mdl"
|
||||
gone := "visualeffects:head_accessories/hat/hfx_removed.mdl"
|
||||
collected := []nativeCollectedDataset{vfxDataset(map[string]int{keep: 1, gone: 2})}
|
||||
got, err := augmentWithAutogeneratedAccessoryVisualeffects(collected, []autogenManifestEntry{vfxEntry()}, accConsumer(), nil)
|
||||
if err != nil {
|
||||
t.Fatalf("augment failed: %v", err)
|
||||
}
|
||||
if _, present := got[0].LockData[gone]; present {
|
||||
t.Fatalf("expected removed model's lock key pruned, got %#v", got[0].LockData)
|
||||
}
|
||||
if got[0].LockData[keep] != 1 {
|
||||
t.Fatalf("expected surviving model ID 1 kept, got %#v", got[0].LockData)
|
||||
}
|
||||
}
|
||||
|
||||
// Cutover: a lock that only has the OLD config-derived key adopts that ID under
|
||||
// the new model-path key, with zero shift, and prunes the old key.
|
||||
func TestCutoverRemapPreservesID(t *testing.T) {
|
||||
oldKey := "visualeffects:head_accessories/hat/bandana" // {dataset}:{group}/{category}/{stem-without-hfx_}
|
||||
newKey := "visualeffects:head_accessories/hat/hfx_bandana.mdl"
|
||||
collected := []nativeCollectedDataset{vfxDataset(map[string]int{oldKey: 4242})}
|
||||
got, err := augmentWithAutogeneratedAccessoryVisualeffects(collected, []autogenManifestEntry{vfxEntry()}, accConsumer(), nil)
|
||||
if err != nil {
|
||||
t.Fatalf("augment failed: %v", err)
|
||||
}
|
||||
if got[0].LockData[newKey] != 4242 {
|
||||
t.Fatalf("expected cutover to carry ID 4242 to new key, got %#v", got[0].LockData)
|
||||
}
|
||||
if _, present := got[0].LockData[oldKey]; present {
|
||||
t.Fatalf("expected old config key pruned after cutover, got %#v", got[0].LockData)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
package topdata
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
|
||||
)
|
||||
|
||||
// The manifest_file branch reads a local autogenManifest JSON, augments the
|
||||
// visualeffects dataset from its entries, and assigns lock IDs — no network.
|
||||
func TestApplyAutogenConsumersReadsManifestFile(t *testing.T) {
|
||||
root := testProjectRoot(t)
|
||||
manifestPath := filepath.Join(root, ".cache", "sow-accessory-vfx-manifest.json")
|
||||
mkdirAll(t, filepath.Join(root, ".cache"))
|
||||
writeFile(t, manifestPath, `{
|
||||
"id": "accessory_visualeffects",
|
||||
"repo": "ShadowsOverWestgate/sow-assets-manifest",
|
||||
"ref": "v1.2.3",
|
||||
"generated_at": "2026-06-21T00:00:00Z",
|
||||
"entries": [
|
||||
{"source": "head_accessories/hat/hfx_bandana.mdl", "model_stem": "hfx_bandana", "group": "head_accessories", "subgroup": "hat"}
|
||||
]
|
||||
}`+"\n")
|
||||
|
||||
p := testProject(root)
|
||||
p.Config.Autogen.Consumers = []project.AutogenConsumerConfig{
|
||||
{
|
||||
ID: "accessory_visualeffects",
|
||||
Producer: "accessory_visualeffects",
|
||||
Dataset: "visualeffects",
|
||||
Mode: "accessory_visualeffects",
|
||||
Optional: true,
|
||||
ManifestFile: ".cache/sow-accessory-vfx-manifest.json",
|
||||
},
|
||||
}
|
||||
|
||||
collected := []nativeCollectedDataset{
|
||||
{
|
||||
Dataset: nativeDataset{Name: "visualeffects", Kind: nativeDatasetBase},
|
||||
Columns: []string{"Label", "Type_FD", "OrientWithGround", "Imp_HeadCon_Node", "OrientWithObject"},
|
||||
Rows: nil,
|
||||
LockData: map[string]int{
|
||||
"visualeffects:existing": 17,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
got, err := applyAutogenConsumers(p, collected, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("applyAutogenConsumers failed: %v", err)
|
||||
}
|
||||
if len(got) != 1 || len(got[0].Rows) != 1 {
|
||||
t.Fatalf("expected one autogenerated row, got %#v", got)
|
||||
}
|
||||
row := got[0].Rows[0]
|
||||
if row["key"] != "visualeffects:head_accessories/hat/bandana" || row["Imp_HeadCon_Node"] != "hfx_bandana" {
|
||||
t.Fatalf("unexpected generated row: %#v", row)
|
||||
}
|
||||
if _, ok := got[0].LockData["visualeffects:head_accessories/hat/hfx_bandana.mdl"]; !ok {
|
||||
t.Fatalf("expected lock id for generated accessory, got %#v", got[0].LockData)
|
||||
}
|
||||
// Authored key untouched.
|
||||
if got[0].LockData["visualeffects:existing"] != 17 {
|
||||
t.Fatalf("expected authored lock id retained at 17, got %#v", got[0].LockData)
|
||||
}
|
||||
}
|
||||
|
||||
// A missing manifest_file is an ignorable-optional condition: the optional
|
||||
// consumer fails open (no rows) and preserves existing lock IDs.
|
||||
func TestApplyAutogenConsumersMissingManifestFileFailsOpen(t *testing.T) {
|
||||
root := testProjectRoot(t)
|
||||
lockPath := filepath.Join(root, "visualeffects-lock.json")
|
||||
writeFile(t, lockPath, `{"visualeffects:head_accessories/hat/bandana":10101}`+"\n")
|
||||
|
||||
p := testProject(root)
|
||||
p.Config.Autogen.Consumers = []project.AutogenConsumerConfig{
|
||||
{
|
||||
ID: "accessory_visualeffects",
|
||||
Producer: "accessory_visualeffects",
|
||||
Dataset: "visualeffects",
|
||||
Mode: "accessory_visualeffects",
|
||||
Optional: true,
|
||||
ManifestFile: ".cache/does-not-exist.json",
|
||||
},
|
||||
}
|
||||
collected := []nativeCollectedDataset{
|
||||
{
|
||||
Dataset: nativeDataset{Name: "visualeffects", Kind: nativeDatasetBase, LockPath: lockPath},
|
||||
Columns: []string{"Label"},
|
||||
Rows: nil,
|
||||
LockData: map[string]int{},
|
||||
},
|
||||
}
|
||||
|
||||
got, err := applyAutogenConsumers(p, collected, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("expected fail-open, got error: %v", err)
|
||||
}
|
||||
if id, ok := got[0].LockData["visualeffects:head_accessories/hat/bandana"]; !ok || id != 10101 {
|
||||
t.Fatalf("expected preserved lock id 10101, got %v (present=%v)", id, ok)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
package topdata
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// validateClassFeatMasterfeatAccessibility warns when a class feat table
|
||||
// references a masterfeat whose member feats are silently dropped from
|
||||
// expansion by classFeatExpansionCanUseFeat. Only the ambiguous case is
|
||||
// flagged: ALLCLASSESCANUSE=0 with no MinLevelClass, where the feat cannot
|
||||
// be obtained by any class at all. Exclusions with an explicit MinLevelClass
|
||||
// are deliberate cross-class restrictions and stay silent.
|
||||
func validateClassFeatMasterfeatAccessibility(dataDir string, report *ValidationReport) {
|
||||
datasets, err := discoverNativeDatasets(dataDir)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
keyToID := map[string]int{}
|
||||
var featRowByKey map[string]map[string]any
|
||||
type classFeatTable struct {
|
||||
path string
|
||||
classKey string
|
||||
rows []map[string]any
|
||||
}
|
||||
classTables := []classFeatTable{}
|
||||
for _, dataset := range datasets {
|
||||
name := filepath.ToSlash(dataset.Name)
|
||||
isClassFeats := strings.HasPrefix(name, "classes/feats/")
|
||||
if name != "feat" && name != "masterfeats" && name != "classes/core" && !isClassFeats {
|
||||
continue
|
||||
}
|
||||
collected, err := collectNativeDataset(dataset)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for key, id := range collected.LockData {
|
||||
keyToID[key] = id
|
||||
}
|
||||
switch {
|
||||
case name == "feat":
|
||||
featRowByKey = rowsByKey(collected.Rows)
|
||||
case isClassFeats:
|
||||
classTables = append(classTables, classFeatTable{
|
||||
path: dataset.BasePath,
|
||||
classKey: "classes:" + name[strings.LastIndex(name, "/")+1:],
|
||||
rows: collected.Rows,
|
||||
})
|
||||
}
|
||||
}
|
||||
if featRowByKey == nil {
|
||||
return
|
||||
}
|
||||
for _, table := range classTables {
|
||||
for _, row := range table.rows {
|
||||
featRef, ok := row["FeatIndex"].(map[string]any)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
refKey, _ := featRef["id"].(string)
|
||||
if !strings.HasPrefix(refKey, "masterfeats:") {
|
||||
continue
|
||||
}
|
||||
masterfeatKey := resolveCanonicalDatasetKey(refKey, keyToID)
|
||||
if _, ok := keyToID[masterfeatKey]; !ok {
|
||||
continue
|
||||
}
|
||||
featKeys := make([]string, 0, len(featRowByKey))
|
||||
for featKey := range featRowByKey {
|
||||
if strings.HasPrefix(featKey, "feat:") {
|
||||
featKeys = append(featKeys, featKey)
|
||||
}
|
||||
}
|
||||
slices.Sort(featKeys)
|
||||
for _, featKey := range featKeys {
|
||||
featRow := featRowByKey[featKey]
|
||||
if !rowReferencesMasterfeat(featRow, masterfeatKey, keyToID) {
|
||||
continue
|
||||
}
|
||||
if classFeatExpansionCanUseFeat(featRow, table.classKey, keyToID) {
|
||||
continue
|
||||
}
|
||||
if minLevelClass, ok := lookupField(featRow, "MinLevelClass"); ok && !isNullishValue(minLevelClass) {
|
||||
continue
|
||||
}
|
||||
report.Diagnostics = append(report.Diagnostics, Diagnostic{
|
||||
Severity: SeverityWarning,
|
||||
Path: table.path,
|
||||
Message: fmt.Sprintf(
|
||||
"%s references %s but %s is excluded from expansion (ALLCLASSESCANUSE=0 with no MinLevelClass makes it unobtainable); override ALLCLASSESCANUSE or set MinLevelClass",
|
||||
table.classKey, masterfeatKey, featKey),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
package topdata
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestValidateClassFeatMasterfeatAccessibilityWarnsOnUnobtainableExpansion(t *testing.T) {
|
||||
root := testProjectRoot(t)
|
||||
mkdirAll(t, filepath.Join(root, "topdata", "data", "classes", "feats"))
|
||||
mkdirAll(t, filepath.Join(root, "topdata", "data", "feat"))
|
||||
mkdirAll(t, filepath.Join(root, "topdata", "data", "masterfeats"))
|
||||
writeFile(t, filepath.Join(root, "topdata", "data", "classes", "base.json"), `{
|
||||
"output": "classes.2da",
|
||||
"columns": ["Label", "Name", "FeatsTable"],
|
||||
"rows": [
|
||||
{"id": 4, "key": "classes:fighter", "Label": "Fighter", "Name": "111", "FeatsTable": "cls_feat_fighter"},
|
||||
{"id": 10, "key": "classes:wizard", "Label": "Wizard", "Name": "112", "FeatsTable": "cls_feat_wizard"}
|
||||
]
|
||||
}`+"\n")
|
||||
writeFile(t, filepath.Join(root, "topdata", "data", "classes", "lock.json"), `{"classes:fighter":4,"classes:wizard":10}`+"\n")
|
||||
writeFile(t, filepath.Join(root, "topdata", "data", "classes", "feats", "wizard.json"), `{
|
||||
"key": "classes/feats:wizard",
|
||||
"output": "cls_feat_wizard.2da",
|
||||
"columns": ["FeatLabel", "FeatIndex", "List", "GrantedOnLevel", "OnMenu"],
|
||||
"rows": [
|
||||
{"FeatIndex": {"id": "masterfeats:spellfocus"}, "List": "1", "GrantedOnLevel": "-1", "OnMenu": "0"},
|
||||
{"FeatIndex": {"id": "masterfeats:weaponspecialization"}, "List": "1", "GrantedOnLevel": "-1", "OnMenu": "0"}
|
||||
]
|
||||
}`+"\n")
|
||||
writeFile(t, filepath.Join(root, "topdata", "data", "feat", "base.json"), `{
|
||||
"output": "feat.2da",
|
||||
"columns": ["LABEL", "FEAT", "DESCRIPTION", "MASTERFEAT", "ALLCLASSESCANUSE", "MinLevelClass"],
|
||||
"rows": [
|
||||
{"id": 100, "key": "feat:spellfocus_abjuration", "LABEL": "SpellFocusAbj", "FEAT": "1000", "DESCRIPTION": "1001", "MASTERFEAT": "1", "ALLCLASSESCANUSE": "0", "MinLevelClass": "****"},
|
||||
{"id": 101, "key": "feat:spellfocus_conjuration", "LABEL": "SpellFocusCon", "FEAT": "1002", "DESCRIPTION": "1003", "MASTERFEAT": "1", "ALLCLASSESCANUSE": "1", "MinLevelClass": "****"},
|
||||
{"id": 200, "key": "feat:weaponspecialization_longsword", "LABEL": "WeaponSpecLongsword", "FEAT": "2000", "DESCRIPTION": "2001", "MASTERFEAT": "2", "ALLCLASSESCANUSE": "0", "MinLevelClass": {"id": "classes:fighter"}}
|
||||
]
|
||||
}`+"\n")
|
||||
writeFile(t, filepath.Join(root, "topdata", "data", "feat", "lock.json"), `{
|
||||
"feat:spellfocus_abjuration": 100,
|
||||
"feat:spellfocus_conjuration": 101,
|
||||
"feat:weaponspecialization_longsword": 200
|
||||
}`+"\n")
|
||||
writeFile(t, filepath.Join(root, "topdata", "data", "masterfeats", "base.json"), `{
|
||||
"output": "masterfeats.2da",
|
||||
"columns": ["LABEL", "STRREF", "DESCRIPTION", "ICON"],
|
||||
"rows": [
|
||||
{"id": 1, "key": "masterfeats:spellfocus", "LABEL": "SpellFocus", "STRREF": "6492", "DESCRIPTION": "426", "ICON": "ife_magic"},
|
||||
{"id": 2, "key": "masterfeats:weaponspecialization", "LABEL": "WeaponSpecialization", "STRREF": "6491", "DESCRIPTION": "437", "ICON": "ife_wepspec"}
|
||||
]
|
||||
}`+"\n")
|
||||
writeFile(t, filepath.Join(root, "topdata", "data", "masterfeats", "lock.json"), `{
|
||||
"masterfeats:spellfocus": 1,
|
||||
"masterfeats:weaponspecialization": 2
|
||||
}`+"\n")
|
||||
|
||||
report := ValidationReport{}
|
||||
validateClassFeatMasterfeatAccessibility(filepath.Join(root, "topdata", "data"), &report)
|
||||
|
||||
warned := map[string]bool{}
|
||||
for _, diagnostic := range report.Diagnostics {
|
||||
if diagnostic.Severity != SeverityWarning {
|
||||
continue
|
||||
}
|
||||
for _, featKey := range []string{"feat:spellfocus_abjuration", "feat:spellfocus_conjuration", "feat:weaponspecialization_longsword"} {
|
||||
if strings.Contains(diagnostic.Message, featKey) {
|
||||
warned[featKey] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if !warned["feat:spellfocus_abjuration"] {
|
||||
t.Fatalf("expected warning for unobtainable ALLCLASSESCANUSE=0 feat, got: %+v", report.Diagnostics)
|
||||
}
|
||||
if warned["feat:spellfocus_conjuration"] {
|
||||
t.Fatalf("did not expect warning for ALLCLASSESCANUSE=1 feat, got: %+v", report.Diagnostics)
|
||||
}
|
||||
if warned["feat:weaponspecialization_longsword"] {
|
||||
t.Fatalf("did not expect warning for feat with explicit MinLevelClass, got: %+v", report.Diagnostics)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,195 @@
|
||||
package topdata
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type globalCondition struct {
|
||||
Field string
|
||||
ID string
|
||||
}
|
||||
|
||||
type globalConditionGroups struct {
|
||||
RequirePresent []globalCondition
|
||||
AnyPresent []globalCondition
|
||||
UnlessPresent []globalCondition
|
||||
}
|
||||
|
||||
var (
|
||||
baseOrPlainRootKeys = []string{
|
||||
"output", "key", "columns", "rows", "compare_reference",
|
||||
}
|
||||
canonicalModuleRootKeys = []string{
|
||||
"output", "key", "columns", "entries", "overrides", "rows",
|
||||
}
|
||||
globalRootKeys = []string{
|
||||
"columns", "entries", "overrides", "defaults", "position", "injections",
|
||||
}
|
||||
globalInjectionKeys = []string{
|
||||
"row", "require_present", "any_present", "unless_present",
|
||||
}
|
||||
globalConditionKeys = []string{"field", "id"}
|
||||
globalDefaultRuleKeys = []string{"match", "values"}
|
||||
globalDefaultMatchKeys = []string{"source"}
|
||||
globalDefaultFormatKeys = []string{"format"}
|
||||
)
|
||||
|
||||
func unsupportedObjectKeys(obj map[string]any, supported ...string) []string {
|
||||
if len(obj) == 0 {
|
||||
return nil
|
||||
}
|
||||
supportedSet := make(map[string]struct{}, len(supported))
|
||||
for _, key := range supported {
|
||||
supportedSet[key] = struct{}{}
|
||||
}
|
||||
unsupported := make([]string, 0, len(obj))
|
||||
for key := range obj {
|
||||
if _, ok := supportedSet[key]; ok {
|
||||
continue
|
||||
}
|
||||
unsupported = append(unsupported, key)
|
||||
}
|
||||
slices.Sort(unsupported)
|
||||
return unsupported
|
||||
}
|
||||
|
||||
func unsupportedObjectKeysError(label string, obj map[string]any, supported ...string) error {
|
||||
unsupported := unsupportedObjectKeys(obj, supported...)
|
||||
if len(unsupported) == 0 {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("%s contains unsupported key %q (supported keys: %s)", label, unsupported[0], strings.Join(supported, ", "))
|
||||
}
|
||||
|
||||
type globalConditionParseOptions struct {
|
||||
Label string
|
||||
Report func(error)
|
||||
}
|
||||
|
||||
func parseGlobalConditionGroups(injection map[string]any, options ...globalConditionParseOptions) (globalConditionGroups, error) {
|
||||
if injection == nil {
|
||||
return globalConditionGroups{}, nil
|
||||
}
|
||||
|
||||
label := "global injection"
|
||||
var report func(error)
|
||||
if len(options) > 0 {
|
||||
if strings.TrimSpace(options[0].Label) != "" {
|
||||
label = options[0].Label
|
||||
}
|
||||
report = options[0].Report
|
||||
}
|
||||
|
||||
var errs []error
|
||||
addError := func(err error) {
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
errs = append(errs, err)
|
||||
if report != nil {
|
||||
report(err)
|
||||
}
|
||||
}
|
||||
if err := unsupportedObjectKeysError(label, injection, globalInjectionKeys...); err != nil {
|
||||
addError(err)
|
||||
}
|
||||
|
||||
var groups globalConditionGroups
|
||||
if raw, ok := injection["require_present"]; ok {
|
||||
parsed, err := parseGlobalConditionList(label+" require_present", raw, false)
|
||||
if err != nil {
|
||||
addError(err)
|
||||
} else {
|
||||
groups.RequirePresent = parsed
|
||||
}
|
||||
}
|
||||
if raw, ok := injection["any_present"]; ok {
|
||||
parsed, err := parseGlobalConditionList(label+" any_present", raw, true)
|
||||
if err != nil {
|
||||
addError(err)
|
||||
} else {
|
||||
groups.AnyPresent = parsed
|
||||
}
|
||||
}
|
||||
if raw, ok := injection["unless_present"]; ok {
|
||||
parsed, err := parseGlobalConditionList(label+" unless_present", raw, false)
|
||||
if err != nil {
|
||||
addError(err)
|
||||
} else {
|
||||
groups.UnlessPresent = parsed
|
||||
}
|
||||
}
|
||||
return groups, errors.Join(errs...)
|
||||
}
|
||||
|
||||
func parseGlobalConditionList(name string, raw any, requireNonEmpty bool) ([]globalCondition, error) {
|
||||
if raw == nil {
|
||||
return nil, fmt.Errorf("%s must be an array", name)
|
||||
}
|
||||
|
||||
conditions, ok := raw.([]any)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("%s must be an array", name)
|
||||
}
|
||||
if requireNonEmpty && len(conditions) == 0 {
|
||||
return nil, fmt.Errorf("%s must contain at least one condition", name)
|
||||
}
|
||||
|
||||
parsed := make([]globalCondition, 0, len(conditions))
|
||||
for index, rawCondition := range conditions {
|
||||
condition, ok := rawCondition.(map[string]any)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("%s[%d] must be an object", name, index)
|
||||
}
|
||||
if err := unsupportedObjectKeysError(fmt.Sprintf("%s[%d]", name, index), condition, globalConditionKeys...); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
field, _ := condition["field"].(string)
|
||||
field = strings.TrimSpace(field)
|
||||
if field == "" {
|
||||
return nil, fmt.Errorf("%s[%d].field is required", name, index)
|
||||
}
|
||||
|
||||
id, _ := condition["id"].(string)
|
||||
id = strings.TrimSpace(id)
|
||||
if id == "" {
|
||||
return nil, fmt.Errorf("%s[%d].id is required", name, index)
|
||||
}
|
||||
|
||||
parsed = append(parsed, globalCondition{Field: field, ID: id})
|
||||
}
|
||||
return parsed, nil
|
||||
}
|
||||
|
||||
func globalConditionGroupsMatch(groups globalConditionGroups, rows []map[string]any) bool {
|
||||
for _, condition := range groups.RequirePresent {
|
||||
if !globalReferencePresent(rows, condition.Field, condition.ID) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
if groups.AnyPresent != nil {
|
||||
if len(groups.AnyPresent) == 0 {
|
||||
return false
|
||||
}
|
||||
matched := false
|
||||
for _, condition := range groups.AnyPresent {
|
||||
if globalReferencePresent(rows, condition.Field, condition.ID) {
|
||||
matched = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !matched {
|
||||
return false
|
||||
}
|
||||
}
|
||||
for _, condition := range groups.UnlessPresent {
|
||||
if globalReferencePresent(rows, condition.Field, condition.ID) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,29 @@
|
||||
package topdata
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestHasLFSPointerStubsDetectsStub(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
if err := os.MkdirAll(filepath.Join(root, "a"), 0o755); err != nil {
|
||||
t.Fatalf("mkdir: %v", err)
|
||||
}
|
||||
stub := "version https://git-lfs.github.com/spec/v1\noid sha256:abc\nsize 12345\n"
|
||||
writeFile(t, filepath.Join(root, "a", "model.mdl"), stub)
|
||||
writeFile(t, filepath.Join(root, "a", "real.txt"), "not a pointer, real bytes here")
|
||||
|
||||
if !hasLFSPointerStubs(root) {
|
||||
t.Fatal("expected stub detection to be true")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHasLFSPointerStubsNoStub(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
writeFile(t, filepath.Join(root, "real.mdl"), "binary-ish content without a pointer header")
|
||||
if hasLFSPointerStubs(root) {
|
||||
t.Fatal("expected no stub detected")
|
||||
}
|
||||
}
|
||||
+31
-42
@@ -460,7 +460,15 @@ func buildNativeUnchecked(p *project.Project, opts NativeBuildOptions, progress
|
||||
if err != nil {
|
||||
return BuildResult{}, err
|
||||
}
|
||||
collected, err = applyPartOverrides(sourceDir, collected)
|
||||
// Single explicit parts sequence: augment (inside applyAutogenConsumers) ->
|
||||
// normalize -> apply overrides exactly once, all under the one configured
|
||||
// parts_rows consumer's policy.
|
||||
partsCfg := partsRowsConfigForProject(p)
|
||||
collected, err = normalizePartsRowsACBonus(collected, partsCfg)
|
||||
if err != nil {
|
||||
return BuildResult{}, err
|
||||
}
|
||||
collected, err = applyPartOverridesWithConfig(sourceDir, collected, partsCfg)
|
||||
if err != nil {
|
||||
return BuildResult{}, err
|
||||
}
|
||||
@@ -3886,48 +3894,11 @@ func globalManifestPosition(module nativeGeneratedModule) (string, error) {
|
||||
}
|
||||
|
||||
func globalInjectionConditionsMatch(injection map[string]any, rows []map[string]any) (bool, error) {
|
||||
if rawRequired, ok := injection["require_present"]; ok {
|
||||
matches, err := globalConditionListMatches("require_present", rawRequired, rows, true)
|
||||
if err != nil || !matches {
|
||||
return matches, err
|
||||
}
|
||||
groups, err := parseGlobalConditionGroups(injection)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if rawBlocked, ok := injection["unless_present"]; ok {
|
||||
matches, err := globalConditionListMatches("unless_present", rawBlocked, rows, false)
|
||||
if err != nil || !matches {
|
||||
return matches, err
|
||||
}
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func globalConditionListMatches(name string, raw any, rows []map[string]any, required bool) (bool, error) {
|
||||
conditions, ok := raw.([]any)
|
||||
if !ok {
|
||||
return false, fmt.Errorf("%s must be an array", name)
|
||||
}
|
||||
for index, rawCondition := range conditions {
|
||||
condition, ok := rawCondition.(map[string]any)
|
||||
if !ok {
|
||||
return false, fmt.Errorf("%s[%d] must be an object", name, index)
|
||||
}
|
||||
field, _ := condition["field"].(string)
|
||||
id, _ := condition["id"].(string)
|
||||
if strings.TrimSpace(field) == "" {
|
||||
return false, fmt.Errorf("%s[%d].field is required", name, index)
|
||||
}
|
||||
if strings.TrimSpace(id) == "" {
|
||||
return false, fmt.Errorf("%s[%d].id is required", name, index)
|
||||
}
|
||||
found := globalReferencePresent(rows, field, id)
|
||||
if required && !found {
|
||||
return false, nil
|
||||
}
|
||||
if !required && found {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
return true, nil
|
||||
return globalConditionGroupsMatch(groups, rows), nil
|
||||
}
|
||||
|
||||
func globalRowIdentityPresent(row map[string]any, rows []map[string]any) bool {
|
||||
@@ -4741,6 +4712,24 @@ func (r *valueResolver) resolveValue(row map[string]any, field string, value any
|
||||
|
||||
switch typed := value.(type) {
|
||||
case map[string]any:
|
||||
if radial, ok := typed["spellradial"].(map[string]any); ok && len(typed) == 1 {
|
||||
if r.dataset.Name != "spells" || field != "FeatID" {
|
||||
return tlkCompiledValue{}, fmt.Errorf("spellradial reference is only valid for spells.FeatID")
|
||||
}
|
||||
radialKey, ok := radial["id"].(string)
|
||||
if !ok || len(radial) != 1 {
|
||||
return tlkCompiledValue{}, fmt.Errorf("spellradial reference requires exactly one id")
|
||||
}
|
||||
featID, ok := r.keyToID[radialKey]
|
||||
if !ok {
|
||||
return tlkCompiledValue{}, fmt.Errorf("unknown spellradial feat reference: %s", radialKey)
|
||||
}
|
||||
spellID, ok := row["id"].(int)
|
||||
if !ok {
|
||||
return tlkCompiledValue{}, fmt.Errorf("spellradial reference requires a numeric spell row id")
|
||||
}
|
||||
return tlkCompiledValue{Value: spellID<<16 + featID}, nil
|
||||
}
|
||||
if encoding, ok := r.valueEncodingForField(field); ok {
|
||||
mode := strings.TrimSpace(encoding.Mode)
|
||||
_, hasList := typed["list"]
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
package topdata
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
|
||||
)
|
||||
|
||||
// PARITY GUARD: a bare project (no manifest_file, no NWN_ROOT, no token) with a
|
||||
// cdn_channel consumer must produce accessory rows. If accessory resolution ever
|
||||
// drifts back out of Crucible into a wrapper, this fails.
|
||||
func TestParityGuardCDNChannelProducesAccessoryRows(t *testing.T) {
|
||||
root := testProjectRoot(t)
|
||||
srv := cdnServer(t, map[string]string{
|
||||
"/releases/haks/channels.json": `{"current":"v1.0.0"}`,
|
||||
"/releases/haks/v1.0.0/vfxs.yml": `assets:
|
||||
- path: vfxs/head_accessories/hat/hfx_bandana.mdl
|
||||
restype: mdl
|
||||
sha256: aaa
|
||||
- path: vfxs/chest_accessories/cape/cfx_cloak.mdl
|
||||
restype: mdl
|
||||
sha256: bbb
|
||||
`,
|
||||
})
|
||||
t.Setenv("BUNNY_CDN_BASE", srv.URL)
|
||||
t.Setenv("SOW_TOPDATA_ASSET_CHANNEL", "current")
|
||||
// Prove R3: no NWN_ROOT in the environment.
|
||||
t.Setenv("NWN_ROOT", "")
|
||||
|
||||
c := cdnConsumer()
|
||||
c.Source.CDNBaseEnv = "BUNNY_CDN_BASE"
|
||||
c.Source.ChannelEnv = "SOW_TOPDATA_ASSET_CHANNEL"
|
||||
c.AccessoryVisualeffects.ModelColumn = "Imp_HeadCon_Node"
|
||||
|
||||
p := testProject(root)
|
||||
p.Config.Autogen.Consumers = []project.AutogenConsumerConfig{c}
|
||||
|
||||
collected := []nativeCollectedDataset{{
|
||||
Dataset: nativeDataset{Name: "visualeffects", Kind: nativeDatasetBase},
|
||||
Columns: []string{"Label", "Imp_HeadCon_Node", "Imp_Root_S_Node"},
|
||||
Rows: nil,
|
||||
LockData: map[string]int{},
|
||||
}}
|
||||
|
||||
got, err := applyAutogenConsumers(p, collected, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("applyAutogenConsumers failed: %v", err)
|
||||
}
|
||||
if len(got[0].Rows) != 2 {
|
||||
t.Fatalf("PARITY GUARD: expected 2 accessory rows, got %d (%#v)", len(got[0].Rows), got[0].Rows)
|
||||
}
|
||||
wantKey := "visualeffects:head_accessories/hat/hfx_bandana.mdl"
|
||||
if _, ok := got[0].LockData[wantKey]; !ok {
|
||||
t.Fatalf("PARITY GUARD: expected lock id for %s, got %#v", wantKey, got[0].LockData)
|
||||
}
|
||||
}
|
||||
|
||||
// PARITY GUARD (parts): a bare project whose only parts source is the offline
|
||||
// CDN-equivalent (SOW_PART_MANIFEST -> a part.yml), with no NWN_ROOT, no token,
|
||||
// no asset checkout, must produce parts rows from the filtered inventory. If
|
||||
// parts resolution ever drifts back into a wrapper, this fails.
|
||||
func TestParityGuardCDNChannelProducesPartsRows(t *testing.T) {
|
||||
root := testProjectRoot(t)
|
||||
checkout := filepath.Join(root, "manifest-checkout")
|
||||
if err := os.MkdirAll(filepath.Join(checkout, "assets"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
writeFile(t, filepath.Join(checkout, "assets", "part.yml"),
|
||||
"assets:\n - {path: part/belt/m/pfa0_belt017.mdl, restype: mdl}\n - {path: part/belt/m/pfa0_belt018.mdl, restype: mdl}\n")
|
||||
t.Setenv("SOW_PART_MANIFEST", checkout)
|
||||
t.Setenv("NWN_ROOT", "")
|
||||
|
||||
c := projectPartsConsumer()
|
||||
c.Source = project.AutogenSourceConfig{
|
||||
Kind: "cdn_channel",
|
||||
ChannelsPath: "releases/haks/channels.json",
|
||||
ManifestPath: "releases/haks/{tag}/part.yml",
|
||||
ReleaseMarkerPath: "releases/haks/{tag}/haks.json",
|
||||
OfflineOverrideEnv: "SOW_PART_MANIFEST",
|
||||
}
|
||||
|
||||
p := testProject(root)
|
||||
p.Config.Autogen.Consumers = []project.AutogenConsumerConfig{c}
|
||||
|
||||
collected := []nativeCollectedDataset{partsDatasetWithIDs("parts/belt", nil)}
|
||||
|
||||
got, err := applyAutogenConsumers(p, collected, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("applyAutogenConsumers failed: %v", err)
|
||||
}
|
||||
ids := map[int]bool{}
|
||||
for _, row := range got[0].Rows {
|
||||
ids[row["id"].(int)] = true
|
||||
}
|
||||
if !ids[17] || !ids[18] {
|
||||
t.Fatalf("PARITY GUARD: expected belt rows 17 and 18 from inventory, got %#v", got[0].Rows)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
package topdata
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
|
||||
)
|
||||
|
||||
func projectPartsConsumer() project.AutogenConsumerConfig {
|
||||
return project.AutogenConsumerConfig{ID: "parts", Producer: "parts", Mode: "parts_rows"}
|
||||
}
|
||||
|
||||
const partYML = `assets:
|
||||
- {path: part/belt/m/pfa0_belt017.mdl, restype: mdl}
|
||||
- {path: part/hand/m/pfh0_handl105.mdl, restype: mdl}
|
||||
- {path: part/hand/m/pfh0_handr105.mdl, restype: mdl}
|
||||
- {path: part/leg/m/pfa0_legl001.mdl, restype: mdl}
|
||||
- {path: part/leg/m/pfa0_legr001.mdl, restype: mdl}
|
||||
- {path: part/cloak/m/cloak001.mdl, restype: mdl}
|
||||
- {path: part/belt/m/readme.txt, restype: txt}
|
||||
`
|
||||
|
||||
func TestFilterPartsKeepsSupportedDedupsVariants(t *testing.T) {
|
||||
consumer := projectPartsConsumer()
|
||||
entries, err := filterCDNChannelEntries([]byte(partYML), consumer)
|
||||
if err != nil {
|
||||
t.Fatalf("filter: %v", err)
|
||||
}
|
||||
inv := autogenPartsInventory(entries)
|
||||
if _, ok := inv["belt"][17]; !ok {
|
||||
t.Errorf("missing belt 17")
|
||||
}
|
||||
if len(inv["hand"]) != 1 {
|
||||
t.Errorf("hand should dedup l/r to 1 row, got %d", len(inv["hand"]))
|
||||
}
|
||||
if _, ok := inv["hand"][105]; !ok {
|
||||
t.Errorf("missing hand 105")
|
||||
}
|
||||
if len(inv["leg"]) != 1 {
|
||||
t.Errorf("leg should dedup l/r to 1 row, got %d", len(inv["leg"]))
|
||||
}
|
||||
if _, ok := inv["cloak"]; ok {
|
||||
t.Errorf("cloak must be ignored")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilterPartsRejectsZeroAccepted(t *testing.T) {
|
||||
_, err := filterCDNChannelEntries([]byte("assets:\n - {path: part/cloak/m/cloak001.mdl, restype: mdl}\n"), projectPartsConsumer())
|
||||
if err == nil {
|
||||
t.Fatal("expected hard error on zero accepted part rows")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilterPartsRejectsRowZeroAndNonNumeric(t *testing.T) {
|
||||
for _, doc := range []string{
|
||||
"assets:\n - {path: part/belt/m/pfa0_belt000.mdl, restype: mdl}\n",
|
||||
"assets:\n - {path: part/belt/m/pfa0_belt.mdl, restype: mdl}\n",
|
||||
} {
|
||||
if _, err := filterCDNChannelEntries([]byte(doc), projectPartsConsumer()); err == nil {
|
||||
t.Fatalf("expected error for %q", doc)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilterPartsRejectsMalformedAndMissingAssets(t *testing.T) {
|
||||
if _, err := filterCDNChannelEntries([]byte("assets:\n - path: [unterminated"), projectPartsConsumer()); err == nil {
|
||||
t.Fatal("expected malformed error")
|
||||
}
|
||||
if _, err := filterCDNChannelEntries([]byte("other: 1\n"), projectPartsConsumer()); err == nil {
|
||||
t.Fatal("expected no-assets error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSupportedPartCategoriesCoverTwelveDatasets(t *testing.T) {
|
||||
want := []string{"belt", "bicep", "chest", "foot", "forearm", "hand", "leg", "neck", "pelvis", "robe", "shin", "shoulder"}
|
||||
for _, c := range want {
|
||||
if !isSupportedPartCategory(c) {
|
||||
t.Errorf("category %q not supported", c)
|
||||
}
|
||||
}
|
||||
if partDatasetToAssetCategory["hand"] != "hand" {
|
||||
t.Errorf("parts/hand must map to asset category hand")
|
||||
}
|
||||
if partDatasetToAssetCategory["legs"] != "leg" {
|
||||
t.Errorf("parts/legs must map to asset category leg")
|
||||
}
|
||||
}
|
||||
@@ -21,6 +21,7 @@ var supportedPartCategories = []string{
|
||||
"chest",
|
||||
"foot",
|
||||
"forearm",
|
||||
"hand",
|
||||
"leg",
|
||||
"neck",
|
||||
"pelvis",
|
||||
@@ -35,6 +36,7 @@ var partDatasetToAssetCategory = map[string]string{
|
||||
"chest": "chest",
|
||||
"foot": "foot",
|
||||
"forearm": "forearm",
|
||||
"hand": "hand",
|
||||
"legs": "leg",
|
||||
"neck": "neck",
|
||||
"pelvis": "pelvis",
|
||||
@@ -390,6 +392,18 @@ func applyPartOverrides(sourceDir string, collected []nativeCollectedDataset) ([
|
||||
return applyPartOverridesWithConfig(sourceDir, collected, project.PartsRowsConfig{})
|
||||
}
|
||||
|
||||
// partsRowsConfigForProject returns the PartsRows policy of the single configured
|
||||
// parts_rows autogen consumer, or the zero value if none. Project validation
|
||||
// guarantees at most one parts_rows consumer, so the first match is canonical.
|
||||
func partsRowsConfigForProject(p *project.Project) project.PartsRowsConfig {
|
||||
for _, c := range p.Config.Autogen.Consumers {
|
||||
if strings.TrimSpace(c.Mode) == "parts_rows" {
|
||||
return c.PartsRows
|
||||
}
|
||||
}
|
||||
return project.PartsRowsConfig{}
|
||||
}
|
||||
|
||||
func applyPartOverridesWithConfig(sourceDir string, collected []nativeCollectedDataset, cfg project.PartsRowsConfig) ([]nativeCollectedDataset, error) {
|
||||
result := make([]nativeCollectedDataset, len(collected))
|
||||
copy(result, collected)
|
||||
@@ -430,9 +444,7 @@ func applyPartOverridesWithConfig(sourceDir string, collected []nativeCollectedD
|
||||
}
|
||||
row, ok := rowByID[rowID]
|
||||
if !ok {
|
||||
row = createDefaultPartRowForDataset(rowID, dataset.Dataset.Name, dataset.Columns, cfg)
|
||||
rows = append(rows, row)
|
||||
rowByID[rowID] = row
|
||||
return nil, fmt.Errorf("parts/%s override %d targets row id %d which is neither a baseline row nor a discovered model row; author the row in data/parts/%s.json instead", category, index, rowID, category)
|
||||
}
|
||||
for field, value := range override {
|
||||
if field == "id" {
|
||||
|
||||
@@ -274,6 +274,19 @@ func writePartsManifestCache(path string, manifest *partsManifest) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// httpGetStatus performs an anonymous GET and returns the status code plus the
|
||||
// body (capped). No auth header: cdn_channel inputs (channels.json, vfxs.yml,
|
||||
// haks.json) are public CDN text — R1.
|
||||
func httpGetStatus(target string) (int, []byte, error) {
|
||||
resp, err := partsManifestHTTPClient.Get(target)
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
|
||||
return resp.StatusCode, body, nil
|
||||
}
|
||||
|
||||
func inventoryFromPartsManifest(manifest *partsManifest) map[string]map[int]struct{} {
|
||||
result := make(map[string]map[int]struct{}, len(supportedPartCategories))
|
||||
for _, category := range supportedPartCategories {
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
package topdata
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
|
||||
)
|
||||
|
||||
func partsDatasetWithIDs(name string, ids []int) nativeCollectedDataset {
|
||||
rows := make([]map[string]any, 0, len(ids))
|
||||
for _, id := range ids {
|
||||
rows = append(rows, map[string]any{"id": id, "COSTMODIFIER": "0", "ACBONUS": "0.00"})
|
||||
}
|
||||
return nativeCollectedDataset{
|
||||
Dataset: nativeDataset{Name: name, Kind: nativeDatasetBase},
|
||||
Columns: []string{"COSTMODIFIER", "ACBONUS"},
|
||||
Rows: rows,
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyPartOverridesRejectsOrphan(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
if err := os.MkdirAll(filepath.Join(dir, "data", "parts", "overrides"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
writeFile(t, filepath.Join(dir, "data", "parts", "overrides", "belt.json"),
|
||||
`{"overrides": [{"id": 999, "COSTMODIFIER": "5"}]}`)
|
||||
collected := []nativeCollectedDataset{partsDatasetWithIDs("parts/belt", []int{17})}
|
||||
if _, err := applyPartOverridesWithConfig(dir, collected, project.PartsRowsConfig{}); err == nil {
|
||||
t.Fatal("expected orphan override to fail, not synthesize a row")
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyPartOverridesUpdatesDiscoveredRow(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
if err := os.MkdirAll(filepath.Join(dir, "data", "parts", "overrides"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
writeFile(t, filepath.Join(dir, "data", "parts", "overrides", "belt.json"),
|
||||
`{"overrides": [{"id": 17, "COSTMODIFIER": "5"}]}`)
|
||||
collected := []nativeCollectedDataset{partsDatasetWithIDs("parts/belt", []int{17})}
|
||||
out, err := applyPartOverridesWithConfig(dir, collected, project.PartsRowsConfig{})
|
||||
if err != nil {
|
||||
t.Fatalf("override: %v", err)
|
||||
}
|
||||
if got := out[0].Rows[0]["COSTMODIFIER"]; got != "5" {
|
||||
t.Fatalf("row 17 COSTMODIFIER want 5, got %v", got)
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
package topdata
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -75,6 +76,8 @@ func BuildAndPackage(p *project.Project, progress func(string)) (PackageResult,
|
||||
type BuildAndPackageOptions struct {
|
||||
Force bool
|
||||
BuildWiki bool
|
||||
// ponytail: no SkipLFS field — CRUCIBLE_SKIP_LFS env is the single skip mechanism;
|
||||
// the CLI flag sets it directly before calling into this package.
|
||||
}
|
||||
|
||||
func BuildAndPackageWithOptions(p *project.Project, opts BuildAndPackageOptions, progress func(string)) (PackageResult, error) {
|
||||
@@ -123,6 +126,9 @@ func packageBuiltTopData(p *project.Project, nativeResult BuildResult, progress
|
||||
if progress == nil {
|
||||
progress = func(string) {}
|
||||
}
|
||||
if err := materializeAssetLFS(p, progress); err != nil {
|
||||
return PackageResult{}, err
|
||||
}
|
||||
outputHAK := p.TopDataPackageHAKPath()
|
||||
outputTLK := p.TopDataPackageTLKPath()
|
||||
|
||||
@@ -205,7 +211,11 @@ func collectTopPackageResources(p *project.Project, compiled2DADir string) ([]er
|
||||
if d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
if strings.HasPrefix(filepath.Base(path), ".") {
|
||||
rel, err := filepath.Rel(assetsDir, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if skipTopPackageAsset(rel) {
|
||||
return nil
|
||||
}
|
||||
resource, err := topPackageResourceFromPath(path)
|
||||
@@ -379,6 +389,25 @@ func newestMatchingAutogenOverrideInput(scanRoot string, include []string) (time
|
||||
return newest, newestPath, nil
|
||||
}
|
||||
|
||||
// skipTopPackageAsset reports whether a file under assets/ is not a HAK
|
||||
// resource and must be ignored by both validation and packing: anything in a
|
||||
// hidden or underscore-prefixed directory (working dirs like _candidates),
|
||||
// hidden files, docs, and any extension that is not a known NWN ResType
|
||||
// (erf.extensionTypes is the whitelist). rel is the path relative to assets/.
|
||||
func skipTopPackageAsset(rel string) bool {
|
||||
for _, part := range strings.Split(filepath.ToSlash(rel), "/") {
|
||||
if strings.HasPrefix(part, ".") || strings.HasPrefix(part, "_") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
if strings.EqualFold(filepath.Ext(rel), ".md") {
|
||||
return true
|
||||
}
|
||||
ext := strings.TrimPrefix(strings.ToLower(filepath.Ext(rel)), ".")
|
||||
_, ok := erf.HAKResourceTypeForExtension(ext)
|
||||
return !ok
|
||||
}
|
||||
|
||||
func topPackageResourceFromPath(path string) (erf.Resource, error) {
|
||||
extension := strings.TrimPrefix(strings.ToLower(filepath.Ext(path)), ".")
|
||||
resourceType, ok := erf.HAKResourceTypeForExtension(extension)
|
||||
@@ -711,3 +740,80 @@ func countCompiledFiles(dir, extension string) (int, time.Time, error) {
|
||||
}
|
||||
return count, oldest, nil
|
||||
}
|
||||
|
||||
// materializeAssetLFS ensures the topdata asset tree holds real bytes, not
|
||||
// Git-LFS pointer stubs, before the hak is packed. A bare CI checkout does not
|
||||
// smudge LFS, so a direct Crucible build would otherwise ship ~130-byte stubs.
|
||||
// It uses the clone's own credentials (R1) and hard-fails if stubs survive the
|
||||
// pull. Set CRUCIBLE_SKIP_LFS=1 to opt out (maintain-tree regenerates the data/
|
||||
// tree and discards the package, so it must not hard-depend on LFS).
|
||||
func materializeAssetLFS(p *project.Project, progress func(string)) error {
|
||||
if progress == nil {
|
||||
progress = func(string) {}
|
||||
}
|
||||
assetsDir := filepath.Join(p.TopDataSourceDir(), "assets")
|
||||
info, err := os.Stat(assetsDir)
|
||||
if err != nil || !info.IsDir() {
|
||||
return nil // no asset tree to materialize
|
||||
}
|
||||
if !hasLFSPointerStubs(assetsDir) {
|
||||
return nil // already real bytes (or repo does not use LFS)
|
||||
}
|
||||
if isTruthyEnv(os.Getenv("CRUCIBLE_SKIP_LFS")) {
|
||||
progress("CRUCIBLE_SKIP_LFS set: leaving Git-LFS pointer stubs unmaterialized (package will be discarded)")
|
||||
return nil
|
||||
}
|
||||
if _, err := gitOutput(p.Root, "lfs", "version"); err != nil {
|
||||
return fmt.Errorf("asset tree has Git-LFS pointer stubs but git-lfs is unavailable; install git-lfs or enter the nix devshell: %w", err)
|
||||
}
|
||||
progress("Materializing Git-LFS assets (git lfs pull)...")
|
||||
// A fresh clone may lack the lfs smudge/clean filters; install them locally
|
||||
// first (scoped to this repo's .git/config, no host mutation). Idempotent.
|
||||
if _, err := gitOutput(p.Root, "lfs", "install", "--local"); err != nil {
|
||||
return fmt.Errorf("git lfs install --local failed: %w", err)
|
||||
}
|
||||
if _, err := gitOutput(p.Root, "lfs", "pull"); err != nil {
|
||||
return fmt.Errorf("git lfs pull failed: %w", err)
|
||||
}
|
||||
if hasLFSPointerStubs(assetsDir) {
|
||||
return fmt.Errorf("refusing to build from pointer stubs: Git-LFS pointer stubs remain under %s after git lfs pull", assetsDir)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// hasLFSPointerStubs reports whether any small text file under root is a Git-LFS
|
||||
// pointer. Real assets are larger than any pointer stub, so the size gate keeps
|
||||
// this cheap on a tree of binaries.
|
||||
func hasLFSPointerStubs(root string) bool {
|
||||
found := false
|
||||
_ = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
|
||||
if err != nil || d.IsDir() || found {
|
||||
return nil
|
||||
}
|
||||
info, ierr := d.Info()
|
||||
if ierr != nil || info.Size() == 0 || info.Size() > 1024 {
|
||||
return nil
|
||||
}
|
||||
f, oerr := os.Open(path)
|
||||
if oerr != nil {
|
||||
return nil
|
||||
}
|
||||
defer f.Close()
|
||||
buf := make([]byte, 256)
|
||||
n, _ := f.Read(buf)
|
||||
if bytes.Contains(buf[:n], []byte("git-lfs.github.com/spec/v1")) {
|
||||
found = true
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return found
|
||||
}
|
||||
|
||||
func isTruthyEnv(v string) bool {
|
||||
switch strings.TrimSpace(strings.ToLower(v)) {
|
||||
case "1", "true", "yes", "on":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
+71
-93
@@ -12,7 +12,6 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf"
|
||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
|
||||
)
|
||||
|
||||
@@ -268,6 +267,7 @@ func ValidateProject(p *project.Project) ValidationReport {
|
||||
validateNativeLockAllocation(dataDir, p.EffectiveConfig().TopData.RowGeneration, &report)
|
||||
validateGeneratedFeatFamilies(dataDir, &report)
|
||||
validateClassSpellbooks(dataDir, &report)
|
||||
validateClassFeatMasterfeatAccessibility(dataDir, &report)
|
||||
validateItempropsRegistryGraph(dataDir, &report)
|
||||
if _, err := os.Stat(statePath); err == nil {
|
||||
report.Files++
|
||||
@@ -333,6 +333,17 @@ func validateDuplicateJSONKeys(path string, raw []byte, report *ValidationReport
|
||||
}
|
||||
}
|
||||
|
||||
func reportTopdataContractError(path string, err error, report *ValidationReport) {
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
report.Diagnostics = append(report.Diagnostics, Diagnostic{
|
||||
Severity: SeverityError,
|
||||
Path: path,
|
||||
Message: err.Error(),
|
||||
})
|
||||
}
|
||||
|
||||
func scanJSONValueForDuplicateKeys(decoder *json.Decoder, path string, report *ValidationReport) error {
|
||||
token, err := decoder.Token()
|
||||
if err != nil {
|
||||
@@ -425,35 +436,40 @@ func validateDataObject(path string, obj map[string]any, report *ValidationRepor
|
||||
return
|
||||
}
|
||||
if base == "base.json" {
|
||||
reportTopdataContractError(path, unsupportedObjectKeysError("base.json root", obj, baseOrPlainRootKeys...), report)
|
||||
validateRowsFile(path, obj, report, true)
|
||||
return
|
||||
}
|
||||
if inModules {
|
||||
if _, hasEntries := obj["entries"]; hasEntries {
|
||||
validateEntriesFile(path, obj, report)
|
||||
return
|
||||
}
|
||||
if _, hasOverrides := obj["overrides"]; hasOverrides {
|
||||
validateOverridesFile(path, obj, report)
|
||||
return
|
||||
}
|
||||
if _, hasRows := obj["rows"]; hasRows {
|
||||
validateRowsFile(path, obj, report, false)
|
||||
return
|
||||
}
|
||||
if _, hasColumns := obj["columns"]; hasColumns {
|
||||
reportTopdataContractError(path, unsupportedObjectKeysError("module root", obj, canonicalModuleRootKeys...), report)
|
||||
_, hasColumns := obj["columns"]
|
||||
_, hasEntries := obj["entries"]
|
||||
_, hasOverrides := obj["overrides"]
|
||||
_, hasRows := obj["rows"]
|
||||
if hasColumns {
|
||||
validateColumnsFile(path, obj, report)
|
||||
return
|
||||
}
|
||||
report.Diagnostics = append(report.Diagnostics, Diagnostic{
|
||||
Severity: SeverityWarning,
|
||||
Path: path,
|
||||
Message: "module file does not use a recognized canonical shape yet; expected one of columns, entries, overrides, or rows",
|
||||
})
|
||||
if hasEntries {
|
||||
validateEntriesFileWithColumns(path, obj, report, false)
|
||||
}
|
||||
if hasOverrides {
|
||||
validateOverridesFileWithColumns(path, obj, report, false)
|
||||
}
|
||||
if hasRows {
|
||||
validateRowsFileWithColumns(path, obj, report, false, false)
|
||||
}
|
||||
if !hasColumns && !hasEntries && !hasOverrides && !hasRows {
|
||||
report.Diagnostics = append(report.Diagnostics, Diagnostic{
|
||||
Severity: SeverityWarning,
|
||||
Path: path,
|
||||
Message: "module file does not use a recognized canonical shape yet; expected one of columns, entries, overrides, or rows",
|
||||
})
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if _, hasRows := obj["rows"]; hasRows {
|
||||
reportTopdataContractError(path, unsupportedObjectKeysError("plain rows root", obj, baseOrPlainRootKeys...), report)
|
||||
validateRowsFile(path, obj, report, false)
|
||||
return
|
||||
}
|
||||
@@ -516,6 +532,7 @@ func datasetInvariantForPath(dataPath topdataDataPath) (datasetValidationInvaria
|
||||
}
|
||||
|
||||
func validateDatasetInvariantBaseFile(path string, obj map[string]any, invariant datasetValidationInvariant, report *ValidationReport) {
|
||||
reportTopdataContractError(path, unsupportedObjectKeysError("base.json root", obj, baseOrPlainRootKeys...), report)
|
||||
validateRowsFile(path, obj, report, true)
|
||||
validateDerivedBaseOutput(path, obj, invariant.Name, report)
|
||||
validateRequiredColumns(path, obj, invariant, report)
|
||||
@@ -826,6 +843,10 @@ func validateLockObject(path string, obj map[string]any, report *ValidationRepor
|
||||
}
|
||||
|
||||
func validateRowsFile(path string, obj map[string]any, report *ValidationReport, requireColumns bool) {
|
||||
validateRowsFileWithColumns(path, obj, report, requireColumns, true)
|
||||
}
|
||||
|
||||
func validateRowsFileWithColumns(path string, obj map[string]any, report *ValidationReport, requireColumns bool, validateColumns bool) {
|
||||
rows, ok := obj["rows"]
|
||||
if !ok {
|
||||
report.Diagnostics = append(report.Diagnostics, Diagnostic{
|
||||
@@ -845,7 +866,7 @@ func validateRowsFile(path string, obj map[string]any, report *ValidationReport,
|
||||
if rowsList, ok := rows.([]any); ok {
|
||||
validateRowCollection(path, obj, rowsList, report)
|
||||
}
|
||||
if _, ok := obj["columns"]; ok {
|
||||
if _, ok := obj["columns"]; ok && validateColumns {
|
||||
validateColumnsFile(path, obj, report)
|
||||
} else if requireColumns {
|
||||
report.Diagnostics = append(report.Diagnostics, Diagnostic{
|
||||
@@ -896,6 +917,10 @@ func validateColumnsFile(path string, obj map[string]any, report *ValidationRepo
|
||||
}
|
||||
|
||||
func validateEntriesFile(path string, obj map[string]any, report *ValidationReport) {
|
||||
validateEntriesFileWithColumns(path, obj, report, true)
|
||||
}
|
||||
|
||||
func validateEntriesFileWithColumns(path string, obj map[string]any, report *ValidationReport, validateColumns bool) {
|
||||
entries, ok := obj["entries"]
|
||||
if !ok {
|
||||
report.Diagnostics = append(report.Diagnostics, Diagnostic{
|
||||
@@ -931,12 +956,16 @@ func validateEntriesFile(path string, obj map[string]any, report *ValidationRepo
|
||||
validateInlineTextUsage(path, key, entry, report)
|
||||
}
|
||||
}
|
||||
if _, ok := obj["columns"]; ok {
|
||||
if _, ok := obj["columns"]; ok && validateColumns {
|
||||
validateColumnsFile(path, obj, report)
|
||||
}
|
||||
}
|
||||
|
||||
func validateOverridesFile(path string, obj map[string]any, report *ValidationReport) {
|
||||
validateOverridesFileWithColumns(path, obj, report, true)
|
||||
}
|
||||
|
||||
func validateOverridesFileWithColumns(path string, obj map[string]any, report *ValidationReport, validateColumns bool) {
|
||||
overrides, ok := obj["overrides"]
|
||||
if !ok {
|
||||
report.Diagnostics = append(report.Diagnostics, Diagnostic{
|
||||
@@ -957,20 +986,21 @@ func validateOverridesFile(path string, obj map[string]any, report *ValidationRe
|
||||
container := map[string]any{"rows": overrideList}
|
||||
validateRowCollection(path, container, overrideList, report)
|
||||
}
|
||||
if _, ok := obj["columns"]; ok {
|
||||
if _, ok := obj["columns"]; ok && validateColumns {
|
||||
validateColumnsFile(path, obj, report)
|
||||
}
|
||||
}
|
||||
|
||||
func validateGlobalJSONFile(path string, obj map[string]any, report *ValidationReport) {
|
||||
reportTopdataContractError(path, unsupportedObjectKeysError("global.json root", obj, globalRootKeys...), report)
|
||||
if _, ok := obj["columns"]; ok {
|
||||
validateColumnsFile(path, obj, report)
|
||||
}
|
||||
if _, ok := obj["entries"]; ok {
|
||||
validateEntriesFile(path, obj, report)
|
||||
validateEntriesFileWithColumns(path, obj, report, false)
|
||||
}
|
||||
if _, ok := obj["overrides"]; ok {
|
||||
validateOverridesFile(path, obj, report)
|
||||
validateOverridesFileWithColumns(path, obj, report, false)
|
||||
}
|
||||
if _, ok := obj["defaults"]; ok {
|
||||
validateGlobalDefaults(path, obj, report)
|
||||
@@ -1029,8 +1059,12 @@ func validateGlobalJSONFile(path string, obj map[string]any, report *ValidationR
|
||||
} else {
|
||||
rows = append(rows, row)
|
||||
}
|
||||
validateGlobalConditionList(path, fmt.Sprintf("global injection %d require_present", index), injection["require_present"], report)
|
||||
validateGlobalConditionList(path, fmt.Sprintf("global injection %d unless_present", index), injection["unless_present"], report)
|
||||
_, _ = parseGlobalConditionGroups(injection, globalConditionParseOptions{
|
||||
Label: fmt.Sprintf("global injection %d", index),
|
||||
Report: func(err error) {
|
||||
reportTopdataContractError(path, err, report)
|
||||
},
|
||||
})
|
||||
}
|
||||
validateRowCollection(path, obj, rows, report)
|
||||
}
|
||||
@@ -1099,6 +1133,7 @@ func validateGlobalDefaults(path string, obj map[string]any, report *ValidationR
|
||||
})
|
||||
continue
|
||||
}
|
||||
reportTopdataContractError(path, unsupportedObjectKeysError(fmt.Sprintf("global default %d", index), defaultRule, globalDefaultRuleKeys...), report)
|
||||
validateGlobalDefaultMatch(path, index, defaultRule["match"], report)
|
||||
rawValues, ok := defaultRule["values"]
|
||||
if !ok {
|
||||
@@ -1155,14 +1190,12 @@ func validateGlobalDefaultMatch(path string, index int, rawMatch any, report *Va
|
||||
})
|
||||
return
|
||||
}
|
||||
for key := range match {
|
||||
if key != "source" {
|
||||
report.Diagnostics = append(report.Diagnostics, Diagnostic{
|
||||
Severity: SeverityError,
|
||||
Path: path,
|
||||
Message: fmt.Sprintf("global default %d match.%s is not supported", index, key),
|
||||
})
|
||||
}
|
||||
for _, key := range unsupportedObjectKeys(match, globalDefaultMatchKeys...) {
|
||||
report.Diagnostics = append(report.Diagnostics, Diagnostic{
|
||||
Severity: SeverityError,
|
||||
Path: path,
|
||||
Message: fmt.Sprintf("global default %d match.%s is not supported (unsupported key; supported keys: %s)", index, key, strings.Join(globalDefaultMatchKeys, ", ")),
|
||||
})
|
||||
}
|
||||
source, ok := match["source"].(string)
|
||||
if !ok || strings.TrimSpace(source) == "" {
|
||||
@@ -1193,14 +1226,7 @@ func validateGlobalDefaultValue(path string, index int, field string, value any,
|
||||
if !hasFormat {
|
||||
return
|
||||
}
|
||||
if len(obj) != 1 {
|
||||
report.Diagnostics = append(report.Diagnostics, Diagnostic{
|
||||
Severity: SeverityError,
|
||||
Path: path,
|
||||
Message: fmt.Sprintf("global default %d values.%s format object must contain only format", index, field),
|
||||
})
|
||||
return
|
||||
}
|
||||
reportTopdataContractError(path, unsupportedObjectKeysError(fmt.Sprintf("global default %d values.%s", index, field), obj, globalDefaultFormatKeys...), report)
|
||||
format, ok := rawFormat.(string)
|
||||
if !ok {
|
||||
report.Diagnostics = append(report.Diagnostics, Diagnostic{
|
||||
@@ -1220,46 +1246,6 @@ func validateGlobalDefaultValue(path string, index int, field string, value any,
|
||||
}
|
||||
}
|
||||
|
||||
func validateGlobalConditionList(path, label string, raw any, report *ValidationReport) {
|
||||
if raw == nil {
|
||||
return
|
||||
}
|
||||
conditions, ok := raw.([]any)
|
||||
if !ok {
|
||||
report.Diagnostics = append(report.Diagnostics, Diagnostic{
|
||||
Severity: SeverityError,
|
||||
Path: path,
|
||||
Message: fmt.Sprintf("%s must be a JSON array", label),
|
||||
})
|
||||
return
|
||||
}
|
||||
for index, rawCondition := range conditions {
|
||||
condition, ok := rawCondition.(map[string]any)
|
||||
if !ok {
|
||||
report.Diagnostics = append(report.Diagnostics, Diagnostic{
|
||||
Severity: SeverityError,
|
||||
Path: path,
|
||||
Message: fmt.Sprintf("%s[%d] must be an object", label, index),
|
||||
})
|
||||
continue
|
||||
}
|
||||
if field, ok := condition["field"].(string); !ok || strings.TrimSpace(field) == "" {
|
||||
report.Diagnostics = append(report.Diagnostics, Diagnostic{
|
||||
Severity: SeverityError,
|
||||
Path: path,
|
||||
Message: fmt.Sprintf("%s[%d].field is required", label, index),
|
||||
})
|
||||
}
|
||||
if id, ok := condition["id"].(string); !ok || strings.TrimSpace(id) == "" {
|
||||
report.Diagnostics = append(report.Diagnostics, Diagnostic{
|
||||
Severity: SeverityError,
|
||||
Path: path,
|
||||
Message: fmt.Sprintf("%s[%d].id is required", label, index),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func validateStateObject(path string, obj map[string]any, report *ValidationReport) {
|
||||
entries, ok := obj["entries"].(map[string]any)
|
||||
if !ok {
|
||||
@@ -2243,19 +2229,11 @@ func validateTopPackageAssets(sourceDir, dataDir string, report *ValidationRepor
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if strings.HasPrefix(filepath.Base(path), ".") {
|
||||
return nil
|
||||
if skipTopPackageAsset(rel) {
|
||||
return nil // not a NWN ResType (script, doc, _work dir, ...): never packed, never checked
|
||||
}
|
||||
base := strings.ToLower(strings.TrimSuffix(filepath.Base(path), filepath.Ext(path)))
|
||||
ext := strings.TrimPrefix(strings.ToLower(filepath.Ext(path)), ".")
|
||||
if _, ok := erf.HAKResourceTypeForExtension(ext); !ok {
|
||||
report.Diagnostics = append(report.Diagnostics, Diagnostic{
|
||||
Severity: SeverityError,
|
||||
Path: path,
|
||||
Message: fmt.Sprintf("unsupported topdata asset HAK resource extension %q", filepath.Ext(path)),
|
||||
})
|
||||
return nil
|
||||
}
|
||||
key := base + "." + ext
|
||||
if previous, ok := seen[key]; ok {
|
||||
report.Diagnostics = append(report.Diagnostics, Diagnostic{
|
||||
|
||||
@@ -7633,7 +7633,7 @@ func TestApplyAutogenConsumersAugmentsPartsFromLocalOverride(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyAutogenConsumersAugmentsHeadVisualeffectsFromLocalOverride(t *testing.T) {
|
||||
func TestApplyAutogenConsumersAugmentsAccessoryVisualeffectsFromLocalOverride(t *testing.T) {
|
||||
root := testProjectRoot(t)
|
||||
overrideRoot := filepath.Join(root, "autogen-assets")
|
||||
mkdirAll(t, filepath.Join(overrideRoot, "vfxs", "head_accessories", "hat"))
|
||||
@@ -7706,18 +7706,18 @@ func TestApplyAutogenConsumersAugmentsHeadVisualeffectsFromLocalOverride(t *test
|
||||
t.Fatalf("unexpected head feature defaults: %#v", hair)
|
||||
}
|
||||
|
||||
if _, ok := got[0].LockData["visualeffects:head_accessories/hat/bandana"]; !ok {
|
||||
if _, ok := got[0].LockData["visualeffects:head_accessories/hat/hfx_bandana.mdl"]; !ok {
|
||||
t.Fatalf("expected head accessory lock id, got %#v", got[0].LockData)
|
||||
}
|
||||
if _, ok := got[0].LockData["visualeffects:head_decorations/laurel/laurel"]; !ok {
|
||||
if _, ok := got[0].LockData["visualeffects:head_decorations/laurel/hfx_laurel.mdl"]; !ok {
|
||||
t.Fatalf("expected head decoration lock id, got %#v", got[0].LockData)
|
||||
}
|
||||
if _, ok := got[0].LockData["visualeffects:head_features/hair/hair_bangs"]; !ok {
|
||||
if _, ok := got[0].LockData["visualeffects:head_features/hair/hfx_hair_bangs.mdl"]; !ok {
|
||||
t.Fatalf("expected head feature lock id, got %#v", got[0].LockData)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyAutogenConsumersUsesFolderDrivenSlashHeadVisualeffectsNames(t *testing.T) {
|
||||
func TestApplyAutogenConsumersUsesFolderDrivenSlashAccessoryVisualeffectsNames(t *testing.T) {
|
||||
root := testProjectRoot(t)
|
||||
overrideRoot := filepath.Join(root, "autogen-assets")
|
||||
mkdirAll(t, filepath.Join(overrideRoot, "vfxs", "head_features", "sinfar", "ears_plt"))
|
||||
@@ -7735,8 +7735,8 @@ func TestApplyAutogenConsumersUsesFolderDrivenSlashHeadVisualeffectsNames(t *tes
|
||||
Derive: project.AutogenDeriveConfig{Kind: "model_stem", GroupFrom: "first_path_segment"},
|
||||
Manifest: project.AutogenManifestConfig{ReleaseTag: "head-vfx-manifest-current", AssetName: "sow-accessory-vfx-manifest.json", CacheName: "sow-accessory-vfx-manifest.json"},
|
||||
LocalOverrideRoot: "autogen-assets",
|
||||
HeadVisualeffects: project.HeadVisualeffectsConfig{
|
||||
Groups: map[string]project.HeadVisualeffectGroupConfig{
|
||||
AccessoryVisualeffects: project.AccessoryVisualeffectsConfig{
|
||||
Groups: map[string]project.AccessoryVisualeffectGroupConfig{
|
||||
"head_features": {
|
||||
ModelColumns: []string{"Imp_HeadCon_Node", "Imp_Root_H_Node"},
|
||||
},
|
||||
@@ -7790,7 +7790,7 @@ func TestApplyAutogenConsumersUsesFolderDrivenSlashHeadVisualeffectsNames(t *tes
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildNativeUsesConfiguredHeadVisualeffectsGroups(t *testing.T) {
|
||||
func TestBuildNativeUsesConfiguredAccessoryVisualeffectsGroups(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
mkdirAll(t, filepath.Join(root, "src"))
|
||||
mkdirAll(t, filepath.Join(root, "build"))
|
||||
@@ -7878,7 +7878,7 @@ autogen:
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildGenerated2DAAssetsAppendsCachedModelsFromHeadVisualeffects(t *testing.T) {
|
||||
func TestBuildGenerated2DAAssetsAppendsCachedModelsFromAccessoryVisualeffects(t *testing.T) {
|
||||
root := testProjectRoot(t)
|
||||
mkdirAll(t, filepath.Join(root, "assets", "vfxs", "chest_accessories"))
|
||||
mkdirAll(t, filepath.Join(root, "assets", "vfxs", "head_accessories"))
|
||||
@@ -11887,7 +11887,7 @@ func TestBuildSupportsCanonicalSpells(t *testing.T) {
|
||||
"spells:disarm": {
|
||||
"Label": "Disarm",
|
||||
"Name": "501",
|
||||
"FeatID": {"id": "feat:disarm"},
|
||||
"FeatID": {"spellradial": {"id": "feat:specialattacks"}},
|
||||
"Master": {"id": "spells:specialattacks"},
|
||||
"Category": {"ref": "spells:specialattacks", "field": "Category"}
|
||||
}
|
||||
@@ -11939,7 +11939,7 @@ func TestBuildSupportsCanonicalSpells(t *testing.T) {
|
||||
!strings.Contains(text, "\t1000\t") ||
|
||||
!strings.Contains(text, "841\tDisarm\t501") ||
|
||||
!strings.Contains(text, "\t840\t") ||
|
||||
!strings.Contains(text, "\t1001\t") ||
|
||||
!strings.Contains(text, "\t55116776\t") ||
|
||||
!strings.Contains(text, "849\tBard_Fascinate\t600") ||
|
||||
!strings.Contains(text, "\t601\t") ||
|
||||
!strings.Contains(text, "\t602\t") ||
|
||||
@@ -13139,7 +13139,7 @@ func TestBuildNativeWithReleasedPartsManifestUsesAssetsServerOverride(t *testing
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildNativeWithReleasedHeadVisualeffectsManifest(t *testing.T) {
|
||||
func TestBuildNativeWithReleasedAccessoryVisualeffectsManifest(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
projRoot := filepath.Join(root, "project")
|
||||
mkdirAll(t, filepath.Join(projRoot, "src"))
|
||||
@@ -13274,7 +13274,7 @@ func TestBuildNativeWithReleasedHeadVisualeffectsManifest(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildNativePrefersRemoteReleasedHeadVisualeffectsManifestOverFreshCache(t *testing.T) {
|
||||
func TestBuildNativePrefersRemoteReleasedAccessoryVisualeffectsManifestOverFreshCache(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
projRoot := filepath.Join(root, "project")
|
||||
mkdirAll(t, filepath.Join(projRoot, "src"))
|
||||
@@ -15021,7 +15021,7 @@ func (t twoDATable) cell(rowID, column string) string {
|
||||
|
||||
func runGitTest(t *testing.T, dir string, args ...string) {
|
||||
t.Helper()
|
||||
cmd := exec.Command("git", args...)
|
||||
cmd := exec.Command("git", append([]string{"-c", "commit.gpgsign=false", "-c", "tag.gpgsign=false"}, args...)...)
|
||||
if dir != "" {
|
||||
cmd.Dir = dir
|
||||
}
|
||||
@@ -15115,3 +15115,77 @@ func writeBytes(t *testing.T, path string, content []byte) {
|
||||
t.Fatalf("write %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSkipTopPackageAsset(t *testing.T) {
|
||||
skipped := []string{
|
||||
"gui/regions/comfyui-generate.sh", // non-NWN extension
|
||||
"gui/regions/_candidates/abyss.opt4.png",
|
||||
"gui/.hidden/banner.png",
|
||||
"gui/.DS_Store",
|
||||
"gui/AGENTS.md",
|
||||
"gui/noextension",
|
||||
}
|
||||
kept := []string{
|
||||
"gui/regions/abyss.png",
|
||||
"2da-src/placeables.2da",
|
||||
"tex/floor01.dds",
|
||||
}
|
||||
for _, rel := range skipped {
|
||||
if !skipTopPackageAsset(rel) {
|
||||
t.Errorf("expected %s to be skipped", rel)
|
||||
}
|
||||
}
|
||||
for _, rel := range kept {
|
||||
if skipTopPackageAsset(rel) {
|
||||
t.Errorf("expected %s to be kept", rel)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateTopPackageAssetsIgnoresNonNWNFiles(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
assets := filepath.Join(dir, "assets", "gui", "regions")
|
||||
candidates := filepath.Join(assets, "_candidates")
|
||||
if err := os.MkdirAll(candidates, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for path, content := range map[string]string{
|
||||
filepath.Join(assets, "comfyui-generate.sh"): "#!/bin/sh",
|
||||
filepath.Join(assets, "abyss.png"): "png",
|
||||
filepath.Join(candidates, "abyss.opt4.png"): "png",
|
||||
filepath.Join(candidates, "notes.txt.backup"): "junk",
|
||||
} {
|
||||
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
dataDir := filepath.Join(dir, "data")
|
||||
if err := os.MkdirAll(dataDir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var report ValidationReport
|
||||
validateTopPackageAssets(dir, dataDir, &report)
|
||||
for _, d := range report.Diagnostics {
|
||||
t.Errorf("unexpected diagnostic: %s: %s", d.Path, d.Message)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateTopPackageAssetsSkipsMarkdown(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
assets := filepath.Join(dir, "assets", "gui")
|
||||
if err := os.MkdirAll(assets, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(assets, "AGENTS.md"), []byte("docs"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dataDir := filepath.Join(dir, "data")
|
||||
if err := os.MkdirAll(dataDir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var report ValidationReport
|
||||
validateTopPackageAssets(dir, dataDir, &report)
|
||||
for _, d := range report.Diagnostics {
|
||||
t.Errorf("unexpected diagnostic: %s: %s", d.Path, d.Message)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,7 +10,6 @@ import (
|
||||
|
||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf"
|
||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/gff"
|
||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/music"
|
||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
|
||||
)
|
||||
|
||||
@@ -101,8 +100,7 @@ func ValidateProject(p *project.Project) Report {
|
||||
}
|
||||
|
||||
for _, rel := range p.Inventory.AssetFiles {
|
||||
musicSource := isMusicSourceAsset(p, rel)
|
||||
if hasUppercaseResourceName(rel) && !musicSource {
|
||||
if hasUppercaseResourceName(rel) {
|
||||
report.add(rel, "resource filenames should be lowercase", SeverityWarning)
|
||||
}
|
||||
base := strings.ToLower(strings.TrimSuffix(filepath.Base(rel), filepath.Ext(rel)))
|
||||
@@ -116,9 +114,6 @@ func ValidateProject(p *project.Project) Report {
|
||||
if _, exists := assetIndex[key]; !exists {
|
||||
assetIndex[key] = rel
|
||||
}
|
||||
if musicSource {
|
||||
continue
|
||||
}
|
||||
|
||||
group, err := resolver.groupFor(rel)
|
||||
if err != nil {
|
||||
@@ -507,30 +502,6 @@ func hasAsset(name string, extensions []string, assets map[string]string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func isMusicSourceAsset(p *project.Project, rel string) bool {
|
||||
rel = filepath.ToSlash(strings.TrimSpace(rel))
|
||||
ext := strings.ToLower(filepath.Ext(rel))
|
||||
effective := p.EffectiveConfig()
|
||||
for _, dataset := range effective.Music.Datasets {
|
||||
if !music.PathIsUnder(dataset.Source, rel) {
|
||||
continue
|
||||
}
|
||||
for _, candidate := range dataset.ConvertExtensions {
|
||||
if ext == strings.ToLower(strings.TrimSpace(candidate)) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
if music.PathIsUnder("envi/music", rel) {
|
||||
for _, candidate := range effective.Music.ConvertExtensions {
|
||||
if ext == strings.ToLower(strings.TrimSpace(candidate)) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isBuiltinScript(name string, builtinScriptPrefixes []string) bool {
|
||||
lower := strings.ToLower(name)
|
||||
for _, prefix := range builtinScriptPrefixes {
|
||||
|
||||
@@ -17,8 +17,8 @@ bin=bin
|
||||
"${bin}/crucible" list >/dev/null
|
||||
|
||||
# Keep in sync with internal/dispatch.Registry (Wired flag).
|
||||
unwired=(depot)
|
||||
wired=(hak module topdata wiki)
|
||||
unwired=()
|
||||
wired=(depot hak module topdata wiki)
|
||||
|
||||
exit_of() { set +e; "$@" >/dev/null 2>&1; local c=$?; set -e; echo "${c}"; }
|
||||
|
||||
|
||||
@@ -3,3 +3,4 @@
|
||||
# wrappers/ changes. Add a repo here AND grant the sync bot write access to it.
|
||||
ShadowsOverWestgate/sow-module
|
||||
ShadowsOverWestgate/sow-topdata
|
||||
ShadowsOverWestgate/sow-assets-manifest
|
||||
|
||||
@@ -29,7 +29,7 @@ os="$(uname -s)"; arch="$(uname -m)"
|
||||
case "$os" in
|
||||
Linux) os=linux ;;
|
||||
Darwin) os=darwin ;;
|
||||
*) echo "crucible: unsupported OS '$os' — install ffmpeg-free crucible manually" >&2; exit 1 ;;
|
||||
*) echo "crucible: unsupported OS '$os' — install crucible manually" >&2; exit 1 ;;
|
||||
esac
|
||||
case "$arch" in
|
||||
x86_64|amd64) arch=amd64 ;;
|
||||
|
||||
Reference in New Issue
Block a user