package nwsync import ( "bytes" "encoding/binary" "encoding/hex" "encoding/json" "fmt" "io" "path" "path/filepath" "sort" "strings" ) // NSYM manifest, version 3, exactly as upstream neverwinter/nwsync.nim writes // it. Everything is little-endian: // // "NSYM", uint32 version, uint32 entry count, uint32 mapping count, // entries: byte[20] raw sha1, uint32 size, char[16] resref, uint16 restype // mappings: uint32 entry index, char[16] resref, uint16 restype // // Entries are sorted by lowercase sha1 hex then resref, and a resource whose // sha1 was already written becomes a mapping instead of a second entry. const ( manifestVersion = 3 hashTreeDepth = 2 resRefBytes = 16 ) // Entry is one resource in a manifest. type Entry struct { SHA1 [20]byte Size uint32 ResRef string // lowercase, no extension, at most 16 characters ResType uint16 } func (e Entry) sha1Hex() string { return hex.EncodeToString(e.SHA1[:]) } // Identity is what a resref resolves by: name plus type. It is the merge key // inside one artifact and across artifacts alike. type Identity struct { ResRef string ResType uint16 } func (e Entry) identity() Identity { return Identity{ResRef: e.ResRef, ResType: e.ResType} } // writeManifest serialises entries into NSYM v3 bytes. func writeManifest(entries []Entry) ([]byte, error) { sorted := make([]Entry, len(entries)) copy(sorted, entries) sort.SliceStable(sorted, func(i, j int) bool { left, right := sorted[i].sha1Hex(), sorted[j].sha1Hex() if left != right { return left < right } return sorted[i].ResRef < sorted[j].ResRef }) var body, mappings bytes.Buffer seen := make(map[string]uint32, len(sorted)) var entryCount, mappingCount uint32 for _, entry := range sorted { padded, err := padResRef(entry.ResRef) if err != nil { return nil, err } if index, ok := seen[entry.sha1Hex()]; ok { _ = binary.Write(&mappings, binary.LittleEndian, index) mappings.Write(padded) _ = binary.Write(&mappings, binary.LittleEndian, entry.ResType) mappingCount++ continue } seen[entry.sha1Hex()] = entryCount entryCount++ body.Write(entry.SHA1[:]) _ = binary.Write(&body, binary.LittleEndian, entry.Size) body.Write(padded) _ = binary.Write(&body, binary.LittleEndian, entry.ResType) } var out bytes.Buffer out.WriteString("NSYM") for _, field := range []uint32{manifestVersion, entryCount, mappingCount} { _ = binary.Write(&out, binary.LittleEndian, field) } out.Write(body.Bytes()) out.Write(mappings.Bytes()) return out.Bytes(), nil } // readManifest parses NSYM v3 bytes. Mappings are expanded back into entries, // the way upstream's reader does, so a caller sees one entry per resref. func readManifest(data []byte) ([]Entry, error) { reader := bytes.NewReader(data) magic := make([]byte, 4) if _, err := io.ReadFull(reader, magic); err != nil || string(magic) != "NSYM" { return nil, fmt.Errorf("not a manifest (invalid magic bytes)") } var version, entryCount, mappingCount uint32 for _, field := range []*uint32{&version, &entryCount, &mappingCount} { if err := binary.Read(reader, binary.LittleEndian, field); err != nil { return nil, fmt.Errorf("truncated manifest header: %w", err) } } if version != manifestVersion { return nil, fmt.Errorf("unsupported manifest version %d", version) } entries := make([]Entry, 0, entryCount+mappingCount) for i := uint32(0); i < entryCount; i++ { var entry Entry if _, err := io.ReadFull(reader, entry.SHA1[:]); err != nil { return nil, fmt.Errorf("truncated entry %d: %w", i, err) } if err := binary.Read(reader, binary.LittleEndian, &entry.Size); err != nil { return nil, fmt.Errorf("truncated entry %d: %w", i, err) } resref, restype, err := readResRef(reader) if err != nil { return nil, fmt.Errorf("truncated entry %d: %w", i, err) } entry.ResRef, entry.ResType = resref, restype entries = append(entries, entry) } for i := uint32(0); i < mappingCount; i++ { var index uint32 if err := binary.Read(reader, binary.LittleEndian, &index); err != nil { return nil, fmt.Errorf("truncated mapping %d: %w", i, err) } if index >= entryCount { return nil, fmt.Errorf("mapping %d references non-existent entry %d", i, index) } resref, restype, err := readResRef(reader) if err != nil { return nil, fmt.Errorf("truncated mapping %d: %w", i, err) } target := entries[index] entries = append(entries, Entry{SHA1: target.SHA1, Size: target.Size, ResRef: resref, ResType: restype}) } return entries, nil } func readResRef(reader *bytes.Reader) (string, uint16, error) { raw := make([]byte, resRefBytes) if _, err := io.ReadFull(reader, raw); err != nil { return "", 0, err } var restype uint16 if err := binary.Read(reader, binary.LittleEndian, &restype); err != nil { return "", 0, err } return strings.ToLower(string(bytes.TrimRight(raw, "\x00"))), restype, nil } func padResRef(resref string) ([]byte, error) { if len(resref) > resRefBytes { return nil, fmt.Errorf("resref %q exceeds %d characters", resref, resRefBytes) } padded := make([]byte, resRefBytes) copy(padded, strings.ToLower(resref)) return padded, nil } // Sidecar is the .json file written next to every manifest. Clients fetch it // (nwn_nwsync_fetch.nim), so the field names and order match upstream. // Upstream omits an integer meta field whose value is 0, hence group_id's // omitempty: 0 means absent, not "group zero". type Sidecar struct { Version int `json:"version"` SHA1 string `json:"sha1"` HashTreeDepth int `json:"hash_tree_depth"` ModuleName string `json:"module_name"` Description string `json:"description"` IncludesModuleContents bool `json:"includes_module_contents"` IncludesClientContents bool `json:"includes_client_contents"` TotalFiles int `json:"total_files"` TotalBytes int64 `json:"total_bytes"` OnDiskBytes int64 `json:"on_disk_bytes"` Created int64 `json:"created"` CreatedWith string `json:"created_with"` EmitterVersion string `json:"emitter_version,omitempty"` GroupID int `json:"group_id,omitempty"` } func marshalSidecar(sidecar Sidecar) ([]byte, error) { body, err := json.MarshalIndent(sidecar, "", " ") if err != nil { return nil, err } // Upstream terminates the file with CRLF; match it. return append(body, '\r', '\n'), nil } // blobKey is where a blob lives in a zone, hash tree depth 2. emit writes it, // verify reads it and the game client requests it, so the rule lives here and // nowhere else. func blobKey(sha1Hex string) string { return path.Join("data", "sha1", sha1Hex[0:2], sha1Hex[2:4], sha1Hex) } // blobPath is the same location inside a local repository tree. func blobPath(root, sha1Hex string) string { return filepath.Join(root, filepath.FromSlash(blobKey(sha1Hex))) }